Merge branch 'master' of https://github.com/zedeus/nitter

Fixes #1410

.github/workflows/build-docker.yml

@@ -7,55 +7,105 @@ on:
     branches:
       - master
 
+concurrency:
+  group: docker-publish-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  IMAGE: zedeus/nitter
+
 jobs:
   tests:
     uses: ./.github/workflows/run-tests.yml
     secrets: inherit
 
-  build-docker-amd64:
+  build:
     needs: [tests]
-    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            platform: linux/amd64
+          - runner: ubuntu-24.04-arm
+            platform: linux/arm64
+    runs-on: ${{ matrix.runner }}
     steps:
+      - name: Prepare platform name
+        run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV"
+        env:
+          platform: ${{ matrix.platform }}
+
       - uses: actions/checkout@v6
+
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
+
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push AMD64 Docker image
-        uses: docker/build-push-action@v3
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          provenance: false
+          sbom: false
 
-  build-docker-arm64:
-    needs: [tests]
-    runs-on: ubuntu-24.04-arm
+      - name: Export digest
+        run: |
+          mkdir -p "${{ runner.temp }}/digests"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # Combine the per-arch digests into one multi-arch manifest so that
+  # `docker pull zedeus/nitter:latest` serves the right image on any CPU.
+  merge:
+    needs: [build]
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v6
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
+
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push ARM64 Docker image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./Dockerfile.arm64
-          platforms: linux/arm64
-          push: true
-          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create \
+            -t ${{ env.IMAGE }}:latest \
+            -t ${{ env.IMAGE }}:latest-arm64 \
+            -t ${{ env.IMAGE }}:${{ github.sha }} \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }}

.github/workflows/run-tests.yml

@@ -32,10 +32,12 @@ jobs:
         id: cache-nimble
         uses: actions/cache@v5
         with:
-          path: ~/.nimble
-          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
+          path: |
+            ~/.nimble/pkgcache
+            ~/.nimble/packages_official.json
+          key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            ${{ matrix.nim }}-nimble-v2-
+            ${{ matrix.nim }}-nimble-v6-
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2
@@ -103,10 +105,12 @@ jobs:
       - name: Cache Nimble Dependencies
         uses: actions/cache@v5
         with:
-          path: ~/.nimble
-          key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
+          path: |
+            ~/.nimble/pkgcache
+            ~/.nimble/packages_official.json
+          key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }}
           restore-keys: |
-            2.2.x-nimble-v2-
+            2.2.x-nimble-v6-
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v2
@@ -115,6 +119,9 @@ jobs:
           use-nightlies: true
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install Nimble dependencies
+        run: nimble install -y --depsOnly
+
       - name: Download 2.2.x build artifact
         uses: actions/download-artifact@v4
         with:
@@ -130,10 +137,8 @@ jobs:
           sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
           sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf
 
-          # Run both Nimble tasks concurrently
-          nim r tools/rendermd.nim &
-          nim r tools/gencss.nim &
-          wait
+          nim r tools/rendermd.nim
+          nim r tools/gencss.nim
 
           echo '${{ secrets.SESSIONS }}' | head -n1
           echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl

.gitignore

@@ -15,3 +15,6 @@ sessions.json*
 dump.rdb
 *.bak
 /tools/*.json*
+nimbledeps/
+nimble.paths
+nimble.develop

Dockerfile

@@ -1,4 +1,4 @@
-FROM nimlang/nim:2.2.0-alpine-regular as nim
+FROM nimlang/nim:2.2.6-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
 
 RUN apk --no-cache add libsass-dev pcre
@@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
 
 FROM alpine:latest
 WORKDIR /src/
-RUN apk --no-cache add pcre ca-certificates
+RUN apk --no-cache add pcre ca-certificates openssl
 COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public

Dockerfile.arm64

@@ -1,25 +0,0 @@
-FROM alpine:3.20.6 as nim
-LABEL maintainer="setenforce@protonmail.com"
-
-RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
-
-WORKDIR /src/nitter
-
-COPY nitter.nimble .
-RUN nimble install -y --depsOnly
-
-COPY . .
-RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
-    && nimble scss \
-    && nimble md
-
-FROM alpine:3.20.6
-WORKDIR /src/
-RUN apk --no-cache add pcre ca-certificates openssl
-COPY --from=nim /src/nitter/nitter ./
-COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
-COPY --from=nim /src/nitter/public ./public
-EXPOSE 8080
-RUN adduser -h /src/ -D -s /bin/sh nitter
-USER nitter
-CMD ./nitter

README.md

@@ -104,9 +104,9 @@ along with the scss and md files.
 # su nitter
 $ git clone https://github.com/zedeus/nitter
 $ cd nitter
-$ nimble build -d:danger --mm:refc
-$ nimble scss
-$ nimble md
+$ nimble -l build -d:danger --mm:refc
+$ nimble -l scss
+$ nimble -l md
 $ cp nitter.example.conf nitter.conf
 ```
 
@@ -123,12 +123,23 @@ performance reasons.
 
 Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
 
-#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
+#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`.
 
 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
 Docker.
 
+First create your config file. The Docker commands mount it into the container,
+so it has to exist on the host beforehand. If you've cloned the repo:
+
+```bash
+cp nitter.example.conf nitter.conf
+```
+
+If you're using the prebuilt image without a local clone, download
+[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf)
+and save it as `nitter.conf` instead.
+
 To build and run Nitter in Docker:
 
 ```bash
@@ -136,8 +147,6 @@ docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
-Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
-
 A prebuilt Docker image is provided as well:
 
 ```bash
@@ -151,8 +160,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run
 docker-compose up -d
 ```
 
-Note the Docker commands expect a `nitter.conf` file in the directory you run
-them.
+Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for
+docker-compose) from the directory you run them in. If a mounted file doesn't
+exist, Docker silently creates a directory in its place and the container fails
+with `not a directory: Are you trying to mount a directory onto a file`. Remove
+that directory and create the file as shown above.
 
 ### systemd
 

config.nims

@@ -11,3 +11,7 @@ warning("HoleEnumConv", off)
 hint("XDeclaredButNotUsed", off)
 hint("XCannotRaiseY", off)
 hint("User", off)
+# begin Nimble config (version 2)
+when withDir(thisDir(), system.fileExists("nimble.paths")):
+  include "nimble.paths"
+# end Nimble config

nitter.nimble

@@ -11,19 +11,18 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 2.0.0"
-requires "jester#baca3f"
-requires "karax#5cf360c"
-requires "sass#7dfdd03"
-requires "nimcrypto#a079df9"
-requires "markdown#158efe3"
+requires "jester == 0.6.0"
+requires "karax == 1.5.0"
+requires "sass == 0.2.0"
+requires "nimcrypto == 0.7.3"
+requires "markdown == 0.8.8"
 requires "packedjson#9e6fbb6"
-requires "supersnappy#6c94198"
-requires "redpool#8b7c1db"
-requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#ca5989a"
-requires "flatty#e668085"
-requires "jsony#1de1f08"
-requires "oauth#b8c163b"
+requires "supersnappy == 2.1.4"
+requires "redpool == 0.2.2"
+requires "zippy == 0.10.19"
+requires "flatty == 0.4.0"
+requires "jsony == 1.1.6"
+requires "oauth == 0.11"
 
 # Tasks
 

src/api.nim

@@ -11,11 +11,11 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
   if fieldToggles.len > 0:
     result.add ("fieldToggles", fieldToggles)
 
-proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
-  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid)
 
-proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
-  let url = apiUrl(endpoint, variables, fieldToggles)
+proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles, skipTid)
   return ApiReq(cookie: url, oauth: url)
 
 proc mediaUrl(id, cursor: string; count=20): ApiReq =
@@ -25,27 +25,22 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
   )
 
 proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  result = ApiReq(
-    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  )
-  # might change this in the future pending testing
-  result.cookie = result.oauth
+  return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # result = ApiReq(
+  #   cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+  #   oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # )
 
 proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-  result = ApiReq(
-    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  )
+  return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], skipTid=true)
 
 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = tweetDetailVars % [id, cursor]
-  result = ApiReq(
-    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
-    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-  )
+  return apiReq(graphTweet, tweetVars % [id, cursor])
+  # let cookieVars = tweetDetailVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+  #   oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  # )
 
 proc userUrl(username: string): ApiReq =
   let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
@@ -184,13 +179,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
+
   if after.len > 0 and maxId.len == 0:
     variables["cursor"] = % after
   let
@@ -212,12 +207,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "People",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
   if after.len > 0:
     variables["cursor"] = % after

src/apiutils.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
-import jsony, packedjson, zippy, oauth1
+import jsony, packedjson, zippy, oauth/oauth1
 import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
@@ -63,7 +63,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 proc getCookieHeader(authToken, ct0: string): string =
   "auth_token=" & authToken & "; ct0=" & ct0
 
-proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
+proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "accept": "*/*",
     "accept-encoding": "gzip",
@@ -84,13 +84,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    result["referer"] = "https://x.com/"
     result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
     result["sec-ch-ua-mobile"] = "?0"
     result["sec-ch-ua-platform"] = "Windows"
     result["sec-fetch-dest"] = "empty"
     result["sec-fetch-mode"] = "cors"
-    result["sec-fetch-site"] = "same-site"
-    if disableTid or "/1.1/" in url.path:
+    result["sec-fetch-site"] = "same-origin"
+    if disableTid or skipTid or "/1.1/" in url.path:
       result["authorization"] = bearerToken2
     else:
       result["authorization"] = bearerToken
@@ -114,7 +115,12 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(await genHeaders(session, url)):
+    let skipTid = case session.kind
+      of oauth: req.oauth.skipTid
+      of cookie: req.cookie.skipTid
+    let headers = await genHeaders(session, url, skipTid)
+
+    pool.use(headers):
       template getContent =
         # TODO: this is a temporary simple implementation
         if apiProxy.len > 0 and "/1.1/" notin url.path:
@@ -130,7 +136,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         raise newException(BadClientError, "Bad client")
 
       if resp.status == $Http404 and result.len == 0:
-        echo "[sessions] transient 404 (empty body), retrying: ", url.path
+        echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
         raise rateLimitError()
 
     if resp.headers.hasKey(rlRemaining):
@@ -147,7 +153,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          echo "Fetch error, API: ", url.path, ", errors: ", errors
+          echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
@@ -162,7 +168,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     fetchBody
 
     if resp.status == $Http400:
-      echo "ERROR 400, ", url.path, ": ", result
+      echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -177,22 +183,34 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   finally:
     release(session)
 
-template retry(bod) =
+template retry(bod) {.dirty.} =
+  var session: Session
+  var retrySuccess = false
   for i in 0 ..< maxRetries:
     try:
+      session = nil
       bod
+      retrySuccess = true
       break
     except RateLimitError:
-      echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
-           " request (", i, "/", maxRetries, ")..."
+      let api = if session.isNil: req.cookie.endpoint
+                else: req.endpoint(session)
+      if session.isNil:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..."
+      else:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..., session: ", session.pretty
+      session = nil
       if retryDelayMs > 0:
         await sleepAsync(retryDelayMs)
+  if not retrySuccess:
+    raise rateLimitError()
 
 proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
-    var 
-      body: string
-      session = await getAndValidateSession(req)
+    var body: string
+    session = await getAndValidateSession(req)
 
     let url = req.toUrl(session.kind)
 
@@ -200,22 +218,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
-        echo resp.status, ": ", body, " --- url: ", url
+        echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
         result = newJNull()
 
       let error = result.getError
       if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", url.path, ", error: ", error
+        echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
         if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
 proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    var session = await getAndValidateSession(req)
+    session = await getAndValidateSession(req)
     let url = req.toUrl(session.kind)
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
+        echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
         result.setLen(0)

src/auth.nim

@@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
 template log(str: varargs[string, `$`]) =
   echo "[sessions] ", str.join("")
 
-proc endpoint(req: ApiReq; session: Session): string =
+proc endpoint*(req: ApiReq; session: Session): string =
   case session.kind
   of oauth: req.oauth.endpoint
   of cookie: req.cookie.endpoint
@@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
     oldest = now.int64
     newest = 0'i64
     average = 0'i64
+    oauthTotal, cookieTotal = 0
+    oauthLimited, cookieLimited = 0
 
   for session in sessionPool:
     let created = snowflakeToEpoch(session.id)
@@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
       oldest = created
     average += created
 
+    case session.kind
+    of oauth: inc oauthTotal
+    of cookie: inc cookieTotal
+
     if session.limited:
       limited.incl session.id
+      case session.kind
+      of oauth: inc oauthLimited
+      of cookie: inc cookieLimited
 
     for api in session.apis.keys:
       let
@@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
     "sessions": %*{
       "total": sessionPool.len,
       "limited": limited.card,
+      "oauth": %*{"total": oauthTotal, "limited": oauthLimited},
+      "cookie": %*{"total": cookieTotal, "limited": cookieLimited},
       "oldest": $fromUnix(oldest),
       "newest": $fromUnix(newest),
       "average": $fromUnix(average)
@@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =
 
   for session in sessionPool:
     let sessionJson = %*{
+      "kind": $session.kind,
       "apis": newJObject(),
       "pending": session.pending,
     }
@@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
   if not result.isNil and result.isReady(req):
     inc result.pending
   else:
-    log "no sessions available for API: ", req.cookie.endpoint
+    if result.isNil:
+      log "no sessions available for API: ", req.cookie.endpoint
+    else:
+      log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
     raise noSessionsError()
 
 proc setLimited*(session: Session; req: ApiReq) =

src/consts.nim

@@ -7,109 +7,70 @@ const
   bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
   bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
-  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
-  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
+  graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
+  graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
+  graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
+  graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
+  graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
+  graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
+  graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
+  graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
+  graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
+  graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
+  graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
+  graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"
 
-  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
+  graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
+  graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
+  graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
+  graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
+  graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
+
+  graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
   restLiveStream* = "1.1/live_video_stream/status/"
 
   gqlFeatures* = """{
-  "android_ad_formats_media_component_render_overlay_enabled": false,
-  "android_graphql_skip_api_media_color_palette": false,
-  "android_professional_link_spotlight_display_enabled": false,
-  "articles_api_enabled": false,
-  "articles_preview_enabled": true,
-  "blue_business_profile_image_shape_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": true,
-  "commerce_android_shop_module_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
-  "creator_subscriptions_quote_tweet_preview_enabled": false,
-  "creator_subscriptions_subscription_count_enabled": false,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "grok_android_analyze_trend_fetch_enabled": false,
-  "grok_translations_community_note_auto_translation_is_enabled": false,
-  "grok_translations_community_note_translation_is_enabled": false,
-  "grok_translations_post_auto_translation_is_enabled": false,
-  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
-  "hidden_profile_likes_enabled": false,
-  "highlights_tweets_tab_ui_enabled": false,
-  "immersive_video_status_linkable_timestamps": false,
-  "interactive_text_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "mobile_app_spotlight_module_enabled": false,
-  "payments_enabled": false,
-  "post_ctas_fetch_enabled": true,
-  "premium_content_api_read_enabled": false,
+  "rweb_video_screen_enabled": false,
+  "rweb_cashtags_enabled": true,
   "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "profile_label_improvements_pcf_label_in_profile_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "responsive_web_enhance_cards_enabled": false,
-  "responsive_web_graphql_exclude_directive_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "rweb_tipjar_consumption_enabled": false,
+  "verified_phone_label_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
   "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_grok_analysis_button_from_backend": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "c9s_tweet_anatomy_moderator_badge_enabled": true,
   "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
   "responsive_web_grok_analyze_post_followups_enabled": true,
+  "rweb_cashtags_composer_attachment_enabled": true,
+  "responsive_web_jetfuel_frame": true,
+  "responsive_web_grok_share_attachment_enabled": true,
   "responsive_web_grok_annotations_enabled": true,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "articles_preview_enabled": true,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "rweb_conversational_replies_downvote_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "view_counts_everywhere_api_enabled": true,
+  "longform_notetweets_consumption_enabled": true,
+  "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "content_disclosure_indicator_enabled": true,
+  "content_disclosure_ai_generated_indicator_enabled": true,
+  "responsive_web_grok_show_grok_translated_post": true,
+  "responsive_web_grok_analysis_button_from_backend": true,
+  "post_ctas_fetch_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
+  "standardized_nudges_misinfo": true,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
   "responsive_web_grok_image_annotation_enabled": true,
   "responsive_web_grok_imagine_annotation_enabled": true,
-  "responsive_web_grok_share_attachment_enabled": true,
-  "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_jetfuel_frame": true,
-  "responsive_web_media_download_video_enabled": false,
-  "responsive_web_profile_redirect_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_twitter_article_notes_tab_enabled": false,
-  "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-  "rweb_lists_timeline_redesign_enabled": true,
-  "rweb_tipjar_consumption_enabled": true,
-  "rweb_video_screen_enabled": false,
-  "rweb_video_timestamps_enabled": false,
-  "spaces_2022_h2_clipping": true,
-  "spaces_2022_h2_spaces_communities": true,
-  "standardized_nudges_misinfo": true,
-  "subscriptions_feature_can_gift_premium": false,
-  "subscriptions_verification_info_enabled": true,
-  "subscriptions_verification_info_is_identity_verified_enabled": false,
-  "subscriptions_verification_info_reason_enabled": true,
-  "subscriptions_verification_info_verified_since_enabled": true,
-  "super_follow_badge_privacy_enabled": false,
-  "super_follow_exclusive_tweet_notifications_enabled": false,
-  "super_follow_tweet_api_enabled": false,
-  "super_follow_user_api_enabled": false,
-  "tweet_awards_web_tipping_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "tweetypie_unmention_optimization_enabled": false,
-  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-  "unified_cards_destination_url_params_enabled": false,
-  "verified_phone_label_enabled": false,
-  "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": true,
-  "hidden_profile_subscriptions_enabled": false
+  "responsive_web_grok_community_note_auto_translation_is_enabled": true,
+  "responsive_web_enhance_cards_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVars* = """{
@@ -143,7 +104,7 @@ const
   restIdVars* = """{
   "rest_id": "$1", $2
   "count": $3
-}"""
+}""".replace(" ", "").replace("\n", "")
 
   userMediaVars* = """{
   "userId": "$1", $2

src/formatters.nim

@@ -140,25 +140,30 @@ proc pageDesc*(user: User): string =
     "The latest tweets from " & user.fullname
 
 proc getJoinDate*(user: User): string =
+  if user.joinDate.year == 0: return ""
   user.joinDate.format("'Joined' MMMM YYYY")
 
 proc getJoinDateFull*(user: User): string =
+  if user.joinDate.year == 0: return ""
   user.joinDate.format("h:mm tt - d MMM YYYY")
 
 proc getTime*(tweet: Tweet): string =
+  if tweet.time.year == 0: return ""
   tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'")
 
 proc getRfc822Time*(tweet: Tweet): string =
+  if tweet.time.year == 0: return ""
   tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'")
 
-proc getShortTime*(tweet: Tweet): string =
+proc getShortTime*(time: DateTime): string =
+  if time.year == 0: return ""
   let now = now()
-  let since = now - tweet.time
+  let since = now - time
 
-  if now.year != tweet.time.year:
-    result = tweet.time.format("d MMM yyyy")
+  if now.year != time.year:
+    result = time.format("d MMM yyyy")
   elif since.inDays >= 1:
-    result = tweet.time.format("MMM d")
+    result = time.format("MMM d")
   elif since.inHours >= 1:
     result = $since.inHours & "h"
   elif since.inMinutes >= 1:
@@ -168,6 +173,9 @@ proc getShortTime*(tweet: Tweet): string =
   else:
     result = "now"
 
+proc getShortTime*(tweet: Tweet): string =
+  getShortTime(tweet.time)
+
 proc getDuration*(ms: int): string =
   let
     sec = int(round(ms / 1000))

src/nitter.nim

@@ -2,7 +2,7 @@
 import asyncdispatch, strformat, logging
 from net import Port
 from htmlgen import a
-from os import getEnv
+from os import getEnv, normalizedPath
 
 import jester
 
@@ -63,12 +63,17 @@ createDebugRouter(cfg)
 
 settings:
   port = Port(cfg.port)
-  staticDir = cfg.staticDir
+  staticDir = normalizedPath(cfg.staticDir)
   bindAddr = cfg.address
   reusePort = true
+  maxBody = 64 * 1024
 
 routes:
   before:
+    # Reject malformed paths
+    if request.path.len == 0 or request.path[0] != '/':
+      halt Http400
+
     # skip all file URLs
     cond "." notin request.path
     applyUrlPrefs()

src/parser.nim

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math, tables
+import strutils, options, times, math, tables, uri
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode): Tweet
+proc parseGraphTweet*(js: JsonNode): Tweet
 
 proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
   try: parseEnum[VerifiedType](s)
@@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
 proc parseGraphUser(js: JsonNode): User =
   var user = js{"user_result", "result"}
   if user.isNull:
-    user = ? js{"user_results", "result"}
+    user = js{"user_results", "result"}
 
   if user.isNull:
-    if js{"core"}.notNull and js{"legacy"}.notNull:
+    if js{"core"}.notNull:
       user = js
     else:
       return
@@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User =
 
   # fallback to support UserMedia/recent GraphQL updates
   if result.username.len == 0:
+    result.id = user{"rest_id"}.getStr
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@@ -228,6 +229,10 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
             result.attribution = some(parseUser(user))
           else:
             result.attribution = some(parseGraphUser(user))
+          # Set attribution link from expanded_url (strip /video/N suffix)
+          let expanded = m{"expanded_url"}.getStr
+          if expanded.len > 0:
+            result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
       of "animated_gif":
         result.media.addMedia(Gif(
           url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
@@ -236,11 +241,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
         ))
       else: discard
 
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
-
 proc parseMediaEntities(js: JsonNode; result: var Tweet) =
   with mediaEntities, js{"media_entities"}:
     var parsedMedia: MediaEntities
@@ -261,6 +261,18 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
             durationMs: mediaInfo{"duration_millis"}.getInt,
             variants: parseVideoVariants(mediaInfo{"variants"})
           ))
+
+          # Parse source user for video attribution
+          with sourceUser, mediaEntity{"source_user_results", "result"}:
+            if result.attribution.isNone:
+              let expanded = mediaEntity{"expanded_url"}.getStr
+              if expanded.len > 0:
+                result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
+              result.attribution = some(User(
+                id: sourceUser{"rest_id"}.getStr,
+                fullname: sourceUser{"core", "name"}.getStr,
+                userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+              ))
         of "ApiGif":
           parsedMedia.addMedia(Gif(
             url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@@ -269,23 +281,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
           ))
         else: discard
 
-      if "expanded_url" in mediaEntity:
-        let expandedUrl = js.getExpandedUrl
-        if result.text.endsWith(expandedUrl):
-          result.text.removeSuffix(expandedUrl)
-          result.text = result.text.strip()
-
     if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
       result.media = parsedMedia
 
-  # Remove media URLs from text
-  with mediaList, js{"legacy", "entities", "media"}:
-    for url in mediaList:
-      let expandedUrl = url.getExpandedUrl
-      if result.text.endsWith(expandedUrl):
-        result.text.removeSuffix(expandedUrl)
-        result.text = result.text.strip()
-
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"player_image_large"}.getImageVal,
@@ -428,13 +426,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
   # graphql
   with rt, js{"retweeted_status_result", "result"}:
     # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt:
+    if "legacy" in rt or "rest_id" in rt:
       result.retweet = some parseGraphTweet(rt)
       return
 
   with reposts, js{"repostedStatusResults"}:
     with rt, reposts{"result"}:
-      if "legacy" in rt:
+      if "legacy" in rt or "rest_id" in rt:
         result.retweet = some parseGraphTweet(rt)
         return
 
@@ -449,7 +447,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.poll = some parsePoll(jsCard)
     elif name == "amplify":
       result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-    else:
+    elif name.len > 0 and jsCard{"binding_values"}.notNull:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
@@ -469,7 +467,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode): Tweet =
+proc parseGraphTweet*(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
@@ -506,7 +504,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
           "binding_values": %bindingObj
         }
 
-  var replyId = 0
+  var replyId: int64 = 0
   with restId, js{"reply_to_results", "rest_id"}:
     replyId = restId.getId
 
@@ -537,10 +535,25 @@ proc parseGraphTweet(js: JsonNode): Tweet =
         result.poll = some parsePoll(jsCard)
       elif name == "amplify":
         result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-      else:
+      elif name.len > 0 and jsCard{"binding_values"}.notNull:
         result.card = some parseCard(jsCard, js{"url_entities"})
 
+    parseMediaEntities(js, result)
+    if result.attribution.isNone:
+      parseLegacyMediaEntities(js{"legacy"}, result)
+
     result.expandTweetEntitiesV2(js)
+
+    # Strip video source URL from text (for videos from other tweets)
+    with mediaEntities, js{"media_entities"}:
+      for m in mediaEntities:
+        if "source_status_id_str" in m:
+          let mediaUrl = m{"url"}.getStr
+          if mediaUrl.len > 0:
+            let idx = result.text.rfind(mediaUrl)
+            if idx >= 0:
+              result.text = result.text[0 ..< idx].strip()
+            break
   else:
     result = parseTweet(js{"legacy"}, jsCard, replyId)
     result.id = js{"rest_id"}.getId
@@ -559,6 +572,20 @@ proc parseGraphTweet(js: JsonNode): Tweet =
 
   parseMediaEntities(js, result)
 
+  # Hide card if it's redundant with attribution (same video shown via embed)
+  if result.attribution.isSome and result.card.isSome:
+    let cardUri = get(result.card).url.parseUri
+    if cardUri.isTwitterUrl:
+      let cardPath = cardUri.path.replace("/video/1", "")
+      if cardPath.len > 0 and cardPath == result.attributionLink:
+        get(result.card).kind = hidden
+
+  # Handle retweets - check both legacy and top-level paths
+  with reposts, js{"legacy", "repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt or "rest_id" in rt:
+        result.retweet = some parseGraphTweet(rt)
+
   with quoted, js{"quoted_status_result", "result"}:
     result.quote = some(parseGraphTweet(quoted))
 

src/parserutils.nim

@@ -202,19 +202,32 @@ proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) =
 
 proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
                   textSlice: Slice[int]): string =
+  let
+    runeLen = runes.len
+    safeStart = max(0, textSlice.a)
+    safeEnd = min(runeLen, textSlice.b)
+
+  var validRepls: seq[ReplaceSlice]
+  for rep in repls:
+    if rep.slice.a >= 0 and rep.slice.b >= 0 and rep.slice.b < runeLen and rep.slice.a <= rep.slice.b:
+      validRepls.add rep
+
   template extractLowerBound(i: int; idx): int =
-    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+    if i > 0: min(validRepls[idx].slice.b.succ, runeLen) else: safeStart
 
   result = newStringOfCap(runes.len)
 
-  for i, rep in repls:
-    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+  for i, rep in validRepls:
+    let lower = extractLowerBound(i, i - 1)
+    if lower < rep.slice.a:
+      result.add $runes[lower ..< rep.slice.a]
     case rep.kind
     of rkHashtag:
-      let
-        name = $runes[rep.slice.a.succ .. rep.slice.b]
-        symbol = $runes[rep.slice.a]
-      result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name)
+      if rep.slice.a.succ <= rep.slice.b:
+        let
+          name = $runes[rep.slice.a.succ .. rep.slice.b]
+          symbol = $runes[rep.slice.a]
+        result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name)
     of rkMention:
       result.add a($runes[rep.slice], href = rep.url, title = rep.display)
     of rkUrl:
@@ -222,8 +235,8 @@ proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
     of rkRemove:
       discard
 
-  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
-  if rest.a <= rest.b:
+  let rest = extractLowerBound(validRepls.len, ^1) ..< safeEnd
+  if rest.a >= 0 and rest.a <= rest.b and rest.b <= runeLen:
     result.add $runes[rest]
 
 proc deduplicate(s: var seq[ReplaceSlice]) =
@@ -326,7 +339,8 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
       replyTo = reply.getStr
       tweet.reply.add replyTo
 
-  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo,
+                           hasQuote or hasJobCard)
 
 proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
                           hasRedundantLink=false) =
@@ -377,16 +391,19 @@ proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) =
     textSlice = textRange{0}.getInt .. textRange{1}.getInt
     hasQuote = "quoted_tweet_results" in js
     hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
+    hasAttribution = tweet.attribution.isSome
 
-  tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard)
+  tweet.expandTextEntitiesV2(js, tweet.text, textSlice,
+                             hasQuote or hasJobCard or hasAttribution)
 
 proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
     entities = ? js{"entity_set"}
     text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
     textSlice = 0..text.runeLen
+    hasAttribution = tweet.attribution.isSome
 
-  tweet.expandTextEntities(entities, text, textSlice)
+  tweet.expandTextEntities(entities, text, textSlice, hasRedundantLink=hasAttribution)
 
   tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
 

src/query.nim

@@ -8,7 +8,7 @@ const
     "media", "images", "twimg", "videos",
     "native_video", "consumer_video", "spaces",
     "links", "news", "quote", "mentions",
-    "replies", "retweets", "nativeretweets"
+    "replies", "retweets", "nativeretweets", "cashtags"
   ]
 
   emptyQuery* = "include:nativeretweets"

src/redis_cache.nim

@@ -144,9 +144,10 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   else:
     let user = await getGraphUserById(userId)
     result = user.username
-    await setEx(key, baseCacheTime, result)
-    if result.len > 0 and user.id.len > 0:
-      await all(cacheUserId(result, user.id), cache(user))
+    if result.len > 0:
+      await setEx(key, baseCacheTime, result)
+      if user.id.len > 0:
+        await all(cacheUserId(result, user.id), cache(user))
 
 # proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
 #   if id == 0: return

src/routes/router_utils.nim

@@ -8,8 +8,9 @@ export utils, prefs, types, uri
 
 template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
+    let sameSite = if cfg.useHttps: None else: Lax
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
-              httpOnly=true, secure=cfg.useHttps, sameSite=None, path="/")
+              httpOnly=true, secure=cfg.useHttps, sameSite=sameSite, path="/")
 
 template requestPrefs*(): untyped {.dirty.} =
   getPrefs(cookies(request), params(request))

src/routes/search.nim

@@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) =
       redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
-      let url = getUrlPrefix(cfg) & "/search?f=tweets&q="
-      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
+      let
+        url = getUrlPrefix(cfg) & "/search?f=tweets&q="
+        headers = {"Content-Type": "application/opensearchdescription+xml"}
+      resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/types.nim

@@ -16,6 +16,7 @@ type
   ApiUrl* = object
     endpoint*: string
     params*: seq[(string, string)]
+    skipTid*: bool
 
   ApiReq* = object
     oauth*: ApiUrl
@@ -264,6 +265,7 @@ type
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
+    attributionLink*: string
     mediaTags*: seq[User]
     quote*: Option[Tweet]
     card*: Option[Card]

src/views/general.nim

@@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
         text cfg.title
 
     meta(name="viewport", content="width=device-width, initial-scale=1.0")
+    meta(name="referrer", content="same-origin")
     meta(name="theme-color", content="#1F1F1F")
     meta(property="og:type", content=ogType)
     meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))

src/views/search.nim

@@ -15,7 +15,8 @@ const toggles = {
   "links": "Links",
   "images": "Images",
   "quote": "Quotes",
-  "spaces": "Spaces"
+  "spaces": "Spaces",
+  "cashtags": "Cashtags"
 }.toOrderedTable
 
 proc renderSearch*(): VNode =

src/views/tweet.nim

@@ -239,8 +239,9 @@ proc renderReply(tweet: Tweet): VNode =
       if i > 0: text " "
       a(href=("/" & u)): text "@" & u
 
-proc renderAttribution(user: User; prefs: Prefs): VNode =
-  buildHtml(a(class="attribution", href=("/" & user.username))):
+proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode =
+  let href = if link.len > 0: link else: "/" & user.username
+  buildHtml(a(class="attribution", href=href)):
     renderMiniAvatar(user, prefs)
     strong: text user.fullname
     verifiedIcon(user)
@@ -386,7 +387,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
 
       if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get(), prefs)
+        renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink)
 
       if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)

tests/test_about_account.py

@@ -0,0 +1,76 @@
+from base import BaseTestCase, Profile
+from parameterized import parameterized
+
+
+class AboutAccount(object):
+    header = '.about-account-header'
+    name = '.about-account-name'
+    body = '.about-account-body'
+    row = '.about-account-row'
+    label = '.about-account-label'
+    value = '.about-account-value'
+
+
+# (username, expected_labels)
+# Each label is checked for presence in the page text
+about_data = [
+    ['jack', ['Date joined', 'Account based in', 'Connected via']],
+    ['NASA', ['Date joined']],
+    ['elonmusk', ['Date joined']],
+]
+
+about_verified = [
+    ['jack', 'Verified', 'Since '],
+]
+
+about_affiliate = [
+    ['jack', 'An affiliate of', 'Square'],
+    ['elonmusk', 'An affiliate of', 'X'],
+]
+
+
+class AboutAccountTest(BaseTestCase):
+    @parameterized.expand(about_data)
+    def test_about_page_has_labels(self, username, expected_labels):
+        """About page shows expected info labels"""
+        self.open_nitter(f'{username}/about')
+        self.assert_element_visible(AboutAccount.header)
+        self.assert_element_visible(AboutAccount.body)
+        for label in expected_labels:
+            self.assert_text(label, AboutAccount.body)
+
+    @parameterized.expand(about_verified)
+    def test_about_verified(self, username, label, value_prefix):
+        """About page shows verification info for verified accounts"""
+        self.open_nitter(f'{username}/about')
+        self.assert_text(label, AboutAccount.body)
+        self.assert_text(value_prefix, AboutAccount.body)
+
+    @parameterized.expand(about_affiliate)
+    def test_about_affiliate(self, username, label, affiliate):
+        """About page shows affiliate info"""
+        self.open_nitter(f'{username}/about')
+        self.assert_text(label, AboutAccount.body)
+        self.assert_text(f'@{affiliate}', AboutAccount.body)
+
+    def test_about_page_title(self):
+        """Title contains account name"""
+        self.open_nitter('jack/about')
+        self.assert_text('jack', AboutAccount.name)
+
+    def test_about_join_date(self):
+        """About page always shows join date"""
+        self.open_nitter('jack/about')
+        self.assert_text('Date joined', AboutAccount.body)
+        self.assert_text('March 2006', AboutAccount.body)
+
+    def test_about_invalid_user(self):
+        """About page for non-existent user shows error"""
+        self.open_nitter('thisprofiledoesntexist/about')
+        self.assert_text('User "thisprofiledoesntexist" not found')
+
+    def test_joindate_links_to_about(self):
+        """Join date on profile page links to about page"""
+        self.open_nitter('jack')
+        link = self.find_element(Profile.joinDate + ' a')
+        self.assertIn('/jack/about', link.get_attribute('href'))

tests/test_card.py

@@ -11,18 +11,18 @@ card = [
     ['voidtarget/status/1094632512926605312',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True]
+     'gist.github.com', True],
+
+    ['NASA/status/2061872347477418301',
+     'Nancy Grace Roman Space Telescope - NASA Science',
+     'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.',
+     'science.nasa.gov', True]
 ]
 
 no_thumb = [
-    ['FluentAI/status/1116417904831029248',
-     'LinkedIn',
-     'This link will take you to a page that’s not on LinkedIn',
-     'lnkd.in'],
-
     ['Thom_Wolf/status/1122466524860702729',
-     'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
-     '',
+     'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model',
+     'PyTorch original implementation of Cross-lingual Language Model Pretraining.',
      'github.com'],
 
     ['brent_p/status/1088857328680488961',
@@ -37,14 +37,9 @@ no_thumb = [
 ]
 
 playable = [
-    ['nim_lang/status/1118234460904919042',
-     'Nim development blog 2019-03',
-     'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...',
-     'youtube.com'],
-
-    ['nim_lang/status/1121090879823986688',
-     'Nim - First natively compiled language w/ hot code-reloading at...',
-     '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...',
+    ['NASA/status/2047048645845897398',
+     'NASA\'s Artemis II News Conference with Moon Astronauts',
+     'Live from NASA\'s Johnson Space Center in Houston',
      'youtube.com']
 ]
 
@@ -72,7 +67,7 @@ class CardTest(BaseTestCase):
         if len(description) > 0:
             self.assert_text(description, c.description)
 
-    @parameterized.expand(playable)
+    @parameterized.expand(playable, skip_on_empty=True)
     def test_card_playable(self, tweet, title, description, destination):
         self.open_nitter(tweet)
         c = Card(Conversation.main + " ")

tests/test_security.py

@@ -0,0 +1,60 @@
+import subprocess
+from parameterized import parameterized
+
+BASE_URL = 'http://localhost:8080'
+
+
+def curl_status(url):
+    """Get HTTP status code using curl to avoid URL normalization by Python libs."""
+    result = subprocess.run(
+        ['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', url],
+        capture_output=True, text=True, timeout=10
+    )
+    return int(result.stdout)
+
+
+class TestMalformedPaths:
+    """Test that malformed paths don't crash the server.
+
+    URLs like //foo are parsed as having 'foo' as the authority (host),
+    resulting in an empty path. Empty paths previously crashed jester's
+    static file handler. Now they return 400.
+
+    URLs like //foo/bar are parsed as authority='foo', path='/bar',
+    so they route normally (not empty path).
+    """
+
+    @parameterized.expand([
+        # These parse to empty paths -> 400
+        ('//lefty_rae', 400),
+        ('//test', 400),
+        ('//anyuser', 400),
+    ])
+    def test_empty_path_returns_400(self, path, expected_status):
+        """URLs that parse to empty paths should return 400, not crash."""
+        status = curl_status(f'{BASE_URL}{path}')
+        assert status == expected_status, \
+            f'Expected {expected_status} for {path}, got {status}'
+
+    @parameterized.expand([
+        ('/jack', 200),
+        ('/about', 200),
+        ('/', 200),
+    ])
+    def test_normal_paths_work(self, path, expected_status):
+        """Normal paths should still work."""
+        status = curl_status(f'{BASE_URL}{path}')
+        assert status == expected_status, \
+            f'Expected {expected_status} for {path}, got {status}'
+
+    def test_server_survives_malformed_requests(self):
+        """Server should handle malformed requests without crashing."""
+        # These all parse to empty paths
+        malformed_paths = ['//a', '//b', '//c', '//user', '//test']
+        for path in malformed_paths:
+            status = curl_status(f'{BASE_URL}{path}')
+            assert status == 400, f'Expected 400 for {path}, got {status}'
+
+        # Verify server is still responding after malformed requests
+        status = curl_status(f'{BASE_URL}/')
+        assert status == 200, 'Server should still be alive'

tests/test_thread.py

@@ -8,7 +8,7 @@ thread = [
         [],
         "Based",
         ["Crystal", "Julia"],
-        [["yeah,"]],
+        [["For", "Then"], ["yeah,"]],
     ],
     ["octonion/status/975254452625002496", ["Based"], "Crystal", ["Julia"], []],
     ["octonion/status/975256058384887808", ["Based", "Crystal"], "Julia", [], []],

tests/test_tweet.py

@@ -71,8 +71,8 @@ emoji = [
 ]
 
 retweet = [
-    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
+    [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test',
+     'Testing. 1234.']
 ]
 
 
@@ -120,7 +120,7 @@ class TweetTest(BaseTestCase):
             link = self.find_link_text(f'@{un}')
             self.assertIn(f'/{un}', link.get_property('href'))
 
-    @parameterized.expand(retweet)
+    @parameterized.expand(retweet, skip_on_empty=True)
     def test_retweet(self, index, url, retweet_by, fullname, username, text):
         self.open_nitter(url)
         tweet = get_timeline_tweet(index)

tests/test_tweet_media.py

@@ -28,14 +28,14 @@ video_m3u8 = [
 ]
 
 gallery = [
-    # ['mobile_test/status/451108446603980803', [
-    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
-    # ]],
+    ['mobile_test/status/451108446603980803', [
+        ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+    ]],
 
-    # ['mobile_test/status/471539824713691137', [
-    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
-    #     ['Bos--IqIQAAav23']
-    # ]],
+    ['mobile_test/status/471539824713691137', [
+        ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+        ['Bos--IqIQAAav23']
+    ]],
 
     ['mobile_test/status/469530783384743936', [
         ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],