Round video duration properly

Fixes #498

.github/workflows/build-docker.yml

@@ -1,4 +1,4 @@
-name: CI/CD
+name: Docker
 
 on:
   push:
@@ -8,31 +8,56 @@ on:
       - master
 
 jobs:
-  build-docker:
+  tests:
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/run-tests.yml
+    secrets: inherit
+  build-docker-amd64:
+    needs: [tests]
+    runs-on: buildjet-2vcpu-ubuntu-2204
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
+      - name: Build and push AMD64 Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+  build-docker-arm64:
+    needs: [tests]
+    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push ARM64 Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile.arm64
+          platforms: linux/arm64
+          push: true
+          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

.github/workflows/run-tests.yml

@@ -0,0 +1,108 @@
+name: Tests
+
+on:
+  push:
+    paths-ignore:
+      - "*.md"
+    branches-ignore:
+      - master
+  workflow_call:
+
+# Ensure that multiple runs on the same branch do not overlap.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-test:
+    name: Build and test
+    runs-on: buildjet-2vcpu-ubuntu-2204
+    strategy:
+      matrix:
+        nim: ["2.0.x", "2.2.x", "devel"]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Cache Nimble Dependencies
+        id: cache-nimble
+        uses: buildjet/cache@v4
+        with:
+          path: ~/.nimble
+          key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
+          restore-keys: |
+            ${{ matrix.nim }}-nimble-v2-
+
+      - name: Setup Nim
+        uses: jiro4989/setup-nim-action@v2
+        with:
+          nim-version: ${{ matrix.nim }}
+          use-nightlies: true
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Project
+        run: nimble build -d:release -Y
+
+  integration-test:
+    needs: [build-test]
+    name: Integration test
+    runs-on: buildjet-2vcpu-ubuntu-2204
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Cache Nimble Dependencies
+        id: cache-nimble
+        uses: buildjet/cache@v4
+        with:
+          path: ~/.nimble
+          key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
+          restore-keys: |
+            devel-nimble-v2-
+
+      - name: Setup Python (3.10) with pip cache
+        uses: buildjet/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: pip
+
+      - name: Setup Nim
+        uses: jiro4989/setup-nim-action@v2
+        with:
+          nim-version: devel
+          use-nightlies: true
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Project
+        run: nimble build -d:release -Y
+
+      - name: Install SeleniumBase and Chromedriver
+        run: |
+          pip install seleniumbase
+          seleniumbase install chromedriver
+
+      - name: Start Redis Service
+        uses: supercharge/redis-github-action@1.5.0
+
+      - name: Prepare Nitter Environment
+        run: |
+          sudo apt-get update && sudo apt-get install -y libsass-dev
+          cp nitter.example.conf nitter.conf
+          sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
+          nimble md
+          nimble scss
+          echo '${{ secrets.SESSIONS }}' | head -n1
+          echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
+
+      - name: Run Tests
+        run: |
+          ./nitter &
+          pytest -n1 tests

.gitignore

@@ -3,9 +3,13 @@ nitter
 *.db
 /tests/__pycache__
 /tests/geckodriver.log
-/tests/downloaded_files/*
+/tests/downloaded_files
+/tests/latest_logs
 /tools/gencss
 /tools/rendermd
 /public/css/style.css
 /public/md/*.html
 nitter.conf
+guest_accounts.json*
+sessions.json*
+dump.rdb

Dockerfile

@@ -1,6 +1,5 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
+FROM nimlang/nim:2.2.0-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
-EXPOSE 8080
 
 RUN apk --no-cache add libsass-dev pcre
 
@@ -10,14 +9,17 @@ COPY nitter.nimble .
 RUN nimble install -y --depsOnly
 
 COPY . .
-RUN nimble build -d:danger -d:lto -d:strip \
+RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
     && nimble scss \
     && nimble md
 
 FROM alpine:latest
 WORKDIR /src/
-RUN apk --no-cache add pcre
+RUN apk --no-cache add pcre ca-certificates
 COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
+EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter

Dockerfile.arm64

@@ -0,0 +1,25 @@
+FROM alpine:3.20.6 as nim
+LABEL maintainer="setenforce@protonmail.com"
+
+RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
+
+WORKDIR /src/nitter
+
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
+    && nimble scss \
+    && nimble md
+
+FROM alpine:3.20.6
+WORKDIR /src/
+RUN apk --no-cache add pcre ca-certificates openssl
+COPY --from=nim /src/nitter/nitter ./
+COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
+COPY --from=nim /src/nitter/public ./public
+EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
+CMD ./nitter

README.md

@@ -1,29 +1,38 @@
 # Nitter
 
-[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
+[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
+[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
 [![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
 
+> [!NOTE]
+> Running a Nitter instance now requires real accounts, since Twitter removed the previous methods. \
+> This does not affect users. \
+> For instructions on how to obtain session tokens, see [Creating session tokens](https://github.com/zedeus/nitter/wiki/Creating-session-tokens).
+
 A free and open source alternative Twitter front-end focused on privacy and
 performance. \
-Inspired by the [Invidious](https://github.com/iv-org/invidious)
+Inspired by the [Invidious](https://github.com/iv-org/invidious) project.
-project.
 
 - No JavaScript or ads
 - All requests go through the backend, client never talks to Twitter
 - Prevents Twitter from tracking your IP or JavaScript fingerprint
-- Uses Twitter's unofficial API (no rate limits or developer account required)
+- Uses Twitter's unofficial API (no developer account required)
 - Lightweight (for [@nim_lang](https://nitter.net/nim_lang), 60KB vs 784KB from twitter.com)
 - RSS feeds
 - Themes
 - Mobile support (responsive design)
 - AGPLv3 licensed, no proprietary instances permitted
 
-Liberapay: https://liberapay.com/zedeus \
+<details>
-Patreon: https://patreon.com/nitter \
+<summary>Donations</summary>
-BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
+Liberapay: https://liberapay.com/zedeus<br>
-ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
+Patreon: https://patreon.com/nitter<br>
-LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
+BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55<br>
-XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
+ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460<br>
+XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL<br>
+SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW<br>
+ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
+</details>
 
 ## Roadmap
 
@@ -41,12 +50,13 @@ maintained by the community.
 
 ## Why?
 
-It's impossible to use Twitter without JavaScript enabled. For privacy-minded
+It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
-folks, preventing JavaScript analytics and IP-based tracking is important, but
+need to sign up. For privacy-minded folks, preventing JavaScript analytics and
-apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
+IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
-a VPN and using heavy-duty adblockers, you can get accurately tracked with your
+it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
-[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
+you can get accurately tracked with your [browser's
-[no JavaScript required](https://noscriptfingerprint.com/). This all became
+fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
+JavaScript required](https://noscriptfingerprint.com/). This all became
 particularly important after Twitter [removed the
 ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
 for users to control whether their data gets sent to advertisers.
@@ -67,21 +77,24 @@ Twitter account.
 ## Installation
 
 ### Dependencies
-* libpcre
+
-* libsass
+- libpcre
-* redis
+- libsass
+- redis/valkey
 
 To compile Nitter you need a Nim installation, see
-[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
+[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible
-install it system-wide or in the user directory you create below.
+to install it system-wide or in the user directory you create below.
 
 To compile the scss files, you need to install `libsass`. On Ubuntu and Debian,
 you can use `libsass-dev`.
 
-Redis is required for caching and in the future for account info. It should be
+Redis is required for caching and in the future for account info. As of 2024
-available on most distros as `redis` or `redis-server` (Ubuntu/Debian).
+Redis is no longer open source, so using the fork Valkey is recommended. It
-Running it with the default config is fine, Nitter's default config is set to
+should be available on most distros as `redis` or `redis-server`
-use the default Redis port and localhost.
+(Ubuntu/Debian), or `valkey`/`valkey-server`. Running it with the default
+config is fine, Nitter's default config is set to use the default port and
+localhost.
 
 Here's how to create a `nitter` user, clone the repo, and build the project
 along with the scss and md files.
@@ -91,7 +104,7 @@ along with the scss and md files.
 # su nitter
 $ git clone https://github.com/zedeus/nitter
 $ cd nitter
-$ nimble build -d:release
+$ nimble build -d:danger --mm:refc
 $ nimble scss
 $ nimble md
 $ cp nitter.example.conf nitter.conf
@@ -108,25 +121,32 @@ performance reasons.
 
 ### Docker
 
-#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
+Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
+
+#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
 
 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
 Docker.
 
 To build and run Nitter in Docker:
+
 ```bash
 docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
+Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
+
 A prebuilt Docker image is provided as well:
+
 ```bash
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
 ```
 
 Using docker-compose to run both Nitter and Redis as different containers:
 Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
+
 ```bash
 docker-compose up -d
 ```

config.nims

@@ -1,17 +1,13 @@
 --define:ssl
 --define:useStdLib
+--threads:off
 
 # workaround httpbeast file upload bug
 --assertions:off
 
 # disable annoying warnings
 warning("GcUnsafe2", off)
+warning("HoleEnumConv", off)
 hint("XDeclaredButNotUsed", off)
 hint("XCannotRaiseY", off)
 hint("User", off)
-
-const
-  nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch)
-
-when nimVersion >= (1, 6, 0):
-  warning("HoleEnumConv", off)

docker-compose.yml

@@ -8,10 +8,22 @@ services:
     ports:
       - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
     volumes:
-      - ./nitter.conf:/src/nitter.conf:ro
+      - ./nitter.conf:/src/nitter.conf:Z,ro
+      - ./sessions.jsonl:/src/sessions.jsonl:Z,ro # Run get_sessions.py to get the credentials
     depends_on:
       - nitter-redis
     restart: unless-stopped
+    healthcheck:
+      test: wget -nv --tries=1 --spider http://127.0.0.1:8080/Jack/status/20 || exit 1
+      interval: 30s
+      timeout: 5s
+      retries: 2
+    user: "998:998"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
   nitter-redis:
     image: redis:6-alpine
@@ -20,6 +32,17 @@ services:
     volumes:
       - nitter-redis:/data
     restart: unless-stopped
+    healthcheck:
+      test: redis-cli ping
+      interval: 30s
+      timeout: 5s
+      retries: 2
+    user: "999:1000"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
 volumes:
   nitter-redis:

nitter.example.conf

@@ -1,11 +1,11 @@
 [Server]
+hostname = "nitter.net"  # for generating links, change this to your own domain/ip
+title = "nitter"
 address = "0.0.0.0"
 port = 8080
 https = false  # disable to enable cookies when not using https
 httpMaxConnections = 100
 staticDir = "./public"
-title = "nitter"
-hostname = "nitter.net"
 
 [Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
@@ -13,9 +13,9 @@ rssMinutes = 10  # how long to cache rss queries
 redisHost = "localhost"  # Change to "nitter-redis" if using docker-compose
 redisPort = 6379
 redisPassword = ""
-redisConnections = 20  # connection pool size
+redisConnections = 20  # minimum open connections in pool
 redisMaxConnections = 30
-# max, new connections are opened when none are available, but if the pool size
+# new connections are opened when none are available, but if the pool size
 # goes above this, they're closed when released. don't worry about this unless
 # you receive tons of requests per second
 
@@ -23,23 +23,17 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints
+enableDebug = false  # enable request logs and debug endpoints (/.sessions)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
-tokenCount = 10
+disableTid = false # enable this if cookie-based auth is failing
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 187 requests.
-# the limit gets reset every 15 minutes, and the pool is filled up so there's
-# always at least $tokenCount usable tokens. again, only increase this if
-# you receive major bursts all the time
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
-replaceYouTube = "piped.kavin.rocks"
+replaceYouTube = "piped.video"
 replaceReddit = "teddit.net"
-replaceInstagram = ""
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false

nitter.nimble

@@ -10,20 +10,20 @@ bin           = @["nitter"]
 
 # Dependencies
 
-requires "nim >= 1.4.8"
+requires "nim >= 2.0.0"
-requires "jester >= 0.5.0"
+requires "jester#baca3f"
-requires "karax#c71bc92"
+requires "karax#5cf360c"
-requires "sass#e683aa1"
+requires "sass#7dfdd03"
-requires "nimcrypto#a5742a9"
+requires "nimcrypto#a079df9"
-requires "markdown#abdbe5e"
+requires "markdown#158efe3"
-requires "packedjson#d11d167"
+requires "packedjson#9e6fbb6"
-requires "supersnappy#2.1.1"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.7.3"
+requires "zippy#ca5989a"
-requires "flatty#0.2.3"
+requires "flatty#e668085"
-requires "jsony#d0e69bd"
+requires "jsony#1de1f08"
-
+requires "oauth#b8c163b"
 
 # Tasks
 

public/css/fontello.css

@@ -1,16 +1,18 @@
 @font-face {
-  font-family: 'fontello';
+  font-family: "fontello";
-  src: url('/fonts/fontello.eot?21002321');
+  src: url("/fonts/fontello.eot?77185648");
-  src: url('/fonts/fontello.eot?21002321#iefix') format('embedded-opentype'),
+  src:
-       url('/fonts/fontello.woff2?21002321') format('woff2'),
+    url("/fonts/fontello.eot?77185648#iefix") format("embedded-opentype"),
-       url('/fonts/fontello.woff?21002321') format('woff'),
+    url("/fonts/fontello.woff2?77185648") format("woff2"),
-       url('/fonts/fontello.ttf?21002321') format('truetype'),
+    url("/fonts/fontello.woff?77185648") format("woff"),
-       url('/fonts/fontello.svg?21002321#fontello') format('svg');
+    url("/fonts/fontello.ttf?77185648") format("truetype"),
+    url("/fonts/fontello.svg?77185648#fontello") format("svg");
   font-weight: normal;
   font-style: normal;
 }
 
- [class^="icon-"]:before, [class*=" icon-"]:before {
+[class^="icon-"]:before,
+[class*=" icon-"]:before {
   font-family: "fontello";
   font-style: normal;
   font-weight: normal;
@@ -19,6 +21,7 @@
   display: inline-block;
   text-decoration: inherit;
   width: 1em;
+  margin-right: 0.2em;
   text-align: center;
 
   /* For safety - reset parent styles, that can break glyph codes*/
@@ -33,21 +36,103 @@
   -moz-osx-font-smoothing: grayscale;
 }
 
-.icon-heart:before { content: '\2665'; } /* '♥' */
+.icon-views:before {
-.icon-quote:before { content: '\275e'; } /* '❞' */
+  content: "\e800";
-.icon-comment:before { content: '\e802'; } /* '' */
+}
-.icon-ok:before { content: '\e803'; } /* '' */
+
-.icon-play:before { content: '\e804'; } /* '' */
+/* '' */
-.icon-link:before { content: '\e805'; } /* '' */
+.icon-heart:before {
-.icon-calendar:before { content: '\e806'; } /* '' */
+  content: "\e801";
-.icon-location:before { content: '\e807'; } /* '' */
+}
-.icon-picture:before { content: '\e809'; } /* '' */
+
-.icon-lock:before { content: '\e80a'; } /* '' */
+/* '' */
-.icon-down:before { content: '\e80b'; } /* '' */
+.icon-quote:before {
-.icon-retweet:before { content: '\e80d'; } /* '' */
+  content: "\e802";
-.icon-search:before { content: '\e80e'; } /* '' */
+}
-.icon-pin:before { content: '\e80f'; } /* '' */
+
-.icon-cog:before { content: '\e812'; } /* '' */
+/* '' */
-.icon-rss-feed:before { content: '\e813'; } /* '' */
+.icon-comment:before {
-.icon-info:before { content: '\f128'; } /* '' */
+  content: "\e803";
-.icon-bird:before { content: '\f309'; } /* '' */
+}
+
+/* '' */
+.icon-play:before {
+  content: "\e805";
+}
+
+/* '' */
+.icon-link:before {
+  content: "\e806";
+}
+
+/* '' */
+.icon-calendar:before {
+  content: "\e807";
+}
+
+/* '' */
+.icon-location:before {
+  content: "\e808";
+}
+
+/* '' */
+.icon-picture:before {
+  content: "\e809";
+}
+
+/* '' */
+.icon-lock:before {
+  content: "\e80a";
+}
+
+/* '' */
+.icon-down:before {
+  content: "\e80b";
+}
+
+/* '' */
+.icon-retweet:before {
+  content: "\e80c";
+}
+
+/* '' */
+.icon-search:before {
+  content: "\e80d";
+}
+
+/* '' */
+.icon-pin:before {
+  content: "\e80e";
+}
+
+/* '' */
+.icon-cog:before {
+  content: "\e80f";
+}
+
+/* '' */
+.icon-rss:before {
+  content: "\e810";
+}
+
+/* '' */
+.icon-ok:before {
+  content: "\e811";
+}
+
+/* '' */
+.icon-circle:before {
+  content: "\f111";
+}
+
+/* '' */
+.icon-info:before {
+  content: "\f128";
+}
+
+/* '' */
+.icon-bird:before {
+  content: "\f309";
+}
+
+/* '' */

public/css/themes/dracula.css

@@ -0,0 +1,41 @@
+body {
+    --bg_color: #282a36;
+    --fg_color: #f8f8f2;
+    --fg_faded: #818eb6;
+    --fg_dark: var(--fg_faded);
+    --fg_nav: var(--accent);
+
+    --bg_panel: #343746;
+    --bg_elements: #292b36;
+    --bg_overlays: #44475a;
+    --bg_hover: #2f323f;
+
+    --grey: var(--fg_faded);
+    --dark_grey: #44475a;
+    --darker_grey: #3d4051;
+    --darkest_grey: #363948;
+    --border_grey: #44475a;
+
+    --accent: #bd93f9;
+    --accent_light: #caa9fa;
+    --accent_dark: var(--accent);
+    --accent_border: #ff79c696;
+
+    --play_button: #ffb86c;
+    --play_button_hover: #ffc689;
+
+    --more_replies_dots: #bd93f9;
+    --error_red: #ff5555;
+
+    --verified_blue: var(--accent);
+    --icon_text: ##F8F8F2;
+
+    --tab: #6272a4;
+    --tab_selected: var(--accent);
+
+    --profile_stat: #919cbf;
+}
+
+.search-bar > form input::placeholder{
+    color: var(--fg_faded);
+}

public/fonts/LICENSE.txt

@@ -1,6 +1,15 @@
 Font license info
 
 
+## Modern Pictograms
+
+   Copyright (c) 2012 by John Caserta. All rights reserved.
+
+   Author:    John Caserta
+   License:   SIL (http://scripts.sil.org/OFL)
+   Homepage:  http://thedesignoffice.org/project/modern-pictograms/
+
+
 ## Entypo
 
    Copyright (C) 2012 by Daniel Bruce
@@ -37,12 +46,3 @@ Font license info
    Homepage:  http://aristeides.com/
 
 
-## Modern Pictograms
-
-   Copyright (c) 2012 by John Caserta. All rights reserved.
-
-   Author:    John Caserta
-   License:   SIL (http://scripts.sil.org/OFL)
-   Homepage:  http://thedesignoffice.org/project/modern-pictograms/
-
-

public/fonts/fontello.svg

@@ -1,26 +1,26 @@
 <?xml version="1.0" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns="http://www.w3.org/2000/svg">
-<metadata>Copyright (C) 2020 by original authors @ fontello.com</metadata>
+<metadata>Copyright (C) 2025 by original authors @ fontello.com</metadata>
 <defs>
 <font id="fontello" horiz-adv-x="1000" >
 <font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
 <missing-glyph horiz-adv-x="1000" />
-<glyph glyph-name="heart" unicode="&#x2665;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
+<glyph glyph-name="views" unicode="&#xe800;" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
 
-<glyph glyph-name="quote" unicode="&#x275e;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
+<glyph glyph-name="heart" unicode="&#xe801;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
 
-<glyph glyph-name="comment" unicode="&#xe802;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
+<glyph glyph-name="quote" unicode="&#xe802;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
 
-<glyph glyph-name="ok" unicode="&#xe803;" d="M0 260l162 162 166-164 508 510 164-164-510-510-162-162-162 164z" horiz-adv-x="1000" />
+<glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
 
-<glyph glyph-name="play" unicode="&#xe804;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
+<glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
 
-<glyph glyph-name="link" unicode="&#xe805;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
+<glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
 
-<glyph glyph-name="calendar" unicode="&#xe806;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
+<glyph glyph-name="calendar" unicode="&#xe807;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
 
-<glyph glyph-name="location" unicode="&#xe807;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
+<glyph glyph-name="location" unicode="&#xe808;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
 
 <glyph glyph-name="picture" unicode="&#xe809;" d="M357 529q0-45-31-76t-76-32-76 32-31 76 31 76 76 31 76-31 31-76z m572-215v-250h-786v107l178 179 90-89 285 285z m53 393h-893q-7 0-12-5t-6-13v-678q0-7 6-13t12-5h893q7 0 13 5t5 13v678q0 8-5 13t-13 5z m89-18v-678q0-37-26-63t-63-27h-893q-36 0-63 27t-26 63v678q0 37 26 63t63 27h893q37 0 63-27t26-63z" horiz-adv-x="1071.4" />
 
@@ -28,15 +28,19 @@
 
 <glyph glyph-name="down" unicode="&#xe80b;" d="M939 399l-414-413q-10-11-25-11t-25 11l-414 413q-11 11-11 26t11 25l93 92q10 11 25 11t25-11l296-296 296 296q11 11 25 11t26-11l92-92q11-11 11-25t-11-26z" horiz-adv-x="1000" />
 
-<glyph glyph-name="retweet" unicode="&#xe80d;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
+<glyph glyph-name="retweet" unicode="&#xe80c;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
 
-<glyph glyph-name="search" unicode="&#xe80e;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
+<glyph glyph-name="search" unicode="&#xe80d;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
 
-<glyph glyph-name="pin" unicode="&#xe80f;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
+<glyph glyph-name="pin" unicode="&#xe80e;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
 
-<glyph glyph-name="cog" unicode="&#xe812;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
+<glyph glyph-name="cog" unicode="&#xe80f;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
 
-<glyph glyph-name="rss-feed" unicode="&#xe813;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
+<glyph glyph-name="rss" unicode="&#xe810;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
+
+<glyph glyph-name="ok" unicode="&#xe811;" d="M933 534q0-22-16-38l-404-404-76-76q-16-15-38-15t-38 15l-76 76-202 202q-15 16-15 38t15 38l76 76q16 16 38 16t38-16l164-165 366 367q16 16 38 16t38-16l76-76q16-15 16-38z" horiz-adv-x="1000" />
+
+<glyph glyph-name="circle" unicode="&#xf111;" d="M857 350q0-117-57-215t-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58 215-58 156-156 57-215z" horiz-adv-x="857.1" />
 
 <glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />
 

public/js/infiniteScroll.js

@@ -5,7 +5,7 @@ function insertBeforeLast(node, elem) {
 }
 
 function getLoadMore(doc) {
-    return doc.querySelector('.show-more:not(.timeline-item)');
+    return doc.querySelector(".show-more:not(.timeline-item)");
 }
 
 function isDuplicate(item, itemClass) {
@@ -19,14 +19,15 @@ window.onload = function() {
     const url = window.location.pathname;
     const isTweet = url.indexOf("/status/") !== -1;
     const containerClass = isTweet ? ".replies" : ".timeline";
-    const itemClass = containerClass + ' > div:not(.top-ref)';
+    const itemClass = containerClass + " > div:not(.top-ref)";
 
     var html = document.querySelector("html");
     var container = document.querySelector(containerClass);
     var loading = false;
 
-    window.addEventListener('scroll', function() {
+    function handleScroll(failed) {
         if (loading) return;
+
         if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
             loading = true;
             var loadMore = getLoadMore(document);
@@ -35,13 +36,15 @@ window.onload = function() {
             loadMore.children[0].text = "Loading...";
 
             var url = new URL(loadMore.children[0].href);
-            url.searchParams.append('scroll', 'true');
+            url.searchParams.append("scroll", "true");
 
             fetch(url.toString()).then(function (response) {
+                if (response.status === 404) throw "error";
+
                 return response.text();
             }).then(function (html) {
                 var parser = new DOMParser();
-                var doc = parser.parseFromString(html, 'text/html');
+                var doc = parser.parseFromString(html, "text/html");
                 loadMore.remove();
 
                 for (var item of doc.querySelectorAll(itemClass)) {
@@ -57,10 +60,18 @@ window.onload = function() {
                 if (isTweet) container.appendChild(newLoadMore);
                 else insertBeforeLast(container, newLoadMore);
             }).catch(function (err) {
-                console.warn('Something went wrong.', err);
+                console.warn("Something went wrong.", err);
-                loading = true;
+                if (failed > 3) {
+                    loadMore.children[0].text = "Error";
+                    return;
+                }
+
+                loading = false;
+                handleScroll((failed || 0) + 1);
             });
         }
-    });
+    }
+
+    window.addEventListener("scroll", () => handleScroll());
 };
 // @license-end

public/md/about.md

@@ -4,15 +4,15 @@ Nitter is a free and open source alternative Twitter front-end focused on
 privacy and performance. The source is available on GitHub at
 <https://github.com/zedeus/nitter>
 
-* No JavaScript or ads
+- No JavaScript or ads
-* All requests go through the backend, client never talks to Twitter
+- All requests go through the backend, client never talks to Twitter
-* Prevents Twitter from tracking your IP or JavaScript fingerprint
+- Prevents Twitter from tracking your IP or JavaScript fingerprint
-* Uses Twitter's unofficial API (no rate limits or developer account required)
+- Uses Twitter's unofficial API (no developer account required)
-* Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
+- Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
-* RSS feeds
+- RSS feeds
-* Themes
+- Themes
-* Mobile support (responsive design)
+- Mobile support (responsive design)
-* AGPLv3 licensed, no proprietary instances permitted
+- AGPLv3 licensed, no proprietary instances permitted
 
 Nitter's GitHub wiki contains
 [instances](https://github.com/zedeus/nitter/wiki/Instances) and
@@ -21,12 +21,13 @@ maintained by the community.
 
 ## Why use Nitter?
 
-It's impossible to use Twitter without JavaScript enabled. For privacy-minded
+It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
-folks, preventing JavaScript analytics and IP-based tracking is important, but
+need to sign up. For privacy-minded folks, preventing JavaScript analytics and
-apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
+IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
-a VPN and using heavy-duty adblockers, you can get accurately tracked with your
+it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
-[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
+you can get accurately tracked with your [browser's
-[no JavaScript required](https://noscriptfingerprint.com/). This all became
+fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
+JavaScript required](https://noscriptfingerprint.com/). This all became
 particularly important after Twitter [removed the
 ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
 for users to control whether their data gets sent to advertisers.
@@ -42,12 +43,13 @@ Twitter account.
 
 ## Donating
 
-Liberapay: <https://liberapay.com/zedeus> \
+Liberapay: https://liberapay.com/zedeus \
-Patreon: <https://patreon.com/nitter> \
+Patreon: https://patreon.com/nitter \
-BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
+BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55 \
-ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
+ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460 \
-LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
+XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL \
-XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
+SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW \
+ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
 
 ## Contact
 

public/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /
+Crawl-delay: 1
+User-agent: Twitterbot
+Disallow:

src/api.nim

@@ -1,124 +1,193 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
+import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc getGraphUser*(id: string): Future[User] {.async.} =
+# Helper to generate params object for GraphQL requests
+proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
+  result.add ("variables", variables)
+  result.add ("features", gqlFeatures)
+  if fieldToggles.len > 0:
+    result.add ("fieldToggles", fieldToggles)
+
+proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+
+proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles)
+  return ApiReq(cookie: url, oauth: url)
+
+proc mediaUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor]),
+    oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
+  )
+
+proc userTweetsUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
+  )
+  # might change this in the future pending testing
+  result.cookie = result.oauth
+
+proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
+  )
+
+proc tweetDetailUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = tweetDetailVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  )
+
+proc userUrl(username: string): ApiReq =
+  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  result = ApiReq(
+    cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
+  )
+
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let js = await fetchRaw(userUrl(username))
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
+    url = apiReq(graphUserById, """{"rest_id": "$1"}""" % id)
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    js = await fetchRaw(url)
   result = parseGraphUser(js)
 
+proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    url = case kind
+      of TimelineKind.tweets: userTweetsUrl(id, cursor)
+      of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
+      of TimelineKind.media: mediaUrl(id, cursor)
+    js = await fetch(url)
+  result = parseGraphTimeline(js, after)
+
+proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    url = apiReq(graphListTweets, restIdVars % [id, cursor])
+    js = await fetch(url)
+  result = parseGraphTimeline(js, after).tweets
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
-    variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
+    variables = %*{"screenName": name, "listSlug": list}
-    url = graphListBySlug ? {"variables": $variables}
+    url = apiReq(graphListBySlug, $variables)
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let 
-    variables = %*{"listId": id, "withHighlightedLabel": false}
+    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
-    url = graphList ? {"variables": $variables}
+    js = await fetch(url)
-  result = parseGraphList(await fetch(url, Api.list))
+  result = parseGraphList(js)
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
-  let
+  var
     variables = %*{
       "listId": list.id,
-      "cursor": after,
-      "withSuperFollowsUserFields": false,
       "withBirdwatchPivots": false,
       "withDownvotePerspective": false,
       "withReactionsMetadata": false,
-      "withReactionsPerspective": false,
+      "withReactionsPerspective": false
-      "withSuperFollowsTweetFields": false
     }
-    url = graphListMembers ? {"variables": $variables}
+  if after.len > 0:
-  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
+    variables["cursor"] = % after
+  let 
+    url = apiReq(graphListMembers, $variables)
+    js = await fetchRaw(url)
+  result = parseGraphListMembers(js, after)
 
-proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
+    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
-    url = listTimeline ? ps
+    js = await fetch(url)
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+  result = parseGraphTweetResult(js)
 
-proc getUser*(username: string): Future[User] {.async.} =
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
-  if username.len == 0: return
-  let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
-
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
-
-proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
-    ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    url = timeline / (id & ".json") ? ps
+    js = await fetch(tweetDetailUrl(id, cursor))
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+  result = parseGraphConversation(js, id)
-
-proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
-  result = parseTimeline(await fetch(url, Api.timeline), after)
-
-proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
-  let
-    ps = genParams({"screen_name": name, "trim_user": "true"},
-                   count="18", ext=false)
-    url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.timeline))
-
-proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
-  when T is User:
-    const
-      searchMode = ("result_filter", "user")
-      parse = parseUsers
-      fetchFunc = fetchRaw
-  else:
-    const
-      searchMode = ("tweet_search_mode", "live")
-      parse = parseTimeline
-      fetchFunc = fetch
-
-  let q = genQueryParam(query)
-  if q.len == 0 or q == emptyQuery:
-    return Result[T](beginning: true, query: query)
-
-  let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
-  try:
-    result = parse(await fetchFunc(url, Api.search), after)
-    result.query = query
-  except InternalError:
-    return Result[T](beginning: true, query: query)
-
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
-  result = parseConversation(await fetch(url, Api.tweet), id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
+  result = (await getGraphTweet(id, after)).replies
   result.beginning = after.len == 0
 
 proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
+  result = await getGraphTweet(id)
   if after.len > 0:
     result.replies = await getReplies(id, after)
 
-proc getStatus*(id: string): Future[Tweet] {.async.} =
+proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
-  let url = status / (id & ".json") ? genParams()
+  let q = genQueryParam(query)
-  result = parseStatus(await fetch(url, Api.status))
+  if q.len == 0 or q == emptyQuery:
+    return Timeline(query: query, beginning: true)
+
+  var
+    variables = %*{
+      "rawQuery": q,
+      "query_source": "typedQuery",
+      "count": 20,
+      "product": "Latest",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+  let 
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[Tweets](js, after)
+  result.query = query
+
+proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
+  if query.text.len == 0:
+    return Result[User](query: query, beginning: true)
+
+  var
+    variables = %*{
+      "rawQuery": query.text,
+      "query_source": "typedQuery",
+      "count": 20,
+      "product": "People",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+    result.beginning = false
+
+  let 
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[User](js, after)
+  result.query = query
+
+proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+  if id.len == 0: return
+  let js = await fetch(mediaUrl(id, ""))
+  result = parseGraphPhotoRail(js)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -1,95 +1,167 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, strutils, uri
+import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
-import jsony, packedjson, zippy
+import jsony, packedjson, zippy, oauth1
-import types, tokens, consts, parserutils, http_pool
+import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
+  rlLimit = "x-rate-limit-limit"
+  errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
-var pool: HttpPool
+var 
+  pool: HttpPool
+  disableTid: bool
 
-proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
+proc setDisableTid*(disable: bool) =
-                count="20"; ext=true): seq[(string, string)] =
+  disableTid = disable
-  result = timelineParams
-  for p in pars:
-    result &= p
-  if ext:
-    result &= ("ext", "mediaStats")
-    result &= ("include_ext_alt_text", "true")
-    result &= ("include_ext_media_availability", "true")
-  if count.len > 0:
-    result &= ("count", count)
-  if cursor.len > 0:
-    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
-    if " " in cursor:
-      result &= ("cursor", cursor.replace(" ", "+"))
-    else:
-      result &= ("cursor", cursor)
 
-proc genHeaders*(token: Token = nil): HttpHeaders =
+proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
+  case sessionKind
+  of oauth:  
+    let o = req.oauth
+    parseUri("https://api.x.com/graphql")   / o.endpoint ? o.params
+  of cookie: 
+    let c = req.cookie
+    parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
+
+proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
+  let
+    encodedUrl = url.replace(",", "%2C").replace("+", "%20")
+    params = OAuth1Parameters(
+      consumerKey: consumerKey,
+      signatureMethod: "HMAC-SHA1",
+      timestamp: $int(round(epochTime())),
+      nonce: "0",
+      isIncludeVersionToHeader: true,
+      token: oauthToken
+    )
+    signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
+
+  params.signature = percentEncode(signature)
+
+  return getOauth1RequestHeader(params)["authorization"]
+
+proc getCookieHeader(authToken, ct0: string): string =
+  "auth_token=" & authToken & "; ct0=" & ct0
+
+proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "connection": "keep-alive",
-    "authorization": auth,
     "content-type": "application/json",
-    "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
-    "authority": "api.twitter.com",
+    "x-twitter-client-language": "en",
+    "origin": "https://x.com",
     "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "en-US,en;q=0.5",
     "accept": "*/*",
-    "DNT": "1"
+    "DNT": "1",
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
   })
 
-template updateToken() =
+  case session.kind
-  if api != Api.search and resp.headers.hasKey(rlRemaining):
+  of SessionKind.oauth:
-    let
+    result["authority"] = "api.x.com"
-      remaining = parseInt(resp.headers[rlRemaining])
+    result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
-      reset = parseInt(resp.headers[rlReset])
+  of SessionKind.cookie:
-    token.setRateLimit(api, remaining, reset)
+    result["x-twitter-auth-type"] = "OAuth2Session"
+    result["x-csrf-token"] = session.ct0
+    result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    if disableTid:
+      result["authorization"] = bearerToken2
+    else:
+      result["authorization"] = bearerToken
+      result["x-client-transaction-id"] = await genTid(url.path)
+
+proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
+  result = await getSession(req)
+  case result.kind
+  of SessionKind.oauth:
+    if result.oauthToken.len == 0:
+      echo "[sessions] Empty oauth token, session: ", result.pretty
+      raise rateLimitError()
+  of SessionKind.cookie:
+    if result.authToken.len == 0 or result.ct0.len == 0:
+      echo "[sessions] Empty cookie credentials, session: ", result.pretty
+      raise rateLimitError()
 
 template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
-  var token = await getToken(api)
-  if token.tok.len == 0:
-    raise rateLimitError()
-
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(token)):
+    pool.use(await genHeaders(session, url)):
+      template getContent =
         resp = await c.get($url)
         result = await resp.body
 
+      getContent()
+
       if resp.status == $Http503:
         badClient = true
-        raise newException(InternalError, result)
+        raise newException(BadClientError, "Bad client")
+
+    if resp.headers.hasKey(rlRemaining):
+      let
+        remaining = parseInt(resp.headers[rlRemaining])
+        reset = parseInt(resp.headers[rlReset])
+        limit = parseInt(resp.headers[rlLimit])
+      session.setRateLimit(req, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
         result = uncompress(result, dfGzip)
-      else:
+
-        echo "non-gzip body, url: ", url, ", body: ", result
+      if result.startsWith("{\"errors"):
+        let errors = result.fromJson(Errors)
+        if errors notin errorsToSkip:
+          echo "Fetch error, API: ", url.path, ", errors: ", errors
+          if errors in {expiredToken, badToken, locked}:
+            invalidate(session)
+            raise rateLimitError()
+          elif errors in {rateLimited}:
+            # rate limit hit, resets after 24 hours
+            setLimited(session, req)
+            raise rateLimitError()
+      elif result.startsWith("429 Too Many Requests"):
+        echo "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
+        raise rateLimitError()
 
     fetchBody
 
-    release(token, used=true)
-
     if resp.status == $Http400:
+      echo "ERROR 400, ", url.path, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
+  except BadClientError as e:
+    raise e
+  except OSError as e:
+    raise e
   except Exception as e:
-    echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
+    let s = session.pretty
-    if "length" notin e.msg and "descriptor" notin e.msg:
+    echo "error: ", e.name, ", msg: ", e.msg, ", session: ", s, ", url: ", url
-      release(token, invalid=true)
     raise rateLimitError()
+  finally:
+    release(session)
+
+template retry(bod) =
+  try:
+    bod
+  except RateLimitError:
+    echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
+    bod
+
+proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
+  retry:
+    var 
+      body: string
+      session = await getAndValidateSession(req)
+
+    let url = req.toUrl(session.kind)
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
-  var body: string
     fetchImpl body:
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
@@ -97,25 +169,19 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
         echo resp.status, ": ", body, " --- url: ", url
         result = newJNull()
 
-    updateToken()
-
       let error = result.getError
-    if error in {invalidToken, forbidden, badToken}:
+      if error != null and error notin errorsToSkip:
-      echo "fetch error: ", result.getError
+        echo "Fetch error, API: ", url.path, ", error: ", error
-      release(token, invalid=true)
+        if error in {expiredToken, badToken, locked}:
+          invalidate(session)
           raise rateLimitError()
 
-proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
+  retry:
+    var session = await getAndValidateSession(req)
+    let url = req.toUrl(session.kind)
+
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
         echo resp.status, ": ", result, " --- url: ", url
         result.setLen(0)
-
-    updateToken()
-
-    if result.startsWith("{\"errors"):
-      let errors = result.fromJson(Errors)
-      if errors in {invalidToken, forbidden, badToken}:
-        echo "fetch error: ", errors
-        release(token, invalid=true)
-        raise rateLimitError()

src/auth.nim

@@ -0,0 +1,210 @@
+#SPDX-License-Identifier: AGPL-3.0-only
+import std/[asyncdispatch, times, json, random, strutils, tables, packedsets, os]
+import types, consts
+import experimental/parser/session
+
+# max requests at a time per session to avoid race conditions
+const
+  maxConcurrentReqs = 2
+  hourInSeconds = 60 * 60
+
+var
+  sessionPool: seq[Session]
+  enableLogging = false
+
+template log(str: varargs[string, `$`]) =
+  echo "[sessions] ", str.join("")
+
+proc endpoint(req: ApiReq; session: Session): string =
+  case session.kind
+  of oauth: req.oauth.endpoint
+  of cookie: req.cookie.endpoint
+
+proc pretty*(session: Session): string =
+  if session.isNil:
+    return "<null>"
+
+  if session.id > 0 and session.username.len > 0:
+    result = $session.id & " (" & session.username & ")"
+  elif session.username.len > 0:
+    result = session.username
+  elif session.id > 0:
+    result = $session.id
+  else:
+    result = "<unknown>"
+  result = $session.kind & " " & result
+
+proc snowflakeToEpoch(flake: int64): int64 =
+  int64(((flake shr 22) + 1288834974657) div 1000)
+
+proc getSessionPoolHealth*(): JsonNode =
+  let now = epochTime().int
+
+  var
+    totalReqs = 0
+    limited: PackedSet[int64]
+    reqsPerApi: Table[string, int]
+    oldest = now.int64
+    newest = 0'i64
+    average = 0'i64
+
+  for session in sessionPool:
+    let created = snowflakeToEpoch(session.id)
+    if created > newest:
+      newest = created
+    if created < oldest:
+      oldest = created
+    average += created
+
+    if session.limited:
+      limited.incl session.id
+
+    for api in session.apis.keys:
+      let
+        apiStatus = session.apis[api]
+        reqs = apiStatus.limit - apiStatus.remaining
+
+      # no requests made with this session and endpoint since the limit reset
+      if apiStatus.reset < now:
+        continue
+
+      reqsPerApi.mgetOrPut($api, 0).inc reqs
+      totalReqs.inc reqs
+
+  if sessionPool.len > 0:
+    average = average div sessionPool.len
+  else:
+    oldest = 0
+    average = 0
+
+  return %*{
+    "sessions": %*{
+      "total": sessionPool.len,
+      "limited": limited.card,
+      "oldest": $fromUnix(oldest),
+      "newest": $fromUnix(newest),
+      "average": $fromUnix(average)
+    },
+    "requests": %*{
+      "total": totalReqs,
+      "apis": reqsPerApi
+    }
+  }
+
+proc getSessionPoolDebug*(): JsonNode =
+  let now = epochTime().int
+  var list = newJObject()
+
+  for session in sessionPool:
+    let sessionJson = %*{
+      "apis": newJObject(),
+      "pending": session.pending,
+    }
+
+    if session.limited:
+      sessionJson["limited"] = %true
+
+    for api in session.apis.keys:
+      let
+        apiStatus = session.apis[api]
+        obj = %*{}
+
+      if apiStatus.reset > now.int:
+        obj["remaining"] = %apiStatus.remaining
+        obj["reset"] = %apiStatus.reset
+
+      if "remaining" notin obj:
+        continue
+
+      sessionJson{"apis", $api} = obj
+      list[$session.id] = sessionJson
+
+  return %list
+
+proc rateLimitError*(): ref RateLimitError =
+  newException(RateLimitError, "rate limited")
+
+proc noSessionsError*(): ref NoSessionsError =
+  newException(NoSessionsError, "no sessions available")
+
+proc isLimited(session: Session; req: ApiReq): bool =
+  if session.isNil:
+    return true
+
+  let api = req.endpoint(session)
+  if session.limited and api != graphUserTweetsV2:
+    if (epochTime().int - session.limitedAt) > hourInSeconds:
+      session.limited = false
+      log "resetting limit: ", session.pretty
+      return false
+    else:
+      return true
+
+  if api in session.apis:
+    let limit = session.apis[api]
+    return limit.remaining <= 10 and limit.reset > epochTime().int
+  else:
+    return false
+
+proc isReady(session: Session; req: ApiReq): bool =
+  not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(req))
+
+proc invalidate*(session: var Session) =
+  if session.isNil: return
+  log "invalidating: ", session.pretty
+
+  # TODO: This isn't sufficient, but it works for now
+  let idx = sessionPool.find(session)
+  if idx > -1: sessionPool.delete(idx)
+  session = nil
+
+proc release*(session: Session) =
+  if session.isNil: return
+  dec session.pending
+
+proc getSession*(req: ApiReq): Future[Session] {.async.} =
+  for i in 0 ..< sessionPool.len:
+    if result.isReady(req): break
+    result = sessionPool.sample()
+
+  if not result.isNil and result.isReady(req):
+    inc result.pending
+  else:
+    log "no sessions available for API: ", req.cookie.endpoint
+    raise noSessionsError()
+
+proc setLimited*(session: Session; req: ApiReq) =
+  let api = req.endpoint(session)
+  session.limited = true
+  session.limitedAt = epochTime().int
+  log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty
+
+proc setRateLimit*(session: Session; req: ApiReq; remaining, reset, limit: int) =
+  # avoid undefined behavior in race conditions
+  let api = req.endpoint(session)
+  if api in session.apis:
+    let rateLimit = session.apis[api]
+    if rateLimit.reset >= reset and rateLimit.remaining < remaining:
+      return
+    if rateLimit.reset == reset and rateLimit.remaining >= remaining:
+      session.apis[api].remaining = remaining
+      return
+
+  session.apis[api] = RateLimit(limit: limit, remaining: remaining, reset: reset)
+
+proc initSessionPool*(cfg: Config; path: string) =
+  enableLogging = cfg.enableDebug
+
+  if path.endsWith(".json"):
+    log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
+    quit 1
+
+  if not fileExists(path):
+    log "ERROR: ", path, " not found. This file is required to authenticate API requests."
+    quit 1
+
+  log "parsing JSONL account sessions file: ", path
+  for line in path.lines:
+    sessionPool.add parseSession(line)
+
+  log "successfully added ", sessionPool.len, " valid account sessions"

src/config.nim

@@ -40,7 +40,8 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     enableRss: cfg.get("Config", "enableRSS", true),
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
-    proxyAuth: cfg.get("Config", "proxyAuth", "")
+    proxyAuth: cfg.get("Config", "proxyAuth", ""),
+    disableTid: cfg.get("Config", "disableTid", false)
   )
 
   return (conf, cfg)

src/consts.nim

@@ -1,59 +1,163 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
+  consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+  bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+  bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  api = parseUri("https://api.twitter.com")
+  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  activate* = $(api / "1.1/guest/activate.json")
+  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
+  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
+  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
+  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
+  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
+  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
+  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
+  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
+  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
+  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
+  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
+  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
+  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
+  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
 
-  userShow* = api / "1.1/users/show.json"
+  gqlFeatures* = """{
-  photoRail* = api / "1.1/statuses/media_timeline.json"
+  "android_ad_formats_media_component_render_overlay_enabled": false,
-  status* = api / "1.1/statuses/show"
+  "android_graphql_skip_api_media_color_palette": false,
-  search* = api / "2/search/adaptive.json"
+  "android_professional_link_spotlight_display_enabled": false,
+  "blue_business_profile_image_shape_enabled": false,
+  "commerce_android_shop_module_enabled": false,
+  "creator_subscriptions_subscription_count_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "hidden_profile_likes_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_inline_media_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "mobile_app_spotlight_module_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": true,
+  "responsive_web_media_download_video_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "unified_cards_destination_url_params_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "rweb_lists_timeline_redesign_enabled": true,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": true,
+  "subscriptions_verification_info_enabled": true,
+  "subscriptions_verification_info_reason_enabled": true,
+  "subscriptions_verification_info_verified_since_enabled": true,
+  "super_follow_badge_privacy_enabled": false,
+  "super_follow_exclusive_tweet_notifications_enabled": false,
+  "super_follow_tweet_api_enabled": false,
+  "super_follow_user_api_enabled": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+  "tweetypie_unmention_optimization_enabled": false,
+  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": true,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "responsive_web_jetfuel_frame": true,
+  "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
+  "responsive_web_grok_image_annotation_enabled": true,
+  "responsive_web_grok_imagine_annotation_enabled": true,
+  "rweb_tipjar_consumption_enabled": true,
+  "profile_label_improvements_pcf_label_in_post_enabled": true,
+  "creator_subscriptions_quote_tweet_preview_enabled": false,
+  "c9s_tweet_anatomy_moderator_badge_enabled": true,
+  "responsive_web_grok_analyze_post_followups_enabled": true,
+  "rweb_video_timestamps_enabled": false,
+  "responsive_web_grok_share_attachment_enabled": true,
+  "articles_preview_enabled": true,
+  "immersive_video_status_linkable_timestamps": false,
+  "articles_api_enabled": false,
+  "responsive_web_grok_analysis_button_from_backend": true,
+  "rweb_video_screen_enabled": false,
+  "payments_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "responsive_web_grok_show_grok_translated_post": false,
+  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "profile_label_improvements_pcf_label_in_profile_enabled": false,
+  "grok_android_analyze_trend_fetch_enabled": false,
+  "grok_translations_community_note_auto_translation_is_enabled": false,
+  "grok_translations_post_auto_translation_is_enabled": false,
+  "grok_translations_community_note_translation_is_enabled": false,
+  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
+  "subscriptions_feature_can_gift_premium": false,
+  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "subscriptions_verification_info_is_identity_verified_enabled": false,
+  "hidden_profile_subscriptions_enabled": false
+}""".replace(" ", "").replace("\n", "")
 
-  timelineApi = api / "2/timeline"
+  tweetVars* = """{
-  timeline* = timelineApi / "profile"
+  "postId": "$1",
-  mediaTimeline* = timelineApi / "media"
+  $2
-  listTimeline* = timelineApi / "list.json"
+  "includeHasBirdwatchNotes": false,
-  tweet* = timelineApi / "conversation"
+  "includePromotedContent": false,
+  "withBirdwatchNotes": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}""".replace(" ", "").replace("\n", "")
 
-  graphql = api / "graphql"
+  tweetDetailVars* = """{
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
+  "focalTweetId": "$1",
-  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
+  $2
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
+  "referrer": "profile",
-  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  "with_rux_injections": false,
+  "rankingMode": "Relevance",
+  "includePromotedContent": true,
+  "withCommunity": true,
+  "withQuickPromoteEligibilityTweetFields": true,
+  "withBirdwatchNotes": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
 
-  timelineParams* = {
+  restIdVars* = """{
-    "include_profile_interstitial_type": "0",
+  "rest_id": "$1", $2
-    "include_blocking": "0",
+  "count": 20
-    "include_blocked_by": "0",
+}"""
-    "include_followed_by": "0",
-    "include_want_retweets": "0",
-    "include_mute_edge": "0",
-    "include_can_dm": "0",
-    "include_can_media_tag": "1",
-    "skip_status": "1",
-    "cards_platform": "Web-12",
-    "include_cards": "1",
-    "include_composer_source": "false",
-    "include_reply_count": "1",
-    "tweet_mode": "extended",
-    "include_entities": "true",
-    "include_user_entities": "true",
-    "include_ext_media_color": "false",
-    "send_error_codes": "true",
-    "simple_quoted_tweet": "true",
-    "include_quote_count": "true"
-  }.toSeq
 
-  searchParams* = {
+  userMediaVars* = """{
-    "query_source": "typed_query",
+  "userId": "$1", $2
-    "pc": "1",
+  "count": 20,
-    "spelling_corrections": "1"
+  "includePromotedContent": false,
-  }.toSeq
+  "withClientEventToken": false,
-  ## top: nothing
+  "withBirdwatchNotes": false,
-  ## latest: "tweet_search_mode: live"
+  "withVoice": true
-  ## user:   "result_filter: user"
+}""".replace(" ", "").replace("\n", "")
-  ## photos: "result_filter: photos"
+
-  ## videos: "result_filter: videos"
+  userTweetsVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withQuickPromoteEligibilityTweetFields": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  userTweetsAndRepliesVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withCommunity": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
+  userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
+  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql, timeline]
+import parser/[user, graphql]
-export user, graphql, timeline
+export user, graphql

src/experimental/parser/graphql.nim

@@ -1,11 +1,53 @@
+import options, strutils
 import jsony
-import user, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers]
-from ../../types import User, Result, Query, QueryKind
+from ../../types import User, VerifiedType, Result, Query, QueryKind
+
+proc parseUserResult*(userResult: UserResult): User =
+  result = userResult.legacy
+
+  if result.verifiedType == none and userResult.isBlueVerified:
+    result.verifiedType = blue
+
+  if result.username.len == 0 and userResult.core.screenName.len > 0:
+    result.id = userResult.restId
+    result.username = userResult.core.screenName
+    result.fullname = userResult.core.name
+    result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
+
+    if userResult.privacy.isSome:
+      result.protected = userResult.privacy.get.protected
+
+    if userResult.location.isSome:
+      result.location = userResult.location.get.location
+
+    if userResult.core.createdAt.len > 0:
+      result.joinDate = parseTwitterDate(userResult.core.createdAt)
+
+    if userResult.verification.isSome:
+      let v = userResult.verification.get
+      if v.verifiedType != VerifiedType.none:
+        result.verifiedType = v.verifiedType
+
+    if userResult.profileBio.isSome and result.bio.len == 0:
+      result.bio = userResult.profileBio.get.description
 
 proc parseGraphUser*(json: string): User =
-  let raw = json.fromJson(GraphUser)
+  if json.len == 0 or json[0] != '{':
-  result = toUser raw.data.user.result.legacy
+    return
-  result.id = raw.data.user.result.restId
+
+  let 
+    raw = json.fromJson(GraphUser)
+    userResult = 
+      if raw.data.userResult.isSome: raw.data.userResult.get.result
+      elif raw.data.user.isSome: raw.data.user.get.result
+      else: UserResult()
+
+  if userResult.unavailableReason.get("") == "Suspended" or
+     userResult.reason.get("") == "Suspended":
+    return User(suspended: true)
+
+  result = parseUserResult(userResult)
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](
@@ -21,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineItem:
           let userResult = entry.content.itemContent.userResults.result
           if userResult.restId.len > 0:
-            result.content.add toUser userResult.legacy
+            result.content.add parseUserResult(userResult)
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value

src/experimental/parser/session.nim

@@ -0,0 +1,30 @@
+import std/strutils
+import jsony
+import ../types/session
+from ../../types import Session, SessionKind
+
+proc parseSession*(raw: string): Session =
+  let session = raw.fromJson(RawSession)
+  let kind = if session.kind == "": "oauth" else: session.kind
+
+  case kind
+  of "oauth":
+    let id = session.oauthToken[0 ..< session.oauthToken.find('-')]
+    result = Session(
+      kind: SessionKind.oauth,
+      id: parseBiggestInt(id),
+      username: session.username,
+      oauthToken: session.oauthToken,
+      oauthSecret: session.oauthTokenSecret
+    )
+  of "cookie":
+    let id = if session.id.len > 0: parseBiggestInt(session.id) else: 0
+    result = Session(
+      kind: SessionKind.cookie,
+      id: id,
+      username: session.username,
+      authToken: session.authToken,
+      ct0: session.ct0
+    )
+  else:
+    raise newException(ValueError, "Unknown session kind: " & kind)

src/experimental/parser/tid.nim

@@ -0,0 +1,8 @@
+import jsony
+import ../types/tid
+export TidPair
+
+proc parseTidPairs*(raw: string): seq[TidPair] =
+  result = raw.fromJson(seq[TidPair])
+  if result.len == 0:
+    raise newException(ValueError, "Parsing pairs failed: " & raw)

src/experimental/parser/timeline.nim

@@ -1,28 +0,0 @@
-import std/[strutils, tables]
-import jsony
-import user, ../types/timeline
-from ../../types import Result, User
-
-proc getId(id: string): string {.inline.} =
-  let start = id.rfind("-")
-  if start < 0: return id
-  id[start + 1 ..< id.len]
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  let raw = json.fromJson(Search)
-  if raw.timeline.instructions.len == 0:
-    return
-
-  for e in raw.timeline.instructions[0].addEntries.entries:
-    let id = e.entryId.getId
-    if e.entryId.startsWith("user"):
-      if id in raw.globalObjects.users:
-        result.content.add toUser raw.globalObjects.users[id]
-    elif e.entryId.startsWith("cursor"):
-      let cursor = e.content.operation.cursor
-      if cursor.cursorType == "Top":
-        result.top = cursor.value
-      elif cursor.cursorType == "Bottom":
-        result.bottom = cursor.value

src/experimental/parser/unifiedcard.nim

@@ -1,6 +1,7 @@
 import std/[options, tables, strutils, strformat, sugar]
 import jsony
-import ../types/unifiedcard
+import user, ../types/unifiedcard
+import ../../formatters
 from ../../types import Card, CardKind, Video
 from ../../utils import twimg, https
 
@@ -27,6 +28,14 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card)
   result.text = data.topicDetail.title
   result.dest = "Topic"
 
+proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  data.destination.parseDestination(card, result)
+
+  result.kind = CardKind.jobDetails
+  result.title = data.title
+  result.text = data.shortDescriptionText
+  result.dest = &"@{data.profileUser.username} · {data.location}"
+
 proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
   let app = card.appStoreData[data.appId][0]
 
@@ -66,6 +75,20 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
+  of model3d:
+    result.title = "Unsupported 3D model ad"
+
+proc parseGrokShare(data: ComponentData; card: UnifiedCard; result: var Card) =
+  result.kind = summaryLarge
+
+  data.destination.parseDestination(card, result)
+  result.dest = "Answer by Grok"
+
+  for msg in data.conversationPreview:
+    if msg.sender == "USER":
+      result.title = msg.message.shorten(70)
+    elif msg.sender == "AGENT":
+      result.text = msg.message.shorten(500)
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -82,6 +105,14 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of grokShare:
+      component.data.parseGrokShare(card, result)
+    of ComponentType.jobDetails:
+      component.data.parseJobDetails(card, result)
+    of ComponentType.hidden:
+      result.kind = CardKind.hidden
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/parser/user.nim

@@ -1,14 +1,14 @@
-import std/[algorithm, unicode, re, strutils, strformat, options]
+import std/[algorithm, unicode, re, strutils, strformat, options, nre]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import User, Error
+from ../../types import Result, User, Error
 
 let
-  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
 
-  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htRegex = nre.re"""(*U)(^|[^\w-_.?])([#＃$])([\w_]*+)(?!</a>|">|#)"""
   htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
 
 proc expandUserEntities(user: var User; raw: RawUser) =
@@ -29,7 +29,7 @@ proc expandUserEntities(user: var User; raw: RawUser) =
 
   user.bio = orig.replacedWith(replacements, 0 .. orig.len)
                  .replacef(unRegex, unReplace)
-                 .replacef(htRegex, htReplace)
+                 .replace(htRegex, htReplace)
 
 proc getBanner(user: RawUser): string =
   if user.profileBannerUrl.len > 0:
@@ -56,23 +56,21 @@ proc toUser*(raw: RawUser): User =
     tweets: raw.statusesCount,
     likes: raw.favouritesCount,
     media: raw.mediaCount,
-    verified: raw.verified,
+    verifiedType: raw.verifiedType,
     protected: raw.protected,
-    joinDate: parseTwitterDate(raw.createdAt),
     banner: getBanner(raw),
     userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
   )
 
+  if raw.createdAt.len > 0:
+    result.joinDate = parseTwitterDate(raw.createdAt)
+
   if raw.pinnedTweetIdsStr.len > 0:
     result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
 
   result.expandUserEntities(raw)
 
-proc parseUser*(json: string; username=""): User =
+proc parseHook*(s: string; i: var int; v: var User) =
-  handleErrors:
+  var u: RawUser
-    case error.code
+  parseHook(s, i, u)
-    of suspended: return User(username: username, suspended: true)
+  v = toUser u
-    of userNotFound: return
-    else: echo "[error - parseUser]: ", error
-
-  result = toUser json.fromJson(RawUser)

src/experimental/types/graphuser.nim

@@ -1,12 +1,48 @@
-import user
+import options, strutils
+from ../../types import User, VerifiedType
 
 type
   GraphUser* = object
-    data*: tuple[user: UserData]
+    data*: tuple[userResult: Option[UserData], user: Option[UserData]]
 
   UserData* = object
     result*: UserResult
 
-  UserResult = object
+  UserCore* = object
-    legacy*: RawUser
+    name*: string
+    screenName*: string
+    createdAt*: string
+
+  UserBio* = object
+    description*: string
+
+  UserAvatar* = object
+    imageUrl*: string
+
+  Verification* = object
+    verifiedType*: VerifiedType
+
+  Location* = object
+    location*: string
+
+  Privacy* = object
+    protected*: bool
+
+  UserResult* = object
+    legacy*: User
     restId*: string
+    isBlueVerified*: bool
+    core*: UserCore
+    avatar*: UserAvatar
+    unavailableReason*: Option[string]
+    reason*: Option[string]
+    privacy*: Option[Privacy]
+    profileBio*: Option[UserBio]
+    verification*: Option[Verification]
+    location*: Option[Location]
+
+proc enumHook*(s: string; v: var VerifiedType) =
+  v = try:
+    parseEnum[VerifiedType](s)
+  except:
+    VerifiedType.none

src/experimental/types/session.nim

@@ -0,0 +1,9 @@
+type
+  RawSession* = object
+    kind*: string
+    id*: string
+    username*: string
+    oauthToken*: string
+    oauthTokenSecret*: string
+    authToken*: string
+    ct0*: string

src/experimental/types/tid.nim

@@ -0,0 +1,4 @@
+type
+  TidPair* = object
+    animationKey*: string
+    verification*: string

src/experimental/types/timeline.nim

@@ -1,23 +0,0 @@
-import std/tables
-import user
-
-type
-  Search* = object
-    globalObjects*: GlobalObjects
-    timeline*: Timeline
-
-  GlobalObjects = object
-    users*: Table[string, RawUser]
-
-  Timeline = object
-    instructions*: seq[Instructions]
-
-  Instructions = object
-    addEntries*: tuple[entries: seq[Entry]]
-
-  Entry = object
-    entryId*: string
-    content*: tuple[operation: Operation]
-
-  Operation = object
-    cursor*: tuple[value, cursorType: string]

src/experimental/types/unifiedcard.nim

@@ -1,7 +1,10 @@
-import options, tables
+import std/[options, tables, times]
-from ../../types import VideoType, VideoVariant
+import jsony
+from ../../types import VideoType, VideoVariant, User
 
 type
+  Text* = distinct string
+
   UnifiedCard* = object
     componentObjects*: Table[string, Component]
     destinationObjects*: Table[string, Destination]
@@ -13,10 +16,14 @@ type
     media
     swipeableMedia
     buttonGroup
+    jobDetails
     appStoreDetails
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
+    hidden
+    grokShare
+    unknown
 
   Component* = object
     kind*: ComponentType
@@ -27,12 +34,16 @@ type
     appId*: string
     mediaId*: string
     destination*: string
+    location*: string
     title*: Text
     subtitle*: Text
     name*: Text
     memberCount*: int
     mediaList*: seq[MediaItem]
     topicDetail*: tuple[title: Text]
+    profileUser*: User
+    shortDescriptionText*: string
+    conversationPreview*: seq[GrokConversation]
 
   MediaItem* = object
     id*: string
@@ -47,7 +58,7 @@ type
     vanity*: string
 
   MediaType* = enum
-    photo, video
+    photo, video, model3d
 
   MediaEntity* = object
     kind*: MediaType
@@ -67,13 +78,58 @@ type
     title*: Text
     category*: Text
 
-  Text = object
+  GrokConversation* = object
-    content: string
+    message*: string
+    sender*: string
 
-  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+  TypeField = Component | Destination | MediaEntity | AppStoreData
 
-converter fromText*(text: Text): string = text.content
+converter fromText*(text: Text): string = string(text)
 
-proc renameHook*(v: var HasTypeField; fieldName: var string) =
+proc renameHook*(v: var TypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "job_details": jobDetails
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      of "commerce_drop_details": hidden
+      of "grok_share": grokShare
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
+
+proc parseHook*(s: string; i: var int; v: var DateTime) =
+  var str: string
+  parseHook(s, i, str)
+  v = parse(str, "yyyy-MM-dd hh:mm:ss")
+
+proc parseHook*(s: string; i: var int; v: var Text) =
+  if s[i] == '"':
+    var str: string
+    parseHook(s, i, str)
+    v = Text(str)
+  else:
+    var t: tuple[content: string]
+    parseHook(s, i, t)
+    v = Text(t.content)

src/experimental/types/user.nim

@@ -1,5 +1,6 @@
 import options
 import common
+from ../../types import VerifiedType
 
 type
   RawUser* = object
@@ -15,7 +16,7 @@ type
     favouritesCount*: int
     statusesCount*: int
     mediaCount*: int
-    verified*: bool
+    verifiedType*: VerifiedType
     protected*: bool
     profileLinkColor*: string
     profileBannerUrl*: string

src/formatters.nim

@@ -1,19 +1,20 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, times, uri, tables, xmltree, htmlparser, htmlgen
+import strutils, strformat, times, uri, tables, xmltree, htmlparser, htmlgen, math
 import std/[enumerate, re]
 import types, utils, query
 
 const
   cards = "cards.twitter.com/cards"
   tco = "https://t.co"
-  twitter = parseUri("https://twitter.com")
+  twitter = parseUri("https://x.com")
 
 let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
+  xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
+  xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
 
-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
-  igRegex = re"(www\.)?instagram\.com"
 
   rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
   rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -32,11 +33,14 @@ proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
   else: "http://" & cfg.hostname
 
-proc shortLink*(text: string; length=28): string =
+proc shorten*(text: string; length=28): string =
-  result = text.replace(wwwRegex, "")
+  result = text
   if result.len > length:
     result = result[0 ..< length] & "…"
 
+proc shortLink*(text: string; length=28): string =
+  result = text.replace(wwwRegex, "").shorten(length)
+    
 proc stripHtml*(text: string; shorten=false): string =
   var html = parseHtml(text)
   for el in html.findAll("a"):
@@ -55,28 +59,32 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   result = body
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
-    result = result.replace(ytRegex, prefs.replaceYouTube)
+    let youtubeHost = strip(prefs.replaceYouTube, chars={'/'})
-    if prefs.replaceYouTube in result:
+    result = result.replace(ytRegex, youtubeHost)
-      result = result.replace("/c/", "/")
 
-  if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
+  if prefs.replaceTwitter.len > 0:
-    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
+    let twitterHost = strip(prefs.replaceTwitter, chars={'/'})
-    result = result.replace(cards, prefs.replaceTwitter & "/cards")
+    if tco in result:
-    result = result.replace(twRegex, prefs.replaceTwitter)
+      result = result.replace(tco, https & twitterHost & "/t.co")
+    if "x.com" in result:
+      result = result.replace(xRegex, twitterHost)
+      result = result.replacef(xLinkRegex, a(
+        twitterHost & "$2", href = https & twitterHost & "$1"))
+    if "twitter.com" in result:
+      result = result.replace(cards, twitterHost & "/cards")
+      result = result.replace(twRegex, twitterHost)
       result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
+        twitterHost & "$2", href = https & twitterHost & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
-    result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
+    let redditHost = strip(prefs.replaceReddit, chars={'/'})
-    result = result.replace(rdRegex, prefs.replaceReddit)
+    result = result.replace(rdShortRegex, redditHost & "/comments/")
-    if prefs.replaceReddit in result and "/gallery/" in result:
+    result = result.replace(rdRegex, redditHost)
+    if redditHost in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
   if absolute.len > 0 and "href" in result:
-    result = result.replace("href=\"/", "href=\"" & absolute & "/")
+    result = result.replace("href=\"/", &"href=\"{absolute}/")
 
 proc getM3u8Url*(content: string): string =
   var matches: array[1, string]
@@ -88,6 +96,8 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
   for line in manifest.splitLines:
     let url =
       if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
+      elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
+        line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
       else: line
     if url.startsWith('/'):
       let path = "https://video.twimg.com" & url
@@ -144,6 +154,17 @@ proc getShortTime*(tweet: Tweet): string =
   else:
     result = "now"
 
+proc getDuration*(video: Video): string =
+  let 
+    ms = video.durationMs
+    sec = int(round(ms / 1000))
+    min = floorDiv(sec, 60)
+    hour = floorDiv(min, 60)
+  if hour > 0:
+    return &"{hour}:{min mod 60}:{sec mod 60:02}"
+  else:
+    return &"{min mod 60}:{sec mod 60:02}"
+
 proc getLink*(tweet: Tweet; focus=true): string =
   if tweet.id == 0: return
   var username = tweet.user.username

src/http_pool.nim

@@ -39,8 +39,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
 
   try:
     body
-  except ProtocolError:
+  except BadClientError, ProtocolError:
-    # Twitter closed the connection, retry
+    # Twitter returned 503 or closed the connection, we need a new client
+    pool.release(c, true)
+    badClient = false
+    c = pool.acquire(heads)
     body
   finally:
     pool.release(c, badClient)

src/nitter.nim

@@ -6,7 +6,7 @@ from os import getEnv
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool, tokens
+import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
@@ -15,8 +15,13 @@ import routes/[
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
 
-let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+let
-let (cfg, fullCfg) = getConfig(configPath)
+  configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
+  (cfg, fullCfg) = getConfig(configPath)
+
+  sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
+
+initSessionPool(cfg, sessionsPath)
 
 if not cfg.enableDebug:
   # Silence Jester's query warning
@@ -32,14 +37,13 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setDisableTid(cfg.disableTid)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)
 stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
 stdout.flushFile
 
-asyncCheck initTokenPool(cfg)
-
 createUnsupportedRouter(cfg)
 createResolverRouter(cfg)
 createPrefRouter(cfg)
@@ -56,6 +60,7 @@ settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir
   bindAddr = cfg.address
+  reusePort = true
 
 routes:
   get "/":
@@ -84,20 +89,28 @@ routes:
     resp Http500, showError(
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
-  error RateLimitError:
+  error BadClientError:
     echo error.exc.name, ": ", error.exc.msg
+    resp Http500, showError("Network error occurred, please try again.", cfg)
+
+  error RateLimitError:
     const link = a("another instance", href = instancesUrl)
     resp Http429, showError(
       &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
 
-  extend unsupported, ""
+  error NoSessionsError:
-  extend preferences, ""
+    const link = a("another instance", href = instancesUrl)
-  extend resolver, ""
+    resp Http429, showError(
+      &"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
+
   extend rss, ""
+  extend status, ""
   extend search, ""
   extend timeline, ""
-  extend list, ""
-  extend status, ""
   extend media, ""
+  extend list, ""
+  extend preferences, ""
+  extend resolver, ""
   extend embed, ""
   extend debug, ""
+  extend unsupported, ""

src/parser.nim

@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, tables, times, math
+import strutils, options, times, math, tables
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -19,13 +21,46 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verified: js{"verified"}.getBool,
+    protected: js{"protected"}.getBool(js{"privacy", "protected"}.getBool),
-    protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
+  if js{"is_blue_verified"}.getBool(false):
+    result.verifiedType = blue
+
+  with verifiedType, js{"verified_type"}:
+    result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
+
   result.expandUserEntities(js)
 
+proc parseGraphUser(js: JsonNode): User =
+  var user = js{"user_result", "result"}
+  if user.isNull:
+    user = ? js{"user_results", "result"}
+
+  if user.isNull:
+    if js{"core"}.notNull and js{"legacy"}.notNull:
+      user = js
+    else:
+      return
+
+  result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
+
+  if result.verifiedType == none and user{"is_blue_verified"}.getBool(false):
+    result.verifiedType = blue
+
+  # fallback to support UserMedia/recent GraphQL updates
+  if result.username.len == 0:
+    result.username = user{"core", "screen_name"}.getStr
+    result.fullname = user{"core", "name"}.getStr
+    result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+
+    if user{"is_blue_verified"}.getBool(false):
+      result.verifiedType = blue
+
+    with verifiedType, user{"verification", "verified_type"}:
+      result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -38,14 +73,13 @@ proc parseGraphList*(js: JsonNode): List =
   result = List(
     id: list{"id_str"}.getStr,
     name: list{"name"}.getStr,
-    username: list{"user", "legacy", "screen_name"}.getStr,
+    username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
-    userId: list{"user", "rest_id"}.getStr,
+    userId: list{"user_results", "result", "rest_id"}.getStr,
     description: list{"description"}.getStr,
     members: list{"member_count"}.getInt,
-    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
+    banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
   )
 
-
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -64,34 +98,96 @@ proc parsePoll(js: JsonNode): Poll =
   result.leader = result.values.find(max(result.values))
   result.votes = result.values.sum
 
-proc parseGif(js: JsonNode): Gif =
+proc parseVideoVariants(variants: JsonNode): seq[VideoVariant] =
-  result = Gif(
+  result = @[]
-    url: js{"video_info", "variants"}[0]{"url"}.getImageStr,
+  for v in variants:
-    thumb: js{"media_url_https"}.getImageStr
+    let
+      url = v{"url"}.getStr
+      contentType = parseEnum[VideoType](v{"content_type"}.getStr("video/mp4"))
+      bitrate = v{"bit_rate"}.getInt(v{"bitrate"}.getInt(0))
+
+    result.add VideoVariant(
+      contentType: contentType,
+      bitrate: bitrate,
+      url: url,
+      resolution: if contentType == mp4: getMp4Resolution(url) else: 0
     )
 
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
+    available: true,
-    available: js{"ext_media_availability", "status"}.getStr == "available",
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
     # playbackType: mp4
   )
 
+  with status, js{"ext_media_availability", "status"}:
+    if status.getStr.len > 0 and status.getStr.toLowerAscii != "available":
+      result.available = false
+
   with title, js{"additional_media_info", "title"}:
     result.title = title.getStr
 
   with description, js{"additional_media_info", "description"}:
     result.description = description.getStr
 
-  for v in js{"video_info", "variants"}:
+  result.variants = parseVideoVariants(js{"video_info", "variants"})
-    result.variants.add VideoVariant(
+
-      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
+proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
-      bitrate: v{"bitrate"}.getInt,
+  with jsMedia, js{"extended_entities", "media"}:
-      url: v{"url"}.getStr
+    for m in jsMedia:
+      case m.getTypeName:
+      of "photo":
+        result.photos.add m{"media_url_https"}.getImageStr
+      of "video":
+        result.video = some(parseVideo(m))
+        with user, m{"additional_media_info", "source_user"}:
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
+      of "animated_gif":
+        result.gif = some Gif(
+          url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
+          thumb: m{"media_url_https"}.getImageStr
         )
+      else: discard
+
+      with url, m{"url"}:
+        if result.text.endsWith(url.getStr):
+          result.text.removeSuffix(url.getStr)
+          result.text = result.text.strip()
+
+proc parseMediaEntities(js: JsonNode; result: var Tweet) =
+  with mediaEntities, js{"media_entities"}:
+    for mediaEntity in mediaEntities:
+      with mediaInfo, mediaEntity{"media_results", "result", "media_info"}:
+        case mediaInfo.getTypeName
+        of "ApiImage":
+          result.photos.add mediaInfo{"original_img_url"}.getImageStr
+        of "ApiVideo":
+          let status = mediaEntity{"media_results", "result", "media_availability_v2", "status"}
+          result.video = some Video(
+            available: status.getStr == "Available",
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr,
+            durationMs: mediaInfo{"duration_millis"}.getInt,
+            variants: parseVideoVariants(mediaInfo{"variants"})
+          )
+        of "ApiGif":
+          result.gif = some Gif(
+            url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
+            thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr
+          )
+        else: discard
+
+  # Remove media URLs from text
+  with mediaList, js{"legacy", "entities", "media"}:
+    for url in mediaList:
+      let expandedUrl = url.getExpandedUrl
+      if result.text.endsWith(expandedUrl):
+        result.text.removeSuffix(expandedUrl)
+        result.text = result.text.strip()
 
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
@@ -171,7 +267,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
 
   for u in ? urls:
     if u{"url"}.getStr == result.url:
-      result.url = u{"expanded_url"}.getStr
+      result.url = u.getExpandedUrl(result.url)
       break
 
   if kind in {videoDirectMessage, imageDirectMessage}:
@@ -181,14 +277,19 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
      result.url.len == 0 or result.url.startsWith("card://"):
     result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
+
+  let time =
+    if js{"created_at"}.notNull: js{"created_at"}.getTime
+    else: js{"created_at_ms"}.getTimeFromMs
+
   result = Tweet(
     id: js{"id_str"}.getId,
     threadId: js{"conversation_id_str"}.getId,
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
-    time: js{"created_at"}.getTime,
+    time: time,
     hasThread: js{"self_thread"}.notNull,
     available: true,
     user: User(id: js{"user_id_str"}.getStr),
@@ -196,20 +297,38 @@ proc parseTweet(js: JsonNode): Tweet =
       replies: js{"reply_count"}.getInt,
       retweets: js{"retweet_count"}.getInt,
       likes: js{"favorite_count"}.getInt,
-      quotes: js{"quote_count"}.getInt
+      views: js{"views_count"}.getInt
     )
   )
 
-  result.expandTweetEntities(js)
+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
 
-  if js{"is_quote_status"}.getBool:
+  if "retweeted_status" in js:
+    result.retweet = some Tweet()
+  elif js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
+  # legacy
   with rt, js{"retweeted_status_id_str"}:
     result.retweet = some Tweet(id: rt.getId)
     return
 
-  with jsCard, js{"card"}:
+  # graphql
+  with rt, js{"retweeted_status_result", "result"}:
+    # needed due to weird edgecase where the actual tweet data isn't included
+    if "legacy" in rt:
+      result.retweet = some parseGraphTweet(rt)
+      return
+
+  with reposts, js{"repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt:
+        result.retweet = some parseGraphTweet(rt)
+        return
+
+  if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
       if "image" in name:
@@ -221,18 +340,8 @@ proc parseTweet(js: JsonNode): Tweet =
     else:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
-  with jsMedia, js{"extended_entities", "media"}:
+  result.expandTweetEntities(js)
-    for m in jsMedia:
+  parseLegacyMediaEntities(js, result)
-      case m{"type"}.getStr
-      of "photo":
-        result.photos.add m{"media_url_https"}.getImageStr
-      of "video":
-        result.video = some(parseVideo(m))
-        with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseUser(user))
-      of "animated_gif":
-        result.gif = some(parseGif(m))
-      else: discard
 
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =
@@ -248,159 +357,265 @@ proc parseTweet(js: JsonNode): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
+proc parseGraphTweet(js: JsonNode): Tweet =
-  let intId = if id.len > 0: parseBiggestInt(id) else: 0
+  if js.kind == JNull:
-  result = global.tweets.getOrDefault(id, Tweet(id: intId))
+    return Tweet()
+
+  case js.getTypeName:
+  of "TweetUnavailable":
+    return Tweet()
+  of "TweetTombstone":
+    with text, select(js{"tombstone", "richText"}, js{"tombstone", "text"}):
+      return Tweet(text: text.getTombstone)
+    return Tweet()
+  of "TweetPreviewDisplay":
+    return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
+  of "TweetWithVisibilityResults":
+    return parseGraphTweet(js{"tweet"})
+  else:
+    discard
+
+  if not js.hasKey("legacy"):
+    return Tweet()
+
+  var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
+  if jsCard.kind != JNull:
+    let legacyCard = jsCard{"legacy"}
+    if legacyCard.kind != JNull:
+      let bindingArray = legacyCard{"binding_values"}
+      if bindingArray.kind == JArray:
+        var bindingObj: seq[(string, JsonNode)]
+        for item in bindingArray:
+          bindingObj.add((item{"key"}.getStr, item{"value"}))
+        # Create a new card object with flattened structure
+        jsCard = %*{
+          "name": legacyCard{"name"},
+          "url": legacyCard{"url"},
+          "binding_values": %bindingObj
+        }
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.id = js{"rest_id"}.getId
+  result.user = parseGraphUser(js{"core"})
+
+  if result.replyId == 0:
+    result.replyId = js{"reply_to_results", "rest_id"}.getId
+
+  with count, js{"views", "count"}:
+    result.stats.views = count.getStr("0").parseInt
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  parseMediaEntities(js, result)
 
   if result.quote.isSome:
-    let quote = get(result.quote).id
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
-    if $quote in global.tweets:
-      result.quote = some global.tweets[$quote]
-    else:
-      result.quote = some Tweet()
 
-  if result.retweet.isSome:
+  with quoted, js{"quotedPostResults", "result"}:
-    let rt = get(result.retweet).id
+    result.quote = some(parseGraphTweet(quoted))
-    if $rt in global.tweets:
-      result.retweet = some finalizeTweet(global, $rt)
-    else:
-      result.retweet = some Tweet()
 
-proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
-  let pin = js{"pinEntry", "entry", "entryId"}.getStr
+  for t in ? js{"content", "items"}:
-  if pin.len == 0: return
+    let entryId = t.getEntryId
-
+    if "cursor-showmore" in entryId:
-  let id = pin.getId
+      let cursor = t{"item", "content", "value"}
-  if id notin global.tweets: return
+      result.thread.cursor = cursor.getStr
-
-  global.tweets[id].pinned = true
-  return finalizeTweet(global, id)
-
-proc parseGlobalObjects(js: JsonNode): GlobalObjects =
-  result = GlobalObjects()
-  let
-    tweets = ? js{"globalObjects", "tweets"}
-    users = ? js{"globalObjects", "users"}
-
-  for k, v in users:
-    result.users[k] = parseUser(v, k)
-
-  for k, v in tweets:
-    var tweet = parseTweet(v)
-    if tweet.user.id in result.users:
-      tweet.user = result.users[tweet.user.id]
-    result.tweets[k] = tweet
-
-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
       result.thread.hasMore = true
+    elif "tweet" in entryId and "promoted" notin entryId:
+      with tweet, t.getTweetResult("item"):
+        result.thread.content.add parseGraphTweet(tweet)
 
-  for t in thread{"conversationComponents"}:
+        let tweetDisplayType = select(
-    let content = t{"conversationTweetComponent", "tweet"}
+          t{"item", "content", "tweet_display_type"},
-
+          t{"item", "itemContent", "tweetDisplayType"}
-    if content{"displayType"}.getStr == "SelfThread":
+        )
+        if tweetDisplayType.getStr == "SelfThread":
           result.self = true
 
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
+proc parseGraphTweetResult*(js: JsonNode): Tweet =
-    if not tweet.available:
+  with tweet, js{"data", "tweet_result", "result"}:
-      tweet.tombstone = getTombstone(content{"tombstone"})
+    result = parseGraphTweet(tweet)
-    result.thread.content.add tweet
 
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
 
-  let instructions = ? js{"timeline", "instructions"}
+  let instructions = ? select(
+    js{"data", "timelineResponse", "instructions"},
+    js{"data", "timeline_response", "instructions"},
+    js{"data", "threaded_conversation_with_injections_v2", "instructions"}
+  )
   if instructions.len == 0:
     return
 
-  for e in instructions[0]{"addEntries", "entries"}:
+  for i in instructions:
-    let entry = e{"entryId"}.getStr
+    if i.getTypeName == "TimelineAddEntries":
-    if "tweet" in entry or "tombstone" in entry:
+      for e in i{"entries"}:
-      let tweet = finalizeTweet(global, e.getEntryId)
+        let entryId = e.getEntryId
-      if $tweet.id != tweetId:
+        if entryId.startsWith("tweet"):
-        result.before.content.add tweet
+          let tweetResult = getTweetResult(e)
-      else:
+          if tweetResult.notNull:
+            let tweet = parseGraphTweet(tweetResult)
+
+            if not tweet.available:
+              tweet.id = entryId.getId
+
+            if $tweet.id == tweetId:
               result.tweet = tweet
-    elif "conversationThread" in entry:
+            else:
-      let (thread, self) = parseThread(e, global)
+              result.before.content.add tweet
-      if thread.content.len > 0:
+        elif entryId.startsWith("conversationthread"):
+          let (thread, self) = parseGraphThread(e)
           if self:
             result.after = thread
-        else:
+          elif thread.content.len > 0:
             result.replies.content.add thread
-    elif "cursor-showMore" in entry:
+        elif entryId.startsWith("tombstone"):
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
-proc parseStatus*(js: JsonNode): Tweet =
-  with e, js{"errors"}:
-    if e.getError == tweetNotFound:
-      return
-
-  result = parseTweet(js)
-  if not result.isNil:
-    result.user = parseUser(js{"user"})
-
-    with quote, js{"quoted_status"}:
-      result.quote = some parseStatus(js{"quoted_status"})
-
-proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
-  if js.kind != JArray or js.len == 0:
-    return
-
-  for i in js:
-    when T is Tweet:
-      if res.beginning and i{"pinEntry"}.notNull:
-        with pin, parsePin(i, global):
-          res.content.add pin
-
-    with r, i{"replaceEntry", "entry"}:
-      if "top" in r{"entryId"}.getStr:
-        res.top = r.getCursor
-      elif "bottom" in r{"entryId"}.getStr:
-        res.bottom = r.getCursor
-
-proc parseTimeline*(js: JsonNode; after=""): Timeline =
-  result = Timeline(beginning: after.len == 0)
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0: return
-
-  result.parseInstructions(global, instructions)
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if not tweet.available: continue
-      result.content.add tweet
-    elif "cursor-top" in entry:
-      result.top = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.bottom = e.getCursor
-    elif entry.startsWith("sq-C"):
-      with cursor, e{"content", "operation", "cursor"}:
-        if cursor{"cursorType"}.getStr == "Bottom":
-          result.bottom = cursor{"value"}.getStr
-        else:
-          result.top = cursor{"value"}.getStr
-
-proc parsePhotoRail*(js: JsonNode): PhotoRail =
-  for tweet in js:
           let
-      t = parseTweet(tweet)
+            content = select(e{"content", "content"}, e{"content", "itemContent"})
-      url = if t.photos.len > 0: t.photos[0]
+            tweet = Tweet(
-            elif t.video.isSome: get(t.video).thumb
+              id: entryId.getId,
-            elif t.gif.isSome: get(t.gif).thumb
+              available: false,
-            elif t.card.isSome: get(t.card).image
+              text: content{"tombstoneInfo", "richText"}.getTombstone
-            else: ""
+            )
 
-    if url.len == 0: continue
+          if $tweet.id == tweetId:
-    result.add GalleryPhoto(url: url, tweetId: $t.id)
+            result.tweet = tweet
+          else:
+            result.before.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          var cursorValue = select(
+            e{"content", "value"},
+            e{"content", "content", "value"},
+            e{"content", "itemContent", "value"}
+          )
+          result.replies.bottom = cursorValue.getStr
+
+proc extractTweetsFromEntry*(e: JsonNode): seq[Tweet] =
+  with tweetResult, getTweetResult(e):
+    var tweet = parseGraphTweet(tweetResult)
+    if not tweet.available:
+      tweet.id = e.getEntryId.getId
+    result.add tweet
+    return
+
+  for item in e{"content", "items"}:
+    with tweetResult, item.getTweetResult("item"):
+      var tweet = parseGraphTweet(tweetResult)
+      if not tweet.available:
+        tweet.id = item.getEntryId.getId
+      result.add tweet
+
+proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
+  result = Profile(tweets: Timeline(beginning: after.len == 0))
+
+  let instructions = ? select(
+    js{"data", "list", "timeline_response", "timeline", "instructions"},
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+  )
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"moduleItems"}.notNull:
+      for item in i{"moduleItems"}:
+        with tweetResult, item.getTweetResult("item"):
+          let tweet = parseGraphTweet(tweetResult)
+          if not tweet.available:
+            tweet.id = item.getEntryId.getId
+          result.tweets.content.add tweet
+      continue
+
+    if i{"entries"}.notNull:
+      for e in i{"entries"}:
+        let entryId = e.getEntryId
+        if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
+          for tweet in extractTweetsFromEntry(e):
+            result.tweets.content.add tweet
+        elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
+          let (thread, self) = parseGraphThread(e)
+          result.tweets.content.add thread.content
+        elif entryId.startsWith("cursor-bottom"):
+          result.tweets.bottom = e{"content", "value"}.getStr
+
+    if after.len == 0:
+      if i.getTypeName == "TimelinePinEntry":
+        let tweets = extractTweetsFromEntry(i{"entry"})
+        if tweets.len > 0:
+          var tweet = tweets[0]
+          tweet.pinned = true
+          result.pinned = some tweet
+
+proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+  result = @[]
+
+  let instructions = select(
+    js{"data", "user", "result", "timeline", "timeline", "instructions"},
+    js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+  )
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"moduleItems"}.notNull:
+      for item in i{"moduleItems"}:
+        with tweetResult, item.getTweetResult("item"):
+          let t = parseGraphTweet(tweetResult)
+          if not t.available:
+            t.id = item.getEntryId.getId
+
+          let photo = extractGalleryPhoto(t)
+          if photo.url.len > 0:
+            result.add photo
+
+          if result.len == 16:
+            return
+      continue
+
+    if i.getTypeName != "TimelineAddEntries":
+      continue
+
+    for e in i{"entries"}:
+      let entryId = e.getEntryId
+      if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
+        for t in extractTweetsFromEntry(e):
+          let photo = extractGalleryPhoto(t)
+          if photo.url.len > 0:
+            result.add photo
+
+          if result.len == 16:
+            return
+
+proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
+  result = Result[T](beginning: after.len == 0)
+
+  let instructions = select(
+    js{"data", "search", "timeline_response", "timeline", "instructions"},
+    js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  )
+  if instructions.len == 0:
+    return
+
+  for instruction in instructions:
+    let typ = getTypeName(instruction)
+    if typ == "TimelineAddEntries":
+      for e in instruction{"entries"}:
+        let entryId = e.getEntryId
+        when T is Tweets:
+          if entryId.startsWith("tweet"):
+            with tweetRes, getTweetResult(e):
+              let tweet = parseGraphTweet(tweetRes)
+              if not tweet.available:
+                tweet.id = entryId.getId
+              result.content.add tweet
+        elif T is User:
+          if entryId.startsWith("user"):
+            with userRes, e{"content", "itemContent"}:
+              result.content.add parseGraphUser(userRes)
+
+        if entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+    elif typ == "TimelineReplaceEntry":
+      if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
+        result.bottom = instruction{"entry", "content", "value"}.getStr

src/parserutils.nim

@@ -1,9 +1,17 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import std/[strutils, times, macros, htmlgen, options, algorithm, re]
+import std/[times, macros, htmlgen, options, algorithm, re]
+import std/strutils except escape
 import std/unicode except strip
+from xmltree import escape
 import packedjson
 import types, utils, formatters
 
+const
+  unicodeOpen = "\uFFFA"
+  unicodeClose = "\uFFFB"
+  xmlOpen = escape("<")
+  xmlClose = escape(">")
+
 let
   unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
   unReplace = "$1<a href=\"/$2\">@$2</a>"
@@ -28,13 +36,19 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template `with`*(ident, value, body): untyped =
+template select*(a, b: JsonNode): untyped =
-  block:
+  if a.notNull: a else: b
+
+template select*(a, b, c: JsonNode): untyped =
+  if a.notNull: a elif b.notNull: b else: c
+
+template with*(ident, value, body): untyped =
+  if true:
     let ident {.inject.} = value
     if ident != nil: body
 
-template `with`*(ident; value: JsonNode; body): untyped =
+template with*(ident; value: JsonNode; body): untyped =
-  block:
+  if true:
     let ident {.inject.} = value
     if value.notNull: body
 
@@ -45,6 +59,20 @@ template getError*(js: JsonNode): Error =
   if js.kind != JArray or js.len == 0: null
   else: Error(js[0]{"code"}.getInt)
 
+proc getTweetResult*(js: JsonNode; root="content"): JsonNode =
+  select(
+    js{root, "content", "tweet_results", "result"},
+    js{root, "itemContent", "tweet_results", "result"},
+    js{root, "content", "tweetResult", "result"}
+  )
+
+template getTypeName*(js: JsonNode): string =
+  js{"__typename"}.getStr(js{"type"}.getStr)
+
+template getEntryId*(e: JsonNode): string =
+  e{"entryId"}.getStr(e{"entry_id"}.getStr)
+
+
 template parseTime(time: string; f: static string; flen: int): DateTime =
   if time.len != flen: return
   parse(time, f, utc())
@@ -55,29 +83,24 @@ proc getDateTime*(js: JsonNode): DateTime =
 proc getTime*(js: JsonNode): DateTime =
   parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
 
-proc getId*(id: string): string {.inline.} =
+proc getTimeFromMs*(js: JsonNode): DateTime =
+  let ms = js.getInt(0)
+  if ms == 0: return
+  let seconds = ms div 1000
+  return fromUnix(seconds).utc()
+
+proc getId*(id: string): int64 {.inline.} =
   let start = id.rfind("-")
-  if start < 0: return id
+  if start < 0:
-  id[start + 1 ..< id.len]
+    return parseBiggestInt(id)
+  return parseBiggestInt(id[start + 1 ..< id.len])
 
 proc getId*(js: JsonNode): int64 {.inline.} =
   case js.kind
-  of JString: return parseBiggestInt(js.getStr("0"))
+  of JString: return js.getStr("0").getId
   of JInt: return js.getBiggestInt()
   else: return 0
 
-proc getEntryId*(js: JsonNode): string {.inline.} =
-  let entry = js{"entryId"}.getStr
-  if entry.len == 0: return
-
-  if "tweet" in entry or "sq-I-t" in entry:
-    return entry.getId
-  elif "tombstone" in entry:
-    return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr
-  else:
-    echo "unknown entry: ", entry
-    return
-
 template getStrVal*(js: JsonNode; default=""): string =
   js{"string_value"}.getStr(default)
 
@@ -89,6 +112,9 @@ proc getImageStr*(js: JsonNode): string =
 template getImageVal*(js: JsonNode): string =
   js{"image_value", "url"}.getImageStr
 
+template getExpandedUrl*(js: JsonNode; fallback=""): string =
+  js{"expanded_url"}.getStr(js{"url"}.getStr(fallback))
+
 proc getCardUrl*(js: JsonNode; kind: CardKind): string =
   result = js{"website_url"}.getStrVal
   if kind == promoVideoConvo:
@@ -130,16 +156,31 @@ proc getBanner*(js: JsonNode): string =
       return
 
 proc getTombstone*(js: JsonNode): string =
-  result = js{"tombstoneInfo", "richText", "text"}.getStr
+  result = js{"text"}.getStr
   result.removeSuffix(" Learn more")
 
+proc getMp4Resolution*(url: string): int =
+  # parses the height out of a URL like this one:
+  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
+  const vidSep = "/vid/"
+  let
+    vidIdx = url.find(vidSep) + vidSep.len
+    resIdx = url.find('x', vidIdx) + 1
+    res = url[resIdx ..< url.find("/", resIdx)]
+
+  try:
+    return parseInt(res)
+  except ValueError:
+    # cannot determine resolution (e.g. m3u8/non-mp4 video)
+    return 0
+
 proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 
 proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
                  textLen: int; hideTwitter = false) =
   let
-    url = js["expanded_url"].getStr
+    url = js.getExpandedUrl
     slice = js.extractSlice
 
   if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
@@ -200,7 +241,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
     ent = ? js{"entities"}
 
   with urls, ent{"url", "urls"}:
-    user.website = urls[0]{"expanded_url"}.getStr
+    user.website = urls[0].getExpandedUrl
 
   var replacements = newSeq[ReplaceSlice]()
 
@@ -215,47 +256,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
   user.bio = user.bio.replacef(unRegex, unReplace)
                      .replacef(htRegex, htReplace)
 
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
-  let
+                        replyTo=""; hasRedundantLink=false) =
-    orig = tweet.text.toRunes
+  let hasCard = tweet.card.isSome
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
 
   var replacements = newSeq[ReplaceSlice]()
 
-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
     for u in urls:
       let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
         continue
-      replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
-      if hasCard and u{"url"}.getStr == get(tweet.card).url:
-        get(tweet.card).url = u{"expanded_url"}.getStr
 
-  with media, ent{"media"}:
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
+
+      if hasCard and u{"url"}.getStr == get(tweet.card).url:
+        get(tweet.card).url = u.getExpandedUrl
+
+  with media, entities{"media"}:
     for m in media:
       replacements.extractUrls(m, textSlice.b, hideTwitter = true)
 
-  if "hashtags" in ent:
+  if "hashtags" in entities:
-    for hashtag in ent["hashtags"]:
+    for hashtag in entities["hashtags"]:
       replacements.extractHashtags(hashtag)
 
-  if "symbols" in ent:
+  if "symbols" in entities:
-    for symbol in ent["symbols"]:
+    for symbol in entities["symbols"]:
       replacements.extractHashtags(symbol)
 
-  if "user_mentions" in ent:
+  if "user_mentions" in entities:
-    for mention in ent["user_mentions"]:
+    for mention in entities["user_mentions"]:
       let
         name = mention{"screen_name"}.getStr
         slice = mention.extractSlice
@@ -272,5 +303,40 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.deduplicate
   replacements.sort(cmp)
 
-  tweet.text = orig.replacedWith(replacements, textSlice)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
-                   .strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+    hasQuote = js{"is_quote_status"}.getBool
+    hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)
+
+  tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
+
+proc extractGalleryPhoto*(t: Tweet): GalleryPhoto =
+  let url =
+    if t.photos.len > 0: t.photos[0]
+    elif t.video.isSome: get(t.video).thumb
+    elif t.gif.isSome: get(t.gif).thumb
+    elif t.card.isSome: get(t.card).image
+    else: ""
+
+  result = GalleryPhoto(url: url, tweetId: $t.id)

src/prefs_impl.nim

@@ -83,7 +83,7 @@ genPrefs:
       "Enable mp4 video playback (only for gifs)"
 
     hlsPlayback(checkbox, false):
-      "Enable hls video streaming (requires JavaScript)"
+      "Enable HLS video streaming (requires JavaScript)"
 
     proxyVideos(checkbox, true):
       "Proxy video streaming through the server (might be slow)"
@@ -107,10 +107,6 @@ genPrefs:
       "Reddit -> Teddit/Libreddit"
       placeholder: "Teddit hostname"
 
-    replaceInstagram(input, ""):
-      "Instagram -> Bibliogram"
-      placeholder: "Bibliogram hostname"
-
 iterator allPrefs*(): Pref =
   for k, v in prefList:
     for pref in v:

src/query.nim

@@ -6,10 +6,9 @@ import types
 const
   validFilters* = @[
     "media", "images", "twimg", "videos",
-    "native_video", "consumer_video", "pro_video",
+    "native_video", "consumer_video", "spaces",
     "links", "news", "quote", "mentions",
-    "replies", "retweets", "nativeretweets",
+    "replies", "retweets", "nativeretweets"
-    "verified", "safe"
   ]
 
   emptyQuery* = "include:nativeretweets"
@@ -18,6 +17,11 @@ template `@`(param: string): untyped =
   if param in pms: pms[param]
   else: ""
 
+proc validateNumber(value: string): string =
+  if value.anyIt(not it.isDigit):
+    return ""
+  return value
+
 proc initQuery*(pms: Table[string, string]; name=""): Query =
   result = Query(
     kind: parseEnum[QueryKind](@"f", tweets),
@@ -26,7 +30,7 @@ proc initQuery*(pms: Table[string, string]; name=""): Query =
     excludes: validFilters.filterIt("e-" & it in pms),
     since: @"since",
     until: @"until",
-    near: @"near"
+    minLikes: validateNumber(@"min_faves")
   )
 
   if name.len > 0:
@@ -78,8 +82,8 @@ proc genQueryParam*(query: Query): string =
     result &= " since:" & query.since
   if query.until.len > 0:
     result &= " until:" & query.until
-  if query.near.len > 0:
+  if query.minLikes.len > 0:
-    result &= &" near:\"{query.near}\" within:15mi"
+    result &= " min_faves:" & query.minLikes
   if query.text.len > 0:
     if result.len > 0:
       result &= " " & query.text
@@ -93,18 +97,18 @@ proc genQueryUrl*(query: Query): string =
   if query.text.len > 0:
     params.add "q=" & encodeUrl(query.text)
   for f in query.filters:
-    params.add "f-" & f & "=on"
+    params.add &"f-{f}=on"
   for e in query.excludes:
-    params.add "e-" & e & "=on"
+    params.add &"e-{e}=on"
   for i in query.includes.filterIt(it != "nativeretweets"):
-    params.add "i-" & i & "=on"
+    params.add &"i-{i}=on"
 
   if query.since.len > 0:
     params.add "since=" & query.since
   if query.until.len > 0:
     params.add "until=" & query.until
-  if query.near.len > 0:
+  if query.minLikes.len > 0:
-    params.add "near=" & query.near
+    params.add "min_faves=" & query.minLikes
 
   if params.len > 0:
     result &= params.join("&")

src/redis_cache.nim

@@ -52,6 +52,7 @@ proc initRedisPool*(cfg: Config) {.async.} =
     await migrate("profileDates", "p:*")
     await migrate("profileStats", "p:*")
     await migrate("userType", "p:*")
+    await migrate("verifiedType", "p:*")
 
     pool.withAcquire(r):
       # optimize memory usage for user ID buckets
@@ -85,7 +86,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
+  await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return
@@ -118,11 +119,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
   pool.withAcquire(r):
     result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
       if user.suspended:
         return "suspended"
       else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
         return user.id
 
 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -130,8 +131,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   if prof != redisNil:
     prof.deserialize(User)
   elif fetch:
-    let userId = await getUserId(username)
+    result = await getGraphUser(username)
-    result = await getGraphUser(userId)
     await cache(result)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,28 +142,30 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   if username != redisNil:
     result = username
   else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
     result = user.username
     await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))
 
-proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
+# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
-  if id == 0: return
+#   if id == 0: return
-  let tweet = await get(id.tweetKey)
+#   let tweet = await get(id.tweetKey)
-  if tweet != redisNil:
+#   if tweet != redisNil:
-    tweet.deserialize(Tweet)
+#     tweet.deserialize(Tweet)
-  else:
+#   else:
-    result = await getStatus($id)
+#     result = await getGraphTweetResult($id)
-    if result.isNil:
+#     if not result.isNil:
-      await cache(result)
+#       await cache(result)
 
-proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
+proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
+  if id.len == 0: return
-  let rail = await get("pr:" & toLower(name))
+  let rail = await get("pr2:" & toLower(id))
   if rail != redisNil:
     rail.deserialize(PhotoRail)
   else:
-    result = await getPhotoRail(name)
+    result = await getPhotoRail(id)
-    await cache(result, name)
+    await cache(result, id)
 
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil

src/routes/debug.nim

@@ -1,10 +1,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import jester
 import router_utils
-import ".."/[tokens, types]
+import ".."/[auth, types]
 
 proc createDebugRouter*(cfg: Config) =
   router debug:
-    get "/.tokens":
+    get "/.health":
+      respJson getSessionPoolHealth()
+
+    get "/.sessions":
       cond cfg.enableDebug
-      respJson getPoolJson()
+      respJson getSessionPoolDebug()

src/routes/embed.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, options
+import asyncdispatch, strutils, strformat, options
 import jester, karax/vdom
 import ".."/[types, api]
 import ../views/[embed, tweet, general]
@@ -10,27 +10,27 @@ export api, embed, vdom, tweet, general, router_utils
 proc createEmbedRouter*(cfg: Config) =
   router embed:
     get "/i/videos/tweet/@id":
-      let convo = await getTweet(@"id")
+      let tweet = await getGraphTweetResult(@"id")
-      if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
+      if tweet == nil or tweet.video.isNone:
         resp Http404
 
-      resp renderVideoEmbed(convo.tweet, cfg, request)
+      resp renderVideoEmbed(tweet, cfg, request)
 
     get "/@user/status/@id/embed":
       let
-        convo = await getTweet(@"id")
+        tweet = await getGraphTweetResult(@"id")
         prefs = cookiePrefs()
         path = getPath()
 
-      if convo == nil or convo.tweet == nil:
+      if tweet == nil:
         resp Http404
 
-      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+      resp renderTweetEmbed(tweet, path, prefs, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"
 
       if id.len > 0:
-        redirect("/i/status/" & id & "/embed")
+        redirect(&"/i/status/{id}/embed")
       else:
         resp Http404

src/routes/list.nim

@@ -1,23 +1,25 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, uri
+import strutils, strformat, uri
 
 import jester
 
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
   if list.id.len == 0 or list.name.len == 0:
-    resp Http404, showError("List " & @"id" & " not found", cfg)
+    resp Http404, showError(&"""List "{@"id"}" not found""", cfg)
 
   let
     html = renderList(vnode, timeline.query, list)
-    rss = "/i/lists/$1/rss" % [@"id"]
+    rss = &"""/i/lists/{@"id"}/rss"""
 
   resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner)
 
+proc title*(list: List): string =
+  &"@{list.username}/{list.name}"
+
 proc createListRouter*(cfg: Config) =
   router list:
     get "/@name/lists/@slug/?":
@@ -28,24 +30,22 @@ proc createListRouter*(cfg: Config) =
         slug = decodeUrl(@"slug")
         list = await getCachedList(@"name", slug)
       if list.id.len == 0:
-        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
+        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
-      redirect("/i/lists/" & list.id)
+      redirect(&"/i/lists/{list.id}")
 
     get "/i/lists/@id/?":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
+        timeline = await getGraphListTweets(list.id, getCursor())
-        timeline = await getListTimeline(list.id, getCursor())
         vnode = renderTimelineTweets(timeline, prefs, request.path)
-      respList(list, timeline, title, vnode)
+      respList(list, timeline, list.title, vnode)
 
     get "/i/lists/@id/members":
       cond '.' notin @"id"
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        title = "@" & list.username & "/" & list.name
         members = await getGraphListMembers(list, getCursor())
-      respList(list, members, title, renderTimelineUsers(members, prefs, request.path))
+      respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path))

src/routes/media.nim

@@ -37,6 +37,8 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   try:
     let res = await client.get(url)
     if res.status != "200 OK":
+      if res.status != "404 Not Found":
+        echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
       return Http404
 
     let hashed = $hash(url)
@@ -65,6 +67,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
         await request.client.send(data)
     data.setLen 0
   except HttpRequestError, ProtocolError, OSError:
+    echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
     result = Http404
   finally:
     client.close()
@@ -88,6 +91,20 @@ proc createMediaRouter*(cfg: Config) =
     get "/pic/?":
       resp Http404
 
+    get re"^\/pic\/orig\/(enc)?\/?(.+)":
+      var url = decoded(request, 1)
+      if "twimg.com" notin url:
+        url.insert(twimg)
+      if not url.startsWith(https):
+        url.insert(https)
+      url.add("?name=orig")
+
+      let uri = parseUri(url)
+      cond isTwitterUrl(uri) == true
+
+      let code = await proxyMedia(request, url)
+      check code
+
     get re"^\/pic\/(enc)?\/?(.+)":
       var url = decoded(request, 1)
       if "twimg.com" notin url:
@@ -106,7 +123,7 @@ proc createMediaRouter*(cfg: Config) =
       cond "http" in url
 
       if getHmac(url) != request.matches[1]:
-        resp showError("Failed to verify signature", cfg)
+        resp Http403, showError("Failed to verify signature", cfg)
 
       if ".mp4" in url or ".ts" in url or ".m4s" in url:
         let code = await proxyMedia(request, url)

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri
 
 import jester
 
@@ -10,6 +10,11 @@ include "../views/rss.nimf"
 
 export times, hashes
 
+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -18,19 +23,17 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     names = getNames(name)
 
   if names.len == 1:
-    profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
+    profile = await fetchProfile(after, query, skipRail=true)
   else:
     var q = query
     q.fromUser = names
-    profile = Profile(
+    profile.tweets = await getGraphTweetSearch(q, after)
-      tweets: await getSearch[Tweet](q, after),
     # this is kinda dumb
-      user: User(
+    profile.user = User(
       username: name,
       fullname: names.join(" | "),
       userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
     )
-    )
 
   if profile.user.suspended:
     return Rss(feed: profile.user.username, cursor: "suspended")
@@ -42,8 +45,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": " \"$1\" " % @"name"
+               of "User": " \"" & @"name" & "\" "
-               of "List": " $1 " % @"id"
+               of "List": " \"" & @"id" & "\" "
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,13 +70,13 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "Search")
 
-      let tweets = await getSearch[Tweet](query, cursor)
+      let tweets = await getGraphTweetSearch(query, cursor)
       rss.cursor = tweets.bottom
       rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 
@@ -84,9 +87,8 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
-        cursor = getCursor()
         name = @"name"
-        key = "twitter:" & name & ":" & cursor
+        key = redisKey("twitter", name, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -101,18 +103,20 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
+      let
-      let query =
+        name = @"name"
-        case @"tab"
+        tab = @"tab"
+        query =
+          case tab
           of "with_replies": getReplyQuery(name)
           of "media": getMediaQuery(name)
           of "search": initQuery(params(request), name=name)
           else: Query(fromUser: @[name])
 
-      var key = @"tab" & ":" & @"name" & ":"
+      let searchKey = if tab != "search": ""
-      if @"tab" == "search":
+                      else: ":" & $hash(genQueryUrl(query))
-        key &= $hash(genQueryUrl(query)) & ":"
+
-      key &= getCursor()
+      let key = redisKey(tab, name & searchKey, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -143,18 +147,17 @@ proc createRssRouter*(cfg: Config) =
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
+        id = @"id"
         cursor = getCursor()
-        key =
+        key = redisKey("lists", id, cursor)
-          if cursor.len == 0: "lists:" & @"id"
-          else: "lists:" & @"id" & ":" & cursor
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=(@"id"))
+        list = await getCachedList(id=id)
-        timeline = await getListTimeline(list.id, cursor)
+        timeline = await getGraphListTweets(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)
 

src/routes/search.nim

@@ -14,25 +14,31 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in @"q":
+        if "," in q:
-          redirect("/" & @"q")
+          redirect("/" & q)
-        let users = await getSearch[User](query, getCursor())
+        var users: Result[User]
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        try:
+          users = await getGraphUserSearch(query, getCursor())
+        except InternalError:
+          users = Result[User](beginning: true, query: query)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
-          tweets = await getSearch[Tweet](query, getCursor())
+          tweets = await getGraphTweetSearch(query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 

src/routes/status.nim

@@ -16,19 +16,21 @@ proc createStatusRouter*(cfg: Config) =
   router status:
     get "/@name/status/@id/?":
       cond '.' notin @"name"
-      cond not @"id".any(c => not c.isDigit)
+      let id = @"id"
+
+      if id.len > 19 or id.any(c => not c.isDigit):
+        resp Http404, showError("Invalid tweet ID", cfg)
+
       let prefs = cookiePrefs()
 
       # used for the infinite scroll feature
       if @"scroll".len > 0:
-        let replies = await getReplies(@"id", getCursor())
+        let replies = await getReplies(id, getCursor())
         if replies.content.len == 0:
           resp Http404, ""
         resp $renderReplies(replies, prefs, getPath())
 
-      let conv = await getTweet(@"id", getCursor())
+      let conv = await getTweet(id, getCursor())
-      if conv == nil:
-        echo "nil conv"
 
       if conv == nil or conv.tweet == nil or conv.tweet.id == 0:
         var error = "Tweet not found"
@@ -64,7 +66,7 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/@name/@s/@id/@m/?@i?":
       cond @"s" in ["status", "statuses"]
-      cond @"m" in ["video", "photo"]
+      cond @"m" in ["video", "photo", "history"]
       redirect("/$1/status/$2" % [@"name", @"id"])
 
     get "/@name/statuses/@id/?":
@@ -72,3 +74,6 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/i/web/status/@id":
       redirect("/i/status/" & @"id")
+
+    get "/@name/thread/@id/?":
+      redirect("/$1/status/$2" % [@"name", @"id"])

src/routes/timeline.nim

@@ -27,8 +27,7 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
   else:
     body
 
-proc fetchProfile*(after: string; query: Query; skipRail=false;
+proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile] {.async.} =
-                   skipPinned=false): Future[Profile] {.async.} =
   let
     name = query.fromUser[0]
     userId = await getUserId(name)
@@ -45,36 +44,21 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
     after.setLen 0
 
   let
-    timeline =
-      case query.kind
-      of posts: getTimeline(userId, after)
-      of replies: getTimeline(userId, after, replies=true)
-      of media: getMediaTimeline(userId, after)
-      else: getSearch[Tweet](query, after)
-
     rail =
       skipIf(skipRail or query.kind == media, @[]):
-        getCachedPhotoRail(name)
+        getCachedPhotoRail(userId)
 
-    user = await getCachedUser(name)
+    user = getCachedUser(name)
 
-  var pinned: Option[Tweet]
+  result =
-  if not skipPinned and user.pinnedTweet > 0 and
+    case query.kind
-     after.len == 0 and query.kind in {posts, replies}:
+    of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
-    let tweet = await getCachedTweet(user.pinnedTweet)
+    of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
-    if not tweet.isNil:
+    of media: await getGraphUserTweets(userId, TimelineKind.media, after)
-      tweet.pinned = true
+    else: Profile(tweets: await getGraphTweetSearch(query, after))
-      pinned = some tweet
 
-  result = Profile(
+  result.user = await user
-    user: user,
+  result.photoRail = await rail
-    pinned: pinned,
-    tweets: await timeline,
-    photoRail: await rail
-  )
-
-  if result.user.protected or result.user.suspended:
-    return
 
   result.tweets.query = query
 
@@ -82,11 +66,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
   if query.fromUser.len != 1:
     let
-      timeline = await getSearch[Tweet](query, after)
+      timeline = await getGraphTweetSearch(query, after)
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
-  var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
+  var profile = await fetchProfile(after, query)
   template u: untyped = profile.user
 
   if u.suspended:
@@ -121,9 +105,15 @@ proc createTimelineRouter*(cfg: Config) =
     get "/intent/user":
       respUserId()
 
+    get "/intent/follow/?":
+      let username = request.params.getOrDefault("screen_name")
+      if username.len == 0:
+        resp Http400, showError("Missing screen_name parameter", cfg)
+      redirect("/" & username)
+
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
-      cond @"name" notin ["pic", "gif", "video"]
+      cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
@@ -137,7 +127,7 @@ proc createTimelineRouter*(cfg: Config) =
       # used for the infinite scroll feature
       if @"scroll".len > 0:
         if query.fromUser.len != 1:
-          var timeline = await getSearch[Tweet](query, after)
+          var timeline = await getGraphTweetSearch(query, after)
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())

src/routes/unsupported.nim

@@ -17,7 +17,7 @@ proc createUnsupportedRouter*(cfg: Config) =
     get "/@name/lists/?": feature()
 
     get "/intent/?@i?": 
-      cond @"i" notin ["user"]
+      cond @"i" notin ["user", "follow"]
       feature()
 
     get "/i/@i?/?@j?":

src/sass/general.scss

@@ -1,5 +1,5 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 .panel-container {
   margin: auto;
@@ -23,6 +23,7 @@
     font-weight: bold;
     width: 30px;
     height: 30px;
+    padding: 0px 5px 1px 8px;
   }
 
   input {

src/sass/include/_mixins.css

@@ -66,18 +66,7 @@
         }
 
         #search-panel-toggle:checked ~ .search-panel {
-            @if $rows == 6 {
+            max-height: 380px !important;
-                max-height: 200px !important;
-            }
-            @if $rows == 5 {
-                max-height: 300px !important;
-            }
-            @if $rows == 4 {
-                max-height: 300px !important;
-            }
-            @if $rows == 3 {
-                max-height: 365px !important;
-            }
         }
     }
 }

src/sass/include/_variables.scss

@@ -1,44 +1,43 @@
 // colors
-$bg_color: #0F0F0F;
+$bg_color: #0f0f0f;
-$fg_color: #F8F8F2;
+$fg_color: #f8f8f2;
-$fg_faded: #F8F8F2CF;
+$fg_faded: #f8f8f2cf;
-$fg_dark: #FF6C60;
+$fg_dark: #ff6c60;
-$fg_nav: #FF6C60;
+$fg_nav: #ff6c60;
 
 $bg_panel: #161616;
 $bg_elements: #121212;
-$bg_overlays: #1F1F1F;
+$bg_overlays: #1f1f1f;
-$bg_hover: #1A1A1A;
+$bg_hover: #1a1a1a;
 
 $grey: #888889;
 $dark_grey: #404040;
 $darker_grey: #282828;
 $darkest_grey: #222222;
-$border_grey: #3E3E35;
+$border_grey: #3e3e35;
 
-$accent: #FF6C60;
+$accent: #ff6c60;
-$accent_light: #FFACA0;
+$accent_light: #ffaca0;
-$accent_dark: #8A3731;
+$accent_dark: #8a3731;
-$accent_border: #FF6C6091;
+$accent_border: #ff6c6091;
 
-$play_button: #D8574D;
+$play_button: #d8574d;
-$play_button_hover: #FF6C60;
+$play_button_hover: #ff6c60;
 
-$more_replies_dots: #AD433B;
+$more_replies_dots: #ad433b;
-$error_red: #420A05;
+$error_red: #420a05;
 
-$verified_blue: #1DA1F2;
+$verified_blue: #1da1f2;
+$verified_business: #fac82b;
+$verified_government: #c1b6a4;
 $icon_text: $fg_color;
 
 $tab: $fg_color;
 $tab_selected: $accent;
 
-$shadow: rgba(0,0,0,.6);
+$shadow: rgba(0, 0, 0, 0.6);
-$shadow_dark: rgba(0,0,0,.2);
+$shadow_dark: rgba(0, 0, 0, 0.2);
 
 //fonts
-$font_0: Helvetica Neue;
+$font_0: sans-serif;
-$font_1: Helvetica;
+$font_1: fontello;
-$font_2: Arial;
-$font_3: sans-serif;
-$font_4: fontello;

src/sass/index.scss

@@ -1,12 +1,12 @@
-@import '_variables';
+@import "_variables";
 
-@import 'tweet/_base';
+@import "tweet/_base";
-@import 'profile/_base';
+@import "profile/_base";
-@import 'general';
+@import "general";
-@import 'navbar';
+@import "navbar";
-@import 'inputs';
+@import "inputs";
-@import 'timeline';
+@import "timeline";
-@import 'search';
+@import "search";
 
 body {
   // colors
@@ -39,6 +39,8 @@ body {
   --error_red: #{$error_red};
 
   --verified_blue: #{$verified_blue};
+  --verified_business: #{$verified_business};
+  --verified_government: #{$verified_government};
   --icon_text: #{$icon_text};
 
   --tab: #{$fg_color};
@@ -48,8 +50,8 @@ body {
 
   background-color: var(--bg_color);
   color: var(--fg_color);
-    font-family: $font_0, $font_1, $font_2, $font_3;
+  font-family: $font_0, $font_1;
-    font-size: 14px;
+  font-size: 15px;
   line-height: 1.3;
   margin: 0;
 }
@@ -64,7 +66,8 @@ h1 {
   display: inline;
 }
 
-h2, h3 {
+h2,
+h3 {
   font-weight: normal;
 }
 
@@ -88,7 +91,7 @@ fieldset {
 
 legend {
   width: 100%;
-    padding: .6em 0 .3em 0;
+  padding: 0.6em 0 0.3em 0;
   border: 0;
   font-size: 16px;
   font-weight: 600;
@@ -140,18 +143,51 @@ ul {
 }
 
 .verified-icon {
-    color: var(--icon_text);
-    background-color: var(--verified_blue);
-    border-radius: 50%;
-    flex-shrink: 0;
-    margin: 2px 0 3px 3px;
-    padding-top: 2px;
-    height: 12px;
-    width: 14px;
-    font-size: 8px;
   display: inline-block;
-    text-align: center;
+  width: 14px;
-    vertical-align: middle;
+  height: 14px;
+  margin-left: 2px;
+
+  .verified-icon-circle {
+    position: absolute;
+    font-size: 15px;
+  }
+
+  .verified-icon-check {
+    position: absolute;
+    font-size: 9px;
+    margin: 5px 3px;
+  }
+
+  &.blue {
+    .verified-icon-circle {
+      color: var(--verified_blue);
+    }
+
+    .verified-icon-check {
+      color: var(--icon_text);
+    }
+  }
+
+  &.business {
+    .verified-icon-circle {
+      color: var(--verified_business);
+    }
+
+    .verified-icon-check {
+      color: var(--bg_panel);
+    }
+  }
+
+  &.government {
+    .verified-icon-circle {
+      color: var(--verified_government);
+    }
+
+    .verified-icon-check {
+      color: var(--bg_panel);
+    }
+  }
 }
 
 @media (max-width: 600px) {
@@ -159,7 +195,8 @@ ul {
     max-width: 95vw;
   }
 
-    .nav-item, .nav-item .icon-container {
+  .nav-item,
+  .nav-item .icon-container {
     font-size: 16px;
   }
 }

src/sass/inputs.scss

@@ -1,5 +1,5 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 button {
   @include input-colors;
@@ -14,6 +14,7 @@ button {
 
 input[type="text"],
 input[type="date"],
+input[type="number"],
 select {
   @include input-colors;
   background-color: var(--bg_elements);
@@ -24,7 +25,12 @@ select {
   font-size: 14px;
 }
 
-input[type="text"] {
+input[type="number"] {
+  -moz-appearance: textfield;
+}
+
+input[type="text"],
+input[type="number"] {
   height: 16px;
 }
 
@@ -38,6 +44,17 @@ input[type="date"]::-webkit-inner-spin-button {
   display: none;
 }
 
+input[type="number"] {
+  -moz-appearance: textfield;
+}
+
+input[type="number"]::-webkit-inner-spin-button,
+input[type="number"]::-webkit-outer-spin-button {
+  display: none;
+  -webkit-appearance: none;
+  margin: 0;
+}
+
 input[type="date"]::-webkit-clear-button {
   margin-left: 17px;
   filter: grayscale(100%);
@@ -136,8 +153,8 @@ input::-webkit-datetime-edit-year-field:focus {
     left: 2px;
     bottom: 0;
     font-size: 13px;
-        font-family: $font_4;
+    font-family: $font_1;
-        content: '\e803';
+    content: "\e811";
   }
 }
 
@@ -164,7 +181,8 @@ input::-webkit-datetime-edit-year-field:focus {
     appearance: none;
   }
 
-    input[type="text"] {
+  input[type="text"],
+  input[type="number"] {
     position: absolute;
     right: 0;
     max-width: 140px;

src/sass/navbar.scss

@@ -1,4 +1,4 @@
-@import '_variables';
+@import "_variables";
 
 nav {
   display: flex;
@@ -12,7 +12,8 @@ nav {
   z-index: 1000;
   font-size: 16px;
 
-    a, .icon-button button {
+  a,
+  .icon-button button {
     color: var(--fg_nav);
   }
 }
@@ -58,20 +59,17 @@ nav {
     justify-content: flex-end;
   }
 
-    &.right a {
+  &.right a:hover {
-        padding-left: 4px;
-
-        &:hover {
     color: var(--accent_light);
     text-decoration: unset;
   }
 }
-}
 
 .lp {
   height: 14px;
-    margin-top: 2px;
+  display: inline-block;
-    display: block;
+  position: relative;
+  top: 2px;
   fill: var(--fg_nav);
 
   &:hover {
@@ -79,10 +77,11 @@ nav {
   }
 }
 
-.icon-info:before {
+.icon-info {
   margin: 0 -3px;
 }
 
 .icon-cog {
   font-size: 15px;
+  padding-left: 0 !important;
 }

src/sass/profile/card.scss

@@ -73,9 +73,9 @@
         }
     }
 
-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
         color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
         width: 100%;
     }
 }
@@ -115,7 +115,7 @@
     }
 
     .profile-card-tabs-name {
-        @include breakable;
+        flex-shrink: 100;
     }
 
     .profile-card-avatar {

src/sass/search.scss

@@ -1,5 +1,5 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 .search-title {
   font-weight: bold;
@@ -13,7 +13,10 @@
 
   button {
     margin: 0 2px 0 0;
+    padding: 0px 1px 1px 4px;
     height: 23px;
+    display: flex;
+    align-items: center;
   }
 
   .pref-input {
@@ -22,7 +25,8 @@
     height: 23px;
   }
 
-    input[type="text"] {
+  input[type="text"],
+  input[type="number"] {
     height: calc(100% - 4px);
     width: calc(100% - 8px);
   }
@@ -32,7 +36,7 @@
     background-color: var(--bg_elements);
     color: var(--fg_color);
     border: 1px solid var(--accent_border);
-        padding: 1px 6px 2px 6px;
+    padding: 1px 1px 2px 4px;
     font-size: 14px;
     cursor: pointer;
     margin-bottom: 2px;
@@ -40,7 +44,7 @@
     @include input-colors;
   }
 
-    @include create-toggle(search-panel, 200px);
+  @include create-toggle(search-panel, 380px);
 }
 
 .search-panel {
@@ -53,20 +57,17 @@
   font-weight: initial;
   text-align: left;
 
-    > div {
-        line-height: 1.7em;
-    }
-
   .checkbox-container {
     display: inline;
     padding-right: unset;
-        margin-bottom: unset;
+    margin-bottom: 5px;
     margin-left: 23px;
   }
 
   .checkbox {
     right: unset;
     left: -22px;
+    line-height: 1.6em;
   }
 
   .checkbox-container .checkbox:after {
@@ -102,19 +103,18 @@
 .search-toggles {
   flex-grow: 1;
   display: grid;
-    grid-template-columns: repeat(6, auto);
+  grid-template-columns: repeat(5, auto);
   grid-column-gap: 10px;
 }
 
 .profile-tabs {
   @include search-resize(820px, 5);
-    @include search-resize(725px, 4);
+  @include search-resize(715px, 4);
-    @include search-resize(600px, 6);
+  @include search-resize(700px, 5);
-    @include search-resize(560px, 5);
+  @include search-resize(485px, 4);
-    @include search-resize(480px, 4);
   @include search-resize(410px, 3);
 }
 
-@include search-resize(560px, 5);
+@include search-resize(700px, 5);
-@include search-resize(480px, 4);
+@include search-resize(485px, 4);
 @include search-resize(410px, 3);

src/sass/timeline.scss

@@ -1,4 +1,4 @@
-@import '_variables';
+@import "_variables";
 
 .timeline-container {
   @include panel(100%, 600px);
@@ -51,7 +51,7 @@
   margin-top: 0;
 
   a {
-        border-bottom: .1rem solid transparent;
+    border-bottom: 0.1rem solid transparent;
     color: var(--tab);
     display: block;
     padding: 8px 0;
@@ -116,7 +116,7 @@
 .show-more {
   background-color: var(--bg_panel);
   text-align: center;
-    padding: .75em 0;
+  padding: 0.75em 0;
   display: block !important;
 
   a {
@@ -156,7 +156,7 @@
   overflow-wrap: break-word;
   border-left-width: 0;
   min-width: 0;
-    padding: .75em;
+  padding: 0.75em;
   display: flex;
   position: relative;
 }

src/sass/tweet/_base.scss

@@ -1,12 +1,12 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
-@import 'thread';
+@import "thread";
-@import 'media';
+@import "media";
-@import 'video';
+@import "video";
-@import 'embed';
+@import "embed";
-@import 'card';
+@import "card";
-@import 'poll';
+@import "poll";
-@import 'quote';
+@import "quote";
 
 .tweet-body {
   flex: 1;
@@ -17,7 +17,6 @@
 }
 
 .tweet-content {
-    font-family: $font_3;
   line-height: 1.3em;
   pointer-events: all;
   display: inline;
@@ -31,7 +30,7 @@
   padding: 0;
   vertical-align: bottom;
   flex-basis: 100%;
-    margin-bottom: .2em;
+  margin-bottom: 0.2em;
 
   a {
     display: inline-block;
@@ -64,7 +63,7 @@
 .username {
   @include ellipsis;
   min-width: 1.6em;
-    margin-left: .4em;
+  margin-left: 0.4em;
   word-wrap: normal;
 }
 
@@ -74,7 +73,9 @@
   margin-left: 4px;
 }
 
-.tweet-date a, .username, .show-more a {
+.tweet-date a,
+.username,
+.show-more a {
   color: var(--fg_dark);
 }
 
@@ -98,10 +99,9 @@
 }
 
 .avatar {
-    position: absolute;
-
   &.round {
     border-radius: 50%;
+    -webkit-user-select: none;
   }
 
   &.mini {
@@ -121,7 +121,7 @@
   background-color: var(--bg_panel);
 
   .tweet-content {
-        font-size: 18px 
+    font-size: 18px;
   }
 
   .tweet-body {
@@ -129,6 +129,14 @@
     flex-direction: column;
     max-height: calc(100vh - 0.75em * 2);
   }
+
+  .card-image img {
+    height: auto;
+  }
+
+  .avatar {
+    position: absolute;
+  }
 }
 
 .attribution {
@@ -150,7 +158,8 @@
     padding-right: 2px;
   }
 
-    .media-tag, .icon-container {
+  .media-tag,
+  .icon-container {
     color: var(--fg_faded);
   }
 }
@@ -172,7 +181,9 @@
   }
 }
 
-.retweet-header, .pinned, .tweet-stats {
+.retweet-header,
+.pinned,
+.tweet-stats {
   align-content: center;
   color: var(--grey);
   display: flex;
@@ -193,6 +204,7 @@
 
 .tweet-stats {
   margin-bottom: -3px;
+  -webkit-user-select: none;
 }
 
 .tweet-stat {
@@ -224,6 +236,7 @@
   left: 0;
   top: 0;
   position: absolute;
+  -webkit-user-select: none;
 
   &:hover {
     background-color: var(--bg_hover);

src/sass/tweet/card.scss

@@ -42,6 +42,7 @@
 
 .card-description {
     margin: 0.3em 0;
+    white-space: pre-wrap;
 }
 
 .card-destination {

src/sass/tweet/embed.scss

@@ -1,5 +1,5 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 .embed-video {
   .gallery-video {

src/sass/tweet/media.scss

@@ -1,4 +1,4 @@
-@import '_variables';
+@import "_variables";
 
 .gallery-row {
   display: flex;
@@ -18,7 +18,7 @@
 }
 
 .attachments {
-    margin-top: .35em;
+  margin-top: 0.35em;
   display: flex;
   flex-direction: row;
   width: 100%;
@@ -39,7 +39,7 @@
   position: relative;
   line-height: 0;
   overflow: hidden;
-    margin: 0 .25em 0 0;
+  margin: 0 0.25em 0 0;
   flex-grow: 1;
   box-sizing: border-box;
   min-width: 2em;

src/sass/tweet/poll.scss

@@ -1,4 +1,4 @@
-@import '_variables';
+@import "_variables";
 
 .poll-meter {
   overflow: hidden;

src/sass/tweet/quote.scss

@@ -1,4 +1,4 @@
-@import '_variables';
+@import "_variables";
 
 .quote {
   margin-top: 10px;
@@ -84,11 +84,12 @@
     }
   }
 
-    .gallery-video, .gallery-gif {
+  .gallery-video,
+  .gallery-gif {
     max-height: 300px;
   }
 
   .still-image img {
-        max-height: 250px
+    max-height: 250px;
   }
 }

src/sass/tweet/thread.scss

@@ -1,5 +1,5 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 .conversation {
   @include panel(100%, 600px);
@@ -14,7 +14,8 @@
   background-color: var(--bg_panel);
 }
 
-.main-tweet, .replies {
+.main-tweet,
+.replies {
   padding-top: 50px;
   margin-top: -50px;
 }
@@ -23,7 +24,6 @@
   font-size: 18px;
 }
 
-
 @media (max-width: 600px) {
   .main-tweet .tweet-content {
     font-size: 16px;
@@ -39,7 +39,7 @@
   .timeline-item::before,
   &.timeline-item::before {
     background: var(--accent_dark);
-        content: '';
+    content: "";
     position: relative;
     min-width: 3px;
     width: 3px;
@@ -54,7 +54,7 @@
 
   .with-header:not(:first-child)::after {
     background: var(--accent_dark);
-        content: '';
+    content: "";
     position: relative;
     float: left;
     min-width: 3px;
@@ -75,7 +75,7 @@
   }
 
   .more-replies::before {
-        content: '...';
+    content: "...";
     background: unset;
     color: var(--more_replies_dots);
     font-weight: bold;
@@ -111,3 +111,29 @@
   margin-left: 58px;
   padding: 7px 0;
 }
+
+.timeline-item.thread.more-replies-thread {
+  padding: 0 0.75em;
+
+  &::before {
+    top: 40px;
+    margin-bottom: 31px;
+  }
+
+  .more-replies {
+    display: flex;
+    padding-top: unset !important;
+    margin-top: 8px;
+
+    &::before {
+      display: inline-block;
+      position: relative;
+      top: -1px;
+      line-height: 0.4em;
+    }
+
+    .more-replies-text {
+      display: inline;
+    }
+  }
+}

src/sass/tweet/video.scss

@@ -1,9 +1,9 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 video {
-    max-height: 100%;
+  height: 100%;
-    max-width: 100%;
+  width: 100%;
 }
 
 .gallery-video {
@@ -13,14 +13,14 @@ video {
 
 .gallery-video.card-container {
   flex-direction: column;
+  width: 100%;
 }
 
 .video-container {
+  min-height: 80px;
+  min-width: 200px;
   max-height: 530px;
   margin: 0;
-    display: flex;
-    align-items: center;
-    justify-content: center;
 
   img {
     max-height: 100%;
@@ -42,7 +42,7 @@ video {
     margin: 0 20px;
   }
 
-    div {
+  .overlay-circle {
     position: relative;
     z-index: 0;
     top: calc(50% - 20px);
@@ -51,6 +51,17 @@ video {
     height: 40px;
   }
 
+  .overlay-duration {
+    position: absolute;
+    bottom: 8px;
+    left: 8px;
+    background-color: #0000007a;
+    line-height: 1em;
+    padding: 4px 6px 4px 6px;
+    border-radius: 5px;
+    font-weight: bold;
+  }
+
   form {
     width: 100%;
     height: 100%;

src/tid.nim

@@ -0,0 +1,62 @@
+import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
+import nimcrypto
+import experimental/parser/tid
+
+randomize()
+
+const defaultKeyword = "obfiowerehiring";
+const pairsUrl =
+  "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
+
+var
+  cachedPairs: seq[TidPair] = @[]
+  lastCached = 0
+  # refresh every hour
+  ttlSec = 60 * 60
+
+proc getPair(): Future[TidPair] {.async.} =
+  if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
+    lastCached = int(epochTime())
+
+    let client = newAsyncHttpClient()
+    defer: client.close()
+
+    let resp = await client.get(pairsUrl)
+    if resp.status == $Http200:
+      cachedPairs = parseTidPairs(await resp.body)
+
+  return sample(cachedPairs)
+
+proc encodeSha256(text: string): array[32, byte] =
+  let
+    data = cast[ptr byte](addr text[0])
+    dataLen = uint(len(text))
+    digest = sha256.digest(data, dataLen)
+  return digest.data
+
+proc encodeBase64[T](data: T): string =
+  return encode(data).replace("=", "")
+
+proc decodeBase64(data: string): seq[byte] =
+  return cast[seq[byte]](decode(data))
+
+proc genTid*(path: string): Future[string] {.async.} =
+  let 
+    pair = await getPair()
+
+    timeNow = int(epochTime() - 1682924400)
+    timeNowBytes = @[
+      byte(timeNow and 0xff),
+      byte((timeNow shr 8) and 0xff),
+      byte((timeNow shr 16) and 0xff),
+      byte((timeNow shr 24) and 0xff)
+    ]
+
+    data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
+    hashBytes = encodeSha256(data)
+    keyBytes = decodeBase64(pair.verification)
+    bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
+    randomNum = byte(rand(256))
+    tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
+
+  return encodeBase64(tid)

src/tokens.nim

@@ -1,154 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, times, sequtils, json, random
-import strutils, tables
-import zippy
-import types, consts, http_pool
-
-const
-  maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
-  maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
-  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
-  failDelay = initDuration(minutes=30)
-
-var
-  clientPool: HttpPool
-  tokenPool: seq[Token]
-  lastFailed: Time
-
-proc getPoolJson*(): JsonNode =
-  var
-    list = newJObject()
-    totalReqs = 0
-    totalPending = 0
-    reqsPerApi: Table[string, int]
-
-  for token in tokenPool:
-    totalPending.inc(token.pending)
-    list[token.tok] = %*{
-      "apis": newJObject(),
-      "pending": token.pending,
-      "init": $token.init,
-      "lastUse": $token.lastUse
-    }
-
-    for api in token.apis.keys:
-      list[token.tok]["apis"][$api] = %token.apis[api]
-
-      let
-        maxReqs =
-          case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
-          of Api.timeline: 187
-          else: 180
-        reqs = maxReqs - token.apis[api].remaining
-
-      reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
-      totalReqs.inc(reqs)
-
-  return %*{
-    "amount": tokenPool.len,
-    "requests": totalReqs,
-    "pending": totalPending,
-    "apis": reqsPerApi,
-    "tokens": list
-  }
-
-proc rateLimitError*(): ref RateLimitError =
-  newException(RateLimitError, "rate limited")
-
-proc fetchToken(): Future[Token] {.async.} =
-  if getTime() - lastFailed < failDelay:
-    raise rateLimitError()
-
-  let headers = newHttpHeaders({
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "connection": "keep-alive",
-    "authorization": auth
-  })
-
-  try:
-    let
-      resp = clientPool.use(headers): await c.postContent(activate)
-      tokNode = parseJson(uncompress(resp))["guest_token"]
-      tok = tokNode.getStr($(tokNode.getInt))
-      time = getTime()
-
-    return Token(tok: tok, init: time, lastUse: time)
-  except Exception as e:
-    lastFailed = getTime()
-    echo "fetching token failed: ", e.msg
-
-proc expired(token: Token): bool =
-  let time = getTime()
-  token.init < time - maxAge or token.lastUse < time - maxLastUse
-
-proc isLimited(token: Token; api: Api): bool =
-  if token.isNil or token.expired:
-    return true
-
-  if api in token.apis:
-    let limit = token.apis[api]
-    return (limit.remaining <= 10 and limit.reset > epochTime().int)
-  else:
-    return false
-
-proc isReady(token: Token; api: Api): bool =
-  not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
-
-proc release*(token: Token; used=false; invalid=false) =
-  if token.isNil: return
-  if invalid or token.expired:
-    let idx = tokenPool.find(token)
-    if idx > -1: tokenPool.delete(idx)
-  elif used:
-    dec token.pending
-    token.lastUse = getTime()
-
-proc getToken*(api: Api): Future[Token] {.async.} =
-  for i in 0 ..< tokenPool.len:
-    if result.isReady(api): break
-    release(result)
-    result = tokenPool.sample()
-
-  if not result.isReady(api):
-    release(result)
-    result = await fetchToken()
-    tokenPool.add result
-
-  if not result.isNil:
-    inc result.pending
-  else:
-    raise rateLimitError()
-
-proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
-  # avoid undefined behavior in race conditions
-  if api in token.apis:
-    let limit = token.apis[api]
-    if limit.reset >= reset and limit.remaining < remaining:
-      return
-
-  token.apis[api] = RateLimit(remaining: remaining, reset: reset)
-
-proc poolTokens*(amount: int) {.async.} =
-  var futs: seq[Future[Token]]
-  for i in 0 ..< amount:
-    futs.add fetchToken()
-
-  for token in futs:
-    var newToken: Token
-
-    try: newToken = await token
-    except: discard
-
-    if not newToken.isNil:
-      tokenPool.add newToken
-
-proc initTokenPool*(cfg: Config) {.async.} =
-  clientPool = HttpPool()
-
-  while true:
-    if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
-      await poolTokens(min(4, cfg.minTokens - tokenPool.len))
-    await sleepAsync(2000)

src/types.nim

@@ -6,45 +6,75 @@ genPrefsType()
 
 type
   RateLimitError* = object of CatchableError
+  NoSessionsError* = object of CatchableError
   InternalError* = object of CatchableError
+  BadClientError* = object of CatchableError
 
-  Api* {.pure.} = enum
+  TimelineKind* {.pure.} = enum
-    userShow
+    tweets, replies, media
-    timeline
+
-    search
+  ApiUrl* = object
-    tweet
+    endpoint*: string
-    list
+    params*: seq[(string, string)]
-    listBySlug
+
-    listMembers
+  ApiReq* = object
-    userRestId
+    oauth*: ApiUrl
-    status
+    cookie*: ApiUrl
 
   RateLimit* = object
+    limit*: int
     remaining*: int
     reset*: int
 
-  Token* = ref object
+  SessionKind* = enum
-    tok*: string
+    oauth
-    init*: Time
+    cookie
-    lastUse*: Time
+
+  Session* = ref object
+    id*: int64
+    username*: string
     pending*: int
-    apis*: Table[Api, RateLimit]
+    limited*: bool
+    limitedAt*: int
+    apis*: Table[string, RateLimit]
+    case kind*: SessionKind
+    of oauth:
+      oauthToken*: string
+      oauthSecret*: string
+    of cookie:
+      authToken*: string
+      ct0*: string
 
   Error* = enum
     null = 0
     noUserMatches = 17
     protectedUser = 22
+    missingParams = 25
+    timeout = 29
     couldntAuth = 32
     doesntExist = 34
+    unauthorized = 37
+    invalidParam = 47
     userNotFound = 50
     suspended = 63
     rateLimited = 88
-    invalidToken = 89
+    expiredToken = 89
     listIdOrSlug = 112
     tweetNotFound = 144
+    tweetNotAuthorized = 179
     forbidden = 200
+    badRequest = 214
     badToken = 239
+    locked = 326
     noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422
+
+  VerifiedType* = enum
+    none = "None"
+    blue = "Blue"
+    business = "Business"
+    government = "Government"
 
   User* = object
     id*: string
@@ -61,7 +91,7 @@ type
     tweets*: int
     likes*: int
     media*: int
-    verified*: bool
+    verifiedType*: VerifiedType
     protected*: bool
     suspended*: bool
     joinDate*: DateTime
@@ -75,12 +105,12 @@ type
     contentType*: VideoType
     url*: string
     bitrate*: int
+    resolution*: int
 
   Video* = object
     durationMs*: int
     url*: string
     thumb*: string
-    views*: string
     available*: bool
     reason*: string
     title*: string
@@ -100,7 +130,7 @@ type
     fromUser*: seq[string]
     since*: string
     until*: string
-    near*: string
+    minLikes*: string
     sep*: string
 
   Gif* = object
@@ -144,6 +174,8 @@ type
     imageDirectMessage = "image_direct_message"
     audiospace = "audiospace"
     newsletterPublication = "newsletter_publication"
+    jobDetails = "job_details"
+    hidden
     unknown
 
   Card* = object
@@ -159,7 +191,7 @@ type
     replies*: int
     retweets*: int
     likes*: int
-    quotes*: int
+    views*: int
 
   Tweet* = ref object
     id*: int64
@@ -174,6 +206,8 @@ type
     available*: bool
     tombstone*: string
     location*: string
+    # Unused, needed for backwards compat
+    source*: string
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
@@ -185,6 +219,8 @@ type
     video*: Option[Video]
     photos*: seq[string]
 
+  Tweets* = seq[Tweet]
+
   Result*[T] = object
     content*: seq[T]
     top*, bottom*: string
@@ -192,7 +228,7 @@ type
     query*: Query
 
   Chain* = object
-    content*: seq[Tweet]
+    content*: Tweets
     hasMore*: bool
     cursor*: string
 
@@ -202,7 +238,7 @@ type
     after*: Chain
     replies*: Result[Chain]
 
-  Timeline* = Result[Tweet]
+  Timeline* = Result[Tweets]
 
   Profile* = object
     user*: User
@@ -239,6 +275,7 @@ type
     enableDebug*: bool
     proxy*: string
     proxyAuth*: string
+    disableTid*: bool
 
     rssCacheTime*: int
     listCacheTime*: int
@@ -254,3 +291,6 @@ type
 
 proc contains*(thread: Chain; tweet: Tweet): bool =
   thread.content.anyIt(it.id == tweet.id)
+
+proc add*(timeline: var seq[Tweets]; tweet: Tweet) =
+  timeline.add @[tweet]

src/utils.nim

@@ -16,7 +16,8 @@ const
     "twimg.com",
     "abs.twimg.com",
     "pbs.twimg.com",
-    "video.twimg.com"
+    "video.twimg.com",
+    "x.com"
   ]
 
 proc setHmacKey*(key: string) =
@@ -42,6 +43,12 @@ proc getPicUrl*(link: string): string =
   else:
     &"/pic/{encodeUrl(link)}"
 
+proc getOrigPicUrl*(link: string): string =
+  if base64Media:
+    &"/pic/orig/enc/{encode(link, safe=true)}"
+  else:
+    &"/pic/orig/{encodeUrl(link)}"
+
 proc filterParams*(params: Table): seq[(string, string)] =
   for p in params.pairs():
     if p[1].len > 0 and p[0] notin nitterParams:
@@ -51,4 +58,4 @@ proc isTwitterUrl*(uri: Uri): bool =
   uri.hostname in twitterDomains
 
 proc isTwitterUrl*(url: string): bool =
-  parseUri(url).hostname in twitterDomains
+  isTwitterUrl(parseUri(url))

src/views/embed.nim

@@ -11,10 +11,11 @@ const doctype = "<!DOCTYPE html>\n"
 proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let thumb = get(tweet.video).thumb
   let vidUrl = getVideoEmbed(cfg, tweet.id)
-  let prefs = Prefs(hlsPlayback: true)
+  let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
+    body:
       tdiv(class="embed-video"):
         renderVideo(get(tweet.video), prefs, "")
 

src/views/general.nim

@@ -30,15 +30,15 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
       tdiv(class="nav-item right"):
         icon "search", title="Search", href="/search"
         if cfg.enableRss and rss.len > 0:
-          icon "rss-feed", title="RSS Feed", href=rss
+          icon "rss", title="RSS Feed", href=rss
-        icon "bird", title="Open in Twitter", href=canonical
+        icon "bird", title="Open in X", href=canonical
         a(href="https://liberapay.com/zedeus"): verbatim lp
         icon "info", title="About", href="/about"
         icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
 
 proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
                  video=""; images: seq[string] = @[]; banner=""; ogTitle="";
-                 rss=""; canonical=""): VNode =
+                 rss=""; alternate=""): VNode =
   var theme = prefs.theme.toTheme
   if "theme" in req.params:
     theme = req.params["theme"].toTheme
@@ -52,8 +52,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=16")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=22")
-    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
+    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=4")
 
     if theme.len > 0:
       link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css"))
@@ -66,14 +66,14 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     link(rel="search", type="application/opensearchdescription+xml", title=cfg.title,
                             href=opensearchUrl)
 
-    if canonical.len > 0:
+    if alternate.len > 0:
-      link(rel="canonical", href=canonical)
+      link(rel="alternate", href=alternate, title="View on X")
 
     if cfg.enableRss and rss.len > 0:
       link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
 
     if prefs.hlsPlayback:
-      script(src="/js/hls.light.min.js", `defer`="")
+      script(src="/js/hls.min.js", `defer`="")
       script(src="/js/hlsPlayback.js", `defer`="")
 
     if prefs.infiniteScroll:
@@ -93,14 +93,13 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     meta(property="og:site_name", content="Nitter")
     meta(property="og:locale", content="en_US")
 
-    if banner.len > 0:
+    if banner.len > 0 and not banner.startsWith('#'):
       let bannerUrl = getPicUrl(banner)
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
-                   else: "?name=small"
+                       else: getSmallPic(url)
-      let preloadUrl = getPicUrl(url & suffix)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
 
       let image = getUrlPrefix(cfg) & getPicUrl(url)
@@ -120,20 +119,20 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     # this is last so images are also preloaded
     # if this is done earlier, Chrome only preloads one image for some reason
     link(rel="preload", type="font/woff2", `as`="font",
-         href="/fonts/fontello.woff2?21002321", crossorigin="anonymous")
+         href="/fonts/fontello.woff2?61663884", crossorigin="anonymous")
 
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
                  images: seq[string] = @[]; banner=""): string =
 
-  let canonical = getTwitterLink(req.path, req.params)
+  let twitterLink = getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
-               rss, canonical)
+               rss, twitterLink)
 
     body:
-      renderNavbar(cfg, req, rss, canonical)
+      renderNavbar(cfg, req, rss, twitterLink)
 
       tdiv(class="container"):
         body

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -78,8 +78,11 @@ proc renderPhotoRail(profile: Profile): VNode =
     tdiv(class="photo-rail-grid"):
       for i, photo in profile.photoRail:
         if i == 16: break
+        let photoSuffix =
+          if "format" in photo.url or "placeholder" in photo.url: ""
+          else: ":thumb"
         a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
-          genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
+          genImg(photo.url & photoSuffix)
 
 proc renderBanner(banner: string): VNode =
   buildHtml():
@@ -105,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -1,11 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils
+import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
-  if class.len > 0: c = c & " " & class
+  if class.len > 0: c = &"{c} {class}"
   buildHtml(tdiv(class="icon-container")):
     if href.len > 0:
       a(class=c, title=title, href=href)
@@ -15,6 +23,15 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
     if text.len > 0:
       text " " & text
 
+template verifiedIcon*(user: User): untyped {.dirty.} =
+  if user.verifiedType != VerifiedType.none:
+    let lower = ($user.verifiedType).toLowerAscii()
+    buildHtml(tdiv(class=(&"verified-icon {lower}"))):
+      icon "circle", class="verified-icon-circle", title=(&"Verified {lower} account")
+      icon "ok", class="verified-icon-check", title=(&"Verified {lower} account")
+  else:
+    text ""
+
 proc linkUser*(user: User, class=""): VNode =
   let
     isName = "username" notin class
@@ -24,9 +41,9 @@ proc linkUser*(user: User, class=""): VNode =
 
   buildHtml(a(href=href, class=class, title=nameText)):
     text nameText
-    if isName and user.verified:
+    if isName:
-      icon "ok", class="verified-icon", title="Verified account"
+      verifiedIcon(user)
-    if isName and user.protected:
+      if user.protected:
         text " "
         icon "lock", title="Protected account"
 
@@ -51,46 +68,44 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
 proc genCheckbox*(pref, label: string; state: bool): VNode =
   buildHtml(label(class="pref-group checkbox-container")):
     text label
-    if state: input(name=pref, `type`="checkbox", checked="")
+    input(name=pref, `type`="checkbox", checked=state)
-    else: input(name=pref, `type`="checkbox")
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    if state.len == 0:
+    input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
-      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
-    else:
-      input(name=pref, `type`="text", placeholder=p, value=state)
 
 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
   buildHtml(tdiv(class="pref-group pref-input")):
     label(`for`=pref): text label
     select(name=pref):
       for opt in options:
-        if opt == state:
+        option(value=opt, selected=(opt == state)):
-          option(value=opt, selected=""): text opt
+          text opt
-        else:
-          option(value=opt): text opt
 
 proc genDate*(pref, state: string): VNode =
   buildHtml(span(class="date-input")):
     input(name=pref, `type`="date", value=state)
     icon "calendar"
 
+proc genNumberInput*(pref, label, state, placeholder: string; class=""; autofocus=true; min="0"): VNode =
+  let p = placeholder
+  buildHtml(tdiv(class=("pref-group pref-input " & class))):
+    if label.len > 0:
+      label(`for`=pref): text label
+    input(name=pref, `type`="number", placeholder=p, value=state, autofocus=(autofocus and state.len == 0), min=min, step="1")
+
 proc genImg*(url: string; class=""): VNode =
   buildHtml():
-    img(src=getPicUrl(url), class=class, alt="")
+    img(src=getPicUrl(url), class=class, alt="", loading="lazy")
 
 proc getTabClass*(query: Query; tab: QueryKind): string =
-  result = "tab-item"
+  if query.kind == tab: "tab-item active"
-  if query.kind == tab:
+  else: "tab-item"
-    result &= " active"
 
 proc getAvatarClass*(prefs: Prefs): string =
-  if prefs.squareAvatars:
+  if prefs.squareAvatars: "avatar"
-    "avatar"
+  else: "avatar round"
-  else:
-    "avatar round"

src/views/rss.nimf

@@ -25,7 +25,25 @@
 #end proc
 #
 #proc getDescription(desc: string; cfg: Config): string =
-Twitter feed for: ${desc}. Generated by ${cfg.hostname}
+Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
+#end proc
+#
+#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
+#result = profile.tweets.content
+#if profile.pinned.isSome and result.len > 0:
+#  let pinnedTweet = profile.pinned.get
+#  var inserted = false
+#  for threadIdx in 0 ..< result.len:
+#    if not inserted:
+#      for tweetIdx in 0 ..< result[threadIdx].len:
+#        if result[threadIdx][tweetIdx].id < pinnedTweet.id:
+#          result[threadIdx].insert(pinnedTweet, tweetIdx)
+#          inserted = true
+#        end if
+#      end for
+#    end if
+#  end for
+#end if
 #end proc
 #
 #proc renderRssTweet(tweet: Tweet; cfg: Config): string =
@@ -33,35 +51,52 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 #let urlPrefix = getUrlPrefix(cfg)
 #let text = replaceUrls(tweet.text, defaultPrefs, absolute=urlPrefix)
 <p>${text.replace("\n", "<br>\n")}</p>
-#if tweet.quote.isSome and get(tweet.quote).available:
-#  let quoteLink = getLink(get(tweet.quote))
-<p><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></p>
-#end if
 #if tweet.photos.len > 0:
 #  for photo in tweet.photos:
 <img src="${urlPrefix}${getPicUrl(photo)}" style="max-width:250px;" />
 #  end for
 #elif tweet.video.isSome:
+<a href="${urlPrefix}${tweet.getLink}">
+<br>Video<br>
   <img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+</a>
 #elif tweet.gif.isSome:
 #  let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
 #  let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
 <video poster="${thumb}" autoplay muted loop style="max-width:250px;">
-  <source src="${url}" type="video/mp4"</source></video>
+  <source src="${url}" type="video/mp4"></video>
 #elif tweet.card.isSome:
 #  let card = tweet.card.get()
 #  if card.image.len > 0:
 <img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
 #  end if
 #end if
+#if tweet.quote.isSome and get(tweet.quote).available:
+#  let quoteTweet = get(tweet.quote)
+#  let quoteLink = urlPrefix & getLink(quoteTweet)
+<hr/>
+<blockquote>
+<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
+<p>
+${renderRssTweet(quoteTweet, cfg)}
+</p>
+<footer>
+— <cite><a href="${quoteLink}">${quoteLink}</a>
+</footer>
+</blockquote>
+#end if
 #end proc
 #
-#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string =
+#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string =
 #let urlPrefix = getUrlPrefix(cfg)
 #var links: seq[string]
-#for t in tweets:
+#for thread in tweets:
-#  let retweet = if t.retweet.isSome: t.user.username else: ""
+#  for tweet in thread:
-#  let tweet = if retweet.len > 0: t.retweet.get else: t
+#    if userId.len > 0 and tweet.user.id != userId: continue
+#    end if
+#
+#    let retweet = if tweet.retweet.isSome: tweet.user.username else: ""
+#    let tweet = if retweet.len > 0: tweet.retweet.get else: tweet
 #    let link = getLink(tweet)
 #    if link in links: continue
 #    end if
@@ -75,6 +110,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
         <link>${urlPrefix & link}</link>
       </item>
 #  end for
+#end for
 #end proc
 #
 #proc renderTimelineRss*(profile: Profile; cfg: Config; multi=false): string =
@@ -101,14 +137,15 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
       <width>128</width>
       <height>128</height>
     </image>
-#if profile.tweets.content.len > 0:
+#let tweetsList = getTweetsWithPinned(profile)
-${renderRssTweets(profile.tweets.content, cfg)}
+#if tweetsList.len > 0:
+${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
 #end if
   </channel>
 </rss>
 #end proc
 #
-#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string =
+#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}"
 #result = ""
 <?xml version="1.0" encoding="UTF-8"?>
@@ -117,7 +154,7 @@ ${renderRssTweets(profile.tweets.content, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>${xmltree.escape(list.name)} / @${list.username}</title>
     <link>${link}</link>
-    <description>${getDescription(list.name & " by @" & list.username, cfg)}</description>
+    <description>${getDescription(&"{list.name} by @{list.username}", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}
@@ -125,7 +162,7 @@ ${renderRssTweets(tweets, cfg)}
 </rss>
 #end proc
 #
-#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string =
+#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string =
 #let link = &"{getUrlPrefix(cfg)}/search"
 #let escName = xmltree.escape(name)
 #result = ""
@@ -135,7 +172,7 @@ ${renderRssTweets(tweets, cfg)}
     <atom:link href="${link}" rel="self" type="application/rss+xml" />
     <title>Search results for "${escName}"</title>
     <link>${link}</link>
-    <description>${getDescription("Search \"" & escName & "\"", cfg)}</description>
+    <description>${getDescription(&"Search \"{escName}\"", cfg)}</description>
     <language>en-us</language>
     <ttl>40</ttl>
 ${renderRssTweets(tweets, cfg)}

src/views/search.nim

@@ -10,23 +10,21 @@ const toggles = {
   "media": "Media",
   "videos": "Videos",
   "news": "News",
-  "verified": "Verified",
   "native_video": "Native videos",
   "replies": "Replies",
   "links": "Links",
   "images": "Images",
-  "safe": "Safe",
   "quote": "Quotes",
-  "pro_video": "Pro videos"
+  "spaces": "Spaces"
 }.toOrderedTable
 
 proc renderSearch*(): VNode =
   buildHtml(tdiv(class="panel-container")):
     tdiv(class="search-bar"):
       form(`method`="get", action="/search", autocomplete="off"):
-        hiddenField("f", "users")
+        hiddenField("f", "tweets")
         input(`type`="text", name="q", autofocus="",
-              placeholder="Enter username...", dir="auto")
+              placeholder="Search...", dir="auto")
         button(`type`="submit"): icon "search"
 
 proc renderProfileTabs*(query: Query; username: string): VNode =
@@ -53,7 +51,7 @@ proc renderSearchTabs*(query: Query): VNode =
 
 proc isPanelOpen(q: Query): bool =
   q.fromUser.len == 0 and (q.filters.len > 0 or q.excludes.len > 0 or
-  @[q.near, q.until, q.since].anyIt(it.len > 0))
+  @[q.minLikes, q.until, q.since].anyIt(it.len > 0))
 
 proc renderSearchPanel*(query: Query): VNode =
   let user = query.fromUser.join(",")
@@ -63,12 +61,10 @@ proc renderSearchPanel*(query: Query): VNode =
     hiddenField("f", "tweets")
     genInput("q", "", query.text, "Enter search...", class="pref-inline")
     button(`type`="submit"): icon "search"
-    if isPanelOpen(query):
+
-      input(id="search-panel-toggle", `type`="checkbox", checked="")
+    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
-    else:
+    label(`for`="search-panel-toggle"): icon "down"
-      input(id="search-panel-toggle", `type`="checkbox")
+
-    label(`for`="search-panel-toggle"):
-      icon "down"
     tdiv(class="search-panel"):
       for f in @["filter", "exclude"]:
         span(class="search-title"): text capitalize(f)
@@ -87,10 +83,10 @@ proc renderSearchPanel*(query: Query): VNode =
             span(class="search-title"): text "-"
             genDate("until", query.until)
         tdiv:
-          span(class="search-title"): text "Near"
+          span(class="search-title"): text "Minimum likes"
-          genInput("near", "", query.near, placeholder="Location...")
+          genNumberInput("min_faves", "", query.minLikes, "Number...", autofocus=false)
 
-proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
+proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =
   let query = results.query
   buildHtml(tdiv(class="timeline-container")):

src/views/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, sequtils, algorithm, uri, options
+import strutils, strformat, algorithm, uri, options
 import karax/[karaxdsl, vdom]
 
 import ".."/[types, query, formatters]
@@ -39,26 +39,24 @@ proc renderNoneFound(): VNode =
     h2(class="timeline-none"):
       text "No items found"
 
-proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
+proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="thread-line")):
     let sortedThread = thread.sortedByIt(it.id)
     for i, tweet in sortedThread:
+      # thread has a gap, display "more replies" link
+      if i > 0 and tweet.replyId != sortedThread[i - 1].id:
+        tdiv(class="timeline-item thread more-replies-thread"):
+          tdiv(class="more-replies"):
+            a(class="more-replies-text", href=getLink(tweet)):
+              text "more replies"
+
       let show = i == thread.high and sortedThread[0].id != tweet.threadId
       let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: ""
       renderTweet(tweet, prefs, path, class=(header & "thread"),
                   index=i, last=(i == thread.high), showThread=show)
 
-proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] =
-  result = @[it]
-  if it.retweet.isSome or it.replyId in threads: return
-  for t in tweets:
-    if t.id == result[0].replyId:
-      result.insert t
-    elif t.replyId == result[0].id:
-      result.add t
-
 proc renderUser(user: User; prefs: Prefs): VNode =
-  buildHtml(tdiv(class="timeline-item")):
+  buildHtml(tdiv(class="timeline-item", data-username=user.username)):
     a(class="tweet-link", href=("/" & user.username))
     tdiv(class="tweet-body profile-result"):
       tdiv(class="tweet-header"):
@@ -89,7 +87,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
     else:
       renderNoMore()
 
-proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
+proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
                            pinned=none(Tweet)): VNode =
   buildHtml(tdiv(class="timeline")):
     if not results.beginning:
@@ -105,26 +103,26 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
       else:
         renderNoneFound()
     else:
-      var
+      var retweets: seq[int64]
-        threads: seq[int64]
-        retweets: seq[int64]
 
-      for tweet in results.content:
+      for thread in results.content:
-        let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
+        if thread.len == 1:
+          let
+            tweet = thread[0]
+            retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
 
-        if tweet.id in threads or rt in retweets or tweet.id in retweets or
+          if retweetId in retweets or tweet.id in retweets or
-           tweet.pinned and prefs.hidePins: continue
+             tweet.pinned and prefs.hidePins:
+            continue
 
-        let thread = results.content.threadFilter(threads, tweet)
-        if thread.len < 2:
           var hasThread = tweet.hasThread
-          if rt != 0:
+          if retweetId != 0 and tweet.retweet.isSome:
-            retweets &= rt
+            retweets &= retweetId
             hasThread = get(tweet.retweet).hasThread
           renderTweet(tweet, prefs, path, showThread=hasThread)
         else:
           renderThread(thread, prefs, path)
-          threads &= thread.mapIt(it.id)
 
+      if results.bottom.len > 0:
         renderMore(results.query, results.bottom)
       renderToTop()

src/views/tweet.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, sequtils, strformat, options
+import strutils, sequtils, strformat, options, algorithm
 import karax/[karaxdsl, vdom, vstyles]
 from jester import Request
 
@@ -7,26 +7,19 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general
 
-proc getSmallPic(url: string): string =
+const doctype = "<!DOCTYPE html>\n"
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= ":small"
-  result = getPicUrl(result)
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
-  let url = getPicUrl(user.getUserPic("_mini"))
+  genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
-  buildHtml():
-    img(class=(prefs.getAvatarClass & " mini"), src=url)
 
-proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
+proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
   buildHtml(tdiv):
-    if retweet.len > 0:
+    if pinned:
-      tdiv(class="retweet-header"):
-        span: icon "retweet", retweet & " retweeted"
-
-    if tweet.pinned:
       tdiv(class="pinned"):
         span: icon "pin", "Pinned Tweet"
+    elif retweet.len > 0:
+      tdiv(class="retweet-header"):
+        span: icon "retweet", retweet & " retweeted"
 
     tdiv(class="tweet-header"):
       a(class="tweet-avatar", href=("/" & tweet.user.username)):
@@ -57,19 +50,21 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              orig = if named: photo else: photo & "?name=orig"
+              small = if named: photo else: photo & smallWebp
-              small = if named: photo else: photo & "?name=small"
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
-            a(href=getPicUrl(orig), class="still-image", target="_blank"):
               genImg(small)
 
-proc isPlaybackEnabled(prefs: Prefs; video: Video): bool =
+proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
-  case video.playbackType
+  case playbackType
   of mp4: prefs.mp4Playback
   of m3u8, vmap: prefs.hlsPlayback
 
-proc renderVideoDisabled(video: Video; path: string): VNode =
+proc hasMp4Url(video: Video): bool =
+  video.variants.anyIt(it.contentType == mp4)
+
+proc renderVideoDisabled(playbackType: VideoType; path: string): VNode =
   buildHtml(tdiv(class="video-overlay")):
-    case video.playbackType
+    case playbackType
     of mp4:
       p: text "mp4 playback disabled in preferences"
     of m3u8, vmap:
@@ -84,35 +79,37 @@ proc renderVideoUnavailable(video: Video): VNode =
       p: text "This media is unavailable"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
-  let container =
+  let
-    if video.description.len > 0 or video.title.len > 0: " card-container"
+    container = if video.description.len == 0 and video.title.len == 0: ""
-    else: ""
+                else: " card-container"
+    playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
+                   else: video.playbackType
 
   buildHtml(tdiv(class="attachments card")):
     tdiv(class="gallery-video" & container):
       tdiv(class="attachment video-container"):
         let thumb = getSmallPic(video.thumb)
         if not video.available:
-          img(src=thumb)
+          img(src=thumb, loading="lazy")
           renderVideoUnavailable(video)
-        elif not prefs.isPlaybackEnabled(video):
+        elif not prefs.isPlaybackEnabled(playbackType):
-          img(src=thumb)
+          img(src=thumb, loading="lazy")
-          renderVideoDisabled(video, path)
+          renderVideoDisabled(playbackType, path)
         else:
-          let vid = video.variants.filterIt(it.contentType == video.playbackType)
+          let
-          let source = getVidUrl(vid[0].url)
+            vars = video.variants.filterIt(it.contentType == playbackType)
-          case video.playbackType
+            vidUrl = vars.sortedByIt(it.resolution)[^1].url
+            source = if prefs.proxyVideos: getVidUrl(vidUrl)
+                     else: vidUrl
+          case playbackType
           of mp4:
-            if prefs.muteVideos:
+            video(poster=thumb, controls="", muted=prefs.muteVideos):
-              video(poster=thumb, controls="", muted=""):
-                source(src=source, `type`="video/mp4")
-            else:
-              video(poster=thumb, controls=""):
               source(src=source, `type`="video/mp4")
           of m3u8, vmap:
-            video(poster=thumb, data-url=source, data-autoload="false")
+            video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
             verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
             tdiv(class="overlay-circle"): span(class="overlay-triangle")
+            tdiv(class="overlay-duration"): text getDuration(video)
             verbatim "</div>"
       if container.len > 0:
         tdiv(class="card-content"):
@@ -124,14 +121,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
   buildHtml(tdiv(class="attachments media-gif")):
     tdiv(class="gallery-gif", style={maxHeight: "unset"}):
       tdiv(class="attachment"):
-        let thumb = getSmallPic(gif.thumb)
+        video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
-        let url = getPicUrl(gif.url)
+              controls="", muted="", loop=""):
-        if prefs.autoplayGifs:
+          source(src=getPicUrl(gif.url), `type`="video/mp4")
-          video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
-            source(src=url, `type`="video/mp4")
-        else:
-          video(class="gif", poster=thumb, controls="", muted="", loop=""):
-            source(src=url, `type`="video/mp4")
 
 proc renderPoll(poll: Poll): VNode =
   buildHtml(tdiv(class="poll")):
@@ -146,12 +138,12 @@ proc renderPoll(poll: Poll): VNode =
         span(class="poll-choice-value"): text percStr
         span(class="poll-choice-option"): text poll.options[i]
     span(class="poll-info"):
-      text insertSep($poll.votes, ',') & " votes • " & poll.status
+      text &"{insertSep($poll.votes, ',')} votes • {poll.status}"
 
 proc renderCardImage(card: Card): VNode =
   buildHtml(tdiv(class="card-image-container")):
     tdiv(class="card-image"):
-      img(src=getPicUrl(card.image), alt="")
+      genImg(card.image)
       if card.kind == player:
         tdiv(class="card-overlay"):
           tdiv(class="overlay-circle"):
@@ -187,14 +179,12 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(stats: TweetStats; views: string): VNode =
+proc renderStats(stats: TweetStats): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
     span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
-    span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
     span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
-    if views.len > 0:
+    span(class="tweet-stat"): icon "views", formatStat(stats.views)
-      span(class="tweet-stat"): icon "play", insertSep(views, ',')
 
 proc renderReply(tweet: Tweet): VNode =
   buildHtml(tdiv(class="replying-to")):
@@ -207,8 +197,7 @@ proc renderAttribution(user: User; prefs: Prefs): VNode =
   buildHtml(a(class="attribution", href=("/" & user.username))):
     renderMiniAvatar(user, prefs)
     strong: text user.fullname
-    if user.verified:
+    verifiedIcon(user)
-      icon "ok", class="verified-icon", title="Verified account"
 
 proc renderMediaTags(tags: seq[User]): VNode =
   buildHtml(tdiv(class="media-tag-block")):
@@ -284,7 +273,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
     divClass = "thread-last " & class
 
   if not tweet.available:
-    return buildHtml(tdiv(class=divClass & "unavailable timeline-item")):
+    return buildHtml(tdiv(class=divClass & "unavailable timeline-item", data-username=tweet.user.username)):
       tdiv(class="unavailable-box"):
         if tweet.tombstone.len > 0:
           text tweet.tombstone
@@ -296,20 +285,22 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.quote.isSome:
         renderQuote(tweet.quote.get(), prefs, path)
 
-  let fullTweet = tweet
+  let
+    fullTweet = tweet
+    pinned = tweet.pinned
+
   var retweet: string
   var tweet = fullTweet
   if tweet.retweet.isSome:
     tweet = tweet.retweet.get
     retweet = fullTweet.user.fullname
 
-  buildHtml(tdiv(class=("timeline-item " & divClass))):
+  buildHtml(tdiv(class=("timeline-item " & divClass), data-username=tweet.user.username)):
     if not mainTweet:
       a(class="tweet-link", href=getLink(tweet))
 
     tdiv(class="tweet-body"):
-      var views = ""
+      renderHeader(tweet, retweet, pinned, prefs)
-      renderHeader(tweet, retweet, prefs)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
          (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
@@ -325,17 +316,15 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.attribution.isSome:
         renderAttribution(tweet.attribution.get(), prefs)
 
-      if tweet.card.isSome:
+      if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)
 
       if tweet.photos.len > 0:
         renderAlbum(tweet)
       elif tweet.video.isSome:
         renderVideo(tweet.video.get(), prefs, path)
-        views = tweet.video.get().views
       elif tweet.gif.isSome:
         renderGif(tweet.gif.get(), prefs)
-        views = "GIF"
 
       if tweet.poll.isSome:
         renderPoll(tweet.poll.get())
@@ -344,19 +333,24 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderQuote(tweet.quote.get(), prefs, path)
 
       if mainTweet:
-        p(class="tweet-published"): text getTime(tweet)
+        p(class="tweet-published"): text &"{getTime(tweet)}"
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)
 
       if not prefs.hideTweetStats:
-        renderStats(tweet.stats, views)
+        renderStats(tweet.stats)
 
       if showThread:
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
-  buildHtml(tdiv(class="tweet-embed")):
+  let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req)
+
+    body:
+      tdiv(class="tweet-embed"):
         renderTweet(tweet, prefs, path, mainTweet=true)
+
+  result = doctype & $node

tests/requirements.txt

@@ -0,0 +1 @@
+seleniumbase

tests/test_card.py

@@ -3,11 +3,6 @@ from parameterized import parameterized
 
 
 card = [
-    ['Thom_Wolf/status/1122466524860702729',
-     'pytorch/fairseq',
-     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
-     'github.com', True],
-
     ['nim_lang/status/1136652293510717440',
      'Version 0.20.0 released',
      'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@@ -16,39 +11,29 @@ card = [
     ['voidtarget/status/1094632512926605312',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
      'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
-     'gist.github.com', True],
+     'gist.github.com', True]
-
-    ['FluentAI/status/1116417904831029248',
-     'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
-     'One of the only ways to improve Alexa is to have human beings check it for errors',
-     'theverge.com', True]
 ]
 
 no_thumb = [
-    ['Bountysource/status/1141879700639215617',
+    ['FluentAI/status/1116417904831029248',
-     'Post a bounty on kivy/plyer!',
+     'LinkedIn',
-     'Automation and Screen Reader Support',
+     'This link will take you to a page that’s not on LinkedIn',
-     'bountysource.com'],
+     'lnkd.in'],
+
+    ['Thom_Wolf/status/1122466524860702729',
+     'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
+     '',
+     'github.com'],
 
     ['brent_p/status/1088857328680488961',
-     'Hts Nim Sugar',
+     'GitHub - brentp/hts-nim: nim wrapper for htslib for parsing genomics data files',
-     'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
+     '',
-     'brentp.github.io'],
+     'github.com'],
 
     ['voidtarget/status/1133028231672582145',
      'sinkingsugar/nimqt-example',
      'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
-     'github.com'],
+     'github.com']
-
-    ['mobile_test/status/490378953744318464',
-     'Nantasket Beach',
-     'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px',
-     '500px.com'],
-
-    ['nim_lang/status/1082989146040340480',
-     'Nim in 2018: A short recap',
-     'Posted in r/programming by u/miran1',
-     'reddit.com']
 ]
 
 playable = [
@@ -63,17 +48,6 @@ playable = [
      'youtube.com']
 ]
 
-# promo = [
-    # ['BangOlufsen/status/1145698701517754368',
-    #  'Upgrade your journey', '',
-    #  'www.bang-olufsen.com'],
-
-    # ['BangOlufsen/status/1154934429900406784',
-    #  'Learn more about Beosound Shape', '',
-    #  'www.bang-olufsen.com']
-# ]
-
-
 class CardTest(BaseTestCase):
     @parameterized.expand(card)
     def test_card(self, tweet, title, description, destination, large):
@@ -81,7 +55,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         if len(description) > 0:
             self.assert_text(description, c.description)
         if large:
@@ -104,17 +78,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         self.assert_element_visible('.card-overlay')
         if len(description) > 0:
             self.assert_text(description, c.description)
-
-    # @parameterized.expand(promo)
-    # def test_card_promo(self, tweet, title, description, destination):
-    #     self.open_nitter(tweet)
-    #     c = Card(Conversation.main + " ")
-    #     self.assert_text(title, c.title)
-    #     self.assert_text(destination, c.destination)
-    #     self.assert_element_visible('.video-overlay')
-    #     if len(description) > 0:
-    #         self.assert_text(description, c.description)

tests/test_profile.py

@@ -4,7 +4,7 @@ from parameterized import parameterized
 profiles = [
         ['mobile_test', 'Test account',
          'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
-         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'],
+         'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '97'],
         ['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
 ]
 
@@ -17,11 +17,6 @@ protected = [
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
-banner_color = [
-    ['nim_lang', '22, 25, 32'],
-    ['rustlang', '35, 31, 32']
-]
-
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -71,14 +66,8 @@ class ProfileTest(BaseTestCase):
         self.assert_text(f'User "{username}" not found')
 
     def test_suspended(self):
-        self.open_nitter('user')
+        self.open_nitter('suspendme')
-        self.assert_text('User "user" has been suspended')
+        self.assert_text('User "suspendme" has been suspended')
-
-    @parameterized.expand(banner_color)
-    def test_banner_color(self, username, color):
-        self.open_nitter(username)
-        banner = self.find_element(Profile.banner + ' a')
-        self.assertIn(color, banner.value_of_css_property('background-color'))
 
     @parameterized.expand(banner_image)
     def test_banner_image(self, username, url):

tests/test_quote.py

@@ -2,15 +2,13 @@ from base import BaseTestCase, Quote, Conversation
 from parameterized import parameterized
 
 text = [
-    ['elonmusk/status/1138136540096319488',
+    ['nim_lang/status/1491461266849808397#m',
-     'Tesla Owners Online', '@Model3Owners',
+     'Nim', '@nim_lang',
-     """As of March 58.4% of new car sales in Norway are electric. 
+     """What's better than Nim 1.6.0?
 
-What are we doing wrong? reuters.com/article/us-norwa…"""],
+Nim 1.6.2 :)
 
-    ['nim_lang/status/924694255364341760',
+nim-lang.org/blog/2021/12/17…"""]
-     'Hacker News', '@newsycombinator',
-     'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…']
 ]
 
 image = [

tests/test_search.py

@@ -2,8 +2,8 @@ from base import BaseTestCase
 from parameterized import parameterized
 
 
-class SearchTest(BaseTestCase):
+#class SearchTest(BaseTestCase):
-    @parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
+    #@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
-    def test_username_search(self, username):
+    #def test_username_search(self, username):
-        self.search_username(username)
+        #self.search_username(username)
-        self.assert_text(f'{username}')
+        #self.assert_text(f'{username}')

tests/test_timeline.py

@@ -1,23 +1,18 @@
 from base import BaseTestCase, Timeline
 from parameterized import parameterized
 
-normal = [['mobile_test'], ['mobile_test_2']]
+normal = [['jack'], ['elonmusk']]
 
-after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
+after = [['jack', '1681686036294803456'],
-         ['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
+         ['elonmusk', '1681686036294803456']]
 
-no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
+no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
 
 empty = [['emptyuser'], ['mobile_test_10']]
 
 protected = [['mobile_test_7'], ['Empty_user']]
 
-photo_rail = [['mobile_test', [
+photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]]
-    'BzUnaDFCUAAmrjs', 'Bo0nDsYIYAIjqVn', 'Bos--KNIQAAA7Li', 'Boq1sDJIYAAxaoi',
-    'BonISmPIEAAhP3G', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG',
-    'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKovdrCUAAEz79',
-    'BkKoe_oCIAASAqr', 'BkKoRLNCAAAYfDf', 'BkKndxoCQAE1vFt', 'BPEmIbYCMAE44dl'
-]]]
 
 
 class TweetTest(BaseTestCase):
@@ -60,10 +55,10 @@ class TweetTest(BaseTestCase):
         self.assert_element_absent(Timeline.older)
         self.assert_element_absent(Timeline.end)
 
-    @parameterized.expand(photo_rail)
+    #@parameterized.expand(photo_rail)
-    def test_photo_rail(self, username, images):
+    #def test_photo_rail(self, username, images):
-        self.open_nitter(username)
+        #self.open_nitter(username)
-        self.assert_element_visible(Timeline.photo_rail)
+        #self.assert_element_visible(Timeline.photo_rail)
-        for i, url in enumerate(images):
+        #for i, url in enumerate(images):
-            img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
+            #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
-            self.assertIn(url, img)
+            #self.assertIn(url, img)

tests/test_tweet.py

@@ -1,4 +1,4 @@
-from base import BaseTestCase, Tweet, get_timeline_tweet
+from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet
 from parameterized import parameterized
 
 # image = tweet + 'div.attachments.media-body > div > div > a > div > img'
@@ -16,7 +16,7 @@ timeline = [
 ]
 
 status = [
-    [20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'],
+    [20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
     [134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
     [105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
     [572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
@@ -28,14 +28,15 @@ invalid = [
 ]
 
 multiline = [
-    [400897186990284800, 'mobile_test_3',
+    [1718660434457239868, 'WebDesignMuseum',
      """
-♔
+Happy 32nd Birthday HTML tags! 
-  KEEP
+
- CALM
+On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags. 
-   AND
+
-CLICHÉ
+The document contained a description of the first 18 HTML tags: <title>, <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language.
-    ON"""]
+
+#WebDesignHistory"""]
 ]
 
 link = [
@@ -71,11 +72,7 @@ emoji = [
 
 retweet = [
     [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
-    [3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr']
+    [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
-]
-
-reply = [
-    ['mobile_test/with_replies', 15]
 ]
 
 
@@ -103,18 +100,18 @@ class TweetTest(BaseTestCase):
     @parameterized.expand(multiline)
     def test_multiline_formatting(self, tid, username, text):
         self.open_nitter(f'{username}/status/{tid}')
-        self.assert_text(text.strip('\n'), '.main-tweet')
+        self.assert_text(text.strip('\n'), Conversation.main)
 
     @parameterized.expand(emoji)
     def test_emoji(self, tweet, text):
         self.open_nitter(tweet)
-        self.assert_text(text, '.main-tweet')
+        self.assert_text(text, Conversation.main)
 
     @parameterized.expand(link)
     def test_link(self, tweet, links):
         self.open_nitter(tweet)
         for link in links:
-            self.assert_text(link, '.main-tweet')
+            self.assert_text(link, Conversation.main)
 
     @parameterized.expand(username)
     def test_username(self, tweet, usernames):
@@ -137,8 +134,8 @@ class TweetTest(BaseTestCase):
         self.open_nitter(tweet)
         self.assert_text('Tweet not found', '.error-panel')
 
-    @parameterized.expand(reply)
+    #@parameterized.expand(reply)
-    def test_thread(self, tweet, num):
+    #def test_thread(self, tweet, num):
-        self.open_nitter(tweet)
+        #self.open_nitter(tweet)
-        thread = self.find_element(f'.timeline > div:nth-child({num})')
+        #thread = self.find_element(f'.timeline > div:nth-child({num})')
-        self.assertIn(thread.get_attribute('class'), 'thread-line')
+        #self.assertIn(thread.get_attribute('class'), 'thread-line')

tests/test_tweet_media.py

@@ -14,7 +14,7 @@ poll = [
 
 image = [
     ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
-    ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
+    #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
 ]
 
 gif = [
@@ -28,14 +28,14 @@ video_m3u8 = [
 ]
 
 gallery = [
-    ['mobile_test/status/451108446603980803', [
+    # ['mobile_test/status/451108446603980803', [
-        ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
+    #     ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
-    ]],
+    # ]],
 
-    ['mobile_test/status/471539824713691137', [
+    # ['mobile_test/status/471539824713691137', [
-        ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
+    #     ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
-        ['Bos--IqIQAAav23']
+    #     ['Bos--IqIQAAav23']
-    ]],
+    # ]],
 
     ['mobile_test/status/469530783384743936', [
         ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],

tools/create_session_browser.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+Requirements:
+  pip install -r tools/requirements.txt
+
+Usage:
+  python3 tools/create_session_browser.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
+
+Examples:
+  # Output to terminal
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET
+
+  # Append to sessions.jsonl
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --append sessions.jsonl
+
+  # Headless mode (may increase detection risk)
+  python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --headless
+
+Output:
+  {"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
+"""
+
+import sys
+import json
+import asyncio
+import pyotp
+import nodriver as uc
+import os
+
+
+async def login_and_get_cookies(username, password, totp_seed=None, headless=False):
+    """Authenticate with X.com and extract session cookies"""
+    # Note: headless mode may increase detection risk from bot-detection systems
+    browser = await uc.start(headless=headless)
+    tab = await browser.get('https://x.com/i/flow/login')
+
+    try:
+        # Enter username
+        print('[*] Entering username...', file=sys.stderr)
+        username_input = await tab.find('input[autocomplete="username"]', timeout=10)
+        await username_input.send_keys(username + '\n')
+        await asyncio.sleep(1)
+
+        # Enter password
+        print('[*] Entering password...', file=sys.stderr)
+        password_input = await tab.find('input[autocomplete="current-password"]', timeout=15)
+        await password_input.send_keys(password + '\n')
+        await asyncio.sleep(2)
+
+        # Handle 2FA if needed
+        page_content = await tab.get_content()
+        if 'verification code' in page_content or 'Enter code' in page_content:
+            if not totp_seed:
+                raise Exception('2FA required but no TOTP seed provided')
+
+            print('[*] 2FA detected, entering code...', file=sys.stderr)
+            totp_code = pyotp.TOTP(totp_seed).now()
+            code_input = await tab.select('input[type="text"]')
+            await code_input.send_keys(totp_code + '\n')
+            await asyncio.sleep(3)
+
+        # Get cookies
+        print('[*] Retrieving cookies...', file=sys.stderr)
+        for _ in range(20):  # 20 second timeout
+            cookies = await browser.cookies.get_all()
+            cookies_dict = {cookie.name: cookie.value for cookie in cookies}
+
+            if 'auth_token' in cookies_dict and 'ct0' in cookies_dict:
+                print('[*] Found both cookies', file=sys.stderr)
+
+                # Extract ID from twid cookie (may be URL-encoded)
+                user_id = None
+                if 'twid' in cookies_dict:
+                    twid = cookies_dict['twid']
+                    # Try to extract the ID from twid (format: u%3D<id> or u=<id>)
+                    if 'u%3D' in twid:
+                        user_id = twid.split('u%3D')[1].split('&')[0].strip('"')
+                    elif 'u=' in twid:
+                        user_id = twid.split('u=')[1].split('&')[0].strip('"')
+
+                cookies_dict['username'] = username
+                if user_id:
+                    cookies_dict['id'] = user_id
+
+                return cookies_dict
+
+            await asyncio.sleep(1)
+
+        raise Exception('Timeout waiting for cookies')
+
+    finally:
+        browser.stop()
+
+
+async def main():
+    if len(sys.argv) < 3:
+        print('Usage: python3 create_session_browser.py username password [totp_seed] [--append file.jsonl] [--headless]')
+        sys.exit(1)
+
+    username = sys.argv[1]
+    password = sys.argv[2]
+    totp_seed = None
+    append_file = None
+    headless = False
+
+    # Parse optional arguments
+    i = 3
+    while i < len(sys.argv):
+        arg = sys.argv[i]
+        if arg == '--append':
+            if i + 1 < len(sys.argv):
+                append_file = sys.argv[i + 1]
+                i += 2  # Skip '--append' and filename
+            else:
+                print('[!] Error: --append requires a filename', file=sys.stderr)
+                sys.exit(1)
+        elif arg == '--headless':
+            headless = True
+            i += 1
+        elif not arg.startswith('--'):
+            if totp_seed is None: 
+                totp_seed = arg
+            i += 1
+        else:
+            # Unkown args
+            print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
+            i += 1
+
+    try:
+        cookies = await login_and_get_cookies(username, password, totp_seed, headless)
+        session = {
+            'kind': 'cookie',
+            'username': cookies['username'],
+            'id': cookies.get('id'),
+            'auth_token': cookies['auth_token'],
+            'ct0': cookies['ct0']
+        }
+        output = json.dumps(session)
+
+        if append_file:
+            with open(append_file, 'a') as f:
+                f.write(output + '\n')
+            print(f'✓ Session appended to {append_file}', file=sys.stderr)
+        else:
+            print(output)
+
+        os._exit(0)
+
+    except Exception as error:
+        print(f'[!] Error: {error}', file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

tools/create_session_curl.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""
+Requirements:
+  pip install curl_cffi pyotp
+
+Usage:
+  python3 tools/create_session_curl.py <username> <password> [totp_seed] [--append sessions.jsonl]
+
+Examples:
+  # Output to terminal
+  python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET
+
+  # Append to sessions.jsonl
+  python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET --append sessions.jsonl
+
+Output:
+  {"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
+"""
+
+import sys
+import json
+import pyotp
+from curl_cffi import requests
+
+BEARER_TOKEN = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
+BASE_URL = "https://api.x.com/1.1/onboarding/task.json"
+GUEST_ACTIVATE_URL = "https://api.x.com/1.1/guest/activate.json"
+
+# Subtask versions required by API
+SUBTASK_VERSIONS = {
+    "action_list": 2, "alert_dialog": 1, "app_download_cta": 1,
+    "check_logged_in_account": 2, "choice_selection": 3,
+    "contacts_live_sync_permission_prompt": 0, "cta": 7, "email_verification": 2,
+    "end_flow": 1, "enter_date": 1, "enter_email": 2, "enter_password": 5,
+    "enter_phone": 2, "enter_recaptcha": 1, "enter_text": 5, "generic_urt": 3,
+    "in_app_notification": 1, "interest_picker": 3, "js_instrumentation": 1,
+    "menu_dialog": 1, "notifications_permission_prompt": 2, "open_account": 2,
+    "open_home_timeline": 1, "open_link": 1, "phone_verification": 4,
+    "privacy_options": 1, "security_key": 3, "select_avatar": 4,
+    "select_banner": 2, "settings_list": 7, "show_code": 1, "sign_up": 2,
+    "sign_up_review": 4, "tweet_selection_urt": 1, "update_users": 1,
+    "upload_media": 1, "user_recommendations_list": 4,
+    "user_recommendations_urt": 1, "wait_spinner": 3, "web_modal": 1
+}
+
+
+def get_base_headers(guest_token=None):
+    """Build base headers for API requests."""
+    headers = {
+        "Authorization": f"Bearer {BEARER_TOKEN}",
+        "Content-Type": "application/json",
+        "Accept": "*/*",
+        "Accept-Language": "en-US",
+        "X-Twitter-Client-Language": "en-US",
+        "Origin": "https://x.com",
+        "Referer": "https://x.com/",
+    }
+    if guest_token:
+        headers["X-Guest-Token"] = guest_token
+    return headers
+
+
+def get_cookies_dict(session):
+    """Extract cookies from session."""
+    return session.cookies.get_dict() if hasattr(session.cookies, 'get_dict') else dict(session.cookies)
+
+
+def make_request(session, headers, flow_token, subtask_data, print_msg):
+    """Generic request handler for flow steps."""
+    print(f"[*] {print_msg}...", file=sys.stderr)
+
+    payload = {
+        "flow_token": flow_token,
+        "subtask_inputs": [subtask_data] if isinstance(subtask_data, dict) else subtask_data
+    }
+
+    response = session.post(BASE_URL, json=payload, headers=headers)
+    response.raise_for_status()
+
+    data = response.json()
+    new_flow_token = data.get('flow_token')
+    if not new_flow_token:
+        raise Exception(f"Failed to get flow token: {print_msg}")
+
+    return new_flow_token, data
+
+
+def get_guest_token(session):
+    """Get guest token for unauthenticated requests."""
+    print("[*] Getting guest token...", file=sys.stderr)
+    response = session.post(GUEST_ACTIVATE_URL, headers={"Authorization": f"Bearer {BEARER_TOKEN}"})
+    response.raise_for_status()
+
+    guest_token = response.json().get('guest_token')
+    if not guest_token:
+        raise Exception("Failed to obtain guest token")
+
+    print(f"[*] Got guest token: {guest_token}", file=sys.stderr)
+    return guest_token
+
+
+def init_flow(session, guest_token):
+    """Initialize the login flow."""
+    print("[*] Initializing login flow...", file=sys.stderr)
+
+    headers = get_base_headers(guest_token)
+    payload = {
+        "input_flow_data": {
+            "flow_context": {
+                "debug_overrides": {},
+                "start_location": {"location": "manual_link"}
+            },
+            "subtask_versions": SUBTASK_VERSIONS
+        }
+    }
+
+    response = session.post(f"{BASE_URL}?flow_name=login", json=payload, headers=headers)
+    response.raise_for_status()
+
+    flow_token = response.json().get('flow_token')
+    if not flow_token:
+        raise Exception("Failed to get initial flow token")
+
+    print("[*] Got initial flow token", file=sys.stderr)
+    return flow_token, headers
+
+
+def submit_username(session, flow_token, headers, guest_token, username):
+    """Submit username."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginEnterUserIdentifierSSO",
+        "settings_list": {
+            "setting_responses": [{
+                "key": "user_identifier",
+                "response_data": {"text_data": {"result": username}}
+            }],
+            "link": "next_link"
+        }
+    }
+
+    flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting username")
+
+    # Check for denial (suspicious activity)
+    if data.get('subtasks') and 'cta' in data['subtasks'][0]:
+        error_msg = data['subtasks'][0]['cta'].get('primary_text', {}).get('text')
+        if error_msg:
+            raise Exception(f"Login denied: {error_msg}")
+
+    return flow_token
+
+
+def submit_password(session, flow_token, headers, guest_token, password):
+    """Submit password and detect if 2FA is needed."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginEnterPassword",
+        "enter_password": {"password": password, "link": "next_link"}
+    }
+
+    flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting password")
+
+    needs_2fa = any(s.get('subtask_id') == 'LoginTwoFactorAuthChallenge' for s in data.get('subtasks', []))
+    if needs_2fa:
+        print("[*] 2FA required", file=sys.stderr)
+
+    return flow_token, needs_2fa
+
+
+def submit_2fa(session, flow_token, headers, guest_token, totp_seed):
+    """Submit 2FA code."""
+    if not totp_seed:
+        raise Exception("2FA required but no TOTP seed provided")
+
+    code = pyotp.TOTP(totp_seed).now()
+    print("[*] Generating 2FA code...", file=sys.stderr)
+
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginTwoFactorAuthChallenge",
+        "enter_text": {"text": code, "link": "next_link"}
+    }
+
+    flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting 2FA code")
+    return flow_token
+
+
+def submit_js_instrumentation(session, flow_token, headers, guest_token):
+    """Submit JS instrumentation response."""
+    headers = headers.copy()
+    headers["X-Guest-Token"] = guest_token
+
+    subtask = {
+        "subtask_id": "LoginJsInstrumentationSubtask",
+        "js_instrumentation": {
+            "response": '{"rf":{"a4fc506d24bb4843c48a1966940c2796bf4fb7617a2d515ad3297b7df6b459b6":121,"bff66e16f1d7ea28c04653dc32479cf416a9c8b67c80cb8ad533b2a44fee82a3":-1,"ac4008077a7e6ca03210159dbe2134dea72a616f03832178314bb9931645e4f7":-22,"c3a8a81a9b2706c6fec42c771da65a9597c537b8e4d9b39e8e58de9fe31ff239":-12},"s":"ZHYaDA9iXRxOl2J3AZ9cc23iJx-Fg5E82KIBA_fgeZFugZGYzRtf8Bl3EUeeYgsK30gLFD2jTQx9fAMsnYCw0j8ahEy4Pb5siM5zD6n7YgOeWmFFaXoTwaGY4H0o-jQnZi5yWZRAnFi4lVuCVouNz_xd2BO2sobCO7QuyOsOxQn2CWx7bjD8vPAzT5BS1mICqUWyjZDjLnRZJU6cSQG5YFIHEPBa8Kj-v1JFgkdAfAMIdVvP7C80HWoOqYivQR7IBuOAI4xCeLQEdxlGeT-JYStlP9dcU5St7jI6ExyMeQnRicOcxXLXsan8i5Joautk2M8dAJFByzBaG4wtrPhQ3QAAAZEi-_t7"}',
+            "link": "next_link"
+        }
+    }
+
+    flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting JS instrumentation")
+    return flow_token
+
+
+def complete_flow(session, flow_token, headers):
+    """Complete the login flow."""
+    cookies = get_cookies_dict(session)
+
+    headers = headers.copy()
+    headers["X-Twitter-Auth-Type"] = "OAuth2Session"
+    if cookies.get('ct0'):
+        headers["X-Csrf-Token"] = cookies['ct0']
+
+    subtask = {
+        "subtask_id": "AccountDuplicationCheck",
+        "check_logged_in_account": {"link": "AccountDuplicationCheck_false"}
+    }
+
+    make_request(session, headers, flow_token, subtask, "Completing login flow")
+
+
+def extract_user_id(cookies_dict):
+    """Extract user ID from twid cookie."""
+    twid = cookies_dict.get('twid', '').strip('"')
+
+    for prefix in ['u=', 'u%3D']:
+        if prefix in twid:
+            return twid.split(prefix)[1].split('&')[0].strip('"')
+
+    return None
+
+
+def login_and_get_cookies(username, password, totp_seed=None):
+    """Authenticate with X.com and extract session cookies."""
+    session = requests.Session(impersonate="chrome")
+
+    try:
+        guest_token = get_guest_token(session)
+        flow_token, headers = init_flow(session, guest_token)
+        flow_token = submit_js_instrumentation(session, flow_token, headers, guest_token)
+        flow_token = submit_username(session, flow_token, headers, guest_token, username)
+        flow_token, needs_2fa = submit_password(session, flow_token, headers, guest_token, password)
+
+        if needs_2fa:
+            flow_token = submit_2fa(session, flow_token, headers, guest_token, totp_seed)
+
+        complete_flow(session, flow_token, headers)
+
+        cookies_dict = get_cookies_dict(session)
+        cookies_dict['username'] = username
+
+        user_id = extract_user_id(cookies_dict)
+        if user_id:
+            cookies_dict['id'] = user_id
+
+        print("[*] Successfully authenticated", file=sys.stderr)
+        return cookies_dict
+
+    finally:
+        session.close()
+
+
+def main():
+    if len(sys.argv) < 3:
+        print('Usage: python3 create_session_curl.py username password [totp_seed] [--append sessions.jsonl]', file=sys.stderr)
+        sys.exit(1)
+
+    username = sys.argv[1]
+    password = sys.argv[2]
+    totp_seed = None
+    append_file = None
+
+    # Parse optional arguments
+    i = 3
+    while i < len(sys.argv):
+        arg = sys.argv[i]
+        if arg == '--append':
+            if i + 1 < len(sys.argv):
+                append_file = sys.argv[i + 1]
+                i += 2
+            else:
+                print('[!] Error: --append requires a filename', file=sys.stderr)
+                sys.exit(1)
+        elif not arg.startswith('--'):
+            if totp_seed is None:
+                totp_seed = arg
+            i += 1
+        else:
+            print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
+            i += 1
+
+    try:
+        cookies = login_and_get_cookies(username, password, totp_seed)
+
+        session = {
+            'kind': 'cookie',
+            'username': cookies['username'],
+            'id': cookies.get('id'),
+            'auth_token': cookies['auth_token'],
+            'ct0': cookies['ct0']
+        }
+
+        output = json.dumps(session)
+
+        if append_file:
+            with open(append_file, 'a') as f:
+                f.write(output + '\n')
+            print(f'✓ Session appended to {append_file}', file=sys.stderr)
+        else:
+            print(output)
+
+        sys.exit(0)
+
+    except Exception as error:
+        print(f'[!] Error: {error}', file=sys.stderr)
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/get_session.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+import requests
+import json
+import sys
+import pyotp
+import cloudscraper
+
+# NOTE: pyotp, requests and cloudscraper are dependencies
+# > pip install pyotp requests cloudscraper
+
+TW_CONSUMER_KEY = '3nVuSoBZnx6U4vzUxf5w'
+TW_CONSUMER_SECRET = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'
+
+def auth(username, password, otp_secret):
+    bearer_token_req = requests.post("https://api.twitter.com/oauth2/token",
+        auth=(TW_CONSUMER_KEY, TW_CONSUMER_SECRET),
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+        data='grant_type=client_credentials'
+    ).json()
+    bearer_token = ' '.join(str(x) for x in bearer_token_req.values())
+
+    guest_token = requests.post(
+        "https://api.twitter.com/1.1/guest/activate.json",
+        headers={'Authorization': bearer_token}
+    ).json().get('guest_token')
+
+    if not guest_token:
+        print("Failed to obtain guest token.")
+        sys.exit(1)
+
+    twitter_header = {
+        'Authorization': bearer_token,
+        "Content-Type": "application/json",
+        "User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)",
+        "X-Twitter-API-Version": '5',
+        "X-Twitter-Client": "TwitterAndroid",
+        "X-Twitter-Client-Version": "10.21.0-release.0",
+        "OS-Version": "28",
+        "System-User-Agent": "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)",
+        "X-Twitter-Active-User": "yes",
+        "X-Guest-Token": guest_token,
+        "X-Twitter-Client-DeviceID": ""
+    }
+
+    scraper = cloudscraper.create_scraper()
+    scraper.headers = twitter_header
+
+    task1 = scraper.post(
+        'https://api.twitter.com/1.1/onboarding/task.json',
+        params={
+            'flow_name': 'login',
+            'api_version': '1',
+            'known_device_token': '',
+            'sim_country_code': 'us'
+        },
+        json={
+            "flow_token": None,
+            "input_flow_data": {
+                "country_code": None,
+                "flow_context": {
+                    "referrer_context": {
+                        "referral_details": "utm_source=google-play&utm_medium=organic",
+                        "referrer_url": ""
+                    },
+                    "start_location": {
+                        "location": "deeplink"
+                    }
+                },
+                "requested_variant": None,
+                "target_user_id": 0
+            }
+        }
+    )
+
+    scraper.headers['att'] = task1.headers.get('att')
+
+    task2 = scraper.post(
+        'https://api.twitter.com/1.1/onboarding/task.json',
+        json={
+            "flow_token": task1.json().get('flow_token'),
+            "subtask_inputs": [{
+                "enter_text": {
+                    "suggestion_id": None,
+                    "text": username,
+                    "link": "next_link"
+                },
+                "subtask_id": "LoginEnterUserIdentifier"
+            }]
+        }
+    )
+
+    task3 = scraper.post(
+        'https://api.twitter.com/1.1/onboarding/task.json',
+        json={
+            "flow_token": task2.json().get('flow_token'),
+            "subtask_inputs": [{
+                "enter_password": {
+                    "password": password,
+                    "link": "next_link"
+                },
+                "subtask_id": "LoginEnterPassword"
+            }],
+        }
+    )
+
+    for t3_subtask in task3.json().get('subtasks', []):
+        if "open_account" in t3_subtask:
+            return t3_subtask["open_account"]
+        elif "enter_text" in t3_subtask:
+            response_text = t3_subtask["enter_text"]["hint_text"]
+            totp = pyotp.TOTP(otp_secret)
+            generated_code = totp.now()
+            task4resp = scraper.post(
+                "https://api.twitter.com/1.1/onboarding/task.json",
+                json={
+                    "flow_token": task3.json().get("flow_token"),
+                    "subtask_inputs": [
+                        {
+                            "enter_text": {
+                                "suggestion_id": None,
+                                "text": generated_code,
+                                "link": "next_link",
+                            },
+                            "subtask_id": "LoginTwoFactorAuthChallenge",
+                        }
+                    ],
+                }
+            )
+            task4 = task4resp.json()
+            for t4_subtask in task4.get("subtasks", []):
+                if "open_account" in t4_subtask:
+                    return t4_subtask["open_account"]
+
+    return None
+
+if __name__ == "__main__":
+    if len(sys.argv) != 5:
+        print("Usage: python3 get_session.py <username> <password> <2fa secret> <path>")
+        sys.exit(1)
+
+    username = sys.argv[1]
+    password = sys.argv[2]
+    otp_secret = sys.argv[3]
+    path = sys.argv[4]
+
+    result = auth(username, password, otp_secret)
+    if result is None:
+        print("Authentication failed.")
+        sys.exit(1)
+
+    session_entry = {
+        "oauth_token": result.get("oauth_token"),
+        "oauth_token_secret": result.get("oauth_token_secret")
+    }
+
+    try:
+        with open(path, "a") as f:
+            f.write(json.dumps(session_entry) + "\n")
+        print("Authentication successful. Session appended to", path)
+    except Exception as e:
+        print(f"Failed to write session information: {e}")
+        sys.exit(1)

tools/requirements.txt

@@ -0,0 +1,3 @@
+nodriver>=0.48.0
+pyotp
+curl_cffi