Salastil/nitter
1
0
Fork 0
mirror of https://github.com/zedeus/nitter.git synced 2025-12-06 03:55:36 -05:00
Code Activity

Compare commits

base: Salastil:3f3196d10357069153dee43718bdd80b16f5b2fc
Branches Tags
Salastil:master Salastil:feature/embed-quotes Salastil:feature/rss-video Salastil:feature/mp4-streaming Salastil:tweets-parser
..
compare: Salastil:tweets-parser
Branches Tags
Salastil:master Salastil:feature/embed-quotes Salastil:feature/rss-video Salastil:feature/mp4-streaming Salastil:tweets-parser

1 Commits
3f3196d103 ... tweets-par

Author SHA1 Message Date
Zed
c9b261a793 WIP tweets/timeline parser 2022-01-30 23:38:39 +01:00
84 changed files with 1501 additions and 2407 deletions
Expand all files Collapse all files

49
.github/workflows/build-docker.yml vendored
View File

@@ -1,4 +1,4 @@
name: Docker
name: CI/CD
on:
push:
@@ -8,56 +8,31 @@ on:
- master
jobs:
tests:
uses: ./.github/workflows/run-tests.yml
secrets: inherit
build-docker-amd64:
needs: [tests]
runs-on: buildjet-2vcpu-ubuntu-2204
build-docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push AMD64 Docker image
uses: docker/build-push-action@v3
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile
platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
build-docker-arm64:
needs: [tests]
runs-on: buildjet-2vcpu-ubuntu-2204-arm
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push ARM64 Docker image
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile.arm64
platforms: linux/arm64
push: true
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

108
.github/workflows/run-tests.yml vendored
View File

@@ -1,108 +0,0 @@
name: Tests
on:
push:
paths-ignore:
- "*.md"
branches-ignore:
- master
workflow_call:
# Ensure that multiple runs on the same branch do not overlap.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
build-test:
name: Build and test
runs-on: buildjet-2vcpu-ubuntu-2204
strategy:
matrix:
nim: ["1.6.x", "2.0.x", "2.2.x", "devel"]
steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Cache Nimble Dependencies
id: cache-nimble
uses: buildjet/cache@v4
with:
path: ~/.nimble
key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
restore-keys: |
${{ matrix.nim }}-nimble-v2-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
with:
nim-version: ${{ matrix.nim }}
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Project
run: nimble build -d:release -Y
integration-test:
needs: [build-test]
name: Integration test
runs-on: buildjet-2vcpu-ubuntu-2204
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Cache Nimble Dependencies
id: cache-nimble
uses: buildjet/cache@v4
with:
path: ~/.nimble
key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
restore-keys: |
devel-nimble-v2-
- name: Setup Python (3.10) with pip cache
uses: buildjet/setup-python@v4
with:
python-version: "3.10"
cache: pip
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
with:
nim-version: devel
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Project
run: nimble build -d:release -Y
- name: Install SeleniumBase and Chromedriver
run: |
pip install seleniumbase
seleniumbase install chromedriver
- name: Start Redis Service
uses: supercharge/redis-github-action@1.5.0
- name: Prepare Nitter Environment
run: |
sudo apt-get update && sudo apt-get install -y libsass-dev
cp nitter.example.conf nitter.conf
sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
nimble md
nimble scss
echo '${{ secrets.SESSIONS }}' | head -n1
echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
- name: Run Tests
run: |
./nitter &
pytest -n1 tests

6
.gitignore vendored
View File

@@ -3,13 +3,9 @@ nitter
*.db
/tests/__pycache__
/tests/geckodriver.log
/tests/downloaded_files
/tests/latest_logs
/tests/downloaded_files/*
/tools/gencss
/tools/rendermd
/public/css/style.css
/public/md/*.html
nitter.conf
guest_accounts.json*
sessions.json*
dump.rdb

10
Dockerfile
View File

@@ -1,5 +1,6 @@
FROM nimlang/nim:2.2.0-alpine-regular as nim
FROM nimlang/nim:1.6.2-alpine-regular as nim
LABEL maintainer="setenforce@protonmail.com"
EXPOSE 8080
RUN apk --no-cache add libsass-dev pcre
@@ -9,17 +10,14 @@ COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
RUN nimble build -d:danger -d:lto -d:strip \
&& nimble scss \
&& nimble md
FROM alpine:latest
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates
RUN apk --no-cache add pcre
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

25
Dockerfile.arm64
View File

@@ -1,25 +0,0 @@
FROM alpine:3.20.6 as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
WORKDIR /src/nitter
COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
&& nimble scss \
&& nimble md
FROM alpine:3.20.6
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates openssl
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

76
README.md
View File

@@ -1,38 +1,29 @@
# Nitter
[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
[![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
> [!NOTE]
> Running a Nitter instance now requires real accounts, since Twitter removed the previous methods. \
> This does not affect users. \
> For instructions on how to obtain session tokens, see [Creating session tokens](https://github.com/zedeus/nitter/wiki/Creating-session-tokens).
A free and open source alternative Twitter front-end focused on privacy and
performance. \
Inspired by the [Invidious](https://github.com/iv-org/invidious) project.
Inspired by the [Invidious](https://github.com/iv-org/invidious)
project.
- No JavaScript or ads
- All requests go through the backend, client never talks to Twitter
- Prevents Twitter from tracking your IP or JavaScript fingerprint
- Uses Twitter's unofficial API (no developer account required)
- Uses Twitter's unofficial API (no rate limits or developer account required)
- Lightweight (for [@nim_lang](https://nitter.net/nim_lang), 60KB vs 784KB from twitter.com)
- RSS feeds
- Themes
- Mobile support (responsive design)
- AGPLv3 licensed, no proprietary instances permitted
<details>
<summary>Donations</summary>
Liberapay: https://liberapay.com/zedeus<br>
Patreon: https://patreon.com/nitter<br>
BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55<br>
ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460<br>
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL<br>
SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW<br>
ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
</details>
Liberapay: https://liberapay.com/zedeus \
Patreon: https://patreon.com/nitter \
BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
## Roadmap
@@ -43,20 +34,19 @@ ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faq
## Resources
The wiki contains
The wiki contains
[a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
[browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
maintained by the community.
## Why?
It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
need to sign up. For privacy-minded folks, preventing JavaScript analytics and
IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
you can get accurately tracked with your [browser's
fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
JavaScript required](https://noscriptfingerprint.com/). This all became
It's impossible to use Twitter without JavaScript enabled. For privacy-minded
folks, preventing JavaScript analytics and IP-based tracking is important, but
apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
a VPN and using heavy-duty adblockers, you can get accurately tracked with your
[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
[no JavaScript required](https://noscriptfingerprint.com/). This all became
particularly important after Twitter [removed the
ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
for users to control whether their data gets sent to advertisers.
@@ -77,24 +67,21 @@ Twitter account.
## Installation
### Dependencies
- libpcre
- libsass
- redis/valkey
* libpcre
* libsass
* redis
To compile Nitter you need a Nim installation, see
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible
to install it system-wide or in the user directory you create below.
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
install it system-wide or in the user directory you create below.
To compile the scss files, you need to install `libsass`. On Ubuntu and Debian,
you can use `libsass-dev`.
Redis is required for caching and in the future for account info. As of 2024
Redis is no longer open source, so using the fork Valkey is recommended. It
should be available on most distros as `redis` or `redis-server`
(Ubuntu/Debian), or `valkey`/`valkey-server`. Running it with the default
config is fine, Nitter's default config is set to use the default port and
localhost.
Redis is required for caching and in the future for account info. It should be
available on most distros as `redis` or `redis-server` (Ubuntu/Debian).
Running it with the default config is fine, Nitter's default config is set to
use the default Redis port and localhost.
Here's how to create a `nitter` user, clone the repo, and build the project
along with the scss and md files.
@@ -104,7 +91,7 @@ along with the scss and md files.
# su nitter
$ git clone https://github.com/zedeus/nitter
$ cd nitter
$ nimble build -d:danger --mm:refc
$ nimble build -d:release
$ nimble scss
$ nimble md
$ cp nitter.example.conf nitter.conf
@@ -121,32 +108,25 @@ performance reasons.
### Docker
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
To run Nitter with Docker, you'll need to install and run Redis separately
before you can run the container. See below for how to also run Redis using
Docker.
To build and run Nitter in Docker:
```bash
docker build -t nitter:latest .
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
```
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
A prebuilt Docker image is provided as well:
```bash
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
```
Using docker-compose to run both Nitter and Redis as different containers:
Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
```bash
docker-compose up -d
```

8
config.nims
View File

@@ -1,13 +1,17 @@
--define:ssl
--define:useStdLib
--threads:off
# workaround httpbeast file upload bug
--assertions:off
# disable annoying warnings
warning("GcUnsafe2", off)
warning("HoleEnumConv", off)
hint("XDeclaredButNotUsed", off)
hint("XCannotRaiseY", off)
hint("User", off)
const
nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch)
when nimVersion >= (1, 6, 0):
warning("HoleEnumConv", off)

25
docker-compose.yml
View File

@@ -8,22 +8,10 @@ services:
ports:
- "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
volumes:
- ./nitter.conf:/src/nitter.conf:Z,ro
- ./sessions.jsonl:/src/sessions.jsonl:Z,ro # Run get_sessions.py to get the credentials
- ./nitter.conf:/src/nitter.conf:ro
depends_on:
- nitter-redis
restart: unless-stopped
healthcheck:
test: wget -nv --tries=1 --spider http://127.0.0.1:8080/Jack/status/20 || exit 1
interval: 30s
timeout: 5s
retries: 2
user: "998:998"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
nitter-redis:
image: redis:6-alpine
@@ -32,17 +20,6 @@ services:
volumes:
- nitter-redis:/data
restart: unless-stopped
healthcheck:
test: redis-cli ping
interval: 30s
timeout: 5s
retries: 2
user: "999:1000"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
volumes:
nitter-redis:

19
nitter.example.conf
View File

@@ -1,11 +1,11 @@
[Server]
hostname = "nitter.net" # for generating links, change this to your own domain/ip
title = "nitter"
address = "0.0.0.0"
port = 8080
https = false # disable to enable cookies when not using https
httpMaxConnections = 100
staticDir = "./public"
title = "nitter"
hostname = "nitter.net"
[Cache]
listMinutes = 240 # how long to cache list info (not the tweets, so keep it high)
@@ -13,9 +13,9 @@ rssMinutes = 10 # how long to cache rss queries
redisHost = "localhost" # Change to "nitter-redis" if using docker-compose
redisPort = 6379
redisPassword = ""
redisConnections = 20 # minimum open connections in pool
redisConnections = 20 # connection pool size
redisMaxConnections = 30
# new connections are opened when none are available, but if the pool size
# max, new connections are opened when none are available, but if the pool size
# goes above this, they're closed when released. don't worry about this unless
# you receive tons of requests per second
@@ -23,16 +23,23 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.sessions)
enableDebug = false # enable request logs and debug endpoints
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 187 requests.
# the limit gets reset every 15 minutes, and the pool is filled up so there's
# always at least $tokenCount usable tokens. again, only increase this if
# you receive major bursts all the time
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]
theme = "Nitter"
replaceTwitter = "nitter.net"
replaceYouTube = "piped.video"
replaceYouTube = "piped.kavin.rocks"
replaceReddit = "teddit.net"
replaceInstagram = ""
proxyVideos = true
hlsPlayback = false
infiniteScroll = false

24
nitter.nimble
View File

@@ -10,20 +10,20 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 1.6.10"
requires "jester#baca3f"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#a079df9"
requires "markdown#158efe3"
requires "packedjson#9e6fbb6"
requires "supersnappy#6c94198"
requires "nim >= 1.4.8"
requires "jester >= 0.5.0"
requires "karax#c71bc92"
requires "sass#e683aa1"
requires "nimcrypto#a5742a9"
requires "markdown#abdbe5e"
requires "packedjson#d11d167"
requires "supersnappy#2.1.1"
requires "redpool#8b7c1db"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#ca5989a"
requires "flatty#e668085"
requires "jsony#1de1f08"
requires "oauth#b8c163b"
requires "zippy#0.7.3"
requires "flatty#0.2.3"
requires "jsony#d0e69bd"
# Tasks

41
public/css/themes/dracula.css
View File

@@ -1,41 +0,0 @@
body {
--bg_color: #282a36;
--fg_color: #f8f8f2;
--fg_faded: #818eb6;
--fg_dark: var(--fg_faded);
--fg_nav: var(--accent);
--bg_panel: #343746;
--bg_elements: #292b36;
--bg_overlays: #44475a;
--bg_hover: #2f323f;
--grey: var(--fg_faded);
--dark_grey: #44475a;
--darker_grey: #3d4051;
--darkest_grey: #363948;
--border_grey: #44475a;
--accent: #bd93f9;
--accent_light: #caa9fa;
--accent_dark: var(--accent);
--accent_border: #ff79c696;
--play_button: #ffb86c;
--play_button_hover: #ffc689;
--more_replies_dots: #bd93f9;
--error_red: #ff5555;
--verified_blue: var(--accent);
--icon_text: ##F8F8F2;
--tab: #6272a4;
--tab_selected: var(--accent);
--profile_stat: #919cbf;
}
.search-bar > form input::placeholder{
color: var(--fg_faded);
}

5
public/js/hls.light.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
public/js/hls.min.js vendored
View File

File diff suppressed because one or more lines are too long

29
public/js/infiniteScroll.js
View File

@@ -5,7 +5,7 @@ function insertBeforeLast(node, elem) {
}
function getLoadMore(doc) {
return doc.querySelector(".show-more:not(.timeline-item)");
return doc.querySelector('.show-more:not(.timeline-item)');
}
function isDuplicate(item, itemClass) {
@@ -15,19 +15,18 @@ function isDuplicate(item, itemClass) {
return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null;
}
window.onload = function () {
window.onload = function() {
const url = window.location.pathname;
const isTweet = url.indexOf("/status/") !== -1;
const containerClass = isTweet ? ".replies" : ".timeline";
const itemClass = containerClass + " > div:not(.top-ref)";
const itemClass = containerClass + ' > div:not(.top-ref)';
var html = document.querySelector("html");
var container = document.querySelector(containerClass);
var loading = false;
function handleScroll(failed) {
window.addEventListener('scroll', function() {
if (loading) return;
if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
loading = true;
var loadMore = getLoadMore(document);
@@ -36,15 +35,13 @@ window.onload = function () {
loadMore.children[0].text = "Loading...";
var url = new URL(loadMore.children[0].href);
url.searchParams.append("scroll", "true");
url.searchParams.append('scroll', 'true');
fetch(url.toString()).then(function (response) {
if (response.status === 404) throw "error";
return response.text();
}).then(function (html) {
var parser = new DOMParser();
var doc = parser.parseFromString(html, "text/html");
var doc = parser.parseFromString(html, 'text/html');
loadMore.remove();
for (var item of doc.querySelectorAll(itemClass)) {
@@ -60,18 +57,10 @@ window.onload = function () {
if (isTweet) container.appendChild(newLoadMore);
else insertBeforeLast(container, newLoadMore);
}).catch(function (err) {
console.warn("Something went wrong.", err);
if (failed > 3) {
loadMore.children[0].text = "Error";
return;
}
loading = false;
handleScroll((failed || 0) + 1);
console.warn('Something went wrong.', err);
loading = true;
});
}
}
window.addEventListener("scroll", () => handleScroll());
});
};
// @license-end

44
public/md/about.md
View File

@@ -4,15 +4,15 @@ Nitter is a free and open source alternative Twitter front-end focused on
privacy and performance. The source is available on GitHub at
<https://github.com/zedeus/nitter>
- No JavaScript or ads
- All requests go through the backend, client never talks to Twitter
- Prevents Twitter from tracking your IP or JavaScript fingerprint
- Uses Twitter's unofficial API (no developer account required)
- Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
- RSS feeds
- Themes
- Mobile support (responsive design)
- AGPLv3 licensed, no proprietary instances permitted
* No JavaScript or ads
* All requests go through the backend, client never talks to Twitter
* Prevents Twitter from tracking your IP or JavaScript fingerprint
* Uses Twitter's unofficial API (no rate limits or developer account required)
* Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
* RSS feeds
* Themes
* Mobile support (responsive design)
* AGPLv3 licensed, no proprietary instances permitted
Nitter's GitHub wiki contains
[instances](https://github.com/zedeus/nitter/wiki/Instances) and
@@ -21,13 +21,12 @@ maintained by the community.
## Why use Nitter?
It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
need to sign up. For privacy-minded folks, preventing JavaScript analytics and
IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
you can get accurately tracked with your [browser's
fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
JavaScript required](https://noscriptfingerprint.com/). This all became
It's impossible to use Twitter without JavaScript enabled. For privacy-minded
folks, preventing JavaScript analytics and IP-based tracking is important, but
apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
a VPN and using heavy-duty adblockers, you can get accurately tracked with your
[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
[no JavaScript required](https://noscriptfingerprint.com/). This all became
particularly important after Twitter [removed the
ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
for users to control whether their data gets sent to advertisers.
@@ -43,13 +42,12 @@ Twitter account.
## Donating
Liberapay: https://liberapay.com/zedeus \
Patreon: https://patreon.com/nitter \
BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55 \
ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460 \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL \
SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW \
ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
Liberapay: <https://liberapay.com/zedeus> \
Patreon: <https://patreon.com/nitter> \
BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
## Contact

5
public/robots.txt
View File

@@ -1,5 +0,0 @@
User-agent: *
Disallow: /
Crawl-delay: 1
User-agent: Twitterbot
Disallow:

194
src/api.nim
View File

@@ -4,155 +4,121 @@ import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser
proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
let
cookieVariables = userMediaVariables % [id, cursor]
oauthVariables = userTweetsVariables % [id, cursor]
result = SessionAwareUrl(
cookieUrl: graphUserMedia ? {"variables": cookieVariables, "features": gqlFeatures},
oauthUrl: graphUserMediaV2 ? {"variables": oauthVariables, "features": gqlFeatures}
)
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
variables = """{"screen_name": "$1"}""" % username
params = {"variables": variables, "features": gqlFeatures}
js = await fetchRaw(graphUser ? params, Api.userScreenName)
result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} =
proc getGraphUser*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = """{"rest_id": "$1"}""" % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetchRaw(graphUserById ? params, Api.userRestId)
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = case kind
of TimelineKind.tweets:
await fetch(graphUserTweets ? params, Api.userTweets)
of TimelineKind.replies:
await fetch(graphUserTweetsAndReplies ? params, Api.userTweetsAndReplies)
of TimelineKind.media:
await fetch(mediaUrl(id, cursor), Api.userMedia)
result = parseGraphTimeline(js, after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = listTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphListTweets ? params, Api.listTweets)
result = parseGraphTimeline(js, after).tweets
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list}
params = {"variables": $variables, "features": gqlFeatures}
url = graphListBySlug ? params
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
url = graphListBySlug ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.listBySlug))
proc getGraphList*(id: string): Future[List] {.async.} =
let
variables = """{"listId": "$1"}""" % id
params = {"variables": variables, "features": gqlFeatures}
url = graphListById ? params
variables = %*{"listId": id, "withHighlightedLabel": false}
url = graphList ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.list))
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
var
let
variables = %*{
"listId": list.id,
"cursor": after,
"withSuperFollowsUserFields": false,
"withBirdwatchPivots": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
url = graphListMembers ? {"variables": $variables}
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
variables = """{"rest_id": "$1"}""" % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js)
ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
url = listTimeline ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
proc getUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
ps = genParams({"screen_name": username})
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json, username)
proc getUserById*(userId: string): Future[User] {.async.} =
if userId.len == 0: return
let
ps = genParams({"user_id": userId})
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = timeline / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline))
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
when T is User:
const
searchMode = ("result_filter", "user")
parse = parseUsers
fetchFunc = fetchRaw
else:
const
searchMode = ("tweet_search_mode", "live")
parse = parseTweets
fetchFunc = fetchRaw
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Result[T](beginning: true, query: query)
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
try:
result = parse(await fetchFunc(url, Api.search), after)
result.query = query
except InternalError:
return Result[T](beginning: true, query: query)
proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
let url = tweet / (id & ".json") ? genParams(cursor=after)
result = parseConversation(await fetch(url, Api.tweet), id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getGraphTweet(id, after)).replies
result = (await getTweetImpl(id, after)).replies
result.beginning = after.len == 0
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
result = await getGraphTweet(id)
result = await getTweetImpl(id)
if after.len > 0:
result.replies = await getReplies(id, after)
proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Timeline(query: query, beginning: true)
var
variables = %*{
"rawQuery": q,
"count": 20,
"product": "Latest",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
result.query = query
proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
if query.text.len == 0:
return Result[User](query: query, beginning: true)
var
variables = %*{
"rawQuery": query.text,
"count": 20,
"product": "People",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
result.beginning = false
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch[User](await fetch(url, Api.search), after)
result.query = query
proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
if id.len == 0: return
let js = await fetch(mediaUrl(id, ""), Api.userMedia)
result = parseGraphPhotoRail(js)
proc getStatus*(id: string): Future[Tweet] {.async.} =
let url = status / (id & ".json") ? genParams()
result = parseStatus(await fetch(url, Api.status))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)

202
src/apiutils.nim
View File

@@ -1,177 +1,121 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1
import types, auth, consts, parserutils, http_pool
import httpclient, asyncdispatch, options, strutils, uri
import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
import experimental/types/common
const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
rlLimit = "x-rate-limit-limit"
errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
var pool: HttpPool
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
let
encodedUrl = url.replace(",", "%2C").replace("+", "%20")
params = OAuth1Parameters(
consumerKey: consumerKey,
signatureMethod: "HMAC-SHA1",
timestamp: $int(round(epochTime())),
nonce: "0",
isIncludeVersionToHeader: true,
token: oauthToken
)
signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
count="20"; ext=true): seq[(string, string)] =
result = timelineParams
for p in pars:
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "true")
result &= ("include_ext_media_availability", "true")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
# The raw cursor often has plus signs, which sometimes get turned into spaces,
# so we need to them back into a plus
if " " in cursor:
result &= ("cursor", cursor.replace(" ", "+"))
else:
result &= ("cursor", cursor)
params.signature = percentEncode(signature)
return getOauth1RequestHeader(params)["authorization"]
proc getCookieHeader(authToken, ct0: string): string =
"auth_token=" & authToken & "; ct0=" & ct0
proc genHeaders*(session: Session, url: string): HttpHeaders =
proc genHeaders*(token: Token = nil): HttpHeaders =
result = newHttpHeaders({
"connection": "keep-alive",
"authorization": auth,
"content-type": "application/json",
"x-guest-token": if token == nil: "" else: token.tok,
"x-twitter-active-user": "yes",
"x-twitter-client-language": "en",
"authority": "api.x.com",
"authority": "api.twitter.com",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.9",
"accept": "*/*",
"DNT": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
"DNT": "1"
})
case session.kind
of SessionKind.oauth:
result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
of SessionKind.cookie:
result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
result["x-twitter-auth-type"] = "OAuth2Session"
result["x-csrf-token"] = session.ct0
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
result = await getSession(api)
case result.kind
of SessionKind.oauth:
if result.oauthToken.len == 0:
echo "[sessions] Empty oauth token, session: ", result.id
raise rateLimitError()
of SessionKind.cookie:
if result.authToken.len == 0 or result.ct0.len == 0:
echo "[sessions] Empty cookie credentials, session: ", result.id
raise rateLimitError()
template updateToken() =
if api != Api.search and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
var token = await getToken(api)
if token.tok.len == 0:
raise rateLimitError()
try:
var resp: AsyncResponse
pool.use(genHeaders(session, $url)):
template getContent =
resp = await c.get($url)
result = await resp.body
getContent()
pool.use(genHeaders(token)):
resp = await c.get($url)
result = await resp.body
if resp.status == $Http503:
badClient = true
raise newException(BadClientError, "Bad client")
if resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
limit = parseInt(resp.headers[rlLimit])
session.setRateLimit(api, remaining, reset, limit)
raise newException(InternalError, result)
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
result = uncompress(result, dfGzip)
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors notin errorsToSkip:
echo "Fetch error, API: ", api, ", errors: ", errors
if errors in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
elif errors in {rateLimited}:
# rate limit hit, resets after 24 hours
setLimited(session, api)
raise rateLimitError()
elif result.startsWith("429 Too Many Requests"):
echo "[sessions] 429 error, API: ", api, ", session: ", session.id
session.apis[api].remaining = 0
# rate limit hit, resets after the 15 minute window
raise rateLimitError()
else:
echo "non-gzip body, url: ", url, ", body: ", result
fetchBody
release(token, used=true)
if resp.status == $Http400:
echo "ERROR 400, ", api, ": ", result
raise newException(InternalError, $url)
except InternalError as e:
raise e
except BadClientError as e:
raise e
except OSError as e:
raise e
except Exception as e:
let id = if session.isNil: "null" else: $session.id
echo "error: ", e.name, ", msg: ", e.msg, ", sessionId: ", id, ", url: ", url
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
if "length" notin e.msg and "descriptor" notin e.msg:
release(token, invalid=true)
raise rateLimitError()
finally:
release(session)
template retry(bod) =
try:
bod
except RateLimitError:
echo "[sessions] Rate limited, retrying ", api, " request..."
bod
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
var body: string
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
result = newJNull()
proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
retry:
var
body: string
session = await getAndValidateSession(api)
updateToken()
when url is SessionAwareUrl:
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
let error = result.getError
if error in {invalidToken, forbidden, badToken}:
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
result = newJNull()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)
let error = result.getError
if error != null and error notin errorsToSkip:
echo "Fetch error, API: ", api, ", error: ", error
if error in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
updateToken()
proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
retry:
var session = await getAndValidateSession(api)
when url is SessionAwareUrl:
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors in {invalidToken, forbidden, badToken}:
echo "fetch error: ", errors
release(token, invalid=true)
raise rateLimitError()

203
src/auth.nim
View File

@@ -1,203 +0,0 @@
#SPDX-License-Identifier: AGPL-3.0-only
import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
import types
import experimental/parser/session
# max requests at a time per session to avoid race conditions
const
maxConcurrentReqs = 2
hourInSeconds = 60 * 60
apiMaxReqs: Table[Api, int] = {
Api.search: 50,
Api.tweetDetail: 500,
Api.userTweets: 500,
Api.userTweetsAndReplies: 500,
Api.userMedia: 500,
Api.userRestId: 500,
Api.userScreenName: 500,
Api.tweetResult: 500,
Api.list: 500,
Api.listTweets: 500,
Api.listMembers: 500,
Api.listBySlug: 500
}.toTable
var
sessionPool: seq[Session]
enableLogging = false
template log(str: varargs[string, `$`]) =
echo "[sessions] ", str.join("")
proc snowflakeToEpoch(flake: int64): int64 =
int64(((flake shr 22) + 1288834974657) div 1000)
proc getSessionPoolHealth*(): JsonNode =
let now = epochTime().int
var
totalReqs = 0
limited: PackedSet[int64]
reqsPerApi: Table[string, int]
oldest = now.int64
newest = 0'i64
average = 0'i64
for session in sessionPool:
let created = snowflakeToEpoch(session.id)
if created > newest:
newest = created
if created < oldest:
oldest = created
average += created
if session.limited:
limited.incl session.id
for api in session.apis.keys:
let
apiStatus = session.apis[api]
limit = if apiStatus.limit > 0: apiStatus.limit else: apiMaxReqs.getOrDefault(api, 0)
reqs = limit - apiStatus.remaining
# no requests made with this session and endpoint since the limit reset
if apiStatus.reset < now:
continue
reqsPerApi.mgetOrPut($api, 0).inc reqs
totalReqs.inc reqs
if sessionPool.len > 0:
average = average div sessionPool.len
else:
oldest = 0
average = 0
return %*{
"sessions": %*{
"total": sessionPool.len,
"limited": limited.card,
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
"average": $fromUnix(average)
},
"requests": %*{
"total": totalReqs,
"apis": reqsPerApi
}
}
proc getSessionPoolDebug*(): JsonNode =
let now = epochTime().int
var list = newJObject()
for session in sessionPool:
let sessionJson = %*{
"apis": newJObject(),
"pending": session.pending,
}
if session.limited:
sessionJson["limited"] = %true
for api in session.apis.keys:
let
apiStatus = session.apis[api]
obj = %*{}
if apiStatus.reset > now.int:
obj["remaining"] = %apiStatus.remaining
obj["reset"] = %apiStatus.reset
if "remaining" notin obj:
continue
sessionJson{"apis", $api} = obj
list[$session.id] = sessionJson
return %list
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
proc noSessionsError*(): ref NoSessionsError =
newException(NoSessionsError, "no sessions available")
proc isLimited(session: Session; api: Api): bool =
if session.isNil:
return true
if session.limited and api != Api.userTweets:
if (epochTime().int - session.limitedAt) > hourInSeconds:
session.limited = false
log "resetting limit: ", session.id
return false
else:
return true
if api in session.apis:
let limit = session.apis[api]
return limit.remaining <= 10 and limit.reset > epochTime().int
else:
return false
proc isReady(session: Session; api: Api): bool =
not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
proc invalidate*(session: var Session) =
if session.isNil: return
log "invalidating: ", session.id
# TODO: This isn't sufficient, but it works for now
let idx = sessionPool.find(session)
if idx > -1: sessionPool.delete(idx)
session = nil
proc release*(session: Session) =
if session.isNil: return
dec session.pending
proc getSession*(api: Api): Future[Session] {.async.} =
for i in 0 ..< sessionPool.len:
if result.isReady(api): break
result = sessionPool.sample()
if not result.isNil and result.isReady(api):
inc result.pending
else:
log "no sessions available for API: ", api
raise noSessionsError()
proc setLimited*(session: Session; api: Api) =
session.limited = true
session.limitedAt = epochTime().int
log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
# avoid undefined behavior in race conditions
if api in session.apis:
let rateLimit = session.apis[api]
if rateLimit.reset >= reset and rateLimit.remaining < remaining:
return
if rateLimit.reset == reset and rateLimit.remaining >= remaining:
session.apis[api].remaining = remaining
return
session.apis[api] = RateLimit(limit: limit, remaining: remaining, reset: reset)
proc initSessionPool*(cfg: Config; path: string) =
enableLogging = cfg.enableDebug
if path.endsWith(".json"):
log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
quit 1
if not fileExists(path):
log "ERROR: ", path, " not found. This file is required to authenticate API requests."
quit 1
log "parsing JSONL account sessions file: ", path
for line in path.lines:
sessionPool.add parseSession(line)
log "successfully added ", sessionPool.len, " valid account sessions"

169
src/consts.nim
View File

@@ -1,128 +1,59 @@
# SPDX-License-Identifier: AGPL-3.0-only
import uri, strutils
import uri, sequtils
const
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
gql = parseUri("https://api.x.com") / "graphql"
api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json")
graphUser* = gql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
graphUserById* = gql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
graphUserTweets* = gql / "JLApJKFY0MxGTzCoK6ps8Q/UserWithProfileTweetsQueryV2"
graphUserTweetsAndReplies* = gql / "Y86LQY7KMvxn5tu3hFTyPg/UserWithProfileTweetsAndRepliesQueryV2"
graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
graphUserMediaV2* = gql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2"
graphTweet* = gql / "Vorskcd2tZ-tc4Gx3zbk4Q/ConversationTimelineV2"
graphTweetResult* = gql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery"
graphSearchTimeline* = gql / "KI9jCXUx3Ymt-hDKLOZb9Q/SearchTimeline"
graphListById* = gql / "oygmAig8kjn0pKsx_bUadQ/ListByRestId"
graphListBySlug* = gql / "88GTz-IPPWLn1EiU8XoNVg/ListBySlug"
graphListMembers* = gql / "kSmxeqEeelqdHSR7jMnb_w/ListMembers"
graphListTweets* = gql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
userShow* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json"
status* = api / "1.1/statuses/show"
search* = api / "2/search/adaptive.json"
gqlFeatures* = """{
"android_graphql_skip_api_media_color_palette": false,
"blue_business_profile_image_shape_enabled": false,
"creator_subscriptions_subscription_count_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"hidden_profile_likes_enabled": false,
"highlights_tweets_tab_ui_enabled": false,
"interactive_text_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"longform_notetweets_richtext_consumption_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_media_download_video_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"rweb_lists_timeline_redesign_enabled": true,
"spaces_2022_h2_clipping": true,
"spaces_2022_h2_spaces_communities": true,
"standardized_nudges_misinfo": true,
"subscriptions_verification_info_enabled": true,
"subscriptions_verification_info_reason_enabled": true,
"subscriptions_verification_info_verified_since_enabled": true,
"super_follow_badge_privacy_enabled": false,
"super_follow_exclusive_tweet_notifications_enabled": false,
"super_follow_tweet_api_enabled": false,
"super_follow_user_api_enabled": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"tweetypie_unmention_optimization_enabled": false,
"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
"verified_phone_label_enabled": false,
"vibe_api_enabled": false,
"view_counts_everywhere_api_enabled": true,
"premium_content_api_read_enabled": false,
"communities_web_enable_tweet_community_results_fetch": true,
"responsive_web_jetfuel_frame": true,
"responsive_web_grok_analyze_button_fetch_trends_enabled": false,
"responsive_web_grok_image_annotation_enabled": true,
"responsive_web_grok_imagine_annotation_enabled": true,
"rweb_tipjar_consumption_enabled": true,
"profile_label_improvements_pcf_label_in_post_enabled": true,
"creator_subscriptions_quote_tweet_preview_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"responsive_web_grok_analyze_post_followups_enabled": true,
"rweb_video_timestamps_enabled": false,
"responsive_web_grok_share_attachment_enabled": true,
"articles_preview_enabled": true,
"immersive_video_status_linkable_timestamps": false,
"articles_api_enabled": false,
"responsive_web_grok_analysis_button_from_backend": true,
"rweb_video_screen_enabled": false,
"payments_enabled": false,
"responsive_web_profile_redirect_enabled": false,
"responsive_web_grok_show_grok_translated_post": false,
"responsive_web_grok_community_note_auto_translation_is_enabled": false
}""".replace(" ", "").replace("\n", "")
timelineApi = api / "2/timeline"
timeline* = timelineApi / "profile"
mediaTimeline* = timelineApi / "media"
listTimeline* = timelineApi / "list.json"
tweet* = timelineApi / "conversation"
tweetVariables* = """{
"focalTweetId": "$1",
$2
"includeHasBirdwatchNotes": false,
"includePromotedContent": false,
"withBirdwatchNotes": false,
"withVoice": false,
"withV2Timeline": true
}""".replace(" ", "").replace("\n", "")
graphql = api / "graphql"
graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
# oldUserTweetsVariables* = """{
# "userId": "$1", $2
# "count": 20,
# "includePromotedContent": false,
# "withDownvotePerspective": false,
# "withReactionsMetadata": false,
# "withReactionsPerspective": false,
# "withVoice": false,
# "withV2Timeline": true
# }
# """
timelineParams* = {
"include_profile_interstitial_type": "0",
"include_blocking": "0",
"include_blocked_by": "0",
"include_followed_by": "0",
"include_want_retweets": "0",
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "false",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "true",
"include_user_entities": "true",
"include_ext_media_color": "false",
"send_error_codes": "true",
"simple_quoted_tweet": "true",
"include_quote_count": "true"
}.toSeq
userTweetsVariables* = """{
"rest_id": "$1", $2
"count": 20
}"""
listTweetsVariables* = """{
"rest_id": "$1", $2
"count": 20
}"""
userMediaVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withClientEventToken": false,
"withBirdwatchNotes": false,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
searchParams* = {
"query_source": "typed_query",
"pc": "1",
"spelling_corrections": "1"
}.toSeq
## top: nothing
## latest: "tweet_search_mode: live"
## user: "result_filter: user"
## photos: "result_filter: photos"
## videos: "result_filter: videos"

4
src/experimental/parser.nim
View File

@@ -1,2 +1,2 @@
import parser/[user, graphql]
export user, graphql
import parser/[user, graphql, timeline]
export user, graphql, timeline

18
src/experimental/parser/graphql.nim
View File

@@ -1,21 +1,11 @@
import options
import jsony
import user, ../types/[graphuser, graphlistmembers]
from ../../types import User, VerifiedType, Result, Query, QueryKind
from ../../types import User, Result, Query, QueryKind
proc parseGraphUser*(json: string): User =
if json.len == 0 or json[0] != '{':
return
let raw = json.fromJson(GraphUser)
if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
return User(suspended: true)
result = raw.data.userResult.result.legacy
result.id = raw.data.userResult.result.restId
if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
result.verifiedType = blue
result = toUser raw.data.user.result.legacy
result.id = raw.data.user.result.restId
proc parseGraphListMembers*(json, cursor: string): Result[User] =
result = Result[User](
@@ -31,7 +21,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
of TimelineTimelineItem:
let userResult = entry.content.itemContent.userResults.result
if userResult.restId.len > 0:
result.content.add userResult.legacy
result.content.add toUser userResult.legacy
of TimelineTimelineCursor:
if entry.content.cursorType == "Bottom":
result.bottom = entry.content.value

44
src/experimental/parser/media.nim Normal file
View File

@@ -0,0 +1,44 @@
import std/[json, strutils, times, math]
import utils
import ".."/types/[media, tweet]
from ../../types import Poll, Gif, Video, VideoVariant, VideoType
proc parseVideo*(entity: Entity): Video =
result = Video(
thumb: entity.mediaUrlHttps.getImageUrl,
views: entity.ext.mediaStats{"r", "ok", "viewCount"}.getStr,
available: entity.extMediaAvailability.status == "available",
title: entity.extAltText,
durationMs: entity.videoInfo.durationMillis,
description: entity.additionalMediaInfo.description,
variants: entity.videoInfo.variants
# playbackType: mp4
)
if entity.additionalMediaInfo.title.len > 0:
result.title = entity.additionalMediaInfo.title
proc parseGif*(entity: Entity): Gif =
result = Gif(
url: entity.videoInfo.variants[0].url.getImageUrl,
thumb: entity.getImageUrl
)
proc parsePoll*(card: Card): Poll =
let vals = card.bindingValues
# name format is pollNchoice_*
for i in '1' .. card.name[4]:
let choice = "choice" & i
result.values.add parseInt(vals{choice & "_count", "string_value"}.getStr("0"))
result.options.add vals{choice & "_label", "string_value"}.getStr
let time = vals{"end_datetime_utc", "string_value"}.getStr.parseIsoDate
if time > now():
let timeLeft = $(time - now())
result.status = timeLeft[0 ..< timeLeft.find(",")]
else:
result.status = "Final results"
result.leader = result.values.find(max(result.values))
result.votes = result.values.sum

28
src/experimental/parser/session.nim
View File

@@ -1,28 +0,0 @@
import std/strutils
import jsony
import ../types/session
from ../../types import Session, SessionKind
proc parseSession*(raw: string): Session =
let session = raw.fromJson(RawSession)
let kind = if session.kind == "": "oauth" else: session.kind
case kind
of "oauth":
let id = session.oauthToken[0 ..< session.oauthToken.find('-')]
result = Session(
kind: SessionKind.oauth,
id: parseBiggestInt(id),
oauthToken: session.oauthToken,
oauthSecret: session.oauthTokenSecret
)
of "cookie":
let id = if session.id.len > 0: parseBiggestInt(session.id) else: 0
result = Session(
kind: SessionKind.cookie,
id: id,
authToken: session.authToken,
ct0: session.ct0
)
else:
raise newException(ValueError, "Unknown session kind: " & kind)

20
src/experimental/parser/slices.nim
View File

@@ -1,15 +1,14 @@
import std/[macros, htmlgen, unicode]
import ../types/common
import ".."/../[formatters, utils]
type
ReplaceSliceKind = enum
ReplaceSliceKind* = enum
rkRemove, rkUrl, rkHashtag, rkMention
ReplaceSlice* = object
slice: Slice[int]
kind: ReplaceSliceKind
url, display: string
slice*: Slice[int]
kind*: ReplaceSliceKind
url*, display*: string
proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
@@ -27,11 +26,14 @@ proc dedupSlices*(s: var seq[ReplaceSlice]) =
inc j
inc i
proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
textLen: int; hideTwitter = false) =
proc extractHashtags*(result: var seq[ReplaceSlice]; slice: Slice[int]) =
result.add ReplaceSlice(kind: rkHashtag, slice: slice)
proc extractUrls*[T](result: var seq[ReplaceSlice]; entity: T;
textLen: int; hideTwitter = false) =
let
link = url.expandedUrl
slice = url.indices[0] ..< url.indices[1]
link = entity.expandedUrl
slice = entity.indices
if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
if slice.a < textLen:

84
src/experimental/parser/timeline.nim Normal file
View File

@@ -0,0 +1,84 @@
import std/[strutils, tables, options]
import jsony
import user, tweet, utils, ../types/timeline
from ../../types import Result, User, Tweet
proc parseHook(s: string; i: var int; v: var Slice[int]) =
var slice: array[2, int]
parseHook(s, i, slice)
v = slice[0] ..< slice[1]
proc getId(id: string): string {.inline.} =
let start = id.rfind("-")
if start < 0: return id
id[start + 1 ..< id.len]
proc processTweet(id: string; objects: GlobalObjects;
userCache: var Table[string, User]): Tweet =
let raw = objects.tweets[id]
result = toTweet raw
let uid = result.user.id
if uid.len > 0 and uid in objects.users:
if uid notin userCache:
userCache[uid] = toUser objects.users[uid]
result.user = userCache[uid]
let rtId = raw.retweetedStatusIdStr
if rtId.len > 0:
if rtId in objects.tweets:
result.retweet = some processTweet(rtId, objects, userCache)
else:
result.retweet = some Tweet(id: rtId.toId)
let qId = raw.quotedStatusIdStr
if qId.len > 0:
if qId in objects.tweets:
result.quote = some processTweet(qId, objects, userCache)
else:
result.quote = some Tweet(id: qId.toId)
proc parseCursor[T](e: Entry; result: var Result[T]) =
let cursor = e.content.operation.cursor
if cursor.cursorType == "Top":
result.top = cursor.value
elif cursor.cursorType == "Bottom":
result.bottom = cursor.value
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
let raw = json.fromJson(Search)
if raw.timeline.instructions.len == 0:
return
for e in raw.timeline.instructions[0].addEntries.entries:
let
eId = e.entryId
id = eId.getId
if eId.startsWith("user") or eId.startsWith("sq-U"):
if id in raw.globalObjects.users:
result.content.add toUser raw.globalObjects.users[id]
elif eId.startsWith("cursor") or eId.startsWith("sq-C"):
parseCursor(e, result)
proc parseTweets*(json: string; after=""): Result[Tweet] =
result = Result[Tweet](beginning: after.len == 0)
let raw = json.fromJson(Search)
if raw.timeline.instructions.len == 0:
return
var userCache: Table[string, User]
for e in raw.timeline.instructions[0].addEntries.entries:
let
eId = e.entryId
id = eId.getId
if eId.startsWith("tweet") or eId.startsWith("sq-I-t"):
if id in raw.globalObjects.tweets:
result.content.add processTweet(id, raw.globalObjects, userCache)
elif eId.startsWith("cursor") or eId.startsWith("sq-C"):
parseCursor(e, result)

97
src/experimental/parser/tweet.nim Normal file
View File

@@ -0,0 +1,97 @@
import std/[strutils, options, algorithm, json]
import std/unicode except strip
import utils, slices, media, user
import ../types/tweet
from ../types/media as mediaTypes import MediaType
from ../../types import Tweet, User, TweetStats
proc expandTweetEntities(tweet: var Tweet; raw: RawTweet) =
let
orig = raw.fullText.toRunes
textRange = raw.displayTextRange
textSlice = textRange[0] .. textRange[1]
hasCard = raw.card.isSome
var replyTo = ""
if tweet.replyId > 0:
tweet.reply.add raw.inReplyToScreenName
replyTo = raw.inReplyToScreenName
var replacements = newSeq[ReplaceSlice]()
for u in raw.entities.urls:
if u.url.len == 0 or u.url notin raw.fullText:
continue
replacements.extractUrls(u, textSlice.b, hideTwitter=raw.isQuoteStatus)
# if hasCard and u.url == get(tweet.card).url:
# get(tweet.card).url = u.expandedUrl
for m in raw.entities.media:
replacements.extractUrls(m, textSlice.b, hideTwitter=true)
for hashtag in raw.entities.hashtags:
replacements.extractHashtags(hashtag.indices)
for symbol in raw.entities.symbols:
replacements.extractHashtags(symbol.indices)
for mention in raw.entities.userMentions:
let
name = mention.screenName
idx = tweet.reply.find(name)
if mention.indices.a >= textSlice.a:
replacements.add ReplaceSlice(kind: rkMention, slice: mention.indices,
url: "/" & name, display: mention.name)
if idx > -1 and name != replyTo:
tweet.reply.delete idx
elif idx == -1 and tweet.replyId != 0:
tweet.reply.add name
replacements.dedupSlices
replacements.sort(cmp)
tweet.text = orig.replacedWith(replacements, textSlice)
.strip(leading=false)
proc toTweet*(raw: RawTweet): Tweet =
result = Tweet(
id: raw.idStr.toId,
threadId: raw.conversationIdStr.toId,
replyId: raw.inReplyToStatusIdStr.toId,
time: parseTwitterDate(raw.createdAt),
hasThread: raw.selfThread.idStr.len > 0,
available: true,
user: User(id: raw.userIdStr),
stats: TweetStats(
replies: raw.replyCount,
retweets: raw.retweetCount,
likes: raw.favoriteCount,
quotes: raw.quoteCount
)
)
result.expandTweetEntities(raw)
if raw.card.isSome:
let card = raw.card.get
if "poll" in card.name:
result.poll = some parsePoll(card)
if "image" in card.name:
result.photos.add card.bindingValues{"image_large", "image_value", "url"}
.getStr.getImageUrl
# elif card.name == "amplify":
# discard
# # result.video = some(parsePromoVideo(jsCard{"binding_values"}))
# else:
# result.card = some parseCard(card, raw.entities.urls)
for m in raw.extendedEntities.media:
case m.kind
of photo: result.photos.add m.getImageUrl
of video:
result.video = some parseVideo(m)
if m.additionalMediaInfo.sourceUser.isSome:
result.attribution = some toUser get(m.additionalMediaInfo.sourceUser)
of animatedGif: result.gif = some parseGif(m)

28
src/experimental/parser/unifiedcard.nim
View File

@@ -1,11 +1,8 @@
import std/[options, tables, strutils, strformat, sugar]
import std/[options, tables, strformat]
import jsony
import user, ../types/unifiedcard
import utils
import ".."/types/[unifiedcard, media]
from ../../types import Card, CardKind, Video
from ../../utils import twimg, https
proc getImageUrl(entity: MediaEntity): string =
entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https))
proc parseDestination(id: string; card: UnifiedCard; result: var Card) =
let destination = card.destinationObjects[id].data
@@ -27,14 +24,6 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card)
result.text = data.topicDetail.title
result.dest = "Topic"
proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
data.destination.parseDestination(card, result)
result.kind = CardKind.jobDetails
result.title = data.title
result.text = data.shortDescriptionText
result.dest = &"@{data.profileUser.username} · {data.location}"
proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
let app = card.appStoreData[data.appId][0]
@@ -74,8 +63,7 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
durationMs: videoInfo.durationMillis,
variants: videoInfo.variants
)
of model3d:
result.title = "Unsupported 3D model ad"
of animatedGif: discard
proc parseUnifiedCard*(json: string): Card =
let card = json.fromJson(UnifiedCard)
@@ -88,16 +76,10 @@ proc parseUnifiedCard*(json: string): Card =
component.data.parseAppDetails(card, result)
of mediaWithDetailsHorizontal:
component.data.parseMediaDetails(card, result)
of media, swipeableMedia:
of ComponentType.media, swipeableMedia:
component.parseMedia(card, result)
of buttonGroup:
discard
of ComponentType.jobDetails:
component.data.parseJobDetails(card, result)
of ComponentType.hidden:
result.kind = CardKind.hidden
of ComponentType.unknown:
echo "ERROR: Unknown component type: ", json
case component.kind
of twitterListDetails:

28
src/experimental/parser/user.nim
View File

@@ -1,14 +1,14 @@
import std/[algorithm, unicode, re, strutils, strformat, options, nre]
import std/[algorithm, unicode, re, strutils, strformat, options]
import jsony
import utils, slices
import ../types/user as userType
from ../../types import Result, User, Error
from ../../types import User, Error
let
unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1<a href=\"/$2\">@$2</a>"
htRegex = nre.re"""(*U)(^|[^\w-_.?])([#＃$])([\w_]*+)(?!</a>|">|#)"""
htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
proc expandUserEntities(user: var User; raw: RawUser) =
@@ -29,7 +29,7 @@ proc expandUserEntities(user: var User; raw: RawUser) =
user.bio = orig.replacedWith(replacements, 0 .. orig.len)
.replacef(unRegex, unReplace)
.replace(htRegex, htReplace)
.replacef(htRegex, htReplace)
proc getBanner(user: RawUser): string =
if user.profileBannerUrl.len > 0:
@@ -56,7 +56,7 @@ proc toUser*(raw: RawUser): User =
tweets: raw.statusesCount,
likes: raw.favouritesCount,
media: raw.mediaCount,
verifiedType: raw.verifiedType,
verified: raw.verified,
protected: raw.protected,
joinDate: parseTwitterDate(raw.createdAt),
banner: getBanner(raw),
@@ -68,11 +68,6 @@ proc toUser*(raw: RawUser): User =
result.expandUserEntities(raw)
proc parseHook*(s: string; i: var int; v: var User) =
var u: RawUser
parseHook(s, i, u)
v = toUser u
proc parseUser*(json: string; username=""): User =
handleErrors:
case error.code
@@ -80,13 +75,4 @@ proc parseUser*(json: string; username=""): User =
of userNotFound: return
else: echo "[error - parseUser]: ", error
result = json.fromJson(User)
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
# starting with '{' means it's an error
if json[0] == '[':
let raw = json.fromJson(seq[RawUser])
for user in raw:
result.content.add user.toUser
result = toUser json.fromJson(RawUser)

9
src/experimental/parser/utils.nim
View File

@@ -1,12 +1,16 @@
# SPDX-License-Identifier: AGPL-3.0-only
import std/[sugar, strutils, times]
import ../types/common
import ".."/types/[common, media, tweet]
import ../../utils as uutils
template parseTime(time: string; f: static string; flen: int): DateTime =
if time.len != flen: return
parse(time, f, utc())
proc toId*(id: string): int64 =
if id.len == 0: 0'i64
else: parseBiggestInt(id)
proc parseIsoDate*(date: string): DateTime =
date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
@@ -16,6 +20,9 @@ proc parseTwitterDate*(date: string): DateTime =
proc getImageUrl*(url: string): string =
url.dup(removePrefix(twimg), removePrefix(https))
proc getImageUrl*(entity: MediaEntity | Entity): string =
entity.mediaUrlHttps.getImageUrl
template handleErrors*(body) =
if json.startsWith("{\"errors"):
for error {.inject.} in json.fromJson(Errors).errors:

8
src/experimental/types/common.nim
View File

@@ -1,3 +1,4 @@
import jsony
from ../../types import Error
type
@@ -5,7 +6,7 @@ type
url*: string
expandedUrl*: string
displayUrl*: string
indices*: array[2, int]
indices*: Slice[int]
ErrorObj* = object
code*: Error
@@ -18,3 +19,8 @@ proc contains*(codes: set[Error]; errors: Errors): bool =
for e in errors.errors:
if e.code in codes:
return true
proc parseHook*(s: string; i: var int; v: var Slice[int]) =
var slice: array[2, int]
parseHook(s, i, slice)
v = slice[0] ..< slice[1]

9
src/experimental/types/graphuser.nim
View File

@@ -1,15 +1,12 @@
import options
from ../../types import User
import user
type
GraphUser* = object
data*: tuple[userResult: UserData]
data*: tuple[user: UserData]
UserData* = object
result*: UserResult
UserResult = object
legacy*: User
legacy*: RawUser
restId*: string
isBlueVerified*: bool
unavailableReason*: Option[string]

15
src/experimental/types/media.nim Normal file
View File

@@ -0,0 +1,15 @@
import options
from ../../types import VideoType, VideoVariant
type
MediaType* = enum
photo, video, animatedGif
MediaEntity* = object
kind*: MediaType
mediaUrlHttps*: string
videoInfo*: Option[VideoInfo]
VideoInfo* = object
durationMillis*: int
variants*: seq[VideoVariant]

9
src/experimental/types/session.nim
View File

@@ -1,9 +0,0 @@
type
RawSession* = object
kind*: string
username*: string
id*: string
oauthToken*: string
oauthTokenSecret*: string
authToken*: string
ct0*: string

13
src/experimental/types/timeline.nim
View File

@@ -1,13 +1,14 @@
import std/tables
from ../../types import User
import user, tweet
type
Search* = object
globalObjects*: GlobalObjects
timeline*: Timeline
GlobalObjects = object
users*: Table[string, User]
GlobalObjects* = object
users*: Table[string, RawUser]
tweets*: Table[string, RawTweet]
Timeline = object
instructions*: seq[Instructions]
@@ -15,9 +16,13 @@ type
Instructions = object
addEntries*: tuple[entries: seq[Entry]]
Entry = object
Entry* = object
entryId*: string
content*: tuple[operation: Operation]
Operation = object
cursor*: tuple[value, cursorType: string]
proc renameHook*(v: var Entity; fieldName: var string) =
if fieldName == "type":
fieldName = "kind"

85
src/experimental/types/tweet.nim Normal file
View File

@@ -0,0 +1,85 @@
import options
import jsony
from json import JsonNode
import user, media, common
type
RawTweet* = object
createdAt*: string
idStr*: string
fullText*: string
displayTextRange*: array[2, int]
entities*: Entities
extendedEntities*: ExtendedEntities
inReplyToStatusIdStr*: string
inReplyToScreenName*: string
userIdStr*: string
isQuoteStatus*: bool
replyCount*: int
retweetCount*: int
favoriteCount*: int
quoteCount*: int
conversationIdStr*: string
favorited*: bool
retweeted*: bool
selfThread*: tuple[idStr: string]
card*: Option[Card]
quotedStatusIdStr*: string
retweetedStatusIdStr*: string
Card* = object
name*: string
url*: string
bindingValues*: JsonNode
Entities* = object
hashtags*: seq[Hashtag]
symbols*: seq[Hashtag]
userMentions*: seq[UserMention]
urls*: seq[Url]
media*: seq[Entity]
Hashtag* = object
indices*: Slice[int]
UserMention* = object
screenName*: string
name*: string
indices*: Slice[int]
ExtendedEntities* = object
media*: seq[Entity]
Entity* = object
kind*: MediaType
indices*: Slice[int]
mediaUrlHttps*: string
url*: string
expandedUrl*: string
videoInfo*: VideoInfo
ext*: Ext
extMediaAvailability*: tuple[status: string]
extAltText*: string
additionalMediaInfo*: AdditionalMediaInfo
sourceStatusIdStr*: string
sourceUserIdStr*: string
AdditionalMediaInfo* = object
sourceUser*: Option[RawUser]
title*: string
description*: string
Ext* = object
mediaStats*: JsonNode
MediaStats* = object
ok*: tuple[viewCount: string]
proc renameHook*(v: var Entity; fieldName: var string) =
if fieldName == "type":
fieldName = "kind"
proc parseHook*(s: string; i: var int; v: var Slice[int]) =
var slice: array[2, int]
parseHook(s, i, slice)
v = slice[0] ..< slice[1]

83
src/experimental/types/unifiedcard.nim
View File

@@ -1,10 +1,7 @@
import std/[options, tables, times]
import jsony
from ../../types import VideoType, VideoVariant, User
import options, tables
import media as mediaTypes
type
Text* = distinct string
UnifiedCard* = object
componentObjects*: Table[string, Component]
destinationObjects*: Table[string, Destination]
@@ -16,13 +13,10 @@ type
media
swipeableMedia
buttonGroup
jobDetails
appStoreDetails
twitterListDetails
communityDetails
mediaWithDetailsHorizontal
hidden
unknown
Component* = object
kind*: ComponentType
@@ -33,39 +27,24 @@ type
appId*: string
mediaId*: string
destination*: string
location*: string
title*: Text
subtitle*: Text
name*: Text
memberCount*: int
mediaList*: seq[MediaItem]
topicDetail*: tuple[title: Text]
profileUser*: User
shortDescriptionText*: string
MediaItem* = object
id*: string
destination*: string
Destination* = object
kind*: string
data*: tuple[urlData: UrlData]
UrlData* = object
url*: string
vanity*: string
MediaType* = enum
photo, video, model3d
MediaEntity* = object
kind*: MediaType
mediaUrlHttps*: string
videoInfo*: Option[VideoInfo]
VideoInfo* = object
durationMillis*: int
variants*: seq[VideoVariant]
Destination* = object
kind*: string
data*: tuple[urlData: UrlData]
AppType* = enum
androidApp, iPhoneApp, iPadApp
@@ -76,53 +55,13 @@ type
title*: Text
category*: Text
TypeField = Component | Destination | MediaEntity | AppStoreData
Text = object
content: string
converter fromText*(text: Text): string = string(text)
HasTypeField = Component | Destination | MediaEntity | AppStoreData
proc renameHook*(v: var TypeField; fieldName: var string) =
converter fromText*(text: Text): string = text.content
proc renameHook*(v: var HasTypeField; fieldName: var string) =
if fieldName == "type":
fieldName = "kind"
proc enumHook*(s: string; v: var ComponentType) =
v = case s
of "details": details
of "media": media
of "swipeable_media": swipeableMedia
of "button_group": buttonGroup
of "job_details": jobDetails
of "app_store_details": appStoreDetails
of "twitter_list_details": twitterListDetails
of "community_details": communityDetails
of "media_with_details_horizontal": mediaWithDetailsHorizontal
of "commerce_drop_details": hidden
else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
proc enumHook*(s: string; v: var AppType) =
v = case s
of "android_app": androidApp
of "iphone_app": iPhoneApp
of "ipad_app": iPadApp
else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
proc enumHook*(s: string; v: var MediaType) =
v = case s
of "video": video
of "photo": photo
of "model3d": model3d
else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
proc parseHook*(s: string; i: var int; v: var DateTime) =
var str: string
parseHook(s, i, str)
v = parse(str, "yyyy-MM-dd hh:mm:ss")
proc parseHook*(s: string; i: var int; v: var Text) =
if s[i] == '"':
var str: string
parseHook(s, i, str)
v = Text(str)
else:
var t: tuple[content: string]
parseHook(s, i, t)
v = Text(t.content)

9
src/experimental/types/user.nim
View File

@@ -1,6 +1,6 @@
import options
import jsony
import common
from ../../types import VerifiedType
type
RawUser* = object
@@ -16,7 +16,7 @@ type
favouritesCount*: int
statusesCount*: int
mediaCount*: int
verifiedType*: VerifiedType
verified*: bool
protected*: bool
profileLinkColor*: string
profileBannerUrl*: string
@@ -42,3 +42,8 @@ type
Color* = object
red*, green*, blue*: int
proc parseHook*(s: string; i: var int; v: var Slice[int]) =
var slice: array[2, int]
parseHook(s, i, slice)
v = slice[0] ..< slice[1]

32
src/formatters.nim
View File

@@ -11,10 +11,9 @@ const
let
twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
igRegex = re"(www\.)?instagram\.com"
rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -57,19 +56,15 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
if prefs.replaceYouTube.len > 0 and "youtu" in result:
result = result.replace(ytRegex, prefs.replaceYouTube)
if prefs.replaceYouTube in result:
result = result.replace("/c/", "/")
if prefs.replaceTwitter.len > 0:
if tco in result:
result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
if "x.com" in result:
result = result.replace(xRegex, prefs.replaceTwitter)
result = result.replacef(xLinkRegex, a(
prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
if "twitter.com" in result:
result = result.replace(cards, prefs.replaceTwitter & "/cards")
result = result.replace(twRegex, prefs.replaceTwitter)
result = result.replacef(twLinkRegex, a(
prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
result = result.replace(cards, prefs.replaceTwitter & "/cards")
result = result.replace(twRegex, prefs.replaceTwitter)
result = result.replacef(twLinkRegex, a(
prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -77,8 +72,11 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
if prefs.replaceReddit in result and "/gallery/" in result:
result = result.replace("/gallery/", "/comments/")
if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
result = result.replace(igRegex, prefs.replaceInstagram)
if absolute.len > 0 and "href" in result:
result = result.replace("href=\"/", &"href=\"{absolute}/")
result = result.replace("href=\"/", "href=\"" & absolute & "/")
proc getM3u8Url*(content: string): string =
var matches: array[1, string]
@@ -90,8 +88,6 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
for line in manifest.splitLines:
let url =
if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
else: line
if url.startsWith('/'):
let path = "https://video.twimg.com" & url

7
src/http_pool.nim
View File

@@ -39,11 +39,8 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
try:
body
except BadClientError, ProtocolError:
# Twitter returned 503 or closed the connection, we need a new client
pool.release(c, true)
badClient = false
c = pool.acquire(heads)
except ProtocolError:
# Twitter closed the connection, retry
body
finally:
pool.release(c, badClient)

38
src/nitter.nim
View File

@@ -6,7 +6,7 @@ from os import getEnv
import jester
import types, config, prefs, formatters, redis_cache, http_pool, auth
import types, config, prefs, formatters, redis_cache, http_pool, tokens
import views/[general, about]
import routes/[
preferences, timeline, status, media, search, rss, list, debug,
@@ -15,13 +15,8 @@ import routes/[
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
const issuesUrl = "https://github.com/zedeus/nitter/issues"
let
configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
(cfg, fullCfg) = getConfig(configPath)
sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
initSessionPool(cfg, sessionsPath)
let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
let (cfg, fullCfg) = getConfig(configPath)
if not cfg.enableDebug:
# Silence Jester's query warning
@@ -43,6 +38,8 @@ waitFor initRedisPool(cfg)
stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
stdout.flushFile
asyncCheck initTokenPool(cfg)
createUnsupportedRouter(cfg)
createResolverRouter(cfg)
createPrefRouter(cfg)
@@ -59,7 +56,6 @@ settings:
port = Port(cfg.port)
staticDir = cfg.staticDir
bindAddr = cfg.address
reusePort = true
routes:
get "/":
@@ -88,28 +84,20 @@ routes:
resp Http500, showError(
&"An error occurred, please {link} with the URL you tried to visit.", cfg)
error BadClientError:
echo error.exc.name, ": ", error.exc.msg
resp Http500, showError("Network error occurred, please try again.", cfg)
error RateLimitError:
echo error.exc.name, ": ", error.exc.msg
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
error NoSessionsError:
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
&"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
extend rss, ""
extend status, ""
extend search, ""
extend timeline, ""
extend media, ""
extend list, ""
extend unsupported, ""
extend preferences, ""
extend resolver, ""
extend rss, ""
extend search, ""
extend timeline, ""
extend list, ""
extend status, ""
extend media, ""
extend embed, ""
extend debug, ""
extend unsupported, ""

422
src/parser.nim
View File

@@ -1,11 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, times, math
import strutils, options, tables, times, math
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet
proc parseUser(js: JsonNode; id=""): User =
if js.isNull: return
result = User(
@@ -21,38 +19,13 @@ proc parseUser(js: JsonNode; id=""): User =
tweets: js{"statuses_count"}.getInt,
likes: js{"favourites_count"}.getInt,
media: js{"media_count"}.getInt,
verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")),
verified: js{"verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
)
result.expandUserEntities(js)
proc parseGraphUser(js: JsonNode): User =
var user = js{"user_result", "result"}
if user.isNull:
user = ? js{"user_results", "result"}
if user.isNull:
if js{"core"}.notNull and js{"legacy"}.notNull:
user = js
else:
return
result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
# fallback to support UserMedia/recent GraphQL updates
if result.username.len == 0 and user{"core", "screen_name"}.notNull:
result.username = user{"core", "screen_name"}.getStr
result.fullname = user{"core", "name"}.getStr
result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
if user{"is_blue_verified"}.getBool(false):
result.verifiedType = blue
elif user{"verification", "verified_type"}.notNull:
let verifiedType = user{"verification", "verified_type"}.getStr("None")
result.verifiedType = parseEnum[VerifiedType](verifiedType)
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
@@ -65,13 +38,14 @@ proc parseGraphList*(js: JsonNode): List =
result = List(
id: list{"id_str"}.getStr,
name: list{"name"}.getStr,
username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
userId: list{"user_results", "result", "rest_id"}.getStr,
username: list{"user", "legacy", "screen_name"}.getStr,
userId: list{"user", "rest_id"}.getStr,
description: list{"description"}.getStr,
members: list{"member_count"}.getInt,
banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
)
proc parsePoll(js: JsonNode): Poll =
let vals = js{"binding_values"}
# name format is pollNchoice_*
@@ -99,17 +73,13 @@ proc parseGif(js: JsonNode): Gif =
proc parseVideo(js: JsonNode): Video =
result = Video(
thumb: js{"media_url_https"}.getImageStr,
views: getVideoViewCount(js),
available: true,
views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
available: js{"ext_media_availability", "status"}.getStr == "available",
title: js{"ext_alt_text"}.getStr,
durationMs: js{"video_info", "duration_millis"}.getInt
# playbackType: mp4
)
with status, js{"ext_media_availability", "status"}:
if status.getStr.len > 0 and status.getStr.toLowerAscii != "available":
result.available = false
with title, js{"additional_media_info", "title"}:
result.title = title.getStr
@@ -117,15 +87,10 @@ proc parseVideo(js: JsonNode): Video =
result.description = description.getStr
for v in js{"video_info", "variants"}:
let
contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
url = v{"url"}.getStr
result.variants.add VideoVariant(
contentType: contentType,
contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
bitrate: v{"bitrate"}.getInt,
url: url,
resolution: if contentType == mp4: getMp4Resolution(url) else: 0
url: v{"url"}.getStr
)
proc parsePromoVideo(js: JsonNode): Video =
@@ -216,7 +181,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
result.url.len == 0 or result.url.startsWith("card://"):
result.url = getPicUrl(result.image)
proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
proc parseTweet(js: JsonNode): Tweet =
if js.isNull: return
result = Tweet(
id: js{"id_str"}.getId,
@@ -235,28 +200,16 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
)
)
# fix for pinned threads
if result.hasThread and result.threadId == 0:
result.threadId = js{"self_thread", "id_str"}.getId
result.expandTweetEntities(js)
if "retweeted_status" in js:
result.retweet = some Tweet()
elif js{"is_quote_status"}.getBool:
if js{"is_quote_status"}.getBool:
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
# legacy
with rt, js{"retweeted_status_id_str"}:
result.retweet = some Tweet(id: rt.getId)
return
# graphql
with rt, js{"retweeted_status_result", "result"}:
# needed due to weird edgecase where the actual tweet data isn't included
if "legacy" in rt:
result.retweet = some parseGraphTweet(rt)
return
if jsCard.kind != JNull:
with jsCard, js{"card"}:
let name = jsCard{"name"}.getStr
if "poll" in name:
if "image" in name:
@@ -268,8 +221,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
else:
result.card = some parseCard(jsCard, js{"entities", "urls"})
result.expandTweetEntities(js)
with jsMedia, js{"extended_entities", "media"}:
for m in jsMedia:
case m{"type"}.getStr
@@ -278,19 +229,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
of "video":
result.video = some(parseVideo(m))
with user, m{"additional_media_info", "source_user"}:
if user{"id"}.getInt > 0:
result.attribution = some(parseUser(user))
else:
result.attribution = some(parseGraphUser(user))
result.attribution = some(parseUser(user))
of "animated_gif":
result.gif = some(parseGif(m))
else: discard
with url, m{"url"}:
if result.text.endsWith(url.getStr):
result.text.removeSuffix(url.getStr)
result.text = result.text.strip()
with jsWithheld, js{"withheld_in_countries"}:
let withheldInCountries: seq[string] =
if jsWithheld.kind != JArray: @[]
@@ -305,250 +248,159 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.text.removeSuffix(" Learn more.")
result.available = false
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
if js.kind == JNull:
return Tweet()
case js{"__typename"}.getStr
of "TweetUnavailable":
return Tweet()
of "TweetTombstone":
with text, js{"tombstone", "richText"}:
return Tweet(text: text.getTombstone)
with text, js{"tombstone", "text"}:
return Tweet(text: text.getTombstone)
return Tweet()
of "TweetPreviewDisplay":
return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"}, isLegacy)
else:
discard
if not js.hasKey("legacy"):
return Tweet()
var jsCard = copy(js{if isLegacy: "card" else: "tweet_card", "legacy"})
if jsCard.kind != JNull:
var values = newJObject()
for val in jsCard["binding_values"]:
values[val["key"].getStr] = val["value"]
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result.id = js{"rest_id"}.getId
result.user = parseGraphUser(js{"core"})
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
result.expandNoteTweetEntities(noteTweet)
proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
let intId = if id.len > 0: parseBiggestInt(id) else: 0
result = global.tweets.getOrDefault(id, Tweet(id: intId))
if result.quote.isSome:
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}, isLegacy))
let quote = get(result.quote).id
if $quote in global.tweets:
result.quote = some global.tweets[$quote]
else:
result.quote = some Tweet()
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
let cursor = t{"item", "content", "value"}
result.thread.cursor = cursor.getStr
result.thread.hasMore = true
elif "tweet" in entryId and "promoted" notin entryId:
let
isLegacy = t{"item"}.hasKey("itemContent")
(contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results")
else: ("content", "tweetResult")
if result.retweet.isSome:
let rt = get(result.retweet).id
if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt)
else:
result.retweet = some Tweet()
with content, t{"item", contentKey}:
result.thread.content.add parseGraphTweet(content{resultKey, "result"}, isLegacy)
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
if content{"tweetDisplayType"}.getStr == "SelfThread":
result.self = true
let id = pin.getId
if id notin global.tweets: return
proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweet_result", "result"}:
result = parseGraphTweet(tweet, false)
proc parseGraphConversation*(js: JsonNode; tweetId: string; v2=true): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
rootKey = if v2: "timeline_response" else: "threaded_conversation_with_injections_v2"
contentKey = if v2: "content" else: "itemContent"
resultKey = if v2: "tweetResult" else: "tweet_results"
tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"}
let instructions = ? js{"data", rootKey, "instructions"}
if instructions.len == 0:
return
for k, v in users:
result.users[k] = parseUser(v, k)
for i in instructions:
if i{"__typename"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", contentKey, resultKey, "result"}:
let tweet = parseGraphTweet(tweetResult, not v2)
for k, v in tweets:
var tweet = parseTweet(v)
if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
result.thread = Chain()
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
if self:
result.after = thread
elif thread.content.len > 0:
result.replies.content.add thread
elif entryId.startsWith("tombstone"):
let id = entryId.getId()
let tweet = Tweet(
id: parseBiggestInt(id),
available: false,
text: e{"content", contentKey, "tombstoneInfo", "richText"}.getTombstone
)
let thread = js{"content", "item", "content", "conversationThread"}
with cursor, thread{"showMoreCursor"}:
result.thread.cursor = cursor{"value"}.getStr
result.thread.hasMore = true
if id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", contentKey, "value"}.getStr
for t in thread{"conversationComponents"}:
let content = t{"conversationTweetComponent", "tweet"}
proc extractTweetsFromEntry*(e: JsonNode; entryId: string): seq[Tweet] =
if e{"content", "items"}.notNull:
for item in e{"content", "items"}:
with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
var tweet = parseGraphTweet(tweetResult, false)
if not tweet.available:
tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
result.add tweet
return
if content{"displayType"}.getStr == "SelfThread":
result.self = true
with tweetResult, e{"content", "content", "tweetResult", "result"}:
var tweet = parseGraphTweet(tweetResult, false)
var tweet = finalizeTweet(global, content{"id"}.getStr)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.add tweet
tweet.tombstone = getTombstone(content{"tombstone"})
result.thread.content.add tweet
proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let instructions =
if js{"data", "list"}.notNull:
? js{"data", "list", "timeline_response", "timeline", "instructions"}
elif js{"data", "user"}.notNull:
? js{"data", "user", "result", "timeline", "timeline", "instructions"}
else:
? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0:
return
for i in instructions:
# TimelineAddToModule instruction is used by UserMedia
if i{"moduleItems"}.notNull:
for item in i{"moduleItems"}:
with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult, false)
if not tweet.available:
tweet.id = parseBiggestInt(item{"entryId"}.getStr.getId())
result.tweets.content.add tweet
continue
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
if "tweet" in entry or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if $tweet.id != tweetId:
result.before.content.add tweet
else:
result.tweet = tweet
elif "conversationThread" in entry:
let (thread, self) = parseThread(e, global)
if thread.content.len > 0:
if self:
result.after = thread
else:
result.replies.content.add thread
elif "cursor-showMore" in entry:
result.replies.bottom = e.getCursor
elif "cursor-bottom" in entry:
result.replies.bottom = e.getCursor
if i{"entries"}.notNull:
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
for tweet in extractTweetsFromEntry(e, entryId):
result.tweets.content.add tweet
elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
let (thread, self) = parseGraphThread(e)
result.tweets.content.add thread.content
elif entryId.startsWith("cursor-bottom"):
result.tweets.bottom = e{"content", "value"}.getStr
proc parseStatus*(js: JsonNode): Tweet =
with e, js{"errors"}:
if e.getError == tweetNotFound:
return
if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult, false)
tweet.pinned = true
if not tweet.available and tweet.tombstone.len == 0:
let entryId = i{"entry", "entryId"}.getEntryId
if entryId.len > 0:
tweet.id = parseBiggestInt(entryId)
result.pinned = some tweet
result = parseTweet(js)
if not result.isNil:
result.user = parseUser(js{"user"})
proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
result = @[]
with quote, js{"quoted_status"}:
result.quote = some parseStatus(js{"quoted_status"})
let instructions =
if js{"data", "user"}.notNull:
? js{"data", "user", "result", "timeline", "timeline", "instructions"}
else:
? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
if instructions.len == 0:
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
return
for i in instructions:
# TimelineAddToModule instruction is used by MediaTimelineV2
if i{"moduleItems"}.notNull:
for item in i{"moduleItems"}:
with tweetResult, item{"item", "itemContent", "tweet_results", "result"}:
let t = parseGraphTweet(tweetResult, false)
if not t.available:
t.id = parseBiggestInt(item{"entryId"}.getStr.getId())
for i in js:
when T is Tweet:
if res.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.content.add pin
let photo = extractGalleryPhoto(t)
if photo.url.len > 0:
result.add photo
with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor
if result.len == 16:
return
continue
proc parseTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let global = parseGlobalObjects(? js)
let instrType = i{"type"}.getStr(i{"__typename"}.getStr)
if instrType != "TimelineAddEntries":
continue
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
for t in extractTweetsFromEntry(e, entryId):
let photo = extractGalleryPhoto(t)
if photo.url.len > 0:
result.add photo
result.parseInstructions(global, instructions)
if result.len == 16:
return
for e in instructions[0]{"addEntries", "entries"}:
let entry = e{"entryId"}.getStr
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue
result.content.add tweet
elif "cursor-top" in entry:
result.top = e.getCursor
elif "cursor-bottom" in entry:
result.bottom = e.getCursor
elif entry.startsWith("sq-C"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.bottom = cursor{"value"}.getStr
else:
result.top = cursor{"value"}.getStr
proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
result = Result[T](beginning: after.len == 0)
proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js:
let
t = parseTweet(tweet)
url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
else: ""
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
if instructions.len == 0:
return
for instruction in instructions:
let typ = instruction{"type"}.getStr
if typ == "TimelineAddEntries":
for e in instruction{"entries"}:
let entryId = e{"entryId"}.getStr
when T is Tweets:
if entryId.startsWith("tweet"):
with tweetRes, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetRes)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif T is User:
if entryId.startsWith("user"):
with userRes, e{"content", "itemContent"}:
result.content.add parseGraphUser(userRes)
if entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
result.bottom = instruction{"entry", "content", "value"}.getStr
if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $t.id)

123
src/parserutils.nim
View File

@@ -1,17 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
import std/[times, macros, htmlgen, options, algorithm, re]
import std/strutils except escape
import std/[strutils, times, macros, htmlgen, options, algorithm, re]
import std/unicode except strip
from xmltree import escape
import packedjson
import types, utils, formatters
const
unicodeOpen = "\uFFFA"
unicodeClose = "\uFFFB"
xmlOpen = escape("<")
xmlClose = escape(">")
let
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1<a href=\"/$2\">@$2</a>"
@@ -36,16 +28,15 @@ template `?`*(js: JsonNode): untyped =
if j.isNull: return
j
template with*(ident, value, body): untyped =
if true:
template `with`*(ident, value, body): untyped =
block:
let ident {.inject.} = value
if ident != nil: body
template with*(ident; value: JsonNode; body): untyped =
if true:
template `with`*(ident; value: JsonNode; body): untyped =
block:
let ident {.inject.} = value
# value.notNull causes a compilation error for versions < 1.6.14
if notNull(value): body
if value.notNull: body
template getCursor*(js: JsonNode): string =
js{"content", "operation", "cursor", "value"}.getStr
@@ -139,30 +130,9 @@ proc getBanner*(js: JsonNode): string =
return
proc getTombstone*(js: JsonNode): string =
result = js{"text"}.getStr
result = js{"tombstoneInfo", "richText", "text"}.getStr
result.removeSuffix(" Learn more")
proc getMp4Resolution*(url: string): int =
# parses the height out of a URL like this one:
# https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
const vidSep = "/vid/"
let
vidIdx = url.find(vidSep) + vidSep.len
resIdx = url.find('x', vidIdx) + 1
res = url[resIdx ..< url.find("/", resIdx)]
try:
return parseInt(res)
except ValueError:
# cannot determine resolution (e.g. m3u8/non-mp4 video)
return 0
proc getVideoViewCount*(js: JsonNode): string =
with stats, js{"ext_media_stats"}:
return stats{"view_count"}.getStr($stats{"viewCount"}.getInt)
return $js{"mediaStats", "viewCount"}.getInt(0)
proc extractSlice(js: JsonNode): Slice[int] =
result = js["indices"][0].getInt ..< js["indices"][1].getInt
@@ -245,37 +215,47 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
user.bio = user.bio.replacef(unRegex, unReplace)
.replacef(htRegex, htReplace)
proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
replyTo=""; hasRedundantLink=false) =
let hasCard = tweet.card.isSome
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let
orig = tweet.text.toRunes
textRange = js{"display_text_range"}
textSlice = textRange{0}.getInt .. textRange{1}.getInt
hasQuote = js{"is_quote_status"}.getBool
hasCard = tweet.card.isSome
var replyTo = ""
if tweet.replyId != 0:
with reply, js{"in_reply_to_screen_name"}:
tweet.reply.add reply.getStr
replyTo = reply.getStr
let ent = ? js{"entities"}
var replacements = newSeq[ReplaceSlice]()
with urls, entities{"urls"}:
with urls, ent{"urls"}:
for u in urls:
let urlStr = u["url"].getStr
if urlStr.len == 0 or urlStr notin text:
if urlStr.len == 0 or urlStr notin tweet.text:
continue
replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
if hasCard and u{"url"}.getStr == get(tweet.card).url:
get(tweet.card).url = u{"expanded_url"}.getStr
with media, entities{"media"}:
with media, ent{"media"}:
for m in media:
replacements.extractUrls(m, textSlice.b, hideTwitter = true)
if "hashtags" in entities:
for hashtag in entities["hashtags"]:
if "hashtags" in ent:
for hashtag in ent["hashtags"]:
replacements.extractHashtags(hashtag)
if "symbols" in entities:
for symbol in entities["symbols"]:
if "symbols" in ent:
for symbol in ent["symbols"]:
replacements.extractHashtags(symbol)
if "user_mentions" in entities:
for mention in entities["user_mentions"]:
if "user_mentions" in ent:
for mention in ent["user_mentions"]:
let
name = mention{"screen_name"}.getStr
slice = mention.extractSlice
@@ -292,40 +272,5 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
replacements.deduplicate
replacements.sort(cmp)
tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let
entities = ? js{"entities"}
textRange = js{"display_text_range"}
textSlice = textRange{0}.getInt .. textRange{1}.getInt
hasQuote = js{"is_quote_status"}.getBool
hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
var replyTo = ""
if tweet.replyId != 0:
with reply, js{"in_reply_to_screen_name"}:
replyTo = reply.getStr
tweet.reply.add replyTo
tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
let
entities = ? js{"entity_set"}
text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
textSlice = 0..text.runeLen
tweet.expandTextEntities(entities, text, textSlice)
tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
proc extractGalleryPhoto*(t: Tweet): GalleryPhoto =
let url =
if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
else: ""
result = GalleryPhoto(url: url, tweetId: $t.id)
tweet.text = orig.replacedWith(replacements, textSlice)
.strip(leading=false)

6
src/prefs_impl.nim
View File

@@ -83,7 +83,7 @@ genPrefs:
"Enable mp4 video playback (only for gifs)"
hlsPlayback(checkbox, false):
"Enable HLS video streaming (requires JavaScript)"
"Enable hls video streaming (requires JavaScript)"
proxyVideos(checkbox, true):
"Proxy video streaming through the server (might be slow)"
@@ -107,6 +107,10 @@ genPrefs:
"Reddit -> Teddit/Libreddit"
placeholder: "Teddit hostname"
replaceInstagram(input, ""):
"Instagram -> Bibliogram"
placeholder: "Bibliogram hostname"
iterator allPrefs*(): Pref =
for k, v in prefList:
for pref in v:

8
src/query.nim
View File

@@ -60,7 +60,7 @@ proc genQueryParam*(query: Query): string =
param &= "OR "
if query.fromUser.len > 0 and query.kind in {posts, media}:
param &= "filter:self_threads OR -filter:replies "
param &= "filter:self_threads OR-filter:replies "
if "nativeretweets" notin query.excludes:
param &= "include:nativeretweets "
@@ -93,11 +93,11 @@ proc genQueryUrl*(query: Query): string =
if query.text.len > 0:
params.add "q=" & encodeUrl(query.text)
for f in query.filters:
params.add &"f-{f}=on"
params.add "f-" & f & "=on"
for e in query.excludes:
params.add &"e-{e}=on"
params.add "e-" & e & "=on"
for i in query.includes.filterIt(it != "nativeretweets"):
params.add &"i-{i}=on"
params.add "i-" & i & "=on"
if query.since.len > 0:
params.add "since=" & query.since

42
src/redis_cache.nim
View File

@@ -52,7 +52,6 @@ proc initRedisPool*(cfg: Config) {.async.} =
await migrate("profileDates", "p:*")
await migrate("profileStats", "p:*")
await migrate("userType", "p:*")
await migrate("verifiedType", "p:*")
pool.withAcquire(r):
# optimize memory usage for user ID buckets
@@ -86,7 +85,7 @@ proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
proc cache*(data: PhotoRail; name: string) {.async.} =
await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
proc cache*(data: User) {.async.} =
if data.username.len == 0: return
@@ -119,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
pool.withAcquire(r):
result = await r.hGet(name.uidKey, name)
if result == redisNil:
let user = await getGraphUser(username)
let user = await getUser(username)
if user.suspended:
return "suspended"
else:
await all(cacheUserId(name, user.id), cache(user))
await cacheUserId(name, user.id)
return user.id
proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -131,7 +130,8 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
if prof != redisNil:
prof.deserialize(User)
elif fetch:
result = await getGraphUser(username)
let userId = await getUserId(username)
result = await getGraphUser(userId)
await cache(result)
proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,30 +142,28 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
if username != redisNil:
result = username
else:
let user = await getGraphUserById(userId)
let user = await getUserById(userId)
result = user.username
await setEx(key, baseCacheTime, result)
if result.len > 0 and user.id.len > 0:
await all(cacheUserId(result, user.id), cache(user))
# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
# if id == 0: return
# let tweet = await get(id.tweetKey)
# if tweet != redisNil:
# tweet.deserialize(Tweet)
# else:
# result = await getGraphTweetResult($id)
# if not result.isNil:
# await cache(result)
proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
if id == 0: return
let tweet = await get(id.tweetKey)
if tweet != redisNil:
tweet.deserialize(Tweet)
else:
result = await getStatus($id)
if result.isNil:
await cache(result)
proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
if id.len == 0: return
let rail = await get("pr2:" & toLower(id))
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let rail = await get("pr:" & toLower(name))
if rail != redisNil:
rail.deserialize(PhotoRail)
else:
result = await getPhotoRail(id)
await cache(result, id)
result = await getPhotoRail(name)
await cache(result, name)
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
let list = if id.len == 0: redisNil

9
src/routes/debug.nim
View File

@@ -1,13 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
import jester
import router_utils
import ".."/[auth, types]
import ".."/[tokens, types]
proc createDebugRouter*(cfg: Config) =
router debug:
get "/.health":
respJson getSessionPoolHealth()
get "/.sessions":
get "/.tokens":
cond cfg.enableDebug
respJson getSessionPoolDebug()
respJson getPoolJson()

16
src/routes/embed.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils, strformat, options
import asyncdispatch, strutils, options
import jester, karax/vdom
import ".."/[types, api]
import ../views/[embed, tweet, general]
@@ -10,27 +10,27 @@ export api, embed, vdom, tweet, general, router_utils
proc createEmbedRouter*(cfg: Config) =
router embed:
get "/i/videos/tweet/@id":
let tweet = await getGraphTweetResult(@"id")
if tweet == nil or tweet.video.isNone:
let convo = await getTweet(@"id")
if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
resp Http404
resp renderVideoEmbed(tweet, cfg, request)
resp renderVideoEmbed(convo.tweet, cfg, request)
get "/@user/status/@id/embed":
let
tweet = await getGraphTweetResult(@"id")
convo = await getTweet(@"id")
prefs = cookiePrefs()
path = getPath()
if tweet == nil:
if convo == nil or convo.tweet == nil:
resp Http404
resp renderTweetEmbed(tweet, path, prefs, cfg, request)
resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
get "/embed/Tweet.html":
let id = @"id"
if id.len > 0:
redirect(&"/i/status/{id}/embed")
redirect("/i/status/" & id & "/embed")
else:
resp Http404

22
src/routes/list.nim
View File

@@ -1,25 +1,23 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, uri
import strutils, uri
import jester
import router_utils
import ".."/[types, redis_cache, api]
import ../views/[general, timeline, list]
export getListTimeline, getGraphList
template respList*(list, timeline, title, vnode: typed) =
if list.id.len == 0 or list.name.len == 0:
resp Http404, showError(&"""List "{@"id"}" not found""", cfg)
resp Http404, showError("List " & @"id" & " not found", cfg)
let
html = renderList(vnode, timeline.query, list)
rss = &"""/i/lists/{@"id"}/rss"""
rss = "/i/lists/$1/rss" % [@"id"]
resp renderMain(html, request, cfg, prefs, titleText=title, rss=rss, banner=list.banner)
proc title*(list: List): string =
&"@{list.username}/{list.name}"
proc createListRouter*(cfg: Config) =
router list:
get "/@name/lists/@slug/?":
@@ -30,22 +28,24 @@ proc createListRouter*(cfg: Config) =
slug = decodeUrl(@"slug")
list = await getCachedList(@"name", slug)
if list.id.len == 0:
resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
redirect(&"/i/lists/{list.id}")
resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
redirect("/i/lists/" & list.id)
get "/i/lists/@id/?":
cond '.' notin @"id"
let
prefs = cookiePrefs()
list = await getCachedList(id=(@"id"))
timeline = await getGraphListTweets(list.id, getCursor())
title = "@" & list.username & "/" & list.name
timeline = await getListTimeline(list.id, getCursor())
vnode = renderTimelineTweets(timeline, prefs, request.path)
respList(list, timeline, list.title, vnode)
respList(list, timeline, title, vnode)
get "/i/lists/@id/members":
cond '.' notin @"id"
let
prefs = cookiePrefs()
list = await getCachedList(id=(@"id"))
title = "@" & list.username & "/" & list.name
members = await getGraphListMembers(list, getCursor())
respList(list, members, list.title, renderTimelineUsers(members, prefs, request.path))
respList(list, members, title, renderTimelineUsers(members, prefs, request.path))

19
src/routes/media.nim
View File

@@ -37,8 +37,6 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
try:
let res = await client.get(url)
if res.status != "200 OK":
if res.status != "404 Not Found":
echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
return Http404
let hashed = $hash(url)
@@ -67,7 +65,6 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
await request.client.send(data)
data.setLen 0
except HttpRequestError, ProtocolError, OSError:
echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
result = Http404
finally:
client.close()
@@ -91,20 +88,6 @@ proc createMediaRouter*(cfg: Config) =
get "/pic/?":
resp Http404
get re"^\/pic\/orig\/(enc)?\/?(.+)":
var url = decoded(request, 1)
if "twimg.com" notin url:
url.insert(twimg)
if not url.startsWith(https):
url.insert(https)
url.add("?name=orig")
let uri = parseUri(url)
cond isTwitterUrl(uri) == true
let code = await proxyMedia(request, url)
check code
get re"^\/pic\/(enc)?\/?(.+)":
var url = decoded(request, 1)
if "twimg.com" notin url:
@@ -123,7 +106,7 @@ proc createMediaRouter*(cfg: Config) =
cond "http" in url
if getHmac(url) != request.matches[1]:
resp Http403, showError("Failed to verify signature", cfg)
resp showError("Failed to verify signature", cfg)
if ".mp4" in url or ".ts" in url or ".m4s" in url:
let code = await proxyMedia(request, url)

67
src/routes/rss.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, tables, times, hashes, uri
import asyncdispatch, strutils, tables, times, hashes, uri
import jester
@@ -10,11 +10,6 @@ include "../views/rss.nimf"
export times, hashes
proc redisKey*(page, name, cursor: string): string =
result = page & ":" & name
if cursor.len > 0:
result &= ":" & cursor
proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
var profile: Profile
let
@@ -23,16 +18,18 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
names = getNames(name)
if names.len == 1:
profile = await fetchProfile(after, query, skipRail=true)
profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
else:
var q = query
q.fromUser = names
profile.tweets = await getGraphTweetSearch(q, after)
# this is kinda dumb
profile.user = User(
username: name,
fullname: names.join(" | "),
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
profile = Profile(
tweets: await getSearch[Tweet](q, after),
# this is kinda dumb
user: User(
username: name,
fullname: names.join(" | "),
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
)
)
if profile.user.suspended:
@@ -45,8 +42,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
template respRss*(rss, page) =
if rss.cursor.len == 0:
let info = case page
of "User": " \"" & @"name" & "\" "
of "List": " \"" & @"id" & "\" "
of "User": " \"$1\" " % @"name"
of "List": " $1 " % @"id"
else: " "
resp Http404, showError(page & info & "not found", cfg)
@@ -70,13 +67,13 @@ proc createRssRouter*(cfg: Config) =
let
cursor = getCursor()
key = redisKey("search", $hash(genQueryUrl(query)), cursor)
key = "search:" & $hash(genQueryUrl(query)) & ":" & cursor
var rss = await getCachedRss(key)
if rss.cursor.len > 0:
respRss(rss, "Search")
let tweets = await getGraphTweetSearch(query, cursor)
let tweets = await getSearch[Tweet](query, cursor)
rss.cursor = tweets.bottom
rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
@@ -87,8 +84,9 @@ proc createRssRouter*(cfg: Config) =
cond cfg.enableRss
cond '.' notin @"name"
let
cursor = getCursor()
name = @"name"
key = redisKey("twitter", name, getCursor())
key = "twitter:" & name & ":" & cursor
var rss = await getCachedRss(key)
if rss.cursor.len > 0:
@@ -103,20 +101,18 @@ proc createRssRouter*(cfg: Config) =
cond cfg.enableRss
cond '.' notin @"name"
cond @"tab" in ["with_replies", "media", "search"]
let
name = @"name"
tab = @"tab"
query =
case tab
of "with_replies": getReplyQuery(name)
of "media": getMediaQuery(name)
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])
let name = @"name"
let query =
case @"tab"
of "with_replies": getReplyQuery(name)
of "media": getMediaQuery(name)
of "search": initQuery(params(request), name=name)
else: Query(fromUser: @[name])
let searchKey = if tab != "search": ""
else: ":" & $hash(genQueryUrl(query))
let key = redisKey(tab, name & searchKey, getCursor())
var key = @"tab" & ":" & @"name" & ":"
if @"tab" == "search":
key &= $hash(genQueryUrl(query)) & ":"
key &= getCursor()
var rss = await getCachedRss(key)
if rss.cursor.len > 0:
@@ -147,17 +143,18 @@ proc createRssRouter*(cfg: Config) =
get "/i/lists/@id/rss":
cond cfg.enableRss
let
id = @"id"
cursor = getCursor()
key = redisKey("lists", id, cursor)
key =
if cursor.len == 0: "lists:" & @"id"
else: "lists:" & @"id" & ":" & cursor
var rss = await getCachedRss(key)
if rss.cursor.len > 0:
respRss(rss, "List")
let
list = await getCachedList(id=id)
timeline = await getGraphListTweets(list.id, cursor)
list = await getCachedList(id=(@"id"))
timeline = await getListTimeline(list.id, cursor)
rss.cursor = timeline.bottom
rss.feed = renderListRss(timeline.content, list, cfg)

22
src/routes/search.nim
View File

@@ -14,31 +14,25 @@ export search
proc createSearchRouter*(cfg: Config) =
router search:
get "/search/?":
let q = @"q"
if q.len > 500:
if @"q".len > 500:
resp Http400, showError("Search input too long.", cfg)
let
prefs = cookiePrefs()
query = initQuery(params(request))
title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
case query.kind
of users:
if "," in q:
redirect("/" & q)
var users: Result[User]
try:
users = await getGraphUserSearch(query, getCursor())
except InternalError:
users = Result[User](beginning: true, query: query)
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
if "," in @"q":
redirect("/" & @"q")
let users = await getSearch[User](query, getCursor())
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
of tweets:
let
tweets = await getGraphTweetSearch(query, getCursor())
tweets = await getSearch[Tweet](query, getCursor())
rss = "/search/rss?" & genQueryUrl(query)
resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
request, cfg, prefs, title, rss=rss)
request, cfg, prefs, rss=rss)
else:
resp Http404, showError("Invalid search", cfg)
@@ -48,4 +42,4 @@ proc createSearchRouter*(cfg: Config) =
get "/opensearch":
let url = getUrlPrefix(cfg) & "/search?q="
resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
generateOpenSearchXML(cfg.title, cfg.hostname, url)
generateOpenSearchXML(cfg.title, cfg.hostname, url)

13
src/routes/status.nim
View File

@@ -16,21 +16,17 @@ proc createStatusRouter*(cfg: Config) =
router status:
get "/@name/status/@id/?":
cond '.' notin @"name"
let id = @"id"
if id.len > 19 or id.any(c => not c.isDigit):
resp Http404, showError("Invalid tweet ID", cfg)
cond not @"id".any(c => not c.isDigit)
let prefs = cookiePrefs()
# used for the infinite scroll feature
if @"scroll".len > 0:
let replies = await getReplies(id, getCursor())
let replies = await getReplies(@"id", getCursor())
if replies.content.len == 0:
resp Http404, ""
resp $renderReplies(replies, prefs, getPath())
let conv = await getTweet(id, getCursor())
let conv = await getTweet(@"id", getCursor())
if conv == nil:
echo "nil conv"
@@ -76,6 +72,3 @@ proc createStatusRouter*(cfg: Config) =
get "/i/web/status/@id":
redirect("/i/status/" & @"id")
get "/@name/thread/@id/?":
redirect("/$1/status/$2" % [@"name", @"id"])

46
src/routes/timeline.nim
View File

@@ -27,7 +27,8 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
else:
body
proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile] {.async.} =
proc fetchProfile*(after: string; query: Query; skipRail=false;
skipPinned=false): Future[Profile] {.async.} =
let
name = query.fromUser[0]
userId = await getUserId(name)
@@ -44,21 +45,36 @@ proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile]
after.setLen 0
let
timeline =
case query.kind
of posts: getTimeline(userId, after)
of replies: getTimeline(userId, after, replies=true)
of media: getMediaTimeline(userId, after)
else: getSearch[Tweet](query, after)
rail =
skipIf(skipRail or query.kind == media, @[]):
getCachedPhotoRail(userId)
getCachedPhotoRail(name)
user = getCachedUser(name)
user = await getCachedUser(name)
result =
case query.kind
of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after)
else: Profile(tweets: await getGraphTweetSearch(query, after))
var pinned: Option[Tweet]
if not skipPinned and user.pinnedTweet > 0 and
after.len == 0 and query.kind in {posts, replies}:
let tweet = await getCachedTweet(user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
pinned = some tweet
result.user = await user
result.photoRail = await rail
result = Profile(
user: user,
pinned: pinned,
tweets: await timeline,
photoRail: await rail
)
if result.user.protected or result.user.suspended:
return
result.tweets.query = query
@@ -66,11 +82,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} =
if query.fromUser.len != 1:
let
timeline = await getGraphTweetSearch(query, after)
timeline = await getSearch[Tweet](query, after)
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
var profile = await fetchProfile(after, query)
var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
template u: untyped = profile.user
if u.suspended:
@@ -107,7 +123,7 @@ proc createTimelineRouter*(cfg: Config) =
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
cond @"name" notin ["pic", "gif", "video"]
cond @"tab" in ["with_replies", "media", "search", ""]
let
prefs = cookiePrefs()
@@ -121,7 +137,7 @@ proc createTimelineRouter*(cfg: Config) =
# used for the infinite scroll feature
if @"scroll".len > 0:
if query.fromUser.len != 1:
var timeline = await getGraphTweetSearch(query, after)
var timeline = await getSearch[Tweet](query, after)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())

2
src/sass/include/_variables.scss
View File

@@ -28,8 +28,6 @@ $more_replies_dots: #AD433B;
$error_red: #420A05;
$verified_blue: #1DA1F2;
$verified_business: #FAC82B;
$verified_government: #C1B6A4;
$icon_text: $fg_color;
$tab: $fg_color;

21
src/sass/index.scss
View File

@@ -39,8 +39,6 @@ body {
--error_red: #{$error_red};
--verified_blue: #{$verified_blue};
--verified_business: #{$verified_business};
--verified_government: #{$verified_government};
--icon_text: #{$icon_text};
--tab: #{$fg_color};
@@ -143,30 +141,17 @@ ul {
.verified-icon {
color: var(--icon_text);
background-color: var(--verified_blue);
border-radius: 50%;
flex-shrink: 0;
margin: 2px 0 3px 3px;
padding-top: 3px;
height: 11px;
padding-top: 2px;
height: 12px;
width: 14px;
font-size: 8px;
display: inline-block;
text-align: center;
vertical-align: middle;
&.blue {
background-color: var(--verified_blue);
}
&.business {
color: var(--bg_panel);
background-color: var(--verified_business);
}
&.government {
color: var(--bg_panel);
background-color: var(--verified_government);
}
}
@media(max-width: 600px) {

5
src/sass/navbar.scss
View File

@@ -70,9 +70,8 @@ nav {
.lp {
height: 14px;
display: inline-block;
position: relative;
top: 2px;
margin-top: 2px;
display: block;
fill: var(--fg_nav);
&:hover {

6
src/sass/profile/card.scss
View File

@@ -73,9 +73,9 @@
}
}
.profile-joindate, .profile-location, .profile-website {
.profile-joindate, .profile-location, profile-website {
color: var(--fg_faded);
margin: 1px 0;
margin: 2px 0;
width: 100%;
}
}
@@ -115,7 +115,7 @@
}
.profile-card-tabs-name {
flex-shrink: 100;
@include breakable;
}
.profile-card-avatar {

2
src/sass/search.scss
View File

@@ -14,8 +14,6 @@
button {
margin: 0 2px 0 0;
height: 23px;
display: flex;
align-items: center;
}
.pref-input {

17
src/sass/tweet/_base.scss
View File

@@ -98,9 +98,10 @@
}
.avatar {
position: absolute;
&.round {
border-radius: 50%;
-webkit-user-select: none;
}
&.mini {
@@ -120,22 +121,14 @@
background-color: var(--bg_panel);
.tweet-content {
font-size: 18px;
font-size: 18px
}
.tweet-body {
.tweet-body {
display: flex;
flex-direction: column;
max-height: calc(100vh - 0.75em * 2);
}
.card-image img {
height: auto;
}
.avatar {
position: absolute;
}
}
.attribution {
@@ -200,7 +193,6 @@
.tweet-stats {
margin-bottom: -3px;
-webkit-user-select: none;
}
.tweet-stat {
@@ -232,7 +224,6 @@
left: 0;
top: 0;
position: absolute;
-webkit-user-select: none;
&:hover {
background-color: var(--bg_hover);

27
src/sass/tweet/thread.scss
View File

@@ -23,6 +23,7 @@
font-size: 18px;
}
@media(max-width: 600px) {
.main-tweet .tweet-content {
font-size: 16px;
@@ -110,29 +111,3 @@
margin-left: 58px;
padding: 7px 0;
}
.timeline-item.thread.more-replies-thread {
padding: 0 0.75em;
&::before {
top: 40px;
margin-bottom: 31px;
}
.more-replies {
display: flex;
padding-top: unset !important;
margin-top: 8px;
&::before {
display: inline-block;
position: relative;
top: -1px;
line-height: 0.4em;
}
.more-replies-text {
display: inline;
}
}
}

4
src/sass/tweet/video.scss
View File

@@ -3,7 +3,7 @@
video {
max-height: 100%;
width: 100%;
max-width: 100%;
}
.gallery-video {
@@ -16,8 +16,6 @@ video {
}
.video-container {
min-height: 80px;
min-width: 200px;
max-height: 530px;
margin: 0;
display: flex;

154
src/tokens.nim Normal file
View File

@@ -0,0 +1,154 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import zippy
import types, consts, http_pool
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
failDelay = initDuration(minutes=30)
var
clientPool: HttpPool
tokenPool: seq[Token]
lastFailed: Time
proc getPoolJson*(): JsonNode =
var
list = newJObject()
totalReqs = 0
totalPending = 0
reqsPerApi: Table[string, int]
for token in tokenPool:
totalPending.inc(token.pending)
list[token.tok] = %*{
"apis": newJObject(),
"pending": token.pending,
"init": $token.init,
"lastUse": $token.lastUse
}
for api in token.apis.keys:
list[token.tok]["apis"][$api] = %token.apis[api]
let
maxReqs =
case api
of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
of Api.timeline: 187
else: 180
reqs = maxReqs - token.apis[api].remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
totalReqs.inc(reqs)
return %*{
"amount": tokenPool.len,
"requests": totalReqs,
"pending": totalPending,
"apis": reqsPerApi,
"tokens": list
}
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < failDelay:
raise rateLimitError()
let headers = newHttpHeaders({
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.5",
"connection": "keep-alive",
"authorization": auth
})
try:
let
resp = clientPool.use(headers): await c.postContent(activate)
tokNode = parseJson(uncompress(resp))["guest_token"]
tok = tokNode.getStr($(tokNode.getInt))
time = getTime()
return Token(tok: tok, init: time, lastUse: time)
except Exception as e:
lastFailed = getTime()
echo "fetching token failed: ", e.msg
proc expired(token: Token): bool =
let time = getTime()
token.init < time - maxAge or token.lastUse < time - maxLastUse
proc isLimited(token: Token; api: Api): bool =
if token.isNil or token.expired:
return true
if api in token.apis:
let limit = token.apis[api]
return (limit.remaining <= 10 and limit.reset > epochTime().int)
else:
return false
proc isReady(token: Token; api: Api): bool =
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
proc release*(token: Token; used=false; invalid=false) =
if token.isNil: return
if invalid or token.expired:
let idx = tokenPool.find(token)
if idx > -1: tokenPool.delete(idx)
elif used:
dec token.pending
token.lastUse = getTime()
proc getToken*(api: Api): Future[Token] {.async.} =
for i in 0 ..< tokenPool.len:
if result.isReady(api): break
release(result)
result = tokenPool.sample()
if not result.isReady(api):
release(result)
result = await fetchToken()
tokenPool.add result
if not result.isNil:
inc result.pending
else:
raise rateLimitError()
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
# avoid undefined behavior in race conditions
if api in token.apis:
let limit = token.apis[api]
if limit.reset >= reset and limit.remaining < remaining:
return
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
proc poolTokens*(amount: int) {.async.} =
var futs: seq[Future[Token]]
for i in 0 ..< amount:
futs.add fetchToken()
for token in futs:
var newToken: Token
try: newToken = await token
except: discard
if not newToken.isNil:
tokenPool.add newToken
proc initTokenPool*(cfg: Config) {.async.} =
clientPool = HttpPool()
while true:
if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
await sleepAsync(2000)

77
src/types.nim
View File

@@ -1,89 +1,50 @@
# SPDX-License-Identifier: AGPL-3.0-only
import times, sequtils, options, tables, uri
import times, sequtils, options, tables
import prefs_impl
genPrefsType()
type
RateLimitError* = object of CatchableError
NoSessionsError* = object of CatchableError
InternalError* = object of CatchableError
BadClientError* = object of CatchableError
TimelineKind* {.pure.} = enum
tweets, replies, media
Api* {.pure.} = enum
tweetDetail
tweetResult
userShow
timeline
search
tweet
list
listBySlug
listMembers
listTweets
userRestId
userScreenName
userTweets
userTweetsAndReplies
userMedia
status
RateLimit* = object
limit*: int
remaining*: int
reset*: int
SessionKind* = enum
oauth
cookie
Session* = ref object
id*: int64
Token* = ref object
tok*: string
init*: Time
lastUse*: Time
pending*: int
limited*: bool
limitedAt*: int
apis*: Table[Api, RateLimit]
case kind*: SessionKind
of oauth:
oauthToken*: string
oauthSecret*: string
of cookie:
authToken*: string
ct0*: string
SessionAwareUrl* = object
oauthUrl*: Uri
cookieUrl*: Uri
Error* = enum
null = 0
noUserMatches = 17
protectedUser = 22
missingParams = 25
timeout = 29
couldntAuth = 32
doesntExist = 34
unauthorized = 37
invalidParam = 47
userNotFound = 50
suspended = 63
rateLimited = 88
expiredToken = 89
invalidToken = 89
listIdOrSlug = 112
tweetNotFound = 144
tweetNotAuthorized = 179
forbidden = 200
badRequest = 214
badToken = 239
locked = 326
noCsrf = 353
tweetUnavailable = 421
tweetCensored = 422
VerifiedType* = enum
none = "None"
blue = "Blue"
business = "Business"
government = "Government"
User* = object
id*: string
@@ -100,7 +61,7 @@ type
tweets*: int
likes*: int
media*: int
verifiedType*: VerifiedType
verified*: bool
protected*: bool
suspended*: bool
joinDate*: DateTime
@@ -114,7 +75,6 @@ type
contentType*: VideoType
url*: string
bitrate*: int
resolution*: int
Video* = object
durationMs*: int
@@ -184,10 +144,8 @@ type
imageDirectMessage = "image_direct_message"
audiospace = "audiospace"
newsletterPublication = "newsletter_publication"
jobDetails = "job_details"
hidden
unknown
Card* = object
kind*: CardKind
url*: string
@@ -216,8 +174,6 @@ type
available*: bool
tombstone*: string
location*: string
# Unused, needed for backwards compat
source*: string
stats*: TweetStats
retweet*: Option[Tweet]
attribution*: Option[User]
@@ -229,8 +185,6 @@ type
video*: Option[Video]
photos*: seq[string]
Tweets* = seq[Tweet]
Result*[T] = object
content*: seq[T]
top*, bottom*: string
@@ -238,7 +192,7 @@ type
query*: Query
Chain* = object
content*: Tweets
content*: seq[Tweet]
hasMore*: bool
cursor*: string
@@ -248,7 +202,7 @@ type
after*: Chain
replies*: Result[Chain]
Timeline* = Result[Tweets]
Timeline* = Result[Tweet]
Profile* = object
user*: User
@@ -300,6 +254,3 @@ type
proc contains*(thread: Chain; tweet: Tweet): bool =
thread.content.anyIt(it.id == tweet.id)
proc add*(timeline: var seq[Tweets]; tweet: Tweet) =
timeline.add @[tweet]

11
src/utils.nim
View File

@@ -16,8 +16,7 @@ const
"twimg.com",
"abs.twimg.com",
"pbs.twimg.com",
"video.twimg.com",
"x.com"
"video.twimg.com"
]
proc setHmacKey*(key: string) =
@@ -43,12 +42,6 @@ proc getPicUrl*(link: string): string =
else:
&"/pic/{encodeUrl(link)}"
proc getOrigPicUrl*(link: string): string =
if base64Media:
&"/pic/orig/enc/{encode(link, safe=true)}"
else:
&"/pic/orig/{encodeUrl(link)}"
proc filterParams*(params: Table): seq[(string, string)] =
for p in params.pairs():
if p[1].len > 0 and p[0] notin nitterParams:
@@ -58,4 +51,4 @@ proc isTwitterUrl*(uri: Uri): bool =
uri.hostname in twitterDomains
proc isTwitterUrl*(url: string): bool =
isTwitterUrl(parseUri(url))
parseUri(url).hostname in twitterDomains

7
src/views/embed.nim
View File

@@ -11,12 +11,11 @@ const doctype = "<!DOCTYPE html>\n"
proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
let thumb = get(tweet.video).thumb
let vidUrl = getVideoEmbed(cfg, tweet.id)
let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
let prefs = Prefs(hlsPlayback: true)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
body:
tdiv(class="embed-video"):
renderVideo(get(tweet.video), prefs, "")
tdiv(class="embed-video"):
renderVideo(get(tweet.video), prefs, "")
result = doctype & $node

11
src/views/general.nim
View File

@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
link(rel="stylesheet", type="text/css", href="/css/style.css?v=19")
link(rel="stylesheet", type="text/css", href="/css/style.css?v=16")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
if theme.len > 0:
@@ -73,7 +73,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
if prefs.hlsPlayback:
script(src="/js/hls.min.js", `defer`="")
script(src="/js/hls.light.min.js", `defer`="")
script(src="/js/hlsPlayback.js", `defer`="")
if prefs.infiniteScroll:
@@ -93,13 +93,14 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
meta(property="og:site_name", content="Nitter")
meta(property="og:locale", content="en_US")
if banner.len > 0 and not banner.startsWith('#'):
if banner.len > 0:
let bannerUrl = getPicUrl(banner)
link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
for url in images:
let preloadUrl = if "400x400" in url: getPicUrl(url)
else: getSmallPic(url)
let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
else: "?name=small"
let preloadUrl = getPicUrl(url & suffix)
link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
let image = getUrlPrefix(cfg) & getPicUrl(url)

9
src/views/profile.nim
View File

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
span:
let url = replaceUrls(user.website, prefs)
icon "link"
a(href=url): text url.shortLink
a(href=url): text shortLink(url)
tdiv(class="profile-joindate"):
span(title=getJoinDateFull(user)):
@@ -78,11 +78,8 @@ proc renderPhotoRail(profile: Profile): VNode =
tdiv(class="photo-rail-grid"):
for i, photo in profile.photoRail:
if i == 16: break
let photoSuffix =
if "format" in photo.url or "placeholder" in photo.url: ""
else: ":thumb"
a(href=(&"/{profile.user.username}/status/{photo.tweetId}#m")):
genImg(photo.url & photoSuffix)
genImg(photo.url & (if "format" in photo.url: "" else: ":thumb"))
proc renderBanner(banner: string): VNode =
buildHtml():
@@ -108,7 +105,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
renderBanner(profile.user.banner)
let sticky = if prefs.stickyProfile: " sticky" else: ""
tdiv(class=("profile-tab" & sticky)):
tdiv(class=(&"profile-tab{sticky}")):
renderUserCard(profile.user, prefs)
if profile.photoRail.len > 0:
renderPhotoRail(profile)

58
src/views/renderutils.nim
View File

@@ -1,19 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat
import strutils
import karax/[karaxdsl, vdom, vstyles]
import ".."/[types, utils]
const smallWebp* = "?name=small&format=webp"
proc getSmallPic*(url: string): string =
result = url
if "?" notin url and not url.endsWith("placeholder.png"):
result &= smallWebp
result = getPicUrl(result)
proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
var c = "icon-" & icon
if class.len > 0: c = &"{c} {class}"
if class.len > 0: c = c & " " & class
buildHtml(tdiv(class="icon-container")):
if href.len > 0:
a(class=c, title=title, href=href)
@@ -23,13 +15,6 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
if text.len > 0:
text " " & text
template verifiedIcon*(user: User): untyped {.dirty.} =
if user.verifiedType != VerifiedType.none:
let lower = ($user.verifiedType).toLowerAscii()
icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
else:
text ""
proc linkUser*(user: User, class=""): VNode =
let
isName = "username" notin class
@@ -39,11 +24,11 @@ proc linkUser*(user: User, class=""): VNode =
buildHtml(a(href=href, class=class, title=nameText)):
text nameText
if isName:
verifiedIcon(user)
if user.protected:
text " "
icon "lock", title="Protected account"
if isName and user.verified:
icon "ok", class="verified-icon", title="Verified account"
if isName and user.protected:
text " "
icon "lock", title="Protected account"
proc linkText*(text: string; class=""): VNode =
let url = if "http" notin text: https & text else: text
@@ -66,23 +51,29 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
proc genCheckbox*(pref, label: string; state: bool): VNode =
buildHtml(label(class="pref-group checkbox-container")):
text label
input(name=pref, `type`="checkbox", checked=state)
if state: input(name=pref, `type`="checkbox", checked="")
else: input(name=pref, `type`="checkbox")
span(class="checkbox")
proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
let p = placeholder
buildHtml(tdiv(class=("pref-group pref-input " & class))):
if label.len > 0:
label(`for`=pref): text label
input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
if state.len == 0:
input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
else:
input(name=pref, `type`="text", placeholder=p, value=state)
proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
buildHtml(tdiv(class="pref-group pref-input")):
label(`for`=pref): text label
select(name=pref):
for opt in options:
option(value=opt, selected=(opt == state)):
text opt
if opt == state:
option(value=opt, selected=""): text opt
else:
option(value=opt): text opt
proc genDate*(pref, state: string): VNode =
buildHtml(span(class="date-input")):
@@ -91,12 +82,15 @@ proc genDate*(pref, state: string): VNode =
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="", loading="lazy")
img(src=getPicUrl(url), class=class, alt="")
proc getTabClass*(query: Query; tab: QueryKind): string =
if query.kind == tab: "tab-item active"
else: "tab-item"
result = "tab-item"
if query.kind == tab:
result &= " active"
proc getAvatarClass*(prefs: Prefs): string =
if prefs.squareAvatars: "avatar"
else: "avatar round"
if prefs.squareAvatars:
"avatar"
else:
"avatar round"

70
src/views/rss.nimf
View File

@@ -28,24 +28,6 @@
Twitter feed for: ${desc}. Generated by ${cfg.hostname}
#end proc
#
#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
#result = profile.tweets.content
#if profile.pinned.isSome and result.len > 0:
# let pinnedTweet = profile.pinned.get
# var inserted = false
# for threadIdx in 0 ..< result.len:
# if not inserted:
# for tweetIdx in 0 ..< result[threadIdx].len:
# if result[threadIdx][tweetIdx].id < pinnedTweet.id:
# result[threadIdx].insert(pinnedTweet, tweetIdx)
# inserted = true
# end if
# end for
# end if
# end for
#end if
#end proc
#
#proc renderRssTweet(tweet: Tweet; cfg: Config): string =
#let tweet = tweet.retweet.get(tweet)
#let urlPrefix = getUrlPrefix(cfg)
@@ -65,7 +47,7 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
# let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
# let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
<video poster="${thumb}" autoplay muted loop style="max-width:250px;">
<source src="${url}" type="video/mp4"></video>
<source src="${url}" type="video/mp4"</source></video>
#elif tweet.card.isSome:
# let card = tweet.card.get()
# if card.image.len > 0:
@@ -74,29 +56,24 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
#end if
#end proc
#
#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string =
#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string =
#let urlPrefix = getUrlPrefix(cfg)
#var links: seq[string]
#for thread in tweets:
# for tweet in thread:
# if userId.len > 0 and tweet.user.id != userId: continue
# end if
#
# let retweet = if tweet.retweet.isSome: tweet.user.username else: ""
# let tweet = if retweet.len > 0: tweet.retweet.get else: tweet
# let link = getLink(tweet)
# if link in links: continue
# end if
# links.add link
<item>
<title>${getTitle(tweet, retweet)}</title>
<dc:creator>@${tweet.user.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>${urlPrefix & link}</guid>
<link>${urlPrefix & link}</link>
</item>
# end for
#for t in tweets:
# let retweet = if t.retweet.isSome: t.user.username else: ""
# let tweet = if retweet.len > 0: t.retweet.get else: t
# let link = getLink(tweet)
# if link in links: continue
# end if
# links.add link
<item>
<title>${getTitle(tweet, retweet)}</title>
<dc:creator>@${tweet.user.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>${urlPrefix & link}</guid>
<link>${urlPrefix & link}</link>
</item>
#end for
#end proc
#
@@ -124,15 +101,14 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
<width>128</width>
<height>128</height>
</image>
#let tweetsList = getTweetsWithPinned(profile)
#if tweetsList.len > 0:
${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
#if profile.tweets.content.len > 0:
${renderRssTweets(profile.tweets.content, cfg)}
#end if
</channel>
</rss>
#end proc
#
#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string =
#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string =
#let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}"
#result = ""
<?xml version="1.0" encoding="UTF-8"?>
@@ -141,7 +117,7 @@ ${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
<atom:link href="${link}" rel="self" type="application/rss+xml" />
<title>${xmltree.escape(list.name)} / @${list.username}</title>
<link>${link}</link>
<description>${getDescription(&"{list.name} by @{list.username}", cfg)}</description>
<description>${getDescription(list.name & " by @" & list.username, cfg)}</description>
<language>en-us</language>
<ttl>40</ttl>
${renderRssTweets(tweets, cfg)}
@@ -149,7 +125,7 @@ ${renderRssTweets(tweets, cfg)}
</rss>
#end proc
#
#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string =
#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string =
#let link = &"{getUrlPrefix(cfg)}/search"
#let escName = xmltree.escape(name)
#result = ""
@@ -159,7 +135,7 @@ ${renderRssTweets(tweets, cfg)}
<atom:link href="${link}" rel="self" type="application/rss+xml" />
<title>Search results for "${escName}"</title>
<link>${link}</link>
<description>${getDescription(&"Search \"{escName}\"", cfg)}</description>
<description>${getDescription("Search \"" & escName & "\"", cfg)}</description>
<language>en-us</language>
<ttl>40</ttl>
${renderRssTweets(tweets, cfg)}

18
src/views/search.nim
View File

@@ -24,9 +24,9 @@ proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"):
form(`method`="get", action="/search", autocomplete="off"):
hiddenField("f", "tweets")
hiddenField("f", "users")
input(`type`="text", name="q", autofocus="",
placeholder="Search...", dir="auto")
placeholder="Enter username...", dir="auto")
button(`type`="submit"): icon "search"
proc renderProfileTabs*(query: Query; username: string): VNode =
@@ -63,10 +63,12 @@ proc renderSearchPanel*(query: Query): VNode =
hiddenField("f", "tweets")
genInput("q", "", query.text, "Enter search...", class="pref-inline")
button(`type`="submit"): icon "search"
input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
label(`for`="search-panel-toggle"): icon "down"
if isPanelOpen(query):
input(id="search-panel-toggle", `type`="checkbox", checked="")
else:
input(id="search-panel-toggle", `type`="checkbox")
label(`for`="search-panel-toggle"):
icon "down"
tdiv(class="search-panel"):
for f in @["filter", "exclude"]:
span(class="search-title"): text capitalize(f)
@@ -86,9 +88,9 @@ proc renderSearchPanel*(query: Query): VNode =
genDate("until", query.until)
tdiv:
span(class="search-title"): text "Near"
genInput("near", "", query.near, "Location...", autofocus=false)
genInput("near", "", query.near, placeholder="Location...")
proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
let query = results.query
buildHtml(tdiv(class="timeline-container")):

48
src/views/timeline.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, algorithm, uri, options
import strutils, strformat, sequtils, algorithm, uri, options
import karax/[karaxdsl, vdom]
import ".."/[types, query, formatters]
@@ -39,22 +39,24 @@ proc renderNoneFound(): VNode =
h2(class="timeline-none"):
text "No items found"
proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
buildHtml(tdiv(class="thread-line")):
let sortedThread = thread.sortedByIt(it.id)
for i, tweet in sortedThread:
# thread has a gap, display "more replies" link
if i > 0 and tweet.replyId != sortedThread[i - 1].id:
tdiv(class="timeline-item thread more-replies-thread"):
tdiv(class="more-replies"):
a(class="more-replies-text", href=getLink(tweet)):
text "more replies"
let show = i == thread.high and sortedThread[0].id != tweet.threadId
let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: ""
renderTweet(tweet, prefs, path, class=(header & "thread"),
index=i, last=(i == thread.high), showThread=show)
proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] =
result = @[it]
if it.retweet.isSome or it.replyId in threads: return
for t in tweets:
if t.id == result[0].replyId:
result.insert t
elif t.replyId == result[0].id:
result.add t
proc renderUser(user: User; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-item")):
a(class="tweet-link", href=("/" & user.username))
@@ -87,7 +89,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
else:
renderNoMore()
proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
buildHtml(tdiv(class="timeline")):
if not results.beginning:
@@ -103,26 +105,26 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
else:
renderNoneFound()
else:
var retweets: seq[int64]
var
threads: seq[int64]
retweets: seq[int64]
for thread in results.content:
if thread.len == 1:
let
tweet = thread[0]
retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
for tweet in results.content:
let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
if retweetId in retweets or tweet.id in retweets or
tweet.pinned and prefs.hidePins:
continue
if tweet.id in threads or rt in retweets or tweet.id in retweets or
tweet.pinned and prefs.hidePins: continue
let thread = results.content.threadFilter(threads, tweet)
if thread.len < 2:
var hasThread = tweet.hasThread
if retweetId != 0 and tweet.retweet.isSome:
retweets &= retweetId
if rt != 0:
retweets &= rt
hasThread = get(tweet.retweet).hasThread
renderTweet(tweet, prefs, path, showThread=hasThread)
else:
renderThread(thread, prefs, path)
threads &= thread.mapIt(it.id)
if results.bottom.len > 0:
renderMore(results.query, results.bottom)
renderMore(results.query, results.bottom)
renderToTop()

114
src/views/tweet.nim
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, sequtils, strformat, options, algorithm
import strutils, sequtils, strformat, options
import karax/[karaxdsl, vdom, vstyles]
from jester import Request
@@ -7,20 +7,27 @@ import renderutils
import ".."/[types, utils, formatters]
import general
const doctype = "<!DOCTYPE html>\n"
proc getSmallPic(url: string): string =
result = url
if "?" notin url and not url.endsWith("placeholder.png"):
result &= ":small"
result = getPicUrl(result)
proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
let url = getPicUrl(user.getUserPic("_mini"))
buildHtml():
img(class=(prefs.getAvatarClass & " mini"), src=url)
proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
buildHtml(tdiv):
if pinned:
tdiv(class="pinned"):
span: icon "pin", "Pinned Tweet"
elif retweet.len > 0:
if retweet.len > 0:
tdiv(class="retweet-header"):
span: icon "retweet", retweet & " retweeted"
if tweet.pinned:
tdiv(class="pinned"):
span: icon "pin", "Pinned Tweet"
tdiv(class="tweet-header"):
a(class="tweet-avatar", href=("/" & tweet.user.username)):
var size = "_bigger"
@@ -50,21 +57,19 @@ proc renderAlbum(tweet: Tweet): VNode =
tdiv(class="attachment image"):
let
named = "name=" in photo
small = if named: photo else: photo & smallWebp
a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
orig = if named: photo else: photo & "?name=orig"
small = if named: photo else: photo & "?name=small"
a(href=getPicUrl(orig), class="still-image", target="_blank"):
genImg(small)
proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
case playbackType
proc isPlaybackEnabled(prefs: Prefs; video: Video): bool =
case video.playbackType
of mp4: prefs.mp4Playback
of m3u8, vmap: prefs.hlsPlayback
proc hasMp4Url(video: Video): bool =
video.variants.anyIt(it.contentType == mp4)
proc renderVideoDisabled(playbackType: VideoType; path: string): VNode =
proc renderVideoDisabled(video: Video; path: string): VNode =
buildHtml(tdiv(class="video-overlay")):
case playbackType
case video.playbackType
of mp4:
p: text "mp4 playback disabled in preferences"
of m3u8, vmap:
@@ -79,34 +84,33 @@ proc renderVideoUnavailable(video: Video): VNode =
p: text "This media is unavailable"
proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
let
container = if video.description.len == 0 and video.title.len == 0: ""
else: " card-container"
playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
else: video.playbackType
let container =
if video.description.len > 0 or video.title.len > 0: " card-container"
else: ""
buildHtml(tdiv(class="attachments card")):
tdiv(class="gallery-video" & container):
tdiv(class="attachment video-container"):
let thumb = getSmallPic(video.thumb)
if not video.available:
img(src=thumb, loading="lazy")
img(src=thumb)
renderVideoUnavailable(video)
elif not prefs.isPlaybackEnabled(playbackType):
img(src=thumb, loading="lazy")
renderVideoDisabled(playbackType, path)
elif not prefs.isPlaybackEnabled(video):
img(src=thumb)
renderVideoDisabled(video, path)
else:
let
vars = video.variants.filterIt(it.contentType == playbackType)
vidUrl = vars.sortedByIt(it.resolution)[^1].url
source = if prefs.proxyVideos: getVidUrl(vidUrl)
else: vidUrl
case playbackType
let vid = video.variants.filterIt(it.contentType == video.playbackType)
let source = getVidUrl(vid[0].url)
case video.playbackType
of mp4:
video(poster=thumb, controls="", muted=prefs.muteVideos):
source(src=source, `type`="video/mp4")
if prefs.muteVideos:
video(poster=thumb, controls="", muted=""):
source(src=source, `type`="video/mp4")
else:
video(poster=thumb, controls=""):
source(src=source, `type`="video/mp4")
of m3u8, vmap:
video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
video(poster=thumb, data-url=source, data-autoload="false")
verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
tdiv(class="overlay-circle"): span(class="overlay-triangle")
verbatim "</div>"
@@ -120,9 +124,14 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
buildHtml(tdiv(class="attachments media-gif")):
tdiv(class="gallery-gif", style={maxHeight: "unset"}):
tdiv(class="attachment"):
video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
controls="", muted="", loop=""):
source(src=getPicUrl(gif.url), `type`="video/mp4")
let thumb = getSmallPic(gif.thumb)
let url = getPicUrl(gif.url)
if prefs.autoplayGifs:
video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
source(src=url, `type`="video/mp4")
else:
video(class="gif", poster=thumb, controls="", muted="", loop=""):
source(src=url, `type`="video/mp4")
proc renderPoll(poll: Poll): VNode =
buildHtml(tdiv(class="poll")):
@@ -137,12 +146,12 @@ proc renderPoll(poll: Poll): VNode =
span(class="poll-choice-value"): text percStr
span(class="poll-choice-option"): text poll.options[i]
span(class="poll-info"):
text &"{insertSep($poll.votes, ',')} votes • {poll.status}"
text insertSep($poll.votes, ',') & " votes • " & poll.status
proc renderCardImage(card: Card): VNode =
buildHtml(tdiv(class="card-image-container")):
tdiv(class="card-image"):
genImg(card.image)
img(src=getPicUrl(card.image), alt="")
if card.kind == player:
tdiv(class="card-overlay"):
tdiv(class="overlay-circle"):
@@ -198,7 +207,8 @@ proc renderAttribution(user: User; prefs: Prefs): VNode =
buildHtml(a(class="attribution", href=("/" & user.username))):
renderMiniAvatar(user, prefs)
strong: text user.fullname
verifiedIcon(user)
if user.verified:
icon "ok", class="verified-icon", title="Verified account"
proc renderMediaTags(tags: seq[User]): VNode =
buildHtml(tdiv(class="media-tag-block")):
@@ -286,10 +296,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if tweet.quote.isSome:
renderQuote(tweet.quote.get(), prefs, path)
let
fullTweet = tweet
pinned = tweet.pinned
let fullTweet = tweet
var retweet: string
var tweet = fullTweet
if tweet.retweet.isSome:
@@ -302,7 +309,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
tdiv(class="tweet-body"):
var views = ""
renderHeader(tweet, retweet, pinned, prefs)
renderHeader(tweet, retweet, prefs)
if not afterTweet and index == 0 and tweet.reply.len > 0 and
(tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
@@ -318,7 +325,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if tweet.attribution.isSome:
renderAttribution(tweet.attribution.get(), prefs)
if tweet.card.isSome and tweet.card.get().kind != hidden:
if tweet.card.isSome:
renderCard(tweet.card.get(), prefs, path)
if tweet.photos.len > 0:
@@ -337,7 +344,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderQuote(tweet.quote.get(), prefs, path)
if mainTweet:
p(class="tweet-published"): text &"{getTime(tweet)}"
p(class="tweet-published"): text getTime(tweet)
if tweet.mediaTags.len > 0:
renderMediaTags(tweet.mediaTags)
@@ -349,12 +356,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
text "Show this thread"
proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
let node = buildHtml(html(lang="en")):
proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
buildHtml(tdiv(class="tweet-embed")):
renderHead(prefs, cfg, req)
body:
tdiv(class="tweet-embed"):
renderTweet(tweet, prefs, path, mainTweet=true)
result = doctype & $node
renderTweet(tweet, prefs, path, mainTweet=true)

1
tests/requirements.txt
View File

@@ -1 +0,0 @@
seleniumbase

69
tests/test_card.py
View File

@@ -3,6 +3,11 @@ from parameterized import parameterized
card = [
['Thom_Wolf/status/1122466524860702729',
'pytorch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - pytorch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
'github.com', True],
['nim_lang/status/1136652293510717440',
'Version 0.20.0 released',
'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@@ -13,32 +18,37 @@ card = [
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True],
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.',
'nim-lang.org', True]
['FluentAI/status/1116417904831029248',
'Amazon’s Alexa isn’t just AI — thousands of humans are listening',
'One of the only ways to improve Alexa is to have human beings check it for errors',
'theverge.com', True]
]
no_thumb = [
['FluentAI/status/1116417904831029248',
'LinkedIn',
'This link will take you to a page that’s not on LinkedIn',
'lnkd.in'],
['Thom_Wolf/status/1122466524860702729',
'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
'',
'github.com'],
['Bountysource/status/1141879700639215617',
'Post a bounty on kivy/plyer!',
'Automation and Screen Reader Support',
'bountysource.com'],
['brent_p/status/1088857328680488961',
'GitHub - brentp/hts-nim: nim wrapper for htslib for parsing genomics data files',
'',
'github.com'],
'Hts Nim Sugar',
'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
'brentp.github.io'],
['voidtarget/status/1133028231672582145',
'sinkingsugar/nimqt-example',
'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
'github.com']
'github.com'],
['mobile_test/status/490378953744318464',
'Nantasket Beach',
'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px',
'500px.com'],
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'Posted in r/programming by u/miran1',
'reddit.com']
]
playable = [
@@ -53,6 +63,17 @@ playable = [
'youtube.com']
]
# promo = [
# ['BangOlufsen/status/1145698701517754368',
# 'Upgrade your journey', '',
# 'www.bang-olufsen.com'],
# ['BangOlufsen/status/1154934429900406784',
# 'Learn more about Beosound Shape', '',
# 'www.bang-olufsen.com']
# ]
class CardTest(BaseTestCase):
@parameterized.expand(card)
def test_card(self, tweet, title, description, destination, large):
@@ -60,7 +81,7 @@ class CardTest(BaseTestCase):
c = Card(Conversation.main + " ")
self.assert_text(title, c.title)
self.assert_text(destination, c.destination)
self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
self.assertIn('_img', self.get_image_url(c.image + ' img'))
if len(description) > 0:
self.assert_text(description, c.description)
if large:
@@ -83,7 +104,17 @@ class CardTest(BaseTestCase):
c = Card(Conversation.main + " ")
self.assert_text(title, c.title)
self.assert_text(destination, c.destination)
self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
self.assertIn('_img', self.get_image_url(c.image + ' img'))
self.assert_element_visible('.card-overlay')
if len(description) > 0:
self.assert_text(description, c.description)
# @parameterized.expand(promo)
# def test_card_promo(self, tweet, title, description, destination):
# self.open_nitter(tweet)
# c = Card(Conversation.main + " ")
# self.assert_text(title, c.title)
# self.assert_text(destination, c.destination)
# self.assert_element_visible('.video-overlay')
# if len(description) > 0:
# self.assert_text(description, c.description)

17
tests/test_profile.py
View File

@@ -4,7 +4,7 @@ from parameterized import parameterized
profiles = [
['mobile_test', 'Test account',
'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '97'],
'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'],
['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
]
@@ -17,6 +17,11 @@ protected = [
invalid = [['thisprofiledoesntexist'], ['%']]
banner_color = [
['nim_lang', '22, 25, 32'],
['rustlang', '35, 31, 32']
]
banner_image = [
['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
]
@@ -66,8 +71,14 @@ class ProfileTest(BaseTestCase):
self.assert_text(f'User "{username}" not found')
def test_suspended(self):
self.open_nitter('suspendme')
self.assert_text('User "suspendme" has been suspended')
self.open_nitter('user')
self.assert_text('User "user" has been suspended')
@parameterized.expand(banner_color)
def test_banner_color(self, username, color):
self.open_nitter(username)
banner = self.find_element(Profile.banner + ' a')
self.assertIn(color, banner.value_of_css_property('background-color'))
@parameterized.expand(banner_image)
def test_banner_image(self, username, url):

12
tests/test_quote.py
View File

@@ -2,13 +2,15 @@ from base import BaseTestCase, Quote, Conversation
from parameterized import parameterized
text = [
['nim_lang/status/1491461266849808397#m',
'Nim', '@nim_lang',
"""What's better than Nim 1.6.0?
['elonmusk/status/1138136540096319488',
'Tesla Owners Online', '@Model3Owners',
"""As of March 58.4% of new car sales in Norway are electric.
Nim 1.6.2 :)
What are we doing wrong? reuters.com/article/us-norwa…"""],
nim-lang.org/blog/2021/12/17…"""]
['nim_lang/status/924694255364341760',
'Hacker News', '@newsycombinator',
'Why Rust fails hard at scientific computing andre-ratsimbazafy.com/why-r…']
]
image = [

10
tests/test_search.py
View File

@@ -2,8 +2,8 @@ from base import BaseTestCase
from parameterized import parameterized
#class SearchTest(BaseTestCase):
#@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
#def test_username_search(self, username):
#self.search_username(username)
#self.assert_text(f'{username}')
class SearchTest(BaseTestCase):
@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
def test_username_search(self, username):
self.search_username(username)
self.assert_text(f'{username}')

29
tests/test_timeline.py
View File

@@ -1,18 +1,23 @@
from base import BaseTestCase, Timeline
from parameterized import parameterized
normal = [['jack'], ['elonmusk']]
normal = [['mobile_test'], ['mobile_test_2']]
after = [['jack', '1681686036294803456'],
['elonmusk', '1681686036294803456']]
after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
empty = [['emptyuser'], ['mobile_test_10']]
protected = [['mobile_test_7'], ['Empty_user']]
photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]]
photo_rail = [['mobile_test', [
'BzUnaDFCUAAmrjs', 'Bo0nDsYIYAIjqVn', 'Bos--KNIQAAA7Li', 'Boq1sDJIYAAxaoi',
'BonISmPIEAAhP3G', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG',
'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKovdrCUAAEz79',
'BkKoe_oCIAASAqr', 'BkKoRLNCAAAYfDf', 'BkKndxoCQAE1vFt', 'BPEmIbYCMAE44dl'
]]]
class TweetTest(BaseTestCase):
@@ -55,10 +60,10 @@ class TweetTest(BaseTestCase):
self.assert_element_absent(Timeline.older)
self.assert_element_absent(Timeline.end)
#@parameterized.expand(photo_rail)
#def test_photo_rail(self, username, images):
#self.open_nitter(username)
#self.assert_element_visible(Timeline.photo_rail)
#for i, url in enumerate(images):
#img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
#self.assertIn(url, img)
@parameterized.expand(photo_rail)
def test_photo_rail(self, username, images):
self.open_nitter(username)
self.assert_element_visible(Timeline.photo_rail)
for i, url in enumerate(images):
img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
self.assertIn(url, img)

41
tests/test_tweet.py
View File

@@ -1,4 +1,4 @@
from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet
from base import BaseTestCase, Tweet, get_timeline_tweet
from parameterized import parameterized
# image = tweet + 'div.attachments.media-body > div > div > a > div > img'
@@ -16,7 +16,7 @@ timeline = [
]
status = [
[20, 'jack', 'jack', '21 Mar 2006', 'just setting up my twttr'],
[20, 'jack⚡️', 'jack', '21 Mar 2006', 'just setting up my twttr'],
[134849778302464000, 'The Twoffice', 'TheTwoffice', '11 Nov 2011', 'test'],
[105685475985080322, 'The Twoffice', 'TheTwoffice', '22 Aug 2011', 'regular tweet'],
[572593440719912960, 'Test account', 'mobile_test', '3 Mar 2015', 'testing test']
@@ -28,15 +28,14 @@ invalid = [
]
multiline = [
[1718660434457239868, 'WebDesignMuseum',
[400897186990284800, 'mobile_test_3',
"""
Happy 32nd Birthday HTML tags!
On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags.
The document contained a description of the first 18 HTML tags: <title>, <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language.
#WebDesignHistory"""]
♔
KEEP
CALM
AND
CLICHÉ
ON"""]
]
link = [
@@ -72,7 +71,11 @@ emoji = [
retweet = [
[7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
[3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
[3, 'mobile_test_8', 'mobile test 8', 'jack⚡️', '@jack', 'twttr']
]
reply = [
['mobile_test/with_replies', 15]
]
@@ -100,18 +103,18 @@ class TweetTest(BaseTestCase):
@parameterized.expand(multiline)
def test_multiline_formatting(self, tid, username, text):
self.open_nitter(f'{username}/status/{tid}')
self.assert_text(text.strip('\n'), Conversation.main)
self.assert_text(text.strip('\n'), '.main-tweet')
@parameterized.expand(emoji)
def test_emoji(self, tweet, text):
self.open_nitter(tweet)
self.assert_text(text, Conversation.main)
self.assert_text(text, '.main-tweet')
@parameterized.expand(link)
def test_link(self, tweet, links):
self.open_nitter(tweet)
for link in links:
self.assert_text(link, Conversation.main)
self.assert_text(link, '.main-tweet')
@parameterized.expand(username)
def test_username(self, tweet, usernames):
@@ -134,8 +137,8 @@ class TweetTest(BaseTestCase):
self.open_nitter(tweet)
self.assert_text('Tweet not found', '.error-panel')
#@parameterized.expand(reply)
#def test_thread(self, tweet, num):
#self.open_nitter(tweet)
#thread = self.find_element(f'.timeline > div:nth-child({num})')
#self.assertIn(thread.get_attribute('class'), 'thread-line')
@parameterized.expand(reply)
def test_thread(self, tweet, num):
self.open_nitter(tweet)
thread = self.find_element(f'.timeline > div:nth-child({num})')
self.assertIn(thread.get_attribute('class'), 'thread-line')

16
tests/test_tweet_media.py
View File

@@ -14,7 +14,7 @@ poll = [
image = [
['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
#['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
]
gif = [
@@ -28,14 +28,14 @@ video_m3u8 = [
]
gallery = [
# ['mobile_test/status/451108446603980803', [
# ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
# ]],
['mobile_test/status/451108446603980803', [
['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
]],
# ['mobile_test/status/471539824713691137', [
# ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
# ['Bos--IqIQAAav23']
# ]],
['mobile_test/status/471539824713691137', [
['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
['Bos--IqIQAAav23']
]],
['mobile_test/status/469530783384743936', [
['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],

162
tools/get_session.py
View File

@@ -1,162 +0,0 @@
#!/usr/bin/env python3
import requests
import json
import sys
import pyotp
import cloudscraper
# NOTE: pyotp, requests and cloudscraper are dependencies
# > pip install pyotp requests cloudscraper
TW_CONSUMER_KEY = '3nVuSoBZnx6U4vzUxf5w'
TW_CONSUMER_SECRET = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'
def auth(username, password, otp_secret):
bearer_token_req = requests.post("https://api.twitter.com/oauth2/token",
auth=(TW_CONSUMER_KEY, TW_CONSUMER_SECRET),
headers={"Content-Type": "application/x-www-form-urlencoded"},
data='grant_type=client_credentials'
).json()
bearer_token = ' '.join(str(x) for x in bearer_token_req.values())
guest_token = requests.post(
"https://api.twitter.com/1.1/guest/activate.json",
headers={'Authorization': bearer_token}
).json().get('guest_token')
if not guest_token:
print("Failed to obtain guest token.")
sys.exit(1)
twitter_header = {
'Authorization': bearer_token,
"Content-Type": "application/json",
"User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)",
"X-Twitter-API-Version": '5',
"X-Twitter-Client": "TwitterAndroid",
"X-Twitter-Client-Version": "10.21.0-release.0",
"OS-Version": "28",
"System-User-Agent": "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)",
"X-Twitter-Active-User": "yes",
"X-Guest-Token": guest_token,
"X-Twitter-Client-DeviceID": ""
}
scraper = cloudscraper.create_scraper()
scraper.headers = twitter_header
task1 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
params={
'flow_name': 'login',
'api_version': '1',
'known_device_token': '',
'sim_country_code': 'us'
},
json={
"flow_token": None,
"input_flow_data": {
"country_code": None,
"flow_context": {
"referrer_context": {
"referral_details": "utm_source=google-play&utm_medium=organic",
"referrer_url": ""
},
"start_location": {
"location": "deeplink"
}
},
"requested_variant": None,
"target_user_id": 0
}
}
)
scraper.headers['att'] = task1.headers.get('att')
task2 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
json={
"flow_token": task1.json().get('flow_token'),
"subtask_inputs": [{
"enter_text": {
"suggestion_id": None,
"text": username,
"link": "next_link"
},
"subtask_id": "LoginEnterUserIdentifier"
}]
}
)
task3 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
json={
"flow_token": task2.json().get('flow_token'),
"subtask_inputs": [{
"enter_password": {
"password": password,
"link": "next_link"
},
"subtask_id": "LoginEnterPassword"
}],
}
)
for t3_subtask in task3.json().get('subtasks', []):
if "open_account" in t3_subtask:
return t3_subtask["open_account"]
elif "enter_text" in t3_subtask:
response_text = t3_subtask["enter_text"]["hint_text"]
totp = pyotp.TOTP(otp_secret)
generated_code = totp.now()
task4resp = scraper.post(
"https://api.twitter.com/1.1/onboarding/task.json",
json={
"flow_token": task3.json().get("flow_token"),
"subtask_inputs": [
{
"enter_text": {
"suggestion_id": None,
"text": generated_code,
"link": "next_link",
},
"subtask_id": "LoginTwoFactorAuthChallenge",
}
],
}
)
task4 = task4resp.json()
for t4_subtask in task4.get("subtasks", []):
if "open_account" in t4_subtask:
return t4_subtask["open_account"]
return None
if __name__ == "__main__":
if len(sys.argv) != 5:
print("Usage: python3 get_session.py <username> <password> <2fa secret> <path>")
sys.exit(1)
username = sys.argv[1]
password = sys.argv[2]
otp_secret = sys.argv[3]
path = sys.argv[4]
result = auth(username, password, otp_secret)
if result is None:
print("Authentication failed.")
sys.exit(1)
session_entry = {
"oauth_token": result.get("oauth_token"),
"oauth_token_secret": result.get("oauth_token_secret")
}
try:
with open(path, "a") as f:
f.write(json.dumps(session_entry) + "\n")
print("Authentication successful. Session appended to", path)
except Exception as e:
print(f"Failed to write session information: {e}")
sys.exit(1)

158
tools/get_web_session.py
View File

@@ -1,158 +0,0 @@
#!/usr/bin/env python3
"""
Authenticates with X.com/Twitter and extracts session cookies for use with Nitter.
Handles 2FA, extracts user info, and outputs clean JSON for sessions.jsonl.
Requirements:
pip install -r tools/requirements.txt
Usage:
python3 tools/get_web_session.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
Examples:
# Output to terminal
python3 tools/get_web_session.py myusername mypassword TOTP_BASE32_SECRET
# Append to sessions.jsonl
python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --append sessions.jsonl
# Headless mode (may increase detection risk)
python3 tools/get_web_session.py myusername mypassword TOTP_SECRET --headless
Output:
{"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
"""
import sys
import json
import asyncio
import pyotp
import nodriver as uc
import os
async def login_and_get_cookies(username, password, totp_seed=None, headless=False):
"""Authenticate with X.com and extract session cookies"""
# Note: headless mode may increase detection risk from bot-detection systems
browser = await uc.start(headless=headless)
tab = await browser.get('https://x.com/i/flow/login')
try:
# Enter username
print('[*] Entering username...', file=sys.stderr)
username_input = await tab.find('input[autocomplete="username"]', timeout=10)
await username_input.send_keys(username + '\n')
await asyncio.sleep(1)
# Enter password
print('[*] Entering password...', file=sys.stderr)
password_input = await tab.find('input[autocomplete="current-password"]', timeout=15)
await password_input.send_keys(password + '\n')
await asyncio.sleep(2)
# Handle 2FA if needed
page_content = await tab.get_content()
if 'verification code' in page_content or 'Enter code' in page_content:
if not totp_seed:
raise Exception('2FA required but no TOTP seed provided')
print('[*] 2FA detected, entering code...', file=sys.stderr)
totp_code = pyotp.TOTP(totp_seed).now()
code_input = await tab.select('input[type="text"]')
await code_input.send_keys(totp_code + '\n')
await asyncio.sleep(3)
# Get cookies
print('[*] Retrieving cookies...', file=sys.stderr)
for _ in range(20): # 20 second timeout
cookies = await browser.cookies.get_all()
cookies_dict = {cookie.name: cookie.value for cookie in cookies}
if 'auth_token' in cookies_dict and 'ct0' in cookies_dict:
print('[*] Found both cookies', file=sys.stderr)
# Extract ID from twid cookie (may be URL-encoded)
user_id = None
if 'twid' in cookies_dict:
twid = cookies_dict['twid']
# Try to extract the ID from twid (format: u%3D<id> or u=<id>)
if 'u%3D' in twid:
user_id = twid.split('u%3D')[1].split('&')[0].strip('"')
elif 'u=' in twid:
user_id = twid.split('u=')[1].split('&')[0].strip('"')
cookies_dict['username'] = username
if user_id:
cookies_dict['id'] = user_id
return cookies_dict
await asyncio.sleep(1)
raise Exception('Timeout waiting for cookies')
finally:
browser.stop()
async def main():
if len(sys.argv) < 3:
print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]')
sys.exit(1)
username = sys.argv[1]
password = sys.argv[2]
totp_seed = None
append_file = None
headless = False
# Parse optional arguments
i = 3
while i < len(sys.argv):
arg = sys.argv[i]
if arg == '--append':
if i + 1 < len(sys.argv):
append_file = sys.argv[i + 1]
i += 2 # Skip '--append' and filename
else:
print('[!] Error: --append requires a filename', file=sys.stderr)
sys.exit(1)
elif arg == '--headless':
headless = True
i += 1
elif not arg.startswith('--'):
if totp_seed is None:
totp_seed = arg
i += 1
else:
# Unkown args
print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
i += 1
try:
cookies = await login_and_get_cookies(username, password, totp_seed, headless)
session = {
'kind': 'cookie',
'username': cookies['username'],
'id': cookies.get('id'),
'auth_token': cookies['auth_token'],
'ct0': cookies['ct0']
}
output = json.dumps(session)
if append_file:
with open(append_file, 'a') as f:
f.write(output + '\n')
print(f'✓ Session appended to {append_file}', file=sys.stderr)
else:
print(output)
os._exit(0)
except Exception as error:
print(f'[!] Error: {error}', file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
asyncio.run(main())

2
tools/requirements.txt
View File

@@ -1,2 +0,0 @@
nodriver>=0.48.0
pyotp