1
0
mirror of https://github.com/zedeus/nitter.git synced 2025-12-06 03:55:36 -05:00

8 Commits

Author SHA1 Message Date
Zed
401aa26464 Improve proxied mp4 caching 2023-05-21 00:47:09 +02:00
Zed
93208908e6 Merge branch 'master' into feature/mp4-streaming 2023-05-20 22:23:12 +02:00
Zed
6e490c2dd9 Improve gif html 2022-06-11 23:27:11 +02:00
Zed
00daab1f15 Disable mp4 preloading 2022-06-11 20:18:45 +02:00
Zed
608c3ca8df Lazy load images 2022-06-11 17:41:59 +02:00
Zed
0610f7b890 Fix image route order 2022-06-08 22:20:28 +02:00
Zed
1dab9c9c61 Update mp4Playback preference text 2022-06-06 22:56:48 +02:00
Zed
651941acd1 Implement experimental mp4 streaming 2022-06-06 22:50:28 +02:00
81 changed files with 1294 additions and 2541 deletions

View File

@@ -10,7 +10,6 @@ on:
jobs:
tests:
uses: ./.github/workflows/run-tests.yml
secrets: inherit
build-docker-amd64:
needs: [tests]
runs-on: buildjet-2vcpu-ubuntu-2204

View File

@@ -8,101 +8,38 @@ on:
- master
workflow_call:
# Ensure that multiple runs on the same branch do not overlap.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
build-test:
name: Build and test
runs-on: buildjet-2vcpu-ubuntu-2204
strategy:
matrix:
nim: ["2.0.x", "2.2.x", "devel"]
test:
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Cache Nimble Dependencies
- name: Cache nimble
id: cache-nimble
uses: buildjet/cache@v4
uses: actions/cache@v3
with:
path: ~/.nimble
key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
restore-keys: |
${{ matrix.nim }}-nimble-v2-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
with:
nim-version: ${{ matrix.nim }}
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Project
run: nimble build -d:release -Y
integration-test:
needs: [build-test]
name: Integration test
runs-on: buildjet-2vcpu-ubuntu-2204
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Cache Nimble Dependencies
id: cache-nimble
uses: buildjet/cache@v4
with:
path: ~/.nimble
key: devel-nimble-v2-${{ hashFiles('*.nimble') }}
restore-keys: |
devel-nimble-v2-
- name: Setup Python (3.10) with pip cache
uses: buildjet/setup-python@v4
key: nimble-${{ hashFiles('*.nimble') }}
restore-keys: "nimble-"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: pip
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
cache: "pip"
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: devel
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build Project
run: nimble build -d:release -Y
- name: Install SeleniumBase and Chromedriver
nim-version: "1.x"
- run: nimble build -d:release -Y
- run: pip install seleniumbase
- run: seleniumbase install chromedriver
- uses: supercharge/redis-github-action@1.5.0
- name: Prepare Nitter
run: |
pip install seleniumbase
seleniumbase install chromedriver
- name: Start Redis Service
uses: supercharge/redis-github-action@1.5.0
- name: Prepare Nitter Environment
run: |
sudo apt-get update && sudo apt-get install -y libsass-dev
sudo apt install libsass-dev -y
cp nitter.example.conf nitter.conf
sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
nimble md
nimble scss
echo '${{ secrets.SESSIONS }}' | head -n1
echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
- name: Run Tests
- name: Run tests
run: |
./nitter &
pytest -n1 tests
pytest -n4 tests

2
.gitignore vendored
View File

@@ -10,6 +10,4 @@ nitter
/public/css/style.css
/public/md/*.html
nitter.conf
guest_accounts.json*
sessions.json*
dump.rdb

View File

@@ -1,4 +1,4 @@
FROM nimlang/nim:2.2.0-alpine-regular as nim
FROM nimlang/nim:1.6.10-alpine-regular as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre
@@ -9,7 +9,7 @@ COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
RUN nimble build -d:danger -d:lto -d:strip \
&& nimble scss \
&& nimble md

View File

@@ -1,7 +1,7 @@
FROM alpine:3.20.6 as nim
FROM alpine:3.17 as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
WORKDIR /src/nitter
@@ -9,17 +9,15 @@ COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
RUN nimble build -d:danger -d:lto -d:strip \
&& nimble scss \
&& nimble md
FROM alpine:3.20.6
FROM alpine:3.17
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates openssl
RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

View File

@@ -4,35 +4,27 @@
[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
[![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
> [!NOTE]
> Running a Nitter instance now requires real accounts, since Twitter removed the previous methods. \
> This does not affect users. \
> For instructions on how to obtain session tokens, see [Creating session tokens](https://github.com/zedeus/nitter/wiki/Creating-session-tokens).
A free and open source alternative Twitter front-end focused on privacy and
performance. \
Inspired by the [Invidious](https://github.com/iv-org/invidious) project.
Inspired by the [Invidious](https://github.com/iv-org/invidious)
project.
- No JavaScript or ads
- All requests go through the backend, client never talks to Twitter
- Prevents Twitter from tracking your IP or JavaScript fingerprint
- Uses Twitter's unofficial API (no developer account required)
- Uses Twitter's unofficial API (no rate limits or developer account required)
- Lightweight (for [@nim_lang](https://nitter.net/nim_lang), 60KB vs 784KB from twitter.com)
- RSS feeds
- Themes
- Mobile support (responsive design)
- AGPLv3 licensed, no proprietary instances permitted
<details>
<summary>Donations</summary>
Liberapay: https://liberapay.com/zedeus<br>
Patreon: https://patreon.com/nitter<br>
BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55<br>
ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460<br>
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL<br>
SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW<br>
ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
</details>
Liberapay: https://liberapay.com/zedeus \
Patreon: https://patreon.com/nitter \
BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
## Roadmap
@@ -50,13 +42,12 @@ maintained by the community.
## Why?
It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
need to sign up. For privacy-minded folks, preventing JavaScript analytics and
IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
you can get accurately tracked with your [browser's
fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
JavaScript required](https://noscriptfingerprint.com/). This all became
It's impossible to use Twitter without JavaScript enabled. For privacy-minded
folks, preventing JavaScript analytics and IP-based tracking is important, but
apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
a VPN and using heavy-duty adblockers, you can get accurately tracked with your
[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
[no JavaScript required](https://noscriptfingerprint.com/). This all became
particularly important after Twitter [removed the
ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
for users to control whether their data gets sent to advertisers.
@@ -80,21 +71,19 @@ Twitter account.
- libpcre
- libsass
- redis/valkey
- redis
To compile Nitter you need a Nim installation, see
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible
to install it system-wide or in the user directory you create below.
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
install it system-wide or in the user directory you create below.
To compile the scss files, you need to install `libsass`. On Ubuntu and Debian,
you can use `libsass-dev`.
Redis is required for caching and in the future for account info. As of 2024
Redis is no longer open source, so using the fork Valkey is recommended. It
should be available on most distros as `redis` or `redis-server`
(Ubuntu/Debian), or `valkey`/`valkey-server`. Running it with the default
config is fine, Nitter's default config is set to use the default port and
localhost.
Redis is required for caching and in the future for account info. It should be
available on most distros as `redis` or `redis-server` (Ubuntu/Debian).
Running it with the default config is fine, Nitter's default config is set to
use the default Redis port and localhost.
Here's how to create a `nitter` user, clone the repo, and build the project
along with the scss and md files.
@@ -104,7 +93,7 @@ along with the scss and md files.
# su nitter
$ git clone https://github.com/zedeus/nitter
$ cd nitter
$ nimble build -d:danger --mm:refc
$ nimble build -d:release
$ nimble scss
$ nimble md
$ cp nitter.example.conf nitter.conf

View File

@@ -7,7 +7,12 @@
# disable annoying warnings
warning("GcUnsafe2", off)
warning("HoleEnumConv", off)
hint("XDeclaredButNotUsed", off)
hint("XCannotRaiseY", off)
hint("User", off)
const
nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch)
when nimVersion >= (1, 6, 0):
warning("HoleEnumConv", off)

View File

@@ -9,7 +9,6 @@ services:
- "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
volumes:
- ./nitter.conf:/src/nitter.conf:Z,ro
- ./sessions.jsonl:/src/sessions.jsonl:Z,ro # Run get_sessions.py to get the credentials
depends_on:
- nitter-redis
restart: unless-stopped

View File

@@ -23,9 +23,15 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.sessions)
enableDebug = false # enable request logs and debug endpoints (/.tokens)
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
# the limits reset every 15 minutes, and the pool is filled up so there's
# always at least `tokenCount` usable tokens. only increase this if you receive
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]

View File

@@ -10,11 +10,11 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 2.0.0"
requires "nim >= 1.4.8"
requires "jester#baca3f"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#a079df9"
requires "nimcrypto#4014ef9"
requires "markdown#158efe3"
requires "packedjson#9e6fbb6"
requires "supersnappy#6c94198"
@@ -22,8 +22,8 @@ requires "redpool#8b7c1db"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#ca5989a"
requires "flatty#e668085"
requires "jsony#1de1f08"
requires "oauth#b8c163b"
requires "jsony#ea811be"
# Tasks

View File

@@ -1,15 +1,16 @@
@font-face {
font-family: 'fontello';
src: url('/fonts/fontello.eot?61663884');
src: url('/fonts/fontello.eot?61663884#iefix') format('embedded-opentype'),
url('/fonts/fontello.woff2?61663884') format('woff2'),
url('/fonts/fontello.woff?61663884') format('woff'),
url('/fonts/fontello.ttf?61663884') format('truetype'),
url('/fonts/fontello.svg?61663884#fontello') format('svg');
src: url('/fonts/fontello.eot?21002321');
src: url('/fonts/fontello.eot?21002321#iefix') format('embedded-opentype'),
url('/fonts/fontello.woff2?21002321') format('woff2'),
url('/fonts/fontello.woff?21002321') format('woff'),
url('/fonts/fontello.ttf?21002321') format('truetype'),
url('/fonts/fontello.svg?21002321#fontello') format('svg');
font-weight: normal;
font-style: normal;
}
[class^="icon-"]:before, [class*=" icon-"]:before {
[class^="icon-"]:before, [class*=" icon-"]:before {
font-family: "fontello";
font-style: normal;
font-weight: normal;
@@ -31,23 +32,22 @@
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
.icon-views:before { content: '\e800'; } /* '' */
.icon-heart:before { content: '\e801'; } /* '' */
.icon-quote:before { content: '\e802'; } /* '' */
.icon-comment:before { content: '\e803'; } /* '' */
.icon-ok:before { content: '\e804'; } /* '' */
.icon-play:before { content: '\e805'; } /* '' */
.icon-link:before { content: '\e806'; } /* '' */
.icon-calendar:before { content: '\e807'; } /* '' */
.icon-location:before { content: '\e808'; } /* '' */
.icon-heart:before { content: '\2665'; } /* '' */
.icon-quote:before { content: '\275e'; } /* '' */
.icon-comment:before { content: '\e802'; } /* '' */
.icon-ok:before { content: '\e803'; } /* '' */
.icon-play:before { content: '\e804'; } /* '' */
.icon-link:before { content: '\e805'; } /* '' */
.icon-calendar:before { content: '\e806'; } /* '' */
.icon-location:before { content: '\e807'; } /* '' */
.icon-picture:before { content: '\e809'; } /* '' */
.icon-lock:before { content: '\e80a'; } /* '' */
.icon-down:before { content: '\e80b'; } /* '' */
.icon-retweet:before { content: '\e80c'; } /* '' */
.icon-search:before { content: '\e80d'; } /* '' */
.icon-pin:before { content: '\e80e'; } /* '' */
.icon-cog:before { content: '\e80f'; } /* '' */
.icon-rss:before { content: '\e810'; } /* '' */
.icon-retweet:before { content: '\e80d'; } /* '' */
.icon-search:before { content: '\e80e'; } /* '' */
.icon-pin:before { content: '\e80f'; } /* '' */
.icon-cog:before { content: '\e812'; } /* '' */
.icon-rss-feed:before { content: '\e813'; } /* '' */
.icon-info:before { content: '\f128'; } /* '' */
.icon-bird:before { content: '\f309'; } /* '' */

View File

@@ -1,15 +1,6 @@
Font license info
## Modern Pictograms
Copyright (c) 2012 by John Caserta. All rights reserved.
Author: John Caserta
License: SIL (http://scripts.sil.org/OFL)
Homepage: http://thedesignoffice.org/project/modern-pictograms/
## Entypo
Copyright (C) 2012 by Daniel Bruce
@@ -46,3 +37,12 @@ Font license info
Homepage: http://aristeides.com/
## Modern Pictograms
Copyright (c) 2012 by John Caserta. All rights reserved.
Author: John Caserta
License: SIL (http://scripts.sil.org/OFL)
Homepage: http://thedesignoffice.org/project/modern-pictograms/

Binary file not shown.

View File

@@ -1,28 +1,26 @@
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns="http://www.w3.org/2000/svg">
<metadata>Copyright (C) 2025 by original authors @ fontello.com</metadata>
<metadata>Copyright (C) 2020 by original authors @ fontello.com</metadata>
<defs>
<font id="fontello" horiz-adv-x="1000" >
<font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
<missing-glyph horiz-adv-x="1000" />
<glyph glyph-name="views" unicode="&#xe800;" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
<glyph glyph-name="heart" unicode="&#x2665;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
<glyph glyph-name="heart" unicode="&#xe801;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
<glyph glyph-name="quote" unicode="&#x275e;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
<glyph glyph-name="quote" unicode="&#xe802;" d="M18 685l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z m558 0l335 0 0-334q0-140-98-238t-237-97l0 111q92 0 158 65t65 159l-223 0 0 334z" horiz-adv-x="928" />
<glyph glyph-name="comment" unicode="&#xe802;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
<glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
<glyph glyph-name="ok" unicode="&#xe803;" d="M0 260l162 162 166-164 508 510 164-164-510-510-162-162-162 164z" horiz-adv-x="1000" />
<glyph glyph-name="ok" unicode="&#xe804;" d="M0 260l162 162 166-164 508 510 164-164-510-510-162-162-162 164z" horiz-adv-x="1000" />
<glyph glyph-name="play" unicode="&#xe804;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
<glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
<glyph glyph-name="link" unicode="&#xe805;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
<glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
<glyph glyph-name="calendar" unicode="&#xe806;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
<glyph glyph-name="calendar" unicode="&#xe807;" d="M800 700q42 0 71-29t29-71l0-600q0-40-29-70t-71-30l-700 0q-40 0-70 30t-30 70l0 600q0 42 30 71t70 29l46 0 0-100 160 0 0 100 290 0 0-100 160 0 0 100 44 0z m0-700l0 400-700 0 0-400 700 0z m-540 800l0-170-70 0 0 170 70 0z m450 0l0-170-70 0 0 170 70 0z" horiz-adv-x="900" />
<glyph glyph-name="location" unicode="&#xe808;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
<glyph glyph-name="location" unicode="&#xe807;" d="M250 750q104 0 177-73t73-177q0-106-62-243t-126-223l-62-84q-10 12-27 35t-60 89-76 130-60 147-27 149q0 104 73 177t177 73z m0-388q56 0 96 40t40 96-40 95-96 39-95-39-39-95 39-96 95-40z" horiz-adv-x="500" />
<glyph glyph-name="picture" unicode="&#xe809;" d="M357 529q0-45-31-76t-76-32-76 32-31 76 31 76 76 31 76-31 31-76z m572-215v-250h-786v107l178 179 90-89 285 285z m53 393h-893q-7 0-12-5t-6-13v-678q0-7 6-13t12-5h893q7 0 13 5t5 13v678q0 8-5 13t-13 5z m89-18v-678q0-37-26-63t-63-27h-893q-36 0-63 27t-26 63v678q0 37 26 63t63 27h893q37 0 63-27t26-63z" horiz-adv-x="1071.4" />
@@ -30,19 +28,19 @@
<glyph glyph-name="down" unicode="&#xe80b;" d="M939 399l-414-413q-10-11-25-11t-25 11l-414 413q-11 11-11 26t11 25l93 92q10 11 25 11t25-11l296-296 296 296q11 11 25 11t26-11l92-92q11-11 11-25t-11-26z" horiz-adv-x="1000" />
<glyph glyph-name="retweet" unicode="&#xe80c;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
<glyph glyph-name="retweet" unicode="&#xe80d;" d="M714 11q0-7-5-13t-13-5h-535q-5 0-8 1t-5 4-3 4-2 7 0 6v335h-107q-15 0-25 11t-11 25q0 13 8 23l179 214q11 12 27 12t28-12l178-214q9-10 9-23 0-15-11-25t-25-11h-107v-214h321q9 0 14-6l89-108q4-5 4-11z m357 232q0-13-8-23l-178-214q-12-13-28-13t-27 13l-179 214q-8 10-8 23 0 14 11 25t25 11h107v214h-322q-9 0-14 7l-89 107q-4 5-4 11 0 7 5 12t13 6h536q4 0 7-1t5-4 3-5 2-6 1-7v-334h107q14 0 25-11t10-25z" horiz-adv-x="1071.4" />
<glyph glyph-name="search" unicode="&#xe80d;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
<glyph glyph-name="search" unicode="&#xe80e;" d="M772 78q30-34 6-62l-46-46q-36-32-68 0l-190 190q-74-42-156-42-128 0-223 95t-95 223 90 219 218 91 224-95 96-223q0-88-46-162z m-678 358q0-88 68-156t156-68 151 63 63 153q0 88-68 155t-156 67-151-63-63-151z" horiz-adv-x="789" />
<glyph glyph-name="pin" unicode="&#xe80e;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
<glyph glyph-name="pin" unicode="&#xe80f;" d="M268 368v250q0 8-5 13t-13 5-13-5-5-13v-250q0-8 5-13t13-5 13 5 5 13z m375-197q0-14-11-25t-25-10h-239l-29-270q-1-7-6-11t-11-5h-1q-15 0-17 15l-43 271h-225q-15 0-25 10t-11 25q0 69 44 124t99 55v286q-29 0-50 21t-22 50 22 50 50 22h357q29 0 50-22t21-50-21-50-50-21v-286q55 0 99-55t44-124z" horiz-adv-x="642.9" />
<glyph glyph-name="cog" unicode="&#xe80f;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
<glyph glyph-name="cog" unicode="&#xe812;" d="M911 295l-133-56q-8-22-12-31l55-133-79-79-135 53q-9-4-31-12l-55-134-112 0-56 133q-11 4-33 13l-132-55-78 79 53 134q-1 3-4 9t-6 12-4 11l-131 55 0 112 131 56 14 33-54 132 78 79 133-54q22 9 33 13l55 132 112 0 56-132q14-5 31-13l133 55 80-79-54-135q6-12 12-30l133-56 0-112z m-447-111q69 0 118 48t49 118-49 119-118 50-119-50-49-119 49-118 119-48z" horiz-adv-x="928" />
<glyph glyph-name="rss" unicode="&#xe810;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
<glyph glyph-name="rss-feed" unicode="&#xe813;" d="M184 93c0-51-43-91-93-91s-91 40-91 91c0 50 41 91 91 91s93-41 93-91z m261-85l-125 0c0 174-140 323-315 323l0 118c231 0 440-163 440-441z m259 0l-136 0c0 300-262 561-563 561l0 129c370 0 699-281 699-690z" horiz-adv-x="704" />
<glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />
<glyph glyph-name="bird" unicode="&#xf309;" d="M920 636q-36-54-94-98l0-24q0-130-60-250t-186-203-290-83q-160 0-290 84 14-2 46-2 132 0 234 80-62 2-110 38t-66 94q10-4 34-4 26 0 50 6-66 14-108 66t-42 120l0 2q36-20 84-24-84 58-84 158 0 48 26 94 154-188 390-196-6 18-6 42 0 78 55 133t135 55q82 0 136-58 60 12 120 44-20-66-82-104 56 8 108 30z" horiz-adv-x="920" />
</font>
</defs>
</svg>
</svg>

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

Binary file not shown.

Binary file not shown.

5
public/js/hls.light.min.js vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -5,7 +5,7 @@ function insertBeforeLast(node, elem) {
}
function getLoadMore(doc) {
return doc.querySelector(".show-more:not(.timeline-item)");
return doc.querySelector('.show-more:not(.timeline-item)');
}
function isDuplicate(item, itemClass) {
@@ -15,19 +15,18 @@ function isDuplicate(item, itemClass) {
return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null;
}
window.onload = function () {
window.onload = function() {
const url = window.location.pathname;
const isTweet = url.indexOf("/status/") !== -1;
const containerClass = isTweet ? ".replies" : ".timeline";
const itemClass = containerClass + " > div:not(.top-ref)";
const itemClass = containerClass + ' > div:not(.top-ref)';
var html = document.querySelector("html");
var container = document.querySelector(containerClass);
var loading = false;
function handleScroll(failed) {
window.addEventListener('scroll', function() {
if (loading) return;
if (html.scrollTop + html.clientHeight >= html.scrollHeight - 3000) {
loading = true;
var loadMore = getLoadMore(document);
@@ -36,15 +35,13 @@ window.onload = function () {
loadMore.children[0].text = "Loading...";
var url = new URL(loadMore.children[0].href);
url.searchParams.append("scroll", "true");
url.searchParams.append('scroll', 'true');
fetch(url.toString()).then(function (response) {
if (response.status === 404) throw "error";
return response.text();
}).then(function (html) {
var parser = new DOMParser();
var doc = parser.parseFromString(html, "text/html");
var doc = parser.parseFromString(html, 'text/html');
loadMore.remove();
for (var item of doc.querySelectorAll(itemClass)) {
@@ -60,18 +57,10 @@ window.onload = function () {
if (isTweet) container.appendChild(newLoadMore);
else insertBeforeLast(container, newLoadMore);
}).catch(function (err) {
console.warn("Something went wrong.", err);
if (failed > 3) {
loadMore.children[0].text = "Error";
return;
}
loading = false;
handleScroll((failed || 0) + 1);
console.warn('Something went wrong.', err);
loading = true;
});
}
}
window.addEventListener("scroll", () => handleScroll());
});
};
// @license-end

View File

@@ -4,15 +4,15 @@ Nitter is a free and open source alternative Twitter front-end focused on
privacy and performance. The source is available on GitHub at
<https://github.com/zedeus/nitter>
- No JavaScript or ads
- All requests go through the backend, client never talks to Twitter
- Prevents Twitter from tracking your IP or JavaScript fingerprint
- Uses Twitter's unofficial API (no developer account required)
- Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
- RSS feeds
- Themes
- Mobile support (responsive design)
- AGPLv3 licensed, no proprietary instances permitted
* No JavaScript or ads
* All requests go through the backend, client never talks to Twitter
* Prevents Twitter from tracking your IP or JavaScript fingerprint
* Uses Twitter's unofficial API (no rate limits or developer account required)
* Lightweight (for [@nim_lang](/nim_lang), 60KB vs 784KB from twitter.com)
* RSS feeds
* Themes
* Mobile support (responsive design)
* AGPLv3 licensed, no proprietary instances permitted
Nitter's GitHub wiki contains
[instances](https://github.com/zedeus/nitter/wiki/Instances) and
@@ -21,13 +21,12 @@ maintained by the community.
## Why use Nitter?
It's impossible to use Twitter without JavaScript enabled, and as of 2024 you
need to sign up. For privacy-minded folks, preventing JavaScript analytics and
IP-based tracking is important, but apart from using a VPN and uBlock/uMatrix,
it's impossible. Despite being behind a VPN and using heavy-duty adblockers,
you can get accurately tracked with your [browser's
fingerprint](https://restoreprivacy.com/browser-fingerprinting/), [no
JavaScript required](https://noscriptfingerprint.com/). This all became
It's impossible to use Twitter without JavaScript enabled. For privacy-minded
folks, preventing JavaScript analytics and IP-based tracking is important, but
apart from using a VPN and uBlock/uMatrix, it's impossible. Despite being behind
a VPN and using heavy-duty adblockers, you can get accurately tracked with your
[browser's fingerprint](https://restoreprivacy.com/browser-fingerprinting/),
[no JavaScript required](https://noscriptfingerprint.com/). This all became
particularly important after Twitter [removed the
ability](https://www.eff.org/deeplinks/2020/04/twitter-removes-privacy-option-and-shows-why-we-need-strong-privacy-laws)
for users to control whether their data gets sent to advertisers.
@@ -43,13 +42,12 @@ Twitter account.
## Donating
Liberapay: https://liberapay.com/zedeus \
Patreon: https://patreon.com/nitter \
BTC: bc1qpqpzjkcpgluhzf7x9yqe7jfe8gpfm5v08mdr55 \
ETH: 0x24a0DB59A923B588c7A5EBd0dBDFDD1bCe9c4460 \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL \
SOL: ANsyGNXFo6osuFwr1YnUqif2RdoYRhc27WdyQNmmETSW \
ZEC: u1vndfqtzyy6qkzhkapxelel7ams38wmfeccu3fdpy2wkuc4erxyjm8ncjhnyg747x6t0kf0faqhh2hxyplgaum08d2wnj4n7cyu9s6zhxkqw2aef4hgd4s6vh5hpqvfken98rg80kgtgn64ff70djy7s8f839z00hwhuzlcggvefhdlyszkvwy3c7yw623vw3rvar6q6evd3xcvveypt
Liberapay: <https://liberapay.com/zedeus> \
Patreon: <https://patreon.com/nitter> \
BTC: bc1qp7q4qz0fgfvftm5hwz3vy284nue6jedt44kxya \
ETH: 0x66d84bc3fd031b62857ad18c62f1ba072b011925 \
LTC: ltc1qhsz5nxw6jw9rdtw9qssjeq2h8hqk2f85rdgpkr \
XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscWGWJCczFLe9RFhM3d1zpL
## Contact

View File

@@ -1,105 +1,58 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables
import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser
# Helper to generate params object for GraphQL requests
proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
result.add ("variables", variables)
result.add ("features", gqlFeatures)
if fieldToggles.len > 0:
result.add ("fieldToggles", fieldToggles)
proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
let
cookieVars = userMediaVars % [id, cursor]
oauthVars = restIdVars % [id, cursor]
result = SessionAwareUrl(
cookieUrl: graphUserMedia ? genParams(cookieVars),
oauthUrl: graphUserMediaV2 ? genParams(oauthVars)
)
proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
let
cookieVars = userTweetsVars % [id, cursor]
oauthVars = restIdVars % [id, cursor]
result = SessionAwareUrl(
# cookieUrl: graphUserTweets ? genParams(cookieVars, userTweetsFieldToggles),
oauthUrl: graphUserTweetsV2 ? genParams(oauthVars)
)
# might change this in the future pending testing
result.cookieUrl = result.oauthUrl
proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl =
let
cookieVars = userTweetsAndRepliesVars % [id, cursor]
oauthVars = restIdVars % [id, cursor]
result = SessionAwareUrl(
cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVars, userTweetsFieldToggles),
oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVars)
)
proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
let
cookieVars = tweetDetailVars % [id, cursor]
oauthVars = tweetVars % [id, cursor]
result = SessionAwareUrl(
cookieUrl: graphTweetDetail ? genParams(cookieVars, tweetDetailFieldToggles),
oauthUrl: graphTweet ? genParams(oauthVars)
)
proc userUrl(username: string): SessionAwareUrl =
let
cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
oauthVars = """{"screen_name": "$1"}""" % username
result = SessionAwareUrl(
cookieUrl: graphUser ? genParams(cookieVars, tweetDetailFieldToggles),
oauthUrl: graphUserV2 ? genParams(oauthVars)
)
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let js = await fetchRaw(userUrl(username), Api.userScreenName)
let
variables = %*{"screen_name": username}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUser ? params, Api.userScreenName)
result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id)
js = await fetchRaw(url, Api.userRestId)
variables = %*{"userId": id}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUserById ? params, Api.userRestId)
result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
js = case kind
of TimelineKind.tweets:
await fetch(userTweetsUrl(id, cursor), Api.userTweets)
of TimelineKind.replies:
await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies)
of TimelineKind.media:
await fetch(mediaUrl(id, cursor), Api.userMedia)
result = parseGraphTimeline(js, after)
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
(url, apiId) = case kind
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
of TimelineKind.media: (graphUserMedia, Api.userMedia)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
url = graphListTweets ? genParams(restIdVars % [id, cursor])
result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets
variables = listTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphListTweets ? params, Api.listTweets)
result = parseGraphTimeline(js, "list", after)
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list}
url = graphListBySlug ? genParams($variables)
result = parseGraphList(await fetch(url, Api.listBySlug))
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
proc getGraphList*(id: string): Future[List] {.async.} =
let
url = graphListById ? genParams("""{"listId": "$1"}""" % id)
result = parseGraphList(await fetch(url, Api.list))
variables = %*{"listId": id}
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListById ? params, Api.list))
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
@@ -113,13 +66,13 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
}
if after.len > 0:
variables["cursor"] = % after
let url = graphListMembers ? genParams($variables)
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
variables = """{"rest_id": "$1"}""" % id
variables = tweetResultVariables % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js)
@@ -128,7 +81,9 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail)
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -140,15 +95,14 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
if after.len > 0:
result.replies = await getReplies(id, after)
proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Timeline(query: query, beginning: true)
return Result[Tweet](query: query, beginning: true)
var
variables = %*{
"rawQuery": q,
"query_source": "typedQuery",
"count": 20,
"product": "Latest",
"withDownvotePerspective": false,
@@ -157,36 +111,35 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
}
if after.len > 0:
variables["cursor"] = % after
let url = graphSearchTimeline ? genParams($variables)
result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch(await fetch(url, Api.search), after)
result.query = query
proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0:
return Result[User](query: query, beginning: true)
var
variables = %*{
"rawQuery": query.text,
"query_source": "typedQuery",
"count": 20,
"product": "People",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
result.beginning = false
var url = userSearch ? {
"q": query.text,
"skip_status": "1",
"count": "20",
"page": page
}
let url = graphSearchTimeline ? genParams($variables)
result = parseGraphSearch[User](await fetch(url, Api.search), after)
result = parseUsers(await fetchRaw(url, Api.userSearch))
result.query = query
if page.len == 0:
result.bottom = "2"
elif page.allCharsInSet(Digits):
result.bottom = $(parseInt(page) + 1)
proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
if id.len == 0: return
let js = await fetch(mediaUrl(id, ""), Api.userMedia)
result = parseGraphPhotoRail(js)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)

View File

@@ -1,79 +1,66 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1
import types, auth, consts, parserutils, http_pool
import httpclient, asyncdispatch, options, strutils, uri
import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
import experimental/types/common
const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
rlLimit = "x-rate-limit-limit"
errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
var pool: HttpPool
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
let
encodedUrl = url.replace(",", "%2C").replace("+", "%20")
params = OAuth1Parameters(
consumerKey: consumerKey,
signatureMethod: "HMAC-SHA1",
timestamp: $int(round(epochTime())),
nonce: "0",
isIncludeVersionToHeader: true,
token: oauthToken
)
signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
count="20"; ext=true): seq[(string, string)] =
result = timelineParams
for p in pars:
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
# The raw cursor often has plus signs, which sometimes get turned into spaces,
# so we need to turn them back into a plus
if " " in cursor:
result &= ("cursor", cursor.replace(" ", "+"))
else:
result &= ("cursor", cursor)
params.signature = percentEncode(signature)
return getOauth1RequestHeader(params)["authorization"]
proc getCookieHeader(authToken, ct0: string): string =
"auth_token=" & authToken & "; ct0=" & ct0
proc genHeaders*(session: Session, url: string): HttpHeaders =
proc genHeaders*(token: Token = nil): HttpHeaders =
result = newHttpHeaders({
"connection": "keep-alive",
"authorization": auth,
"content-type": "application/json",
"x-guest-token": if token == nil: "" else: token.tok,
"x-twitter-active-user": "yes",
"x-twitter-client-language": "en",
"authority": "api.x.com",
"authority": "api.twitter.com",
"accept-encoding": "gzip",
"accept-language": "en-US,en;q=0.9",
"accept": "*/*",
"DNT": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
"DNT": "1"
})
case session.kind
of SessionKind.oauth:
result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
of SessionKind.cookie:
result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
result["x-twitter-auth-type"] = "OAuth2Session"
result["x-csrf-token"] = session.ct0
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
result = await getSession(api)
case result.kind
of SessionKind.oauth:
if result.oauthToken.len == 0:
echo "[sessions] Empty oauth token, session: ", result.pretty
raise rateLimitError()
of SessionKind.cookie:
if result.authToken.len == 0 or result.ct0.len == 0:
echo "[sessions] Empty cookie credentials, session: ", result.pretty
raise rateLimitError()
template updateToken() =
if resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
var token = await getToken(api)
if token.tok.len == 0:
raise rateLimitError()
try:
var resp: AsyncResponse
pool.use(genHeaders(session, $url)):
pool.use(genHeaders(token)):
template getContent =
resp = await c.get($url)
result = await resp.body
@@ -84,94 +71,57 @@ template fetchImpl(result, fetchBody) {.dirty.} =
badClient = true
raise newException(BadClientError, "Bad client")
if resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
limit = parseInt(resp.headers[rlLimit])
session.setRateLimit(api, remaining, reset, limit)
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
result = uncompress(result, dfGzip)
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors notin errorsToSkip:
echo "Fetch error, API: ", api, ", errors: ", errors
if errors in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
elif errors in {rateLimited}:
# rate limit hit, resets after 24 hours
setLimited(session, api)
raise rateLimitError()
elif result.startsWith("429 Too Many Requests"):
echo "[sessions] 429 error, API: ", api, ", session: ", session.pretty
session.apis[api].remaining = 0
# rate limit hit, resets after the 15 minute window
raise rateLimitError()
else:
echo "non-gzip body, url: ", url, ", body: ", result
fetchBody
release(token, used=true)
if resp.status == $Http400:
echo "ERROR 400, ", api, ": ", result
raise newException(InternalError, $url)
except InternalError as e:
raise e
except BadClientError as e:
raise e
except OSError as e:
release(token, used=true)
raise e
except Exception as e:
let s = session.pretty
echo "error: ", e.name, ", msg: ", e.msg, ", session: ", s, ", url: ", url
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
if "length" notin e.msg and "descriptor" notin e.msg:
release(token, invalid=true)
raise rateLimitError()
finally:
release(session)
template retry(bod) =
try:
bod
except RateLimitError:
echo "[sessions] Rate limited, retrying ", api, " request..."
bod
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
var body: string
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
result = newJNull()
proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
retry:
var
body: string
session = await getAndValidateSession(api)
updateToken()
when url is SessionAwareUrl:
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
let error = result.getError
if error in {invalidToken, badToken}:
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
result = newJNull()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)
let error = result.getError
if error != null and error notin errorsToSkip:
echo "Fetch error, API: ", api, ", error: ", error
if error in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
updateToken()
proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
retry:
var session = await getAndValidateSession(api)
when url is SessionAwareUrl:
let url = case session.kind
of SessionKind.oauth: url.oauthUrl
of SessionKind.cookie: url.cookieUrl
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
result.setLen(0)
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors in {invalidToken, badToken}:
echo "fetch error: ", errors
release(token, invalid=true)
raise rateLimitError()

View File

@@ -1,202 +0,0 @@
#SPDX-License-Identifier: AGPL-3.0-only
import std/[asyncdispatch, times, json, random, sequtils, strutils, tables, packedsets, os]
import types
import experimental/parser/session
# max requests at a time per session to avoid race conditions
const
maxConcurrentReqs = 2
hourInSeconds = 60 * 60
var
sessionPool: seq[Session]
enableLogging = false
template log(str: varargs[string, `$`]) =
echo "[sessions] ", str.join("")
proc pretty*(session: Session): string =
if session.isNil:
return "<null>"
if session.id > 0 and session.username.len > 0:
result = $session.id & " (" & session.username & ")"
elif session.username.len > 0:
result = session.username
elif session.id > 0:
result = $session.id
else:
result = "<unknown>"
result = $session.kind & " " & result
proc snowflakeToEpoch(flake: int64): int64 =
int64(((flake shr 22) + 1288834974657) div 1000)
proc getSessionPoolHealth*(): JsonNode =
let now = epochTime().int
var
totalReqs = 0
limited: PackedSet[int64]
reqsPerApi: Table[string, int]
oldest = now.int64
newest = 0'i64
average = 0'i64
for session in sessionPool:
let created = snowflakeToEpoch(session.id)
if created > newest:
newest = created
if created < oldest:
oldest = created
average += created
if session.limited:
limited.incl session.id
for api in session.apis.keys:
let
apiStatus = session.apis[api]
reqs = apiStatus.limit - apiStatus.remaining
# no requests made with this session and endpoint since the limit reset
if apiStatus.reset < now:
continue
reqsPerApi.mgetOrPut($api, 0).inc reqs
totalReqs.inc reqs
if sessionPool.len > 0:
average = average div sessionPool.len
else:
oldest = 0
average = 0
return %*{
"sessions": %*{
"total": sessionPool.len,
"limited": limited.card,
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
"average": $fromUnix(average)
},
"requests": %*{
"total": totalReqs,
"apis": reqsPerApi
}
}
proc getSessionPoolDebug*(): JsonNode =
let now = epochTime().int
var list = newJObject()
for session in sessionPool:
let sessionJson = %*{
"apis": newJObject(),
"pending": session.pending,
}
if session.limited:
sessionJson["limited"] = %true
for api in session.apis.keys:
let
apiStatus = session.apis[api]
obj = %*{}
if apiStatus.reset > now.int:
obj["remaining"] = %apiStatus.remaining
obj["reset"] = %apiStatus.reset
if "remaining" notin obj:
continue
sessionJson{"apis", $api} = obj
list[$session.id] = sessionJson
return %list
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
proc noSessionsError*(): ref NoSessionsError =
newException(NoSessionsError, "no sessions available")
proc isLimited(session: Session; api: Api): bool =
if session.isNil:
return true
if session.limited and api != Api.userTweets:
if (epochTime().int - session.limitedAt) > hourInSeconds:
session.limited = false
log "resetting limit: ", session.pretty
return false
else:
return true
if api in session.apis:
let limit = session.apis[api]
return limit.remaining <= 10 and limit.reset > epochTime().int
else:
return false
proc isReady(session: Session; api: Api): bool =
not (session.isNil or session.pending > maxConcurrentReqs or session.isLimited(api))
proc invalidate*(session: var Session) =
if session.isNil: return
log "invalidating: ", session.pretty
# TODO: This isn't sufficient, but it works for now
let idx = sessionPool.find(session)
if idx > -1: sessionPool.delete(idx)
session = nil
proc release*(session: Session) =
if session.isNil: return
dec session.pending
proc getSession*(api: Api): Future[Session] {.async.} =
for i in 0 ..< sessionPool.len:
if result.isReady(api): break
result = sessionPool.sample()
if not result.isNil and result.isReady(api):
inc result.pending
else:
log "no sessions available for API: ", api
raise noSessionsError()
proc setLimited*(session: Session; api: Api) =
session.limited = true
session.limitedAt = epochTime().int
log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", ", session.pretty
proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
# avoid undefined behavior in race conditions
if api in session.apis:
let rateLimit = session.apis[api]
if rateLimit.reset >= reset and rateLimit.remaining < remaining:
return
if rateLimit.reset == reset and rateLimit.remaining >= remaining:
session.apis[api].remaining = remaining
return
session.apis[api] = RateLimit(limit: limit, remaining: remaining, reset: reset)
proc initSessionPool*(cfg: Config; path: string) =
enableLogging = cfg.enableDebug
if path.endsWith(".json"):
log "ERROR: .json is not supported, the file must be a valid JSONL file ending in .jsonl"
quit 1
if not fileExists(path):
log "ERROR: ", path, " not found. This file is required to authenticate API requests."
quit 1
log "parsing JSONL account sessions file: ", path
for line in path.lines:
sessionPool.add parseSession(line)
log "successfully added ", sessionPool.len, " valid account sessions"

View File

@@ -1,163 +1,121 @@
# SPDX-License-Identifier: AGPL-3.0-only
import uri, strutils
import uri, sequtils, strutils
const
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
gql = parseUri("https://api.x.com") / "graphql"
api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json")
graphUser* = gql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
graphUserV2* = gql / "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
graphUserById* = gql / "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
graphUserTweetsV2* = gql / "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
graphUserTweetsAndRepliesV2* = gql / "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
graphUserTweets* = gql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
graphUserTweetsAndReplies* = gql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
graphUserMedia* = gql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
graphUserMediaV2* = gql / "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
graphTweet* = gql / "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
graphTweetDetail* = gql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
graphTweetResult* = gql / "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
graphSearchTimeline* = gql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
graphListById* = gql / "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
graphListBySlug* = gql / "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
graphListMembers* = gql / "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
graphListTweets* = gql / "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
photoRail* = api / "1.1/statuses/media_timeline.json"
userSearch* = api / "1.1/users/search.json"
graphql = api / "graphql"
graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName"
graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId"
graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets"
graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies"
graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia"
graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail"
graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId"
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
timelineParams* = {
"include_profile_interstitial_type": "0",
"include_blocking": "0",
"include_blocked_by": "0",
"include_followed_by": "0",
"include_want_retweets": "0",
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "0",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "1",
"include_user_entities": "1",
"include_ext_media_color": "0",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"include_quote_count": "1"
}.toSeq
gqlFeatures* = """{
"android_ad_formats_media_component_render_overlay_enabled": false,
"android_graphql_skip_api_media_color_palette": false,
"android_professional_link_spotlight_display_enabled": false,
"blue_business_profile_image_shape_enabled": false,
"commerce_android_shop_module_enabled": false,
"creator_subscriptions_subscription_count_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"hidden_profile_likes_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"highlights_tweets_tab_ui_enabled": false,
"interactive_text_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"longform_notetweets_richtext_consumption_enabled": true,
"mobile_app_spotlight_module_enabled": false,
"responsive_web_edit_tweet_api_enabled": true,
"longform_notetweets_rich_text_read_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_media_download_video_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"unified_cards_destination_url_params_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"rweb_lists_timeline_redesign_enabled": true,
"spaces_2022_h2_clipping": true,
"spaces_2022_h2_spaces_communities": true,
"standardized_nudges_misinfo": true,
"subscriptions_verification_info_enabled": true,
"subscriptions_verification_info_reason_enabled": true,
"subscriptions_verification_info_verified_since_enabled": true,
"super_follow_badge_privacy_enabled": false,
"super_follow_exclusive_tweet_notifications_enabled": false,
"super_follow_tweet_api_enabled": false,
"super_follow_user_api_enabled": false,
"standardized_nudges_misinfo": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
"verified_phone_label_enabled": false,
"vibe_api_enabled": false,
"view_counts_everywhere_api_enabled": true,
"premium_content_api_read_enabled": false,
"communities_web_enable_tweet_community_results_fetch": true,
"responsive_web_jetfuel_frame": true,
"responsive_web_grok_analyze_button_fetch_trends_enabled": false,
"responsive_web_grok_image_annotation_enabled": true,
"responsive_web_grok_imagine_annotation_enabled": true,
"rweb_tipjar_consumption_enabled": true,
"profile_label_improvements_pcf_label_in_post_enabled": true,
"creator_subscriptions_quote_tweet_preview_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"responsive_web_grok_analyze_post_followups_enabled": true,
"rweb_video_timestamps_enabled": false,
"responsive_web_grok_share_attachment_enabled": true,
"articles_preview_enabled": true,
"immersive_video_status_linkable_timestamps": false,
"articles_api_enabled": false,
"responsive_web_grok_analysis_button_from_backend": true,
"rweb_video_screen_enabled": false,
"payments_enabled": false,
"responsive_web_profile_redirect_enabled": false,
"responsive_web_grok_show_grok_translated_post": false,
"responsive_web_grok_community_note_auto_translation_is_enabled": false,
"profile_label_improvements_pcf_label_in_profile_enabled": false,
"grok_android_analyze_trend_fetch_enabled": false,
"grok_translations_community_note_auto_translation_is_enabled": false,
"grok_translations_post_auto_translation_is_enabled": false,
"grok_translations_community_note_translation_is_enabled": false,
"grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
"subscriptions_feature_can_gift_premium": false,
"responsive_web_twitter_article_notes_tab_enabled": false,
"subscriptions_verification_info_is_identity_verified_enabled": false,
"hidden_profile_subscriptions_enabled": false
"view_counts_everywhere_api_enabled": false
}""".replace(" ", "").replace("\n", "")
tweetVars* = """{
"postId": "$1",
$2
"includeHasBirdwatchNotes": false,
"includePromotedContent": false,
"withBirdwatchNotes": false,
"withVoice": false,
"withV2Timeline": true
}""".replace(" ", "").replace("\n", "")
tweetDetailVars* = """{
tweetVariables* = """{
"focalTweetId": "$1",
$2
"referrer": "profile",
"with_rux_injections": false,
"rankingMode": "Relevance",
"includePromotedContent": true,
"withCommunity": true,
"withQuickPromoteEligibilityTweetFields": true,
"withBirdwatchNotes": true,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
restIdVars* = """{
"rest_id": "$1", $2
"count": 20
"withBirdwatchNotes": false,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""
userMediaVars* = """{
tweetResultVariables* = """{
"tweetId": "$1",
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withCommunity": false
}"""
userTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withClientEventToken": false,
"withBirdwatchNotes": false,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
}"""
userTweetsVars* = """{
"userId": "$1", $2
listTweetsVariables* = """{
"listId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withQuickPromoteEligibilityTweetFields": true,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
userTweetsAndRepliesVars* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withCommunity": true,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""

View File

@@ -1,53 +1,17 @@
import options, strutils
import options
import jsony
import user, utils, ../types/[graphuser, graphlistmembers]
from ../../types import User, VerifiedType, Result, Query, QueryKind
proc parseUserResult*(userResult: UserResult): User =
result = userResult.legacy
if result.verifiedType == none and userResult.isBlueVerified:
result.verifiedType = blue
if result.username.len == 0 and userResult.core.screenName.len > 0:
result.id = userResult.restId
result.username = userResult.core.screenName
result.fullname = userResult.core.name
result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
if userResult.privacy.isSome:
result.protected = userResult.privacy.get.protected
if userResult.location.isSome:
result.location = userResult.location.get.location
if userResult.core.createdAt.len > 0:
result.joinDate = parseTwitterDate(userResult.core.createdAt)
if userResult.verification.isSome:
let v = userResult.verification.get
if v.verifiedType != VerifiedType.none:
result.verifiedType = v.verifiedType
if userResult.profileBio.isSome and result.bio.len == 0:
result.bio = userResult.profileBio.get.description
import user, ../types/[graphuser, graphlistmembers]
from ../../types import User, Result, Query, QueryKind
proc parseGraphUser*(json: string): User =
if json.len == 0 or json[0] != '{':
return
let raw = json.fromJson(GraphUser)
let
raw = json.fromJson(GraphUser)
userResult =
if raw.data.userResult.isSome: raw.data.userResult.get.result
elif raw.data.user.isSome: raw.data.user.get.result
else: UserResult()
if userResult.unavailableReason.get("") == "Suspended" or
userResult.reason.get("") == "Suspended":
if raw.data.user.result.reason.get("") == "Suspended":
return User(suspended: true)
result = parseUserResult(userResult)
result = toUser raw.data.user.result.legacy
result.id = raw.data.user.result.restId
result.verified = result.verified or raw.data.user.result.isBlueVerified
proc parseGraphListMembers*(json, cursor: string): Result[User] =
result = Result[User](
@@ -63,7 +27,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
of TimelineTimelineItem:
let userResult = entry.content.itemContent.userResults.result
if userResult.restId.len > 0:
result.content.add parseUserResult(userResult)
result.content.add toUser userResult.legacy
of TimelineTimelineCursor:
if entry.content.cursorType == "Bottom":
result.bottom = entry.content.value

View File

@@ -1,30 +0,0 @@
import std/strutils
import jsony
import ../types/session
from ../../types import Session, SessionKind
proc parseSession*(raw: string): Session =
let session = raw.fromJson(RawSession)
let kind = if session.kind == "": "oauth" else: session.kind
case kind
of "oauth":
let id = session.oauthToken[0 ..< session.oauthToken.find('-')]
result = Session(
kind: SessionKind.oauth,
id: parseBiggestInt(id),
username: session.username,
oauthToken: session.oauthToken,
oauthSecret: session.oauthTokenSecret
)
of "cookie":
let id = if session.id.len > 0: parseBiggestInt(session.id) else: 0
result = Session(
kind: SessionKind.cookie,
id: id,
username: session.username,
authToken: session.authToken,
ct0: session.ct0
)
else:
raise newException(ValueError, "Unknown session kind: " & kind)

View File

@@ -1,7 +1,6 @@
import std/[options, tables, strutils, strformat, sugar]
import jsony
import user, ../types/unifiedcard
import ../../formatters
import ../types/unifiedcard
from ../../types import Card, CardKind, Video
from ../../utils import twimg, https
@@ -28,14 +27,6 @@ proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card)
result.text = data.topicDetail.title
result.dest = "Topic"
proc parseJobDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
data.destination.parseDestination(card, result)
result.kind = CardKind.jobDetails
result.title = data.title
result.text = data.shortDescriptionText
result.dest = &"@{data.profileUser.username} · {data.location}"
proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
let app = card.appStoreData[data.appId][0]
@@ -78,18 +69,6 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
of model3d:
result.title = "Unsupported 3D model ad"
proc parseGrokShare(data: ComponentData; card: UnifiedCard; result: var Card) =
result.kind = summaryLarge
data.destination.parseDestination(card, result)
result.dest = "Answer by Grok"
for msg in data.conversationPreview:
if msg.sender == "USER":
result.title = msg.message.shorten(70)
elif msg.sender == "AGENT":
result.text = msg.message.shorten(500)
proc parseUnifiedCard*(json: string): Card =
let card = json.fromJson(UnifiedCard)
@@ -105,10 +84,6 @@ proc parseUnifiedCard*(json: string): Card =
component.parseMedia(card, result)
of buttonGroup:
discard
of grokShare:
component.data.parseGrokShare(card, result)
of ComponentType.jobDetails:
component.data.parseJobDetails(card, result)
of ComponentType.hidden:
result.kind = CardKind.hidden
of ComponentType.unknown:

View File

@@ -56,21 +56,32 @@ proc toUser*(raw: RawUser): User =
tweets: raw.statusesCount,
likes: raw.favouritesCount,
media: raw.mediaCount,
verifiedType: raw.verifiedType,
verified: raw.verified,
protected: raw.protected,
joinDate: parseTwitterDate(raw.createdAt),
banner: getBanner(raw),
userPic: getImageUrl(raw.profileImageUrlHttps).replace("_normal", "")
)
if raw.createdAt.len > 0:
result.joinDate = parseTwitterDate(raw.createdAt)
if raw.pinnedTweetIdsStr.len > 0:
result.pinnedTweet = parseBiggestInt(raw.pinnedTweetIdsStr[0])
result.expandUserEntities(raw)
proc parseHook*(s: string; i: var int; v: var User) =
var u: RawUser
parseHook(s, i, u)
v = toUser u
proc parseUser*(json: string; username=""): User =
handleErrors:
case error.code
of suspended: return User(username: username, suspended: true)
of userNotFound: return
else: echo "[error - parseUser]: ", error
result = toUser json.fromJson(RawUser)
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
# starting with '{' means it's an error
if json[0] == '[':
let raw = json.fromJson(seq[RawUser])
for user in raw:
result.content.add user.toUser

View File

@@ -1,48 +1,15 @@
import options, strutils
from ../../types import User, VerifiedType
import options
import user
type
GraphUser* = object
data*: tuple[userResult: Option[UserData], user: Option[UserData]]
data*: tuple[user: UserData]
UserData* = object
result*: UserResult
UserCore* = object
name*: string
screenName*: string
createdAt*: string
UserBio* = object
description*: string
UserAvatar* = object
imageUrl*: string
Verification* = object
verifiedType*: VerifiedType
Location* = object
location*: string
Privacy* = object
protected*: bool
UserResult* = object
legacy*: User
UserResult = object
legacy*: RawUser
restId*: string
isBlueVerified*: bool
core*: UserCore
avatar*: UserAvatar
unavailableReason*: Option[string]
reason*: Option[string]
privacy*: Option[Privacy]
profileBio*: Option[UserBio]
verification*: Option[Verification]
location*: Option[Location]
proc enumHook*(s: string; v: var VerifiedType) =
v = try:
parseEnum[VerifiedType](s)
except:
VerifiedType.none

View File

@@ -1,9 +0,0 @@
type
RawSession* = object
kind*: string
id*: string
username*: string
oauthToken*: string
oauthTokenSecret*: string
authToken*: string
ct0*: string

View File

@@ -0,0 +1,23 @@
import std/tables
import user
type
Search* = object
globalObjects*: GlobalObjects
timeline*: Timeline
GlobalObjects = object
users*: Table[string, RawUser]
Timeline = object
instructions*: seq[Instructions]
Instructions = object
addEntries*: tuple[entries: seq[Entry]]
Entry = object
entryId*: string
content*: tuple[operation: Operation]
Operation = object
cursor*: tuple[value, cursorType: string]

View File

@@ -1,10 +1,7 @@
import std/[options, tables, times]
import jsony
from ../../types import VideoType, VideoVariant, User
import options, tables
from ../../types import VideoType, VideoVariant
type
Text* = distinct string
UnifiedCard* = object
componentObjects*: Table[string, Component]
destinationObjects*: Table[string, Destination]
@@ -16,13 +13,11 @@ type
media
swipeableMedia
buttonGroup
jobDetails
appStoreDetails
twitterListDetails
communityDetails
mediaWithDetailsHorizontal
hidden
grokShare
unknown
Component* = object
@@ -34,16 +29,12 @@ type
appId*: string
mediaId*: string
destination*: string
location*: string
title*: Text
subtitle*: Text
name*: Text
memberCount*: int
mediaList*: seq[MediaItem]
topicDetail*: tuple[title: Text]
profileUser*: User
shortDescriptionText*: string
conversationPreview*: seq[GrokConversation]
MediaItem* = object
id*: string
@@ -78,13 +69,12 @@ type
title*: Text
category*: Text
GrokConversation* = object
message*: string
sender*: string
Text = object
content: string
TypeField = Component | Destination | MediaEntity | AppStoreData
converter fromText*(text: Text): string = string(text)
converter fromText*(text: Text): string = text.content
proc renameHook*(v: var TypeField; fieldName: var string) =
if fieldName == "type":
@@ -96,13 +86,11 @@ proc enumHook*(s: string; v: var ComponentType) =
of "media": media
of "swipeable_media": swipeableMedia
of "button_group": buttonGroup
of "job_details": jobDetails
of "app_store_details": appStoreDetails
of "twitter_list_details": twitterListDetails
of "community_details": communityDetails
of "media_with_details_horizontal": mediaWithDetailsHorizontal
of "commerce_drop_details": hidden
of "grok_share": grokShare
else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
proc enumHook*(s: string; v: var AppType) =
@@ -118,18 +106,3 @@ proc enumHook*(s: string; v: var MediaType) =
of "photo": photo
of "model3d": model3d
else: echo "ERROR: Unknown enum value (MediaType): ", s; photo
proc parseHook*(s: string; i: var int; v: var DateTime) =
var str: string
parseHook(s, i, str)
v = parse(str, "yyyy-MM-dd hh:mm:ss")
proc parseHook*(s: string; i: var int; v: var Text) =
if s[i] == '"':
var str: string
parseHook(s, i, str)
v = Text(str)
else:
var t: tuple[content: string]
parseHook(s, i, t)
v = Text(t.content)

View File

@@ -1,6 +1,5 @@
import options
import common
from ../../types import VerifiedType
type
RawUser* = object
@@ -16,7 +15,7 @@ type
favouritesCount*: int
statusesCount*: int
mediaCount*: int
verifiedType*: VerifiedType
verified*: bool
protected*: bool
profileLinkColor*: string
profileBannerUrl*: string

View File

@@ -6,13 +6,11 @@ import types, utils, query
const
cards = "cards.twitter.com/cards"
tco = "https://t.co"
twitter = parseUri("https://x.com")
twitter = parseUri("https://twitter.com")
let
twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
xRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?x\.com"
xLinkRegex = re"""<a href="https:\/\/x.com([^"]+)">x\.com(\S+)</a>"""
ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
@@ -33,13 +31,10 @@ proc getUrlPrefix*(cfg: Config): string =
if cfg.useHttps: https & cfg.hostname
else: "http://" & cfg.hostname
proc shorten*(text: string; length=28): string =
result = text
proc shortLink*(text: string; length=28): string =
result = text.replace(wwwRegex, "")
if result.len > length:
result = result[0 ..< length] & ""
proc shortLink*(text: string; length=28): string =
result = text.replace(wwwRegex, "").shorten(length)
proc stripHtml*(text: string; shorten=false): string =
var html = parseHtml(text)
@@ -59,28 +54,19 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
result = body
if prefs.replaceYouTube.len > 0 and "youtu" in result:
let youtubeHost = strip(prefs.replaceYouTube, chars={'/'})
result = result.replace(ytRegex, youtubeHost)
result = result.replace(ytRegex, prefs.replaceYouTube)
if prefs.replaceTwitter.len > 0:
let twitterHost = strip(prefs.replaceTwitter, chars={'/'})
if tco in result:
result = result.replace(tco, https & twitterHost & "/t.co")
if "x.com" in result:
result = result.replace(xRegex, twitterHost)
result = result.replacef(xLinkRegex, a(
twitterHost & "$2", href = https & twitterHost & "$1"))
if "twitter.com" in result:
result = result.replace(cards, twitterHost & "/cards")
result = result.replace(twRegex, twitterHost)
result = result.replacef(twLinkRegex, a(
twitterHost & "$2", href = https & twitterHost & "$1"))
if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
result = result.replace(cards, prefs.replaceTwitter & "/cards")
result = result.replace(twRegex, prefs.replaceTwitter)
result = result.replacef(twLinkRegex, a(
prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
let redditHost = strip(prefs.replaceReddit, chars={'/'})
result = result.replace(rdShortRegex, redditHost & "/comments/")
result = result.replace(rdRegex, redditHost)
if redditHost in result and "/gallery/" in result:
result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
result = result.replace(rdRegex, prefs.replaceReddit)
if prefs.replaceReddit in result and "/gallery/" in result:
result = result.replace("/gallery/", "/comments/")
if absolute.len > 0 and "href" in result:
@@ -96,8 +82,6 @@ proc proxifyVideo*(manifest: string; proxy: bool): string =
for line in manifest.splitLines:
let url =
if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2]
elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line:
line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))]
else: line
if url.startsWith('/'):
let path = "https://video.twimg.com" & url

View File

@@ -39,8 +39,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
try:
body
except BadClientError, ProtocolError:
# Twitter returned 503 or closed the connection, we need a new client
except ProtocolError:
# Twitter closed the connection, retry
body
except BadClientError:
# Twitter returned 503, we need a new client
pool.release(c, true)
badClient = false
c = pool.acquire(heads)

View File

@@ -6,7 +6,7 @@ from os import getEnv
import jester
import types, config, prefs, formatters, redis_cache, http_pool, auth
import types, config, prefs, formatters, redis_cache, http_pool, tokens
import views/[general, about]
import routes/[
preferences, timeline, status, media, search, rss, list, debug,
@@ -15,13 +15,8 @@ import routes/[
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
const issuesUrl = "https://github.com/zedeus/nitter/issues"
let
configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
(cfg, fullCfg) = getConfig(configPath)
sessionsPath = getEnv("NITTER_SESSIONS_FILE", "./sessions.jsonl")
initSessionPool(cfg, sessionsPath)
let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf")
let (cfg, fullCfg) = getConfig(configPath)
if not cfg.enableDebug:
# Silence Jester's query warning
@@ -43,6 +38,8 @@ waitFor initRedisPool(cfg)
stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
stdout.flushFile
asyncCheck initTokenPool(cfg)
createUnsupportedRouter(cfg)
createResolverRouter(cfg)
createPrefRouter(cfg)
@@ -90,18 +87,13 @@ routes:
error BadClientError:
echo error.exc.name, ": ", error.exc.msg
resp Http500, showError("Network error occurred, please try again.", cfg)
resp Http500, showError("Network error occured, please try again.", cfg)
error RateLimitError:
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
error NoSessionsError:
const link = a("another instance", href = instancesUrl)
resp Http429, showError(
&"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
extend rss, ""
extend status, ""
extend search, ""

View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, times, math, tables
import strutils, options, tables, times, math
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
@@ -21,45 +21,19 @@ proc parseUser(js: JsonNode; id=""): User =
tweets: js{"statuses_count"}.getInt,
likes: js{"favourites_count"}.getInt,
media: js{"media_count"}.getInt,
protected: js{"protected"}.getBool(js{"privacy", "protected"}.getBool),
verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
protected: js{"protected"}.getBool,
joinDate: js{"created_at"}.getTime
)
if js{"is_blue_verified"}.getBool(false):
result.verifiedType = blue
with verifiedType, js{"verified_type"}:
result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
result.expandUserEntities(js)
proc parseGraphUser(js: JsonNode): User =
var user = js{"user_result", "result"}
if user.isNull:
user = ? js{"user_results", "result"}
let user = ? js{"user_results", "result"}
result = parseUser(user{"legacy"})
if user.isNull:
if js{"core"}.notNull and js{"legacy"}.notNull:
user = js
else:
return
result = parseUser(user{"legacy"}, user{"rest_id"}.getStr)
if result.verifiedType == none and user{"is_blue_verified"}.getBool(false):
result.verifiedType = blue
# fallback to support UserMedia/recent GraphQL updates
if result.username.len == 0:
result.username = user{"core", "screen_name"}.getStr
result.fullname = user{"core", "name"}.getStr
result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
if user{"is_blue_verified"}.getBool(false):
result.verifiedType = blue
with verifiedType, user{"verification", "verified_type"}:
result.verifiedType = parseEnum[VerifiedType](verifiedType.getStr)
if "is_blue_verified" in user:
result.verified = true
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
@@ -98,96 +72,42 @@ proc parsePoll(js: JsonNode): Poll =
result.leader = result.values.find(max(result.values))
result.votes = result.values.sum
proc parseVideoVariants(variants: JsonNode): seq[VideoVariant] =
result = @[]
for v in variants:
let
url = v{"url"}.getStr
contentType = parseEnum[VideoType](v{"content_type"}.getStr("video/mp4"))
bitrate = v{"bit_rate"}.getInt(v{"bitrate"}.getInt(0))
result.add VideoVariant(
contentType: contentType,
bitrate: bitrate,
url: url,
resolution: if contentType == mp4: getMp4Resolution(url) else: 0
)
proc parseGif(js: JsonNode): Gif =
result = Gif(
url: js{"video_info", "variants"}[0]{"url"}.getImageStr,
thumb: js{"media_url_https"}.getImageStr
)
proc parseVideo(js: JsonNode): Video =
result = Video(
thumb: js{"media_url_https"}.getImageStr,
available: true,
views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
title: js{"ext_alt_text"}.getStr,
durationMs: js{"video_info", "duration_millis"}.getInt
# playbackType: mp4
durationMs: js{"video_info", "duration_millis"}.getInt,
playbackType: m3u8
)
with status, js{"ext_media_availability", "status"}:
if status.getStr.len > 0 and status.getStr.toLowerAscii != "available":
result.available = false
with title, js{"additional_media_info", "title"}:
result.title = title.getStr
with description, js{"additional_media_info", "description"}:
result.description = description.getStr
result.variants = parseVideoVariants(js{"video_info", "variants"})
for v in js{"video_info", "variants"}:
let
contentType = parseEnum[VideoType](v{"content_type"}.getStr("summary"))
url = v{"url"}.getStr
proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
with jsMedia, js{"extended_entities", "media"}:
for m in jsMedia:
case m.getTypeName:
of "photo":
result.photos.add m{"media_url_https"}.getImageStr
of "video":
result.video = some(parseVideo(m))
with user, m{"additional_media_info", "source_user"}:
if user{"id"}.getInt > 0:
result.attribution = some(parseUser(user))
else:
result.attribution = some(parseGraphUser(user))
of "animated_gif":
result.gif = some Gif(
url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
thumb: m{"media_url_https"}.getImageStr
)
else: discard
if contentType == mp4:
result.playbackType = mp4
with url, m{"url"}:
if result.text.endsWith(url.getStr):
result.text.removeSuffix(url.getStr)
result.text = result.text.strip()
proc parseMediaEntities(js: JsonNode; result: var Tweet) =
with mediaEntities, js{"media_entities"}:
for mediaEntity in mediaEntities:
with mediaInfo, mediaEntity{"media_results", "result", "media_info"}:
case mediaInfo.getTypeName
of "ApiImage":
result.photos.add mediaInfo{"original_img_url"}.getImageStr
of "ApiVideo":
let status = mediaEntity{"media_results", "result", "media_availability_v2", "status"}
result.video = some Video(
available: status.getStr == "Available",
thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr,
durationMs: mediaInfo{"duration_millis"}.getInt,
variants: parseVideoVariants(mediaInfo{"variants"})
)
of "ApiGif":
result.gif = some Gif(
url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
thumb: mediaInfo{"preview_image", "original_img_url"}.getImageStr
)
else: discard
# Remove media URLs from text
with mediaList, js{"legacy", "entities", "media"}:
for url in mediaList:
let expandedUrl = url{"expanded_url"}.getStr
if result.text.endsWith(expandedUrl):
result.text.removeSuffix(expandedUrl)
result.text = result.text.strip()
result.variants.add VideoVariant(
contentType: contentType,
bitrate: v{"bitrate"}.getInt,
url: url,
resolution: if contentType == mp4: getMp4Resolution(url) else: 0
)
proc parsePromoVideo(js: JsonNode): Video =
result = Video(
@@ -279,17 +199,12 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
if js.isNull: return
let time =
if js{"created_at"}.notNull: js{"created_at"}.getTime
else: js{"created_at_ms"}.getTimeFromMs
result = Tweet(
id: js{"id_str"}.getId,
threadId: js{"conversation_id_str"}.getId,
replyId: js{"in_reply_to_status_id_str"}.getId,
text: js{"full_text"}.getStr,
time: time,
time: js{"created_at"}.getTime,
hasThread: js{"self_thread"}.notNull,
available: true,
user: User(id: js{"user_id_str"}.getStr),
@@ -297,17 +212,17 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
replies: js{"reply_count"}.getInt,
retweets: js{"retweet_count"}.getInt,
likes: js{"favorite_count"}.getInt,
views: js{"views_count"}.getInt
quotes: js{"quote_count"}.getInt
)
)
result.expandTweetEntities(js)
# fix for pinned threads
if result.hasThread and result.threadId == 0:
result.threadId = js{"self_thread", "id_str"}.getId
if "retweeted_status" in js:
result.retweet = some Tweet()
elif js{"is_quote_status"}.getBool:
if js{"is_quote_status"}.getBool:
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
# legacy
@@ -322,12 +237,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.retweet = some parseGraphTweet(rt)
return
with reposts, js{"repostedStatusResults"}:
with rt, reposts{"result"}:
if "legacy" in rt:
result.retweet = some parseGraphTweet(rt)
return
if jsCard.kind != JNull:
let name = jsCard{"name"}.getStr
if "poll" in name:
@@ -340,8 +249,21 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
else:
result.card = some parseCard(jsCard, js{"entities", "urls"})
result.expandTweetEntities(js)
parseLegacyMediaEntities(js, result)
with jsMedia, js{"extended_entities", "media"}:
for m in jsMedia:
case m{"type"}.getStr
of "photo":
result.photos.add m{"media_url_https"}.getImageStr
of "video":
result.video = some(parseVideo(m))
with user, m{"additional_media_info", "source_user"}:
if user{"id"}.getInt > 0:
result.attribution = some(parseUser(user))
else:
result.attribution = some(parseGraphUser(user))
of "animated_gif":
result.gif = some(parseGif(m))
else: discard
with jsWithheld, js{"withheld_in_countries"}:
let withheldInCountries: seq[string] =
@@ -357,264 +279,242 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
result.text.removeSuffix(" Learn more.")
result.available = false
proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
let intId = if id.len > 0: parseBiggestInt(id) else: 0
result = global.tweets.getOrDefault(id, Tweet(id: intId))
if result.quote.isSome:
let quote = get(result.quote).id
if $quote in global.tweets:
result.quote = some global.tweets[$quote]
else:
result.quote = some Tweet()
if result.retweet.isSome:
let rt = get(result.retweet).id
if $rt in global.tweets:
result.retweet = some finalizeTweet(global, $rt)
else:
result.retweet = some Tweet()
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
let pin = js{"pinEntry", "entry", "entryId"}.getStr
if pin.len == 0: return
let id = pin.getId
if id notin global.tweets: return
global.tweets[id].pinned = true
return finalizeTweet(global, id)
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
result = GlobalObjects()
let
tweets = ? js{"globalObjects", "tweets"}
users = ? js{"globalObjects", "users"}
for k, v in users:
result.users[k] = parseUser(v, k)
for k, v in tweets:
var tweet = parseTweet(v, v{"card"})
if tweet.user.id in result.users:
tweet.user = result.users[tweet.user.id]
result.tweets[k] = tweet
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
if js.kind != JArray or js.len == 0:
return
for i in js:
when T is Tweet:
if res.beginning and i{"pinEntry"}.notNull:
with pin, parsePin(i, global):
res.content.add pin
with r, i{"replaceEntry", "entry"}:
if "top" in r{"entryId"}.getStr:
res.top = r.getCursor
elif "bottom" in r{"entryId"}.getStr:
res.bottom = r.getCursor
proc parseTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let global = parseGlobalObjects(? js)
let instructions = ? js{"timeline", "instructions"}
if instructions.len == 0: return
result.parseInstructions(global, instructions)
var entries: JsonNode
for i in instructions:
if "addEntries" in i:
entries = i{"addEntries", "entries"}
for e in ? entries:
let entry = e{"entryId"}.getStr
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
let tweet = finalizeTweet(global, e.getEntryId)
if not tweet.available: continue
result.content.add tweet
elif "cursor-top" in entry:
result.top = e.getCursor
elif "cursor-bottom" in entry:
result.bottom = e.getCursor
elif entry.startsWith("sq-cursor"):
with cursor, e{"content", "operation", "cursor"}:
if cursor{"cursorType"}.getStr == "Bottom":
result.bottom = cursor{"value"}.getStr
else:
result.top = cursor{"value"}.getStr
proc parsePhotoRail*(js: JsonNode): PhotoRail =
for tweet in js:
let
t = parseTweet(tweet, js{"card"})
url = if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
else: ""
if url.len == 0: continue
result.add GalleryPhoto(url: url, tweetId: $t.id)
proc parseGraphTweet(js: JsonNode): Tweet =
if js.kind == JNull:
return Tweet()
case js.getTypeName:
case js{"__typename"}.getStr
of "TweetUnavailable":
return Tweet()
of "TweetTombstone":
with text, select(js{"tombstone", "richText"}, js{"tombstone", "text"}):
return Tweet(text: text.getTombstone)
return Tweet()
return Tweet(text: js{"tombstone", "text"}.getTombstone)
of "TweetPreviewDisplay":
return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
of "TweetWithVisibilityResults":
return parseGraphTweet(js{"tweet"})
else:
discard
if not js.hasKey("legacy"):
return Tweet()
var jsCard = select(js{"card"}, js{"tweet_card"}, js{"legacy", "tweet_card"})
var jsCard = copy(js{"card", "legacy"})
if jsCard.kind != JNull:
let legacyCard = jsCard{"legacy"}
if legacyCard.kind != JNull:
let bindingArray = legacyCard{"binding_values"}
if bindingArray.kind == JArray:
var bindingObj: seq[(string, JsonNode)]
for item in bindingArray:
bindingObj.add((item{"key"}.getStr, item{"value"}))
# Create a new card object with flattened structure
jsCard = %*{
"name": legacyCard{"name"},
"url": legacyCard{"url"},
"binding_values": %bindingObj
}
var values = newJObject()
for val in jsCard["binding_values"]:
values[val["key"].getStr] = val["value"]
jsCard["binding_values"] = values
result = parseTweet(js{"legacy"}, jsCard)
result.id = js{"rest_id"}.getId
result.user = parseGraphUser(js{"core"})
if result.replyId == 0:
result.replyId = js{"reply_to_results", "rest_id"}.getId
with count, js{"views", "count"}:
result.stats.views = count.getStr("0").parseInt
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
result.expandNoteTweetEntities(noteTweet)
parseMediaEntities(js, result)
if result.quote.isSome:
result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
with quoted, js{"quotedPostResults", "result"}:
result.quote = some(parseGraphTweet(quoted))
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
for t in ? js{"content", "items"}:
let entryId = t.getEntryId
let thread = js{"content", "items"}
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
if "cursor-showmore" in entryId:
let cursor = t{"item", "content", "value"}
let cursor = t{"item", "itemContent", "value"}
result.thread.cursor = cursor.getStr
result.thread.hasMore = true
elif "tweet" in entryId and "promoted" notin entryId:
with tweet, t.getTweetResult("item"):
result.thread.content.add parseGraphTweet(tweet)
elif "tweet" in entryId:
let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
result.thread.content.add tweet
let tweetDisplayType = select(
t{"item", "content", "tweet_display_type"},
t{"item", "itemContent", "tweetDisplayType"}
)
if tweetDisplayType.getStr == "SelfThread":
result.self = true
if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
result.self = true
proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweet_result", "result"}:
with tweet, js{"data", "tweetResult", "result"}:
result = parseGraphTweet(tweet)
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
let instructions = ? select(
js{"data", "timelineResponse", "instructions"},
js{"data", "timeline_response", "instructions"},
js{"data", "threaded_conversation_with_injections_v2", "instructions"}
)
let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
if instructions.len == 0:
return
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
# echo entryId
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("tombstone"):
let id = entryId.getId()
let tweet = Tweet(
id: parseBiggestInt(id),
available: false,
text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
)
if id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
if self:
result.after = thread
else:
result.replies.content.add thread
elif entryId.startsWith("cursor-bottom"):
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let instructions =
if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
if instructions.len == 0:
return
for i in instructions:
if i.getTypeName == "TimelineAddEntries":
if i{"type"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e.getEntryId
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
let tweetResult = getTweetResult(e)
if tweetResult.notNull:
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = entryId.getId
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
if self:
result.after = thread
elif thread.content.len > 0:
result.replies.content.add thread
elif entryId.startsWith("tombstone"):
let
content = select(e{"content", "content"}, e{"content", "itemContent"})
tweet = Tweet(
id: entryId.getId,
available: false,
text: content{"tombstoneInfo", "richText"}.getTombstone
)
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
var cursorValue = select(
e{"content", "value"},
e{"content", "content", "value"},
e{"content", "itemContent", "value"}
)
result.replies.bottom = cursorValue.getStr
result.bottom = e{"content", "value"}.getStr
proc extractTweetsFromEntry*(e: JsonNode): seq[Tweet] =
with tweetResult, getTweetResult(e):
var tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = e.getEntryId.getId
result.add tweet
return
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
for item in e{"content", "items"}:
with tweetResult, item.getTweetResult("item"):
var tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = item.getEntryId.getId
result.add tweet
proc parseGraphTimeline*(js: JsonNode; after=""): Profile =
result = Profile(tweets: Timeline(beginning: after.len == 0))
let instructions = ? select(
js{"data", "list", "timeline_response", "timeline", "instructions"},
js{"data", "user", "result", "timeline", "timeline", "instructions"},
js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
)
if instructions.len == 0:
return
for i in instructions:
if i{"moduleItems"}.notNull:
for item in i{"moduleItems"}:
with tweetResult, item.getTweetResult("item"):
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = item.getEntryId.getId
result.tweets.content.add tweet
continue
if i{"entries"}.notNull:
for e in i{"entries"}:
let entryId = e.getEntryId
if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
for tweet in extractTweetsFromEntry(e):
result.tweets.content.add tweet
elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
let (thread, self) = parseGraphThread(e)
result.tweets.content.add thread.content
elif entryId.startsWith("cursor-bottom"):
result.tweets.bottom = e{"content", "value"}.getStr
if after.len == 0:
if i.getTypeName == "TimelinePinEntry":
let tweets = extractTweetsFromEntry(i{"entry"})
if tweets.len > 0:
var tweet = tweets[0]
tweet.pinned = true
result.pinned = some tweet
proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
result = @[]
let instructions = select(
js{"data", "user", "result", "timeline", "timeline", "instructions"},
js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
)
if instructions.len == 0:
return
for i in instructions:
if i{"moduleItems"}.notNull:
for item in i{"moduleItems"}:
with tweetResult, item.getTweetResult("item"):
let t = parseGraphTweet(tweetResult)
if not t.available:
t.id = item.getEntryId.getId
let photo = extractGalleryPhoto(t)
if photo.url.len > 0:
result.add photo
if result.len == 16:
return
continue
if i.getTypeName != "TimelineAddEntries":
continue
for e in i{"entries"}:
let entryId = e.getEntryId
if entryId.startsWith("tweet") or entryId.startsWith("profile-grid"):
for t in extractTweetsFromEntry(e):
let photo = extractGalleryPhoto(t)
if photo.url.len > 0:
result.add photo
if result.len == 16:
return
proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
result = Result[T](beginning: after.len == 0)
let instructions = select(
js{"data", "search", "timeline_response", "timeline", "instructions"},
js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
)
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
if instructions.len == 0:
return
for instruction in instructions:
let typ = getTypeName(instruction)
let typ = instruction{"type"}.getStr
if typ == "TimelineAddEntries":
for e in instruction{"entries"}:
let entryId = e.getEntryId
when T is Tweets:
if entryId.startsWith("tweet"):
with tweetRes, getTweetResult(e):
let tweet = parseGraphTweet(tweetRes)
if not tweet.available:
tweet.id = entryId.getId
result.content.add tweet
elif T is User:
if entryId.startsWith("user"):
with userRes, e{"content", "itemContent"}:
result.content.add parseGraphUser(userRes)
if entryId.startsWith("cursor-bottom"):
for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):

View File

@@ -1,17 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-only
import std/[times, macros, htmlgen, options, algorithm, re]
import std/strutils except escape
import std/[strutils, times, macros, htmlgen, options, algorithm, re]
import std/unicode except strip
from xmltree import escape
import packedjson
import types, utils, formatters
const
unicodeOpen = "\uFFFA"
unicodeClose = "\uFFFB"
xmlOpen = escape("<")
xmlClose = escape(">")
let
unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
unReplace = "$1<a href=\"/$2\">@$2</a>"
@@ -36,12 +28,6 @@ template `?`*(js: JsonNode): untyped =
if j.isNull: return
j
template select*(a, b: JsonNode): untyped =
if a.notNull: a else: b
template select*(a, b, c: JsonNode): untyped =
if a.notNull: a elif b.notNull: b else: c
template with*(ident, value, body): untyped =
if true:
let ident {.inject.} = value
@@ -59,20 +45,6 @@ template getError*(js: JsonNode): Error =
if js.kind != JArray or js.len == 0: null
else: Error(js[0]{"code"}.getInt)
proc getTweetResult*(js: JsonNode; root="content"): JsonNode =
select(
js{root, "content", "tweet_results", "result"},
js{root, "itemContent", "tweet_results", "result"},
js{root, "content", "tweetResult", "result"}
)
template getTypeName*(js: JsonNode): string =
js{"__typename"}.getStr(js{"type"}.getStr)
template getEntryId*(e: JsonNode): string =
e{"entryId"}.getStr(e{"entry_id"}.getStr)
template parseTime(time: string; f: static string; flen: int): DateTime =
if time.len != flen: return
parse(time, f, utc())
@@ -83,24 +55,29 @@ proc getDateTime*(js: JsonNode): DateTime =
proc getTime*(js: JsonNode): DateTime =
parseTime(js.getStr, "ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
proc getTimeFromMs*(js: JsonNode): DateTime =
let ms = js.getInt(0)
if ms == 0: return
let seconds = ms div 1000
return fromUnix(seconds).utc()
proc getId*(id: string): int64 {.inline.} =
proc getId*(id: string): string {.inline.} =
let start = id.rfind("-")
if start < 0:
return parseBiggestInt(id)
return parseBiggestInt(id[start + 1 ..< id.len])
if start < 0: return id
id[start + 1 ..< id.len]
proc getId*(js: JsonNode): int64 {.inline.} =
case js.kind
of JString: return js.getStr("0").getId
of JString: return parseBiggestInt(js.getStr("0"))
of JInt: return js.getBiggestInt()
else: return 0
proc getEntryId*(js: JsonNode): string {.inline.} =
let entry = js{"entryId"}.getStr
if entry.len == 0: return
if "tweet" in entry or "sq-I-t" in entry:
return entry.getId
elif "tombstone" in entry:
return js{"content", "item", "content", "tombstone", "tweet", "id"}.getStr
else:
echo "unknown entry: ", entry
return
template getStrVal*(js: JsonNode; default=""): string =
js{"string_value"}.getStr(default)
@@ -254,7 +231,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
.replacef(htRegex, htReplace)
proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
replyTo=""; hasRedundantLink=false) =
replyTo=""; hasQuote=false) =
let hasCard = tweet.card.isSome
var replacements = newSeq[ReplaceSlice]()
@@ -265,7 +242,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
if urlStr.len == 0 or urlStr notin text:
continue
replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
if hasCard and u{"url"}.getStr == get(tweet.card).url:
get(tweet.card).url = u{"expanded_url"}.getStr
@@ -305,10 +282,9 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
let
entities = ? js{"entities"}
hasQuote = js{"is_quote_status"}.getBool
textRange = js{"display_text_range"}
textSlice = textRange{0}.getInt .. textRange{1}.getInt
hasQuote = js{"is_quote_status"}.getBool
hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
var replyTo = ""
if tweet.replyId != 0:
@@ -316,24 +292,12 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
replyTo = reply.getStr
tweet.reply.add replyTo
tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
let
entities = ? js{"entity_set"}
text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
text = js{"text"}.getStr
textSlice = 0..text.runeLen
tweet.expandTextEntities(entities, text, textSlice)
tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
proc extractGalleryPhoto*(t: Tweet): GalleryPhoto =
let url =
if t.photos.len > 0: t.photos[0]
elif t.video.isSome: get(t.video).thumb
elif t.gif.isSome: get(t.gif).thumb
elif t.card.isSome: get(t.card).image
else: ""
result = GalleryPhoto(url: url, tweetId: $t.id)

View File

@@ -80,7 +80,7 @@ genPrefs:
Media:
mp4Playback(checkbox, true):
"Enable mp4 video playback (only for gifs)"
"Enable mp4 video playback"
hlsPlayback(checkbox, false):
"Enable HLS video streaming (requires JavaScript)"

View File

@@ -6,9 +6,10 @@ import types
const
validFilters* = @[
"media", "images", "twimg", "videos",
"native_video", "consumer_video", "spaces",
"native_video", "consumer_video", "pro_video",
"links", "news", "quote", "mentions",
"replies", "retweets", "nativeretweets"
"replies", "retweets", "nativeretweets",
"verified", "safe"
]
emptyQuery* = "include:nativeretweets"
@@ -17,11 +18,6 @@ template `@`(param: string): untyped =
if param in pms: pms[param]
else: ""
proc validateNumber(value: string): string =
if value.anyIt(not it.isDigit):
return ""
return value
proc initQuery*(pms: Table[string, string]; name=""): Query =
result = Query(
kind: parseEnum[QueryKind](@"f", tweets),
@@ -30,7 +26,7 @@ proc initQuery*(pms: Table[string, string]; name=""): Query =
excludes: validFilters.filterIt("e-" & it in pms),
since: @"since",
until: @"until",
minLikes: validateNumber(@"min_faves")
near: @"near"
)
if name.len > 0:
@@ -64,7 +60,7 @@ proc genQueryParam*(query: Query): string =
param &= "OR "
if query.fromUser.len > 0 and query.kind in {posts, media}:
param &= "filter:self_threads OR -filter:replies "
param &= "filter:self_threads OR-filter:replies "
if "nativeretweets" notin query.excludes:
param &= "include:nativeretweets "
@@ -82,8 +78,8 @@ proc genQueryParam*(query: Query): string =
result &= " since:" & query.since
if query.until.len > 0:
result &= " until:" & query.until
if query.minLikes.len > 0:
result &= " min_faves:" & query.minLikes
if query.near.len > 0:
result &= &" near:\"{query.near}\" within:15mi"
if query.text.len > 0:
if result.len > 0:
result &= " " & query.text
@@ -107,8 +103,8 @@ proc genQueryUrl*(query: Query): string =
params.add "since=" & query.since
if query.until.len > 0:
params.add "until=" & query.until
if query.minLikes.len > 0:
params.add "min_faves=" & query.minLikes
if query.near.len > 0:
params.add "near=" & query.near
if params.len > 0:
result &= params.join("&")

View File

@@ -52,7 +52,6 @@ proc initRedisPool*(cfg: Config) {.async.} =
await migrate("profileDates", "p:*")
await migrate("profileStats", "p:*")
await migrate("userType", "p:*")
await migrate("verifiedType", "p:*")
pool.withAcquire(r):
# optimize memory usage for user ID buckets
@@ -86,7 +85,7 @@ proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
proc cache*(data: PhotoRail; name: string) {.async.} =
await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
await setEx("pr:" & toLower(name), baseCacheTime, compress(toFlatty(data)))
proc cache*(data: User) {.async.} =
if data.username.len == 0: return
@@ -148,24 +147,24 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
if result.len > 0 and user.id.len > 0:
await all(cacheUserId(result, user.id), cache(user))
# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
# if id == 0: return
# let tweet = await get(id.tweetKey)
# if tweet != redisNil:
# tweet.deserialize(Tweet)
# else:
# result = await getGraphTweetResult($id)
# if not result.isNil:
# await cache(result)
proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
if id == 0: return
let tweet = await get(id.tweetKey)
if tweet != redisNil:
tweet.deserialize(Tweet)
else:
result = await getGraphTweetResult($id)
if not result.isNil:
await cache(result)
proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
if id.len == 0: return
let rail = await get("pr2:" & toLower(id))
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let rail = await get("pr:" & toLower(name))
if rail != redisNil:
rail.deserialize(PhotoRail)
else:
result = await getPhotoRail(id)
await cache(result, id)
result = await getPhotoRail(name)
await cache(result, name)
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
let list = if id.len == 0: redisNil

View File

@@ -1,13 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
import jester
import router_utils
import ".."/[auth, types]
import ".."/[tokens, types]
proc createDebugRouter*(cfg: Config) =
router debug:
get "/.health":
respJson getSessionPoolHealth()
get "/.sessions":
get "/.tokens":
cond cfg.enableDebug
respJson getSessionPoolDebug()
respJson getPoolJson()

View File

@@ -10,22 +10,22 @@ export api, embed, vdom, tweet, general, router_utils
proc createEmbedRouter*(cfg: Config) =
router embed:
get "/i/videos/tweet/@id":
let tweet = await getGraphTweetResult(@"id")
if tweet == nil or tweet.video.isNone:
let convo = await getTweet(@"id")
if convo == nil or convo.tweet == nil or convo.tweet.video.isNone:
resp Http404
resp renderVideoEmbed(tweet, cfg, request)
resp renderVideoEmbed(convo.tweet, cfg, request)
get "/@user/status/@id/embed":
let
tweet = await getGraphTweetResult(@"id")
convo = await getTweet(@"id")
prefs = cookiePrefs()
path = getPath()
if tweet == nil:
if convo == nil or convo.tweet == nil:
resp Http404
resp renderTweetEmbed(tweet, path, prefs, cfg, request)
resp renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
get "/embed/Tweet.html":
let id = @"id"

View File

@@ -12,7 +12,8 @@ export httpclient, os, strutils, asyncstreams, base64, re
const
m3u8Mime* = "application/vnd.apple.mpegurl"
maxAge* = "max-age=604800"
mp4Mime* = "video/mp4"
maxAge* = "public, max-age=604800, must-revalidate"
proc safeFetch*(url: string): Future[string] {.async.} =
let client = newAsyncHttpClient()
@@ -20,59 +21,84 @@ proc safeFetch*(url: string): Future[string] {.async.} =
except: discard
finally: client.close()
template respond*(req: asynchttpserver.Request; headers) =
var msg = "HTTP/1.1 200 OK\c\L"
for k, v in headers:
template respond*(req: asynchttpserver.Request; code: HttpCode;
headers: seq[(string, string)]) =
var msg = "HTTP/1.1 " & $code & "\c\L"
for (k, v) in headers:
msg.add(k & ": " & v & "\c\L")
msg.add "\c\L"
yield req.client.send(msg)
yield req.client.send(msg, flags={})
proc getContentLength(res: AsyncResponse): string =
result = "0"
if res.headers.hasKey("content-length"):
result = $res.contentLength
elif res.headers.hasKey("content-range"):
result = res.headers["content-range"]
result = result[result.find('/') + 1 .. ^1]
if result == "*":
result.setLen(0)
proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
result = Http200
let
request = req.getNativeReq()
client = newAsyncHttpClient()
hashed = $hash(url)
if request.headers.getOrDefault("If-None-Match") == hashed:
return Http304
let c = newAsyncHttpClient(headers=newHttpHeaders({
"accept": "*/*",
"range": $req.headers.getOrDefault("range")
}))
try:
let res = await client.get(url)
if res.status != "200 OK":
if res.status != "404 Not Found":
echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
var res = await c.get(url)
if not res.status.startsWith("20"):
return Http404
let hashed = $hash(url)
if request.headers.getOrDefault("If-None-Match") == hashed:
return Http304
var headers = @{
"accept-ranges": "bytes",
"content-type": $res.headers.getOrDefault("content-type"),
"cache-control": maxAge,
"age": $res.headers.getOrDefault("age"),
"date": $res.headers.getOrDefault("date"),
"last-modified": $res.headers.getOrDefault("last-modified")
}
let contentLength =
if res.headers.hasKey("content-length"):
res.headers["content-length", 0]
else:
""
var tries = 0
while tries <= 10 and res.headers.hasKey("transfer-encoding"):
await sleepAsync(100 + tries * 200)
res = await c.get(url)
tries.inc
let headers = newHttpHeaders({
"Content-Type": res.headers["content-type", 0],
"Content-Length": contentLength,
"Cache-Control": maxAge,
"ETag": hashed
})
let contentLength = res.getContentLength
if contentLength.len > 0:
headers.add ("content-length", contentLength)
respond(request, headers)
if res.headers.hasKey("content-range"):
headers.add ("content-range", $res.headers.getOrDefault("content-range"))
respond(request, Http206, headers)
else:
respond(request, Http200, headers)
var (hasValue, data) = (true, "")
while hasValue:
(hasValue, data) = await res.bodyStream.read()
if hasValue:
await request.client.send(data)
await request.client.send(data, flags={})
data.setLen 0
except HttpRequestError, ProtocolError, OSError:
echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
except OSError: discard
except ProtocolError, HttpRequestError:
result = Http404
finally:
client.close()
c.close()
template check*(code): untyped =
template check*(c): untyped =
let code = c
if code != Http200:
resp code
else:
@@ -86,48 +112,37 @@ proc decoded*(req: jester.Request; index: int): string =
if based: decode(encoded)
else: decodeUrl(encoded)
proc getPicUrl*(req: jester.Request): string =
result = decoded(req, 1)
if "twimg.com" notin result:
result.insert(twimg)
if not result.startsWith(https):
result.insert(https)
proc createMediaRouter*(cfg: Config) =
router media:
get "/pic/?":
resp Http404
get re"^\/pic\/orig\/(enc)?\/?(.+)":
var url = decoded(request, 1)
if "twimg.com" notin url:
url.insert(twimg)
if not url.startsWith(https):
url.insert(https)
url.add("?name=orig")
let uri = parseUri(url)
cond isTwitterUrl(uri) == true
let code = await proxyMedia(request, url)
check code
let url = getPicUrl(request)
cond isTwitterUrl(parseUri(url)) == true
check await proxyMedia(request, url & "?name=orig")
get re"^\/pic\/(enc)?\/?(.+)":
var url = decoded(request, 1)
if "twimg.com" notin url:
url.insert(twimg)
if not url.startsWith(https):
url.insert(https)
let uri = parseUri(url)
cond isTwitterUrl(uri) == true
let code = await proxyMedia(request, url)
check code
let url = getPicUrl(request)
cond isTwitterUrl(parseUri(url)) == true
check await proxyMedia(request, url)
get re"^\/video\/(enc)?\/?(.+)\/(.+)$":
let url = decoded(request, 2)
cond "http" in url
if getHmac(url) != request.matches[1]:
resp Http403, showError("Failed to verify signature", cfg)
resp showError("Failed to verify signature", cfg)
if ".mp4" in url or ".ts" in url or ".m4s" in url:
let code = await proxyMedia(request, url)
check code
check await proxyMedia(request, url)
var content: string
if ".vmap" in url:

View File

@@ -23,16 +23,18 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
names = getNames(name)
if names.len == 1:
profile = await fetchProfile(after, query, skipRail=true)
profile = await fetchProfile(after, query, skipRail=true, skipPinned=true)
else:
var q = query
q.fromUser = names
profile.tweets = await getGraphTweetSearch(q, after)
# this is kinda dumb
profile.user = User(
username: name,
fullname: names.join(" | "),
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
profile = Profile(
tweets: await getGraphSearch(q, after),
# this is kinda dumb
user: User(
username: name,
fullname: names.join(" | "),
userpic: "https://abs.twimg.com/sticky/default_profile_images/default_profile.png"
)
)
if profile.user.suspended:
@@ -76,7 +78,7 @@ proc createRssRouter*(cfg: Config) =
if rss.cursor.len > 0:
respRss(rss, "Search")
let tweets = await getGraphTweetSearch(query, cursor)
let tweets = await getGraphSearch(query, cursor)
rss.cursor = tweets.bottom
rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)

View File

@@ -29,13 +29,13 @@ proc createSearchRouter*(cfg: Config) =
redirect("/" & q)
var users: Result[User]
try:
users = await getGraphUserSearch(query, getCursor())
users = await getUserSearch(query, getCursor())
except InternalError:
users = Result[User](beginning: true, query: query)
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
of tweets:
let
tweets = await getGraphTweetSearch(query, getCursor())
tweets = await getGraphSearch(query, getCursor())
rss = "/search/rss?" & genQueryUrl(query)
resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
request, cfg, prefs, title, rss=rss)

View File

@@ -31,6 +31,8 @@ proc createStatusRouter*(cfg: Config) =
resp $renderReplies(replies, prefs, getPath())
let conv = await getTweet(id, getCursor())
if conv == nil:
echo "nil conv"
if conv == nil or conv.tweet == nil or conv.tweet.id == 0:
var error = "Tweet not found"
@@ -66,7 +68,7 @@ proc createStatusRouter*(cfg: Config) =
get "/@name/@s/@id/@m/?@i?":
cond @"s" in ["status", "statuses"]
cond @"m" in ["video", "photo", "history"]
cond @"m" in ["video", "photo"]
redirect("/$1/status/$2" % [@"name", @"id"])
get "/@name/statuses/@id/?":
@@ -74,6 +76,6 @@ proc createStatusRouter*(cfg: Config) =
get "/i/web/status/@id":
redirect("/i/status/" & @"id")
get "/@name/thread/@id/?":
redirect("/$1/status/$2" % [@"name", @"id"])

View File

@@ -27,7 +27,8 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] =
else:
body
proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile] {.async.} =
proc fetchProfile*(after: string; query: Query; skipRail=false;
skipPinned=false): Future[Profile] {.async.} =
let
name = query.fromUser[0]
userId = await getUserId(name)
@@ -44,21 +45,37 @@ proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile]
after.setLen 0
let
timeline =
case query.kind
of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
of media: getGraphUserTweets(userId, TimelineKind.media, after)
else: getGraphSearch(query, after)
rail =
skipIf(skipRail or query.kind == media, @[]):
getCachedPhotoRail(userId)
getCachedPhotoRail(name)
user = getCachedUser(name)
user = await getCachedUser(name)
result =
case query.kind
of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
of media: await getGraphUserTweets(userId, TimelineKind.media, after)
else: Profile(tweets: await getGraphTweetSearch(query, after))
var pinned: Option[Tweet]
if not skipPinned and user.pinnedTweet > 0 and
after.len == 0 and query.kind in {posts, replies}:
let tweet = await getCachedTweet(user.pinnedTweet)
if not tweet.isNil:
tweet.pinned = true
tweet.user = user
pinned = some tweet
result.user = await user
result.photoRail = await rail
result = Profile(
user: user,
pinned: pinned,
tweets: await timeline,
photoRail: await rail
)
if result.user.protected or result.user.suspended:
return
result.tweets.query = query
@@ -66,11 +83,11 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
rss, after: string): Future[string] {.async.} =
if query.fromUser.len != 1:
let
timeline = await getGraphTweetSearch(query, after)
timeline = await getGraphSearch(query, after)
html = renderTweetSearch(timeline, prefs, getPath())
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
var profile = await fetchProfile(after, query)
var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins)
template u: untyped = profile.user
if u.suspended:
@@ -105,12 +122,6 @@ proc createTimelineRouter*(cfg: Config) =
get "/intent/user":
respUserId()
get "/intent/follow/?":
let username = request.params.getOrDefault("screen_name")
if username.len == 0:
resp Http400, showError("Missing screen_name parameter", cfg)
redirect("/" & username)
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
@@ -127,7 +138,7 @@ proc createTimelineRouter*(cfg: Config) =
# used for the infinite scroll feature
if @"scroll".len > 0:
if query.fromUser.len != 1:
var timeline = await getGraphTweetSearch(query, after)
var timeline = await getGraphSearch(query, after)
if timeline.content.len == 0: resp Http404
timeline.beginning = true
resp $renderTweetSearch(timeline, prefs, getPath())

View File

@@ -17,7 +17,7 @@ proc createUnsupportedRouter*(cfg: Config) =
get "/@name/lists/?": feature()
get "/intent/?@i?":
cond @"i" notin ["user", "follow"]
cond @"i" notin ["user"]
feature()
get "/i/@i?/?@j?":

View File

@@ -66,7 +66,18 @@
}
#search-panel-toggle:checked ~ .search-panel {
max-height: 380px !important;
@if $rows == 6 {
max-height: 200px !important;
}
@if $rows == 5 {
max-height: 300px !important;
}
@if $rows == 4 {
max-height: 300px !important;
}
@if $rows == 3 {
max-height: 365px !important;
}
}
}
}

View File

@@ -28,8 +28,6 @@ $more_replies_dots: #AD433B;
$error_red: #420A05;
$verified_blue: #1DA1F2;
$verified_business: #FAC82B;
$verified_government: #C1B6A4;
$icon_text: $fg_color;
$tab: $fg_color;

View File

@@ -39,8 +39,6 @@ body {
--error_red: #{$error_red};
--verified_blue: #{$verified_blue};
--verified_business: #{$verified_business};
--verified_government: #{$verified_government};
--icon_text: #{$icon_text};
--tab: #{$fg_color};
@@ -51,7 +49,7 @@ body {
background-color: var(--bg_color);
color: var(--fg_color);
font-family: $font_0, $font_1, $font_2, $font_3;
font-size: 15px;
font-size: 14px;
line-height: 1.3;
margin: 0;
}
@@ -143,30 +141,17 @@ ul {
.verified-icon {
color: var(--icon_text);
background-color: var(--verified_blue);
border-radius: 50%;
flex-shrink: 0;
margin: 2px 0 3px 3px;
padding-top: 3px;
height: 11px;
padding-top: 2px;
height: 12px;
width: 14px;
font-size: 8px;
display: inline-block;
text-align: center;
vertical-align: middle;
&.blue {
background-color: var(--verified_blue);
}
&.business {
color: var(--bg_panel);
background-color: var(--verified_business);
}
&.government {
color: var(--bg_panel);
background-color: var(--verified_government);
}
}
@media(max-width: 600px) {

View File

@@ -14,7 +14,6 @@ button {
input[type="text"],
input[type="date"],
input[type="number"],
select {
@include input-colors;
background-color: var(--bg_elements);
@@ -25,12 +24,7 @@ select {
font-size: 14px;
}
input[type="number"] {
-moz-appearance: textfield;
}
input[type="text"],
input[type="number"] {
input[type="text"] {
height: 16px;
}
@@ -44,17 +38,6 @@ input[type="date"]::-webkit-inner-spin-button {
display: none;
}
input[type="number"] {
-moz-appearance: textfield;
}
input[type="number"]::-webkit-inner-spin-button,
input[type="number"]::-webkit-outer-spin-button {
display: none;
-webkit-appearance: none;
margin: 0;
}
input[type="date"]::-webkit-clear-button {
margin-left: 17px;
filter: grayscale(100%);
@@ -181,8 +164,7 @@ input::-webkit-datetime-edit-year-field:focus {
appearance: none;
}
input[type="text"],
input[type="number"] {
input[type="text"] {
position: absolute;
right: 0;
max-width: 140px;

View File

@@ -70,9 +70,8 @@ nav {
.lp {
height: 14px;
display: inline-block;
position: relative;
top: 2px;
margin-top: 2px;
display: block;
fill: var(--fg_nav);
&:hover {

View File

@@ -115,7 +115,7 @@
}
.profile-card-tabs-name {
flex-shrink: 100;
@include breakable;
}
.profile-card-avatar {

View File

@@ -14,8 +14,6 @@
button {
margin: 0 2px 0 0;
height: 23px;
display: flex;
align-items: center;
}
.pref-input {
@@ -24,8 +22,7 @@
height: 23px;
}
input[type="text"],
input[type="number"] {
input[type="text"] {
height: calc(100% - 4px);
width: calc(100% - 8px);
}
@@ -43,7 +40,7 @@
@include input-colors;
}
@include create-toggle(search-panel, 380px);
@include create-toggle(search-panel, 200px);
}
.search-panel {
@@ -105,18 +102,19 @@
.search-toggles {
flex-grow: 1;
display: grid;
grid-template-columns: repeat(5, auto);
grid-template-columns: repeat(6, auto);
grid-column-gap: 10px;
}
.profile-tabs {
@include search-resize(820px, 5);
@include search-resize(715px, 4);
@include search-resize(700px, 5);
@include search-resize(485px, 4);
@include search-resize(725px, 4);
@include search-resize(600px, 6);
@include search-resize(560px, 5);
@include search-resize(480px, 4);
@include search-resize(410px, 3);
}
@include search-resize(700px, 5);
@include search-resize(485px, 4);
@include search-resize(560px, 5);
@include search-resize(480px, 4);
@include search-resize(410px, 3);

View File

@@ -42,7 +42,6 @@
.card-description {
margin: 0.3em 0;
white-space: pre-wrap;
}
.card-destination {

View File

@@ -110,29 +110,3 @@
margin-left: 58px;
padding: 7px 0;
}
.timeline-item.thread.more-replies-thread {
padding: 0 0.75em;
&::before {
top: 40px;
margin-bottom: 31px;
}
.more-replies {
display: flex;
padding-top: unset !important;
margin-top: 8px;
&::before {
display: inline-block;
position: relative;
top: -1px;
line-height: 0.4em;
}
.more-replies-text {
display: inline;
}
}
}

View File

@@ -16,8 +16,6 @@ video {
}
.video-container {
min-height: 80px;
min-width: 200px;
max-height: 530px;
margin: 0;
display: flex;

164
src/tokens.nim Normal file
View File

@@ -0,0 +1,164 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, httpclient, times, sequtils, json, random
import strutils, tables
import types, consts
const
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
failDelay = initDuration(minutes=30)
var
tokenPool: seq[Token]
lastFailed: Time
enableLogging = false
let headers = newHttpHeaders({"authorization": auth})
template log(str) =
if enableLogging: echo "[tokens] ", str
proc getPoolJson*(): JsonNode =
var
list = newJObject()
totalReqs = 0
totalPending = 0
reqsPerApi: Table[string, int]
for token in tokenPool:
totalPending.inc(token.pending)
list[token.tok] = %*{
"apis": newJObject(),
"pending": token.pending,
"init": $token.init,
"lastUse": $token.lastUse
}
for api in token.apis.keys:
list[token.tok]["apis"][$api] = %token.apis[api]
let
maxReqs =
case api
of Api.timeline: 187
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
Api.tweetDetail, Api.tweetResult, Api.search: 500
of Api.userSearch: 900
reqs = maxReqs - token.apis[api].remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
totalReqs.inc(reqs)
return %*{
"amount": tokenPool.len,
"requests": totalReqs,
"pending": totalPending,
"apis": reqsPerApi,
"tokens": list
}
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
proc fetchToken(): Future[Token] {.async.} =
if getTime() - lastFailed < failDelay:
raise rateLimitError()
let client = newAsyncHttpClient(headers=headers)
try:
let
resp = await client.postContent(activate)
tokNode = parseJson(resp)["guest_token"]
tok = tokNode.getStr($(tokNode.getInt))
time = getTime()
return Token(tok: tok, init: time, lastUse: time)
except Exception as e:
echo "[tokens] fetching token failed: ", e.msg
if "Try again" notin e.msg:
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
lastFailed = getTime()
finally:
client.close()
proc expired(token: Token): bool =
let time = getTime()
token.init < time - maxAge or token.lastUse < time - maxLastUse
proc isLimited(token: Token; api: Api): bool =
if token.isNil or token.expired:
return true
if api in token.apis:
let limit = token.apis[api]
return (limit.remaining <= 10 and limit.reset > epochTime().int)
else:
return false
proc isReady(token: Token; api: Api): bool =
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
proc release*(token: Token; used=false; invalid=false) =
if token.isNil: return
if invalid or token.expired:
if invalid: log "discarding invalid token"
elif token.expired: log "discarding expired token"
let idx = tokenPool.find(token)
if idx > -1: tokenPool.delete(idx)
elif used:
dec token.pending
token.lastUse = getTime()
proc getToken*(api: Api): Future[Token] {.async.} =
for i in 0 ..< tokenPool.len:
if result.isReady(api): break
release(result)
result = tokenPool.sample()
if not result.isReady(api):
release(result)
result = await fetchToken()
log "added new token to pool"
tokenPool.add result
if not result.isNil:
inc result.pending
else:
raise rateLimitError()
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
# avoid undefined behavior in race conditions
if api in token.apis:
let limit = token.apis[api]
if limit.reset >= reset and limit.remaining < remaining:
return
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
proc poolTokens*(amount: int) {.async.} =
var futs: seq[Future[Token]]
for i in 0 ..< amount:
futs.add fetchToken()
for token in futs:
var newToken: Token
try: newToken = await token
except: discard
if not newToken.isNil:
log "added new token to pool"
tokenPool.add newToken
proc initTokenPool*(cfg: Config) {.async.} =
enableLogging = cfg.enableDebug
while true:
if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
await sleepAsync(2000)

View File

@@ -1,22 +1,25 @@
# SPDX-License-Identifier: AGPL-3.0-only
import times, sequtils, options, tables, uri
import times, sequtils, options, tables
import prefs_impl
genPrefsType()
type
RateLimitError* = object of CatchableError
NoSessionsError* = object of CatchableError
InternalError* = object of CatchableError
BadClientError* = object of CatchableError
TimelineKind* {.pure.} = enum
tweets, replies, media
tweets
replies
media
Api* {.pure.} = enum
tweetDetail
tweetResult
timeline
search
userSearch
list
listBySlug
listMembers
@@ -28,64 +31,37 @@ type
userMedia
RateLimit* = object
limit*: int
remaining*: int
reset*: int
SessionKind* = enum
oauth
cookie
Session* = ref object
id*: int64
username*: string
Token* = ref object
tok*: string
init*: Time
lastUse*: Time
pending*: int
limited*: bool
limitedAt*: int
apis*: Table[Api, RateLimit]
case kind*: SessionKind
of oauth:
oauthToken*: string
oauthSecret*: string
of cookie:
authToken*: string
ct0*: string
SessionAwareUrl* = object
oauthUrl*: Uri
cookieUrl*: Uri
Error* = enum
null = 0
noUserMatches = 17
protectedUser = 22
missingParams = 25
timeout = 29
couldntAuth = 32
doesntExist = 34
unauthorized = 37
invalidParam = 47
userNotFound = 50
suspended = 63
rateLimited = 88
expiredToken = 89
invalidToken = 89
listIdOrSlug = 112
tweetNotFound = 144
tweetNotAuthorized = 179
forbidden = 200
badRequest = 214
badToken = 239
locked = 326
noCsrf = 353
tweetUnavailable = 421
tweetCensored = 422
VerifiedType* = enum
none = "None"
blue = "Blue"
business = "Business"
government = "Government"
User* = object
id*: string
username*: string
@@ -101,7 +77,7 @@ type
tweets*: int
likes*: int
media*: int
verifiedType*: VerifiedType
verified*: bool
protected*: bool
suspended*: bool
joinDate*: DateTime
@@ -121,6 +97,7 @@ type
durationMs*: int
url*: string
thumb*: string
views*: string
available*: bool
reason*: string
title*: string
@@ -140,7 +117,7 @@ type
fromUser*: seq[string]
since*: string
until*: string
minLikes*: string
near*: string
sep*: string
Gif* = object
@@ -184,10 +161,9 @@ type
imageDirectMessage = "image_direct_message"
audiospace = "audiospace"
newsletterPublication = "newsletter_publication"
jobDetails = "job_details"
hidden
unknown
Card* = object
kind*: CardKind
url*: string
@@ -201,7 +177,7 @@ type
replies*: int
retweets*: int
likes*: int
views*: int
quotes*: int
Tweet* = ref object
id*: int64
@@ -229,8 +205,6 @@ type
video*: Option[Video]
photos*: seq[string]
Tweets* = seq[Tweet]
Result*[T] = object
content*: seq[T]
top*, bottom*: string
@@ -238,7 +212,7 @@ type
query*: Query
Chain* = object
content*: Tweets
content*: seq[Tweet]
hasMore*: bool
cursor*: string
@@ -248,7 +222,7 @@ type
after*: Chain
replies*: Result[Chain]
Timeline* = Result[Tweets]
Timeline* = Result[Tweet]
Profile* = object
user*: User
@@ -300,6 +274,3 @@ type
proc contains*(thread: Chain; tweet: Tweet): bool =
thread.content.anyIt(it.id == tweet.id)
proc add*(timeline: var seq[Tweets]; tweet: Tweet) =
timeline.add @[tweet]

View File

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, uri, tables, base64
import nimcrypto
import types
var
hmacKey: string
@@ -16,8 +17,7 @@ const
"twimg.com",
"abs.twimg.com",
"pbs.twimg.com",
"video.twimg.com",
"x.com"
"video.twimg.com"
]
proc setHmacKey*(key: string) =
@@ -29,6 +29,20 @@ proc setProxyEncoding*(state: bool) =
proc getHmac*(data: string): string =
($hmac(sha256, hmacKey, data))[0 .. 12]
proc getBestMp4VidVariant(video: Video): VideoVariant =
for v in video.variants:
if v.bitrate >= result.bitrate:
result = v
proc getVidVariant*(video: Video; playbackType: VideoType): VideoVariant =
case playbackType
of mp4:
return video.getBestMp4VidVariant
of m3u8, vmap:
for variant in video.variants:
if variant.contentType == playbackType:
return variant
proc getVidUrl*(link: string): string =
if link.len == 0: return
let sig = getHmac(link)
@@ -58,4 +72,4 @@ proc isTwitterUrl*(uri: Uri): bool =
uri.hostname in twitterDomains
proc isTwitterUrl*(url: string): bool =
isTwitterUrl(parseUri(url))
parseUri(url).hostname in twitterDomains

View File

@@ -11,7 +11,7 @@ const doctype = "<!DOCTYPE html>\n"
proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
let thumb = get(tweet.video).thumb
let vidUrl = getVideoEmbed(cfg, tweet.id)
let prefs = Prefs(hlsPlayback: true, mp4Playback: true)
let prefs = Prefs(hlsPlayback: true)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))

View File

@@ -30,15 +30,15 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode =
tdiv(class="nav-item right"):
icon "search", title="Search", href="/search"
if cfg.enableRss and rss.len > 0:
icon "rss", title="RSS Feed", href=rss
icon "bird", title="Open in X", href=canonical
icon "rss-feed", title="RSS Feed", href=rss
icon "bird", title="Open in Twitter", href=canonical
a(href="https://liberapay.com/zedeus"): verbatim lp
icon "info", title="About", href="/about"
icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path))
proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
video=""; images: seq[string] = @[]; banner=""; ogTitle="";
rss=""; alternate=""): VNode =
rss=""; canonical=""): VNode =
var theme = prefs.theme.toTheme
if "theme" in req.params:
theme = req.params["theme"].toTheme
@@ -52,8 +52,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
link(rel="stylesheet", type="text/css", href="/css/style.css?v=21")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=3")
link(rel="stylesheet", type="text/css", href="/css/style.css?v=18")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
if theme.len > 0:
link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css"))
@@ -66,14 +66,14 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
link(rel="search", type="application/opensearchdescription+xml", title=cfg.title,
href=opensearchUrl)
if alternate.len > 0:
link(rel="alternate", href=alternate, title="View on X")
if canonical.len > 0:
link(rel="canonical", href=canonical)
if cfg.enableRss and rss.len > 0:
link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed")
if prefs.hlsPlayback:
script(src="/js/hls.min.js", `defer`="")
script(src="/js/hls.light.min.js", `defer`="")
script(src="/js/hlsPlayback.js", `defer`="")
if prefs.infiniteScroll:
@@ -119,20 +119,20 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
# this is last so images are also preloaded
# if this is done earlier, Chrome only preloads one image for some reason
link(rel="preload", type="font/woff2", `as`="font",
href="/fonts/fontello.woff2?61663884", crossorigin="anonymous")
href="/fonts/fontello.woff2?21002321", crossorigin="anonymous")
proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
titleText=""; desc=""; ogTitle=""; rss=""; video="";
images: seq[string] = @[]; banner=""): string =
let twitterLink = getTwitterLink(req.path, req.params)
let canonical = getTwitterLink(req.path, req.params)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,
rss, twitterLink)
rss, canonical)
body:
renderNavbar(cfg, req, rss, twitterLink)
renderNavbar(cfg, req, rss, canonical)
tdiv(class="container"):
body

View File

@@ -23,13 +23,6 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
if text.len > 0:
text " " & text
template verifiedIcon*(user: User): untyped {.dirty.} =
if user.verifiedType != VerifiedType.none:
let lower = ($user.verifiedType).toLowerAscii()
icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
else:
text ""
proc linkUser*(user: User, class=""): VNode =
let
isName = "username" notin class
@@ -39,11 +32,11 @@ proc linkUser*(user: User, class=""): VNode =
buildHtml(a(href=href, class=class, title=nameText)):
text nameText
if isName:
verifiedIcon(user)
if user.protected:
text " "
icon "lock", title="Protected account"
if isName and user.verified:
icon "ok", class="verified-icon", title="Verified account"
if isName and user.protected:
text " "
icon "lock", title="Protected account"
proc linkText*(text: string; class=""): VNode =
let url = if "http" notin text: https & text else: text
@@ -89,16 +82,9 @@ proc genDate*(pref, state: string): VNode =
input(name=pref, `type`="date", value=state)
icon "calendar"
proc genNumberInput*(pref, label, state, placeholder: string; class=""; autofocus=true; min="0"): VNode =
let p = placeholder
buildHtml(tdiv(class=("pref-group pref-input " & class))):
if label.len > 0:
label(`for`=pref): text label
input(name=pref, `type`="number", placeholder=p, value=state, autofocus=(autofocus and state.len == 0), min=min, step="1")
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="", loading="lazy")
img(src=getPicUrl(url), class=class, alt="", loading="lazy", decoding="async")
proc getTabClass*(query: Query; tab: QueryKind): string =
if query.kind == tab: "tab-item active"

View File

@@ -25,25 +25,7 @@
#end proc
#
#proc getDescription(desc: string; cfg: Config): string =
Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
#end proc
#
#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
#result = profile.tweets.content
#if profile.pinned.isSome and result.len > 0:
# let pinnedTweet = profile.pinned.get
# var inserted = false
# for threadIdx in 0 ..< result.len:
# if not inserted:
# for tweetIdx in 0 ..< result[threadIdx].len:
# if result[threadIdx][tweetIdx].id < pinnedTweet.id:
# result[threadIdx].insert(pinnedTweet, tweetIdx)
# inserted = true
# end if
# end for
# end if
# end for
#end if
Twitter feed for: ${desc}. Generated by ${cfg.hostname}
#end proc
#
#proc renderRssTweet(tweet: Tweet; cfg: Config): string =
@@ -51,15 +33,16 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
#let urlPrefix = getUrlPrefix(cfg)
#let text = replaceUrls(tweet.text, defaultPrefs, absolute=urlPrefix)
<p>${text.replace("\n", "<br>\n")}</p>
#if tweet.quote.isSome and get(tweet.quote).available:
# let quoteLink = getLink(get(tweet.quote))
<p><a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></p>
#end if
#if tweet.photos.len > 0:
# for photo in tweet.photos:
<img src="${urlPrefix}${getPicUrl(photo)}" style="max-width:250px;" />
# end for
#elif tweet.video.isSome:
<a href="${urlPrefix}${tweet.getLink}">
<br>Video<br>
<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
</a>
<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
#elif tweet.gif.isSome:
# let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
# let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
@@ -71,45 +54,26 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
# end if
#end if
#if tweet.quote.isSome and get(tweet.quote).available:
# let quoteTweet = get(tweet.quote)
# let quoteLink = urlPrefix & getLink(quoteTweet)
<hr/>
<blockquote>
<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
<p>
${renderRssTweet(quoteTweet, cfg)}
</p>
<footer>
— <cite><a href="${quoteLink}">${quoteLink}</a>
</footer>
</blockquote>
#end if
#end proc
#
#proc renderRssTweets(tweets: seq[Tweets]; cfg: Config; userId=""): string =
#proc renderRssTweets(tweets: seq[Tweet]; cfg: Config): string =
#let urlPrefix = getUrlPrefix(cfg)
#var links: seq[string]
#for thread in tweets:
# for tweet in thread:
# if userId.len > 0 and tweet.user.id != userId: continue
# end if
#
# let retweet = if tweet.retweet.isSome: tweet.user.username else: ""
# let tweet = if retweet.len > 0: tweet.retweet.get else: tweet
# let link = getLink(tweet)
# if link in links: continue
# end if
# links.add link
<item>
<title>${getTitle(tweet, retweet)}</title>
<dc:creator>@${tweet.user.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>${urlPrefix & link}</guid>
<link>${urlPrefix & link}</link>
</item>
# end for
#for t in tweets:
# let retweet = if t.retweet.isSome: t.user.username else: ""
# let tweet = if retweet.len > 0: t.retweet.get else: t
# let link = getLink(tweet)
# if link in links: continue
# end if
# links.add link
<item>
<title>${getTitle(tweet, retweet)}</title>
<dc:creator>@${tweet.user.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>${urlPrefix & link}</guid>
<link>${urlPrefix & link}</link>
</item>
#end for
#end proc
#
@@ -137,15 +101,14 @@ ${renderRssTweet(quoteTweet, cfg)}
<width>128</width>
<height>128</height>
</image>
#let tweetsList = getTweetsWithPinned(profile)
#if tweetsList.len > 0:
${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
#if profile.tweets.content.len > 0:
${renderRssTweets(profile.tweets.content, cfg)}
#end if
</channel>
</rss>
#end proc
#
#proc renderListRss*(tweets: seq[Tweets]; list: List; cfg: Config): string =
#proc renderListRss*(tweets: seq[Tweet]; list: List; cfg: Config): string =
#let link = &"{getUrlPrefix(cfg)}/i/lists/{list.id}"
#result = ""
<?xml version="1.0" encoding="UTF-8"?>
@@ -162,7 +125,7 @@ ${renderRssTweets(tweets, cfg)}
</rss>
#end proc
#
#proc renderSearchRss*(tweets: seq[Tweets]; name, param: string; cfg: Config): string =
#proc renderSearchRss*(tweets: seq[Tweet]; name, param: string; cfg: Config): string =
#let link = &"{getUrlPrefix(cfg)}/search"
#let escName = xmltree.escape(name)
#result = ""

View File

@@ -10,21 +10,23 @@ const toggles = {
"media": "Media",
"videos": "Videos",
"news": "News",
"verified": "Verified",
"native_video": "Native videos",
"replies": "Replies",
"links": "Links",
"images": "Images",
"safe": "Safe",
"quote": "Quotes",
"spaces": "Spaces"
"pro_video": "Pro videos"
}.toOrderedTable
proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"):
form(`method`="get", action="/search", autocomplete="off"):
hiddenField("f", "tweets")
hiddenField("f", "users")
input(`type`="text", name="q", autofocus="",
placeholder="Search...", dir="auto")
placeholder="Enter username...", dir="auto")
button(`type`="submit"): icon "search"
proc renderProfileTabs*(query: Query; username: string): VNode =
@@ -51,7 +53,7 @@ proc renderSearchTabs*(query: Query): VNode =
proc isPanelOpen(q: Query): bool =
q.fromUser.len == 0 and (q.filters.len > 0 or q.excludes.len > 0 or
@[q.minLikes, q.until, q.since].anyIt(it.len > 0))
@[q.near, q.until, q.since].anyIt(it.len > 0))
proc renderSearchPanel*(query: Query): VNode =
let user = query.fromUser.join(",")
@@ -83,10 +85,10 @@ proc renderSearchPanel*(query: Query): VNode =
span(class="search-title"): text "-"
genDate("until", query.until)
tdiv:
span(class="search-title"): text "Minimum likes"
genNumberInput("min_faves", "", query.minLikes, "Number...", autofocus=false)
span(class="search-title"): text "Near"
genInput("near", "", query.near, "Location...", autofocus=false)
proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string;
proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
let query = results.query
buildHtml(tdiv(class="timeline-container")):

View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, algorithm, uri, options
import strutils, strformat, sequtils, algorithm, uri, options
import karax/[karaxdsl, vdom]
import ".."/[types, query, formatters]
@@ -39,24 +39,26 @@ proc renderNoneFound(): VNode =
h2(class="timeline-none"):
text "No items found"
proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode =
proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
buildHtml(tdiv(class="thread-line")):
let sortedThread = thread.sortedByIt(it.id)
for i, tweet in sortedThread:
# thread has a gap, display "more replies" link
if i > 0 and tweet.replyId != sortedThread[i - 1].id:
tdiv(class="timeline-item thread more-replies-thread"):
tdiv(class="more-replies"):
a(class="more-replies-text", href=getLink(tweet)):
text "more replies"
let show = i == thread.high and sortedThread[0].id != tweet.threadId
let header = if tweet.pinned or tweet.retweet.isSome: "with-header " else: ""
renderTweet(tweet, prefs, path, class=(header & "thread"),
index=i, last=(i == thread.high), showThread=show)
proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] =
result = @[it]
if it.retweet.isSome or it.replyId in threads: return
for t in tweets:
if t.id == result[0].replyId:
result.insert t
elif t.replyId == result[0].id:
result.add t
proc renderUser(user: User; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-item", data-username=user.username)):
buildHtml(tdiv(class="timeline-item")):
a(class="tweet-link", href=("/" & user.username))
tdiv(class="tweet-body profile-result"):
tdiv(class="tweet-header"):
@@ -87,7 +89,7 @@ proc renderTimelineUsers*(results: Result[User]; prefs: Prefs; path=""): VNode =
else:
renderNoMore()
proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string;
pinned=none(Tweet)): VNode =
buildHtml(tdiv(class="timeline")):
if not results.beginning:
@@ -103,26 +105,26 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
else:
renderNoneFound()
else:
var retweets: seq[int64]
var
threads: seq[int64]
retweets: seq[int64]
for thread in results.content:
if thread.len == 1:
let
tweet = thread[0]
retweetId = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
for tweet in results.content:
let rt = if tweet.retweet.isSome: get(tweet.retweet).id else: 0
if retweetId in retweets or tweet.id in retweets or
tweet.pinned and prefs.hidePins:
continue
if tweet.id in threads or rt in retweets or tweet.id in retweets or
tweet.pinned and prefs.hidePins: continue
let thread = results.content.threadFilter(threads, tweet)
if thread.len < 2:
var hasThread = tweet.hasThread
if retweetId != 0 and tweet.retweet.isSome:
retweets &= retweetId
if rt != 0:
retweets &= rt
hasThread = get(tweet.retweet).hasThread
renderTweet(tweet, prefs, path, showThread=hasThread)
else:
renderThread(thread, prefs, path)
threads &= thread.mapIt(it.id)
if results.bottom.len > 0:
renderMore(results.query, results.bottom)
renderMore(results.query, results.bottom)
renderToTop()

View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, sequtils, strformat, options, algorithm
import strutils, sequtils, strformat, options
import karax/[karaxdsl, vdom, vstyles]
from jester import Request
@@ -10,17 +10,20 @@ import general
const doctype = "<!DOCTYPE html>\n"
proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
let url = getPicUrl(user.getUserPic("_mini"))
buildHtml():
img(class=(prefs.getAvatarClass & " mini"), src=url, loading="lazy")
proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
buildHtml(tdiv):
if pinned:
tdiv(class="pinned"):
span: icon "pin", "Pinned Tweet"
elif retweet.len > 0:
if retweet.len > 0:
tdiv(class="retweet-header"):
span: icon "retweet", retweet & " retweeted"
if tweet.pinned:
tdiv(class="pinned"):
span: icon "pin", "Pinned Tweet"
tdiv(class="tweet-header"):
a(class="tweet-avatar", href=("/" & tweet.user.username)):
var size = "_bigger"
@@ -82,34 +85,35 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
let
container = if video.description.len == 0 and video.title.len == 0: ""
else: " card-container"
playbackType = if not prefs.proxyVideos and video.hasMp4Url: mp4
playbackType = if prefs.proxyVideos and video.hasMp4Url: mp4
else: video.playbackType
buildHtml(tdiv(class="attachments card")):
tdiv(class="gallery-video" & container):
tdiv(class="attachment video-container"):
let thumb = getSmallPic(video.thumb)
if not video.available:
img(src=thumb, loading="lazy")
renderVideoUnavailable(video)
elif not prefs.isPlaybackEnabled(playbackType):
img(src=thumb, loading="lazy")
renderVideoDisabled(playbackType, path)
else:
let canPlay = prefs.isPlaybackEnabled(playbackType)
if video.available and canPlay:
let
vars = video.variants.filterIt(it.contentType == playbackType)
vidUrl = vars.sortedByIt(it.resolution)[^1].url
vidUrl = video.getVidVariant(playbackType).url
source = if prefs.proxyVideos: getVidUrl(vidUrl)
else: vidUrl
case playbackType
of mp4:
video(poster=thumb, controls="", muted=prefs.muteVideos):
source(src=source, `type`="video/mp4")
video(src=source, poster=thumb, controls="", muted=prefs.muteVideos, preload="metadata")
of m3u8, vmap:
video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
tdiv(class="overlay-circle"): span(class="overlay-triangle")
verbatim "</div>"
else:
img(src=thumb, loading="lazy", decoding="async")
if not canPlay:
renderVideoDisabled(playbackType, path)
else:
renderVideoUnavailable(video)
if container.len > 0:
tdiv(class="card-content"):
h2(class="card-title"): text video.title
@@ -142,7 +146,7 @@ proc renderPoll(poll: Poll): VNode =
proc renderCardImage(card: Card): VNode =
buildHtml(tdiv(class="card-image-container")):
tdiv(class="card-image"):
genImg(card.image)
img(src=getPicUrl(card.image), alt="", loading="lazy")
if card.kind == player:
tdiv(class="card-overlay"):
tdiv(class="overlay-circle"):
@@ -178,12 +182,14 @@ func formatStat(stat: int): string =
if stat > 0: insertSep($stat, ',')
else: ""
proc renderStats(stats: TweetStats): VNode =
proc renderStats(stats: TweetStats; views: string): VNode =
buildHtml(tdiv(class="tweet-stats")):
span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
span(class="tweet-stat"): icon "quote", formatStat(stats.quotes)
span(class="tweet-stat"): icon "heart", formatStat(stats.likes)
span(class="tweet-stat"): icon "views", formatStat(stats.views)
if views.len > 0:
span(class="tweet-stat"): icon "play", insertSep(views, ',')
proc renderReply(tweet: Tweet): VNode =
buildHtml(tdiv(class="replying-to")):
@@ -196,7 +202,8 @@ proc renderAttribution(user: User; prefs: Prefs): VNode =
buildHtml(a(class="attribution", href=("/" & user.username))):
renderMiniAvatar(user, prefs)
strong: text user.fullname
verifiedIcon(user)
if user.verified:
icon "ok", class="verified-icon", title="Verified account"
proc renderMediaTags(tags: seq[User]): VNode =
buildHtml(tdiv(class="media-tag-block")):
@@ -272,7 +279,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
divClass = "thread-last " & class
if not tweet.available:
return buildHtml(tdiv(class=divClass & "unavailable timeline-item", data-username=tweet.user.username)):
return buildHtml(tdiv(class=divClass & "unavailable timeline-item")):
tdiv(class="unavailable-box"):
if tweet.tombstone.len > 0:
text tweet.tombstone
@@ -284,22 +291,20 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
if tweet.quote.isSome:
renderQuote(tweet.quote.get(), prefs, path)
let
fullTweet = tweet
pinned = tweet.pinned
let fullTweet = tweet
var retweet: string
var tweet = fullTweet
if tweet.retweet.isSome:
tweet = tweet.retweet.get
retweet = fullTweet.user.fullname
buildHtml(tdiv(class=("timeline-item " & divClass), data-username=tweet.user.username)):
buildHtml(tdiv(class=("timeline-item " & divClass))):
if not mainTweet:
a(class="tweet-link", href=getLink(tweet))
tdiv(class="tweet-body"):
renderHeader(tweet, retweet, pinned, prefs)
var views = ""
renderHeader(tweet, retweet, prefs)
if not afterTweet and index == 0 and tweet.reply.len > 0 and
(tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username):
@@ -322,8 +327,10 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderAlbum(tweet)
elif tweet.video.isSome:
renderVideo(tweet.video.get(), prefs, path)
views = tweet.video.get().views
elif tweet.gif.isSome:
renderGif(tweet.gif.get(), prefs)
views = "GIF"
if tweet.poll.isSome:
renderPoll(tweet.poll.get())
@@ -338,7 +345,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
renderMediaTags(tweet.mediaTags)
if not prefs.hideTweetStats:
renderStats(tweet.stats)
renderStats(tweet.stats, views)
if showThread:
a(class="show-thread", href=("/i/status/" & $tweet.threadId)):

View File

@@ -13,32 +13,32 @@ card = [
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True],
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'There were several big news in the Nim world in 2018 two new major releases, partnership with Status, and much more. But let us go chronologically.',
'nim-lang.org', True]
['FluentAI/status/1116417904831029248',
'Amazons Alexa isnt just AI — thousands of humans are listening',
'One of the only ways to improve Alexa is to have human beings check it for errors',
'theverge.com', True]
]
no_thumb = [
['FluentAI/status/1116417904831029248',
'LinkedIn',
'This link will take you to a page thats not on LinkedIn',
'lnkd.in'],
['Thom_Wolf/status/1122466524860702729',
'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
'',
'facebookresearch/fairseq',
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
'github.com'],
['brent_p/status/1088857328680488961',
'GitHub - brentp/hts-nim: nim wrapper for htslib for parsing genomics data files',
'',
'github.com'],
'Hts Nim Sugar',
'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
'brentp.github.io'],
['voidtarget/status/1133028231672582145',
'sinkingsugar/nimqt-example',
'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
'github.com']
'github.com'],
['nim_lang/status/1082989146040340480',
'Nim in 2018: A short recap',
'Posted by u/miran1 - 36 votes and 46 comments',
'reddit.com']
]
playable = [
@@ -53,6 +53,17 @@ playable = [
'youtube.com']
]
# promo = [
# ['BangOlufsen/status/1145698701517754368',
# 'Upgrade your journey', '',
# 'www.bang-olufsen.com'],
# ['BangOlufsen/status/1154934429900406784',
# 'Learn more about Beosound Shape', '',
# 'www.bang-olufsen.com']
# ]
class CardTest(BaseTestCase):
@parameterized.expand(card)
def test_card(self, tweet, title, description, destination, large):
@@ -87,3 +98,13 @@ class CardTest(BaseTestCase):
self.assert_element_visible('.card-overlay')
if len(description) > 0:
self.assert_text(description, c.description)
# @parameterized.expand(promo)
# def test_card_promo(self, tweet, title, description, destination):
# self.open_nitter(tweet)
# c = Card(Conversation.main + " ")
# self.assert_text(title, c.title)
# self.assert_text(destination, c.destination)
# self.assert_element_visible('.video-overlay')
# if len(description) > 0:
# self.assert_text(description, c.description)

View File

@@ -4,7 +4,7 @@ from parameterized import parameterized
profiles = [
['mobile_test', 'Test account',
'Test Account. test test Testing username with @mobile_test_2 and a #hashtag',
'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '97'],
'San Francisco, CA', 'example.com/foobar', 'Joined October 2009', '100'],
['mobile_test_2', 'mobile test 2', '', '', '', 'Joined January 2011', '13']
]
@@ -66,8 +66,8 @@ class ProfileTest(BaseTestCase):
self.assert_text(f'User "{username}" not found')
def test_suspended(self):
self.open_nitter('suspendme')
self.assert_text('User "suspendme" has been suspended')
self.open_nitter('user')
self.assert_text('User "user" has been suspended')
@parameterized.expand(banner_image)
def test_banner_image(self, username, url):

View File

@@ -2,8 +2,14 @@ from base import BaseTestCase, Quote, Conversation
from parameterized import parameterized
text = [
['elonmusk/status/1138136540096319488',
'TREV PAGE', '@Model3Owners',
"""As of March 58.4% of new car sales in Norway are electric.
What are we doing wrong? reuters.com/article/us-norwa…"""],
['nim_lang/status/1491461266849808397#m',
'Nim', '@nim_lang',
'Nim language', '@nim_lang',
"""What's better than Nim 1.6.0?
Nim 1.6.2 :)

View File

@@ -2,8 +2,8 @@ from base import BaseTestCase
from parameterized import parameterized
#class SearchTest(BaseTestCase):
#@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
#def test_username_search(self, username):
#self.search_username(username)
#self.assert_text(f'{username}')
class SearchTest(BaseTestCase):
@parameterized.expand([['@mobile_test'], ['@mobile_test_2']])
def test_username_search(self, username):
self.search_username(username)
self.assert_text(f'{username}')

View File

@@ -1,18 +1,23 @@
from base import BaseTestCase, Timeline
from parameterized import parameterized
normal = [['jack'], ['elonmusk']]
normal = [['mobile_test'], ['mobile_test_2']]
after = [['jack', '1681686036294803456'],
['elonmusk', '1681686036294803456']]
after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']]
no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
empty = [['emptyuser'], ['mobile_test_10']]
protected = [['mobile_test_7'], ['Empty_user']]
photo_rail = [['mobile_test', ['Bo0nDsYIYAIjqVn', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG']]]
photo_rail = [['mobile_test', [
'BzUnaDFCUAAmrjs', 'Bo0nDsYIYAIjqVn', 'Bos--KNIQAAA7Li', 'Boq1sDJIYAAxaoi',
'BonISmPIEAAhP3G', 'BoQbwJAIUAA0QCY', 'BoQbRQxIIAA3FWD', 'Bn8Qh8iIIAABXrG',
'Bn8QIG3IYAA0IGT', 'Bn8O3QeIUAAONai', 'Bn8NGViIAAATNG4', 'BkKovdrCUAAEz79',
'BkKoe_oCIAASAqr', 'BkKoRLNCAAAYfDf', 'BkKndxoCQAE1vFt', 'BPEmIbYCMAE44dl'
]]]
class TweetTest(BaseTestCase):
@@ -55,10 +60,10 @@ class TweetTest(BaseTestCase):
self.assert_element_absent(Timeline.older)
self.assert_element_absent(Timeline.end)
#@parameterized.expand(photo_rail)
#def test_photo_rail(self, username, images):
#self.open_nitter(username)
#self.assert_element_visible(Timeline.photo_rail)
#for i, url in enumerate(images):
#img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
#self.assertIn(url, img)
@parameterized.expand(photo_rail)
def test_photo_rail(self, username, images):
self.open_nitter(username)
self.assert_element_visible(Timeline.photo_rail)
for i, url in enumerate(images):
img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
self.assertIn(url, img)

View File

@@ -1,4 +1,4 @@
from base import BaseTestCase, Tweet, Conversation, get_timeline_tweet
from base import BaseTestCase, Tweet, get_timeline_tweet
from parameterized import parameterized
# image = tweet + 'div.attachments.media-body > div > div > a > div > img'
@@ -28,15 +28,14 @@ invalid = [
]
multiline = [
[1718660434457239868, 'WebDesignMuseum',
[400897186990284800, 'mobile_test_3',
"""
Happy 32nd Birthday HTML tags!
On October 29, 1991, the internet pioneer, Tim Berners-Lee, published a document entitled HTML Tags.
The document contained a description of the first 18 HTML tags: <title>, <nextid>, <a>, <isindex>, <plaintext>, <listing>, <p>, <h1>…<h6>, <address>, <hp1>, <hp2>…, <dl>, <dt>, <dd>, <ul>, <li>,<menu> and <dir>. The design of the first version of HTML language was influenced by the SGML universal markup language.
#WebDesignHistory"""]
KEEP
CALM
AND
CLICHÉ
ON"""]
]
link = [
@@ -75,6 +74,10 @@ retweet = [
[3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
]
reply = [
['mobile_test/with_replies', 15]
]
class TweetTest(BaseTestCase):
@parameterized.expand(timeline)
@@ -100,18 +103,18 @@ class TweetTest(BaseTestCase):
@parameterized.expand(multiline)
def test_multiline_formatting(self, tid, username, text):
self.open_nitter(f'{username}/status/{tid}')
self.assert_text(text.strip('\n'), Conversation.main)
self.assert_text(text.strip('\n'), '.main-tweet')
@parameterized.expand(emoji)
def test_emoji(self, tweet, text):
self.open_nitter(tweet)
self.assert_text(text, Conversation.main)
self.assert_text(text, '.main-tweet')
@parameterized.expand(link)
def test_link(self, tweet, links):
self.open_nitter(tweet)
for link in links:
self.assert_text(link, Conversation.main)
self.assert_text(link, '.main-tweet')
@parameterized.expand(username)
def test_username(self, tweet, usernames):
@@ -134,8 +137,8 @@ class TweetTest(BaseTestCase):
self.open_nitter(tweet)
self.assert_text('Tweet not found', '.error-panel')
#@parameterized.expand(reply)
#def test_thread(self, tweet, num):
#self.open_nitter(tweet)
#thread = self.find_element(f'.timeline > div:nth-child({num})')
#self.assertIn(thread.get_attribute('class'), 'thread-line')
@parameterized.expand(reply)
def test_thread(self, tweet, num):
self.open_nitter(tweet)
thread = self.find_element(f'.timeline > div:nth-child({num})')
self.assertIn(thread.get_attribute('class'), 'thread-line')

View File

@@ -14,7 +14,7 @@ poll = [
image = [
['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'],
#['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj']
]
gif = [
@@ -28,14 +28,14 @@ video_m3u8 = [
]
gallery = [
# ['mobile_test/status/451108446603980803', [
# ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
# ]],
['mobile_test/status/451108446603980803', [
['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
]],
# ['mobile_test/status/471539824713691137', [
# ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
# ['Bos--IqIQAAav23']
# ]],
['mobile_test/status/471539824713691137', [
['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
['Bos--IqIQAAav23']
]],
['mobile_test/status/469530783384743936', [
['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],

View File

@@ -1,155 +0,0 @@
#!/usr/bin/env python3
"""
Requirements:
pip install -r tools/requirements.txt
Usage:
python3 tools/create_session_browser.py <username> <password> [totp_seed] [--append sessions.jsonl] [--headless]
Examples:
# Output to terminal
python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET
# Append to sessions.jsonl
python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --append sessions.jsonl
# Headless mode (may increase detection risk)
python3 tools/create_session_browser.py myusername mypassword TOTP_SECRET --headless
Output:
{"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
"""
import sys
import json
import asyncio
import pyotp
import nodriver as uc
import os
async def login_and_get_cookies(username, password, totp_seed=None, headless=False):
"""Authenticate with X.com and extract session cookies"""
# Note: headless mode may increase detection risk from bot-detection systems
browser = await uc.start(headless=headless)
tab = await browser.get('https://x.com/i/flow/login')
try:
# Enter username
print('[*] Entering username...', file=sys.stderr)
username_input = await tab.find('input[autocomplete="username"]', timeout=10)
await username_input.send_keys(username + '\n')
await asyncio.sleep(1)
# Enter password
print('[*] Entering password...', file=sys.stderr)
password_input = await tab.find('input[autocomplete="current-password"]', timeout=15)
await password_input.send_keys(password + '\n')
await asyncio.sleep(2)
# Handle 2FA if needed
page_content = await tab.get_content()
if 'verification code' in page_content or 'Enter code' in page_content:
if not totp_seed:
raise Exception('2FA required but no TOTP seed provided')
print('[*] 2FA detected, entering code...', file=sys.stderr)
totp_code = pyotp.TOTP(totp_seed).now()
code_input = await tab.select('input[type="text"]')
await code_input.send_keys(totp_code + '\n')
await asyncio.sleep(3)
# Get cookies
print('[*] Retrieving cookies...', file=sys.stderr)
for _ in range(20): # 20 second timeout
cookies = await browser.cookies.get_all()
cookies_dict = {cookie.name: cookie.value for cookie in cookies}
if 'auth_token' in cookies_dict and 'ct0' in cookies_dict:
print('[*] Found both cookies', file=sys.stderr)
# Extract ID from twid cookie (may be URL-encoded)
user_id = None
if 'twid' in cookies_dict:
twid = cookies_dict['twid']
# Try to extract the ID from twid (format: u%3D<id> or u=<id>)
if 'u%3D' in twid:
user_id = twid.split('u%3D')[1].split('&')[0].strip('"')
elif 'u=' in twid:
user_id = twid.split('u=')[1].split('&')[0].strip('"')
cookies_dict['username'] = username
if user_id:
cookies_dict['id'] = user_id
return cookies_dict
await asyncio.sleep(1)
raise Exception('Timeout waiting for cookies')
finally:
browser.stop()
async def main():
if len(sys.argv) < 3:
print('Usage: python3 twitter-auth.py username password [totp_seed] [--append sessions.jsonl] [--headless]')
sys.exit(1)
username = sys.argv[1]
password = sys.argv[2]
totp_seed = None
append_file = None
headless = False
# Parse optional arguments
i = 3
while i < len(sys.argv):
arg = sys.argv[i]
if arg == '--append':
if i + 1 < len(sys.argv):
append_file = sys.argv[i + 1]
i += 2 # Skip '--append' and filename
else:
print('[!] Error: --append requires a filename', file=sys.stderr)
sys.exit(1)
elif arg == '--headless':
headless = True
i += 1
elif not arg.startswith('--'):
if totp_seed is None:
totp_seed = arg
i += 1
else:
# Unkown args
print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
i += 1
try:
cookies = await login_and_get_cookies(username, password, totp_seed, headless)
session = {
'kind': 'cookie',
'username': cookies['username'],
'id': cookies.get('id'),
'auth_token': cookies['auth_token'],
'ct0': cookies['ct0']
}
output = json.dumps(session)
if append_file:
with open(append_file, 'a') as f:
f.write(output + '\n')
print(f'✓ Session appended to {append_file}', file=sys.stderr)
else:
print(output)
os._exit(0)
except Exception as error:
print(f'[!] Error: {error}', file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
asyncio.run(main())

View File

@@ -1,328 +0,0 @@
#!/usr/bin/env python3
"""
Requirements:
pip install curl_cffi pyotp
Usage:
python3 tools/create_session_curl.py <username> <password> [totp_seed] [--append sessions.jsonl]
Examples:
# Output to terminal
python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET
# Append to sessions.jsonl
python3 tools/create_session_curl.py myusername mypassword TOTP_SECRET --append sessions.jsonl
Output:
{"kind": "cookie", "username": "...", "id": "...", "auth_token": "...", "ct0": "..."}
"""
import sys
import json
import pyotp
from curl_cffi import requests
BEARER_TOKEN = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
BASE_URL = "https://api.x.com/1.1/onboarding/task.json"
GUEST_ACTIVATE_URL = "https://api.x.com/1.1/guest/activate.json"
# Subtask versions required by API
SUBTASK_VERSIONS = {
"action_list": 2, "alert_dialog": 1, "app_download_cta": 1,
"check_logged_in_account": 2, "choice_selection": 3,
"contacts_live_sync_permission_prompt": 0, "cta": 7, "email_verification": 2,
"end_flow": 1, "enter_date": 1, "enter_email": 2, "enter_password": 5,
"enter_phone": 2, "enter_recaptcha": 1, "enter_text": 5, "generic_urt": 3,
"in_app_notification": 1, "interest_picker": 3, "js_instrumentation": 1,
"menu_dialog": 1, "notifications_permission_prompt": 2, "open_account": 2,
"open_home_timeline": 1, "open_link": 1, "phone_verification": 4,
"privacy_options": 1, "security_key": 3, "select_avatar": 4,
"select_banner": 2, "settings_list": 7, "show_code": 1, "sign_up": 2,
"sign_up_review": 4, "tweet_selection_urt": 1, "update_users": 1,
"upload_media": 1, "user_recommendations_list": 4,
"user_recommendations_urt": 1, "wait_spinner": 3, "web_modal": 1
}
def get_base_headers(guest_token=None):
"""Build base headers for API requests."""
headers = {
"Authorization": f"Bearer {BEARER_TOKEN}",
"Content-Type": "application/json",
"Accept": "*/*",
"Accept-Language": "en-US",
"X-Twitter-Client-Language": "en-US",
"Origin": "https://x.com",
"Referer": "https://x.com/",
}
if guest_token:
headers["X-Guest-Token"] = guest_token
return headers
def get_cookies_dict(session):
"""Extract cookies from session."""
return session.cookies.get_dict() if hasattr(session.cookies, 'get_dict') else dict(session.cookies)
def make_request(session, headers, flow_token, subtask_data, print_msg):
"""Generic request handler for flow steps."""
print(f"[*] {print_msg}...", file=sys.stderr)
payload = {
"flow_token": flow_token,
"subtask_inputs": [subtask_data] if isinstance(subtask_data, dict) else subtask_data
}
response = session.post(BASE_URL, json=payload, headers=headers)
response.raise_for_status()
data = response.json()
new_flow_token = data.get('flow_token')
if not new_flow_token:
raise Exception(f"Failed to get flow token: {print_msg}")
return new_flow_token, data
def get_guest_token(session):
"""Get guest token for unauthenticated requests."""
print("[*] Getting guest token...", file=sys.stderr)
response = session.post(GUEST_ACTIVATE_URL, headers={"Authorization": f"Bearer {BEARER_TOKEN}"})
response.raise_for_status()
guest_token = response.json().get('guest_token')
if not guest_token:
raise Exception("Failed to obtain guest token")
print(f"[*] Got guest token: {guest_token}", file=sys.stderr)
return guest_token
def init_flow(session, guest_token):
"""Initialize the login flow."""
print("[*] Initializing login flow...", file=sys.stderr)
headers = get_base_headers(guest_token)
payload = {
"input_flow_data": {
"flow_context": {
"debug_overrides": {},
"start_location": {"location": "manual_link"}
},
"subtask_versions": SUBTASK_VERSIONS
}
}
response = session.post(f"{BASE_URL}?flow_name=login", json=payload, headers=headers)
response.raise_for_status()
flow_token = response.json().get('flow_token')
if not flow_token:
raise Exception("Failed to get initial flow token")
print("[*] Got initial flow token", file=sys.stderr)
return flow_token, headers
def submit_username(session, flow_token, headers, guest_token, username):
"""Submit username."""
headers = headers.copy()
headers["X-Guest-Token"] = guest_token
subtask = {
"subtask_id": "LoginEnterUserIdentifierSSO",
"settings_list": {
"setting_responses": [{
"key": "user_identifier",
"response_data": {"text_data": {"result": username}}
}],
"link": "next_link"
}
}
flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting username")
# Check for denial (suspicious activity)
if data.get('subtasks') and 'cta' in data['subtasks'][0]:
error_msg = data['subtasks'][0]['cta'].get('primary_text', {}).get('text')
if error_msg:
raise Exception(f"Login denied: {error_msg}")
return flow_token
def submit_password(session, flow_token, headers, guest_token, password):
"""Submit password and detect if 2FA is needed."""
headers = headers.copy()
headers["X-Guest-Token"] = guest_token
subtask = {
"subtask_id": "LoginEnterPassword",
"enter_password": {"password": password, "link": "next_link"}
}
flow_token, data = make_request(session, headers, flow_token, subtask, "Submitting password")
needs_2fa = any(s.get('subtask_id') == 'LoginTwoFactorAuthChallenge' for s in data.get('subtasks', []))
if needs_2fa:
print("[*] 2FA required", file=sys.stderr)
return flow_token, needs_2fa
def submit_2fa(session, flow_token, headers, guest_token, totp_seed):
"""Submit 2FA code."""
if not totp_seed:
raise Exception("2FA required but no TOTP seed provided")
code = pyotp.TOTP(totp_seed).now()
print("[*] Generating 2FA code...", file=sys.stderr)
headers = headers.copy()
headers["X-Guest-Token"] = guest_token
subtask = {
"subtask_id": "LoginTwoFactorAuthChallenge",
"enter_text": {"text": code, "link": "next_link"}
}
flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting 2FA code")
return flow_token
def submit_js_instrumentation(session, flow_token, headers, guest_token):
"""Submit JS instrumentation response."""
headers = headers.copy()
headers["X-Guest-Token"] = guest_token
subtask = {
"subtask_id": "LoginJsInstrumentationSubtask",
"js_instrumentation": {
"response": '{"rf":{"a4fc506d24bb4843c48a1966940c2796bf4fb7617a2d515ad3297b7df6b459b6":121,"bff66e16f1d7ea28c04653dc32479cf416a9c8b67c80cb8ad533b2a44fee82a3":-1,"ac4008077a7e6ca03210159dbe2134dea72a616f03832178314bb9931645e4f7":-22,"c3a8a81a9b2706c6fec42c771da65a9597c537b8e4d9b39e8e58de9fe31ff239":-12},"s":"ZHYaDA9iXRxOl2J3AZ9cc23iJx-Fg5E82KIBA_fgeZFugZGYzRtf8Bl3EUeeYgsK30gLFD2jTQx9fAMsnYCw0j8ahEy4Pb5siM5zD6n7YgOeWmFFaXoTwaGY4H0o-jQnZi5yWZRAnFi4lVuCVouNz_xd2BO2sobCO7QuyOsOxQn2CWx7bjD8vPAzT5BS1mICqUWyjZDjLnRZJU6cSQG5YFIHEPBa8Kj-v1JFgkdAfAMIdVvP7C80HWoOqYivQR7IBuOAI4xCeLQEdxlGeT-JYStlP9dcU5St7jI6ExyMeQnRicOcxXLXsan8i5Joautk2M8dAJFByzBaG4wtrPhQ3QAAAZEi-_t7"}',
"link": "next_link"
}
}
flow_token, _ = make_request(session, headers, flow_token, subtask, "Submitting JS instrumentation")
return flow_token
def complete_flow(session, flow_token, headers):
"""Complete the login flow."""
cookies = get_cookies_dict(session)
headers = headers.copy()
headers["X-Twitter-Auth-Type"] = "OAuth2Session"
if cookies.get('ct0'):
headers["X-Csrf-Token"] = cookies['ct0']
subtask = {
"subtask_id": "AccountDuplicationCheck",
"check_logged_in_account": {"link": "AccountDuplicationCheck_false"}
}
make_request(session, headers, flow_token, subtask, "Completing login flow")
def extract_user_id(cookies_dict):
"""Extract user ID from twid cookie."""
twid = cookies_dict.get('twid', '').strip('"')
for prefix in ['u=', 'u%3D']:
if prefix in twid:
return twid.split(prefix)[1].split('&')[0].strip('"')
return None
def login_and_get_cookies(username, password, totp_seed=None):
"""Authenticate with X.com and extract session cookies."""
session = requests.Session(impersonate="chrome")
try:
guest_token = get_guest_token(session)
flow_token, headers = init_flow(session, guest_token)
flow_token = submit_js_instrumentation(session, flow_token, headers, guest_token)
flow_token = submit_username(session, flow_token, headers, guest_token, username)
flow_token, needs_2fa = submit_password(session, flow_token, headers, guest_token, password)
if needs_2fa:
flow_token = submit_2fa(session, flow_token, headers, guest_token, totp_seed)
complete_flow(session, flow_token, headers)
cookies_dict = get_cookies_dict(session)
cookies_dict['username'] = username
user_id = extract_user_id(cookies_dict)
if user_id:
cookies_dict['id'] = user_id
print("[*] Successfully authenticated", file=sys.stderr)
return cookies_dict
finally:
session.close()
def main():
if len(sys.argv) < 3:
print('Usage: python3 create_session_curl.py username password [totp_seed] [--append sessions.jsonl]', file=sys.stderr)
sys.exit(1)
username = sys.argv[1]
password = sys.argv[2]
totp_seed = None
append_file = None
# Parse optional arguments
i = 3
while i < len(sys.argv):
arg = sys.argv[i]
if arg == '--append':
if i + 1 < len(sys.argv):
append_file = sys.argv[i + 1]
i += 2
else:
print('[!] Error: --append requires a filename', file=sys.stderr)
sys.exit(1)
elif not arg.startswith('--'):
if totp_seed is None:
totp_seed = arg
i += 1
else:
print(f'[!] Warning: Unknown argument: {arg}', file=sys.stderr)
i += 1
try:
cookies = login_and_get_cookies(username, password, totp_seed)
session = {
'kind': 'cookie',
'username': cookies['username'],
'id': cookies.get('id'),
'auth_token': cookies['auth_token'],
'ct0': cookies['ct0']
}
output = json.dumps(session)
if append_file:
with open(append_file, 'a') as f:
f.write(output + '\n')
print(f'✓ Session appended to {append_file}', file=sys.stderr)
else:
print(output)
sys.exit(0)
except Exception as error:
print(f'[!] Error: {error}', file=sys.stderr)
import traceback
traceback.print_exc(file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()

View File

@@ -1,162 +0,0 @@
#!/usr/bin/env python3
import requests
import json
import sys
import pyotp
import cloudscraper
# NOTE: pyotp, requests and cloudscraper are dependencies
# > pip install pyotp requests cloudscraper
TW_CONSUMER_KEY = '3nVuSoBZnx6U4vzUxf5w'
TW_CONSUMER_SECRET = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'
def auth(username, password, otp_secret):
bearer_token_req = requests.post("https://api.twitter.com/oauth2/token",
auth=(TW_CONSUMER_KEY, TW_CONSUMER_SECRET),
headers={"Content-Type": "application/x-www-form-urlencoded"},
data='grant_type=client_credentials'
).json()
bearer_token = ' '.join(str(x) for x in bearer_token_req.values())
guest_token = requests.post(
"https://api.twitter.com/1.1/guest/activate.json",
headers={'Authorization': bearer_token}
).json().get('guest_token')
if not guest_token:
print("Failed to obtain guest token.")
sys.exit(1)
twitter_header = {
'Authorization': bearer_token,
"Content-Type": "application/json",
"User-Agent": "TwitterAndroid/10.21.0-release.0 (310210000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)",
"X-Twitter-API-Version": '5',
"X-Twitter-Client": "TwitterAndroid",
"X-Twitter-Client-Version": "10.21.0-release.0",
"OS-Version": "28",
"System-User-Agent": "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)",
"X-Twitter-Active-User": "yes",
"X-Guest-Token": guest_token,
"X-Twitter-Client-DeviceID": ""
}
scraper = cloudscraper.create_scraper()
scraper.headers = twitter_header
task1 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
params={
'flow_name': 'login',
'api_version': '1',
'known_device_token': '',
'sim_country_code': 'us'
},
json={
"flow_token": None,
"input_flow_data": {
"country_code": None,
"flow_context": {
"referrer_context": {
"referral_details": "utm_source=google-play&utm_medium=organic",
"referrer_url": ""
},
"start_location": {
"location": "deeplink"
}
},
"requested_variant": None,
"target_user_id": 0
}
}
)
scraper.headers['att'] = task1.headers.get('att')
task2 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
json={
"flow_token": task1.json().get('flow_token'),
"subtask_inputs": [{
"enter_text": {
"suggestion_id": None,
"text": username,
"link": "next_link"
},
"subtask_id": "LoginEnterUserIdentifier"
}]
}
)
task3 = scraper.post(
'https://api.twitter.com/1.1/onboarding/task.json',
json={
"flow_token": task2.json().get('flow_token'),
"subtask_inputs": [{
"enter_password": {
"password": password,
"link": "next_link"
},
"subtask_id": "LoginEnterPassword"
}],
}
)
for t3_subtask in task3.json().get('subtasks', []):
if "open_account" in t3_subtask:
return t3_subtask["open_account"]
elif "enter_text" in t3_subtask:
response_text = t3_subtask["enter_text"]["hint_text"]
totp = pyotp.TOTP(otp_secret)
generated_code = totp.now()
task4resp = scraper.post(
"https://api.twitter.com/1.1/onboarding/task.json",
json={
"flow_token": task3.json().get("flow_token"),
"subtask_inputs": [
{
"enter_text": {
"suggestion_id": None,
"text": generated_code,
"link": "next_link",
},
"subtask_id": "LoginTwoFactorAuthChallenge",
}
],
}
)
task4 = task4resp.json()
for t4_subtask in task4.get("subtasks", []):
if "open_account" in t4_subtask:
return t4_subtask["open_account"]
return None
if __name__ == "__main__":
if len(sys.argv) != 5:
print("Usage: python3 get_session.py <username> <password> <2fa secret> <path>")
sys.exit(1)
username = sys.argv[1]
password = sys.argv[2]
otp_secret = sys.argv[3]
path = sys.argv[4]
result = auth(username, password, otp_secret)
if result is None:
print("Authentication failed.")
sys.exit(1)
session_entry = {
"oauth_token": result.get("oauth_token"),
"oauth_token_secret": result.get("oauth_token_secret")
}
try:
with open(path, "a") as f:
f.write(json.dumps(session_entry) + "\n")
print("Authentication successful. Session appended to", path)
except Exception as e:
print(f"Failed to write session information: {e}")
sys.exit(1)

View File

@@ -1,3 +0,0 @@
nodriver>=0.48.0
pyotp
curl_cffi