From 5cb9e6e4cd0571e650d8d0dd75296f6b1198ecc5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 21:54:37 +0000 Subject: [PATCH 1/9] Add "Nitter" source type: tweets rendered as a distinct embed card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nitter list/user RSS feeds are ingested as their own source type, enriched via fxtwitter (author name/handle/avatar, cleaner text, attached photo) with a graceful RSS-only fallback when that enrichment fails. Tweets always publish directly, one per article, and never enter the LLM clustering/synthesis pipeline — the same bypass already used for YouTube, since merging unrelated tweets together makes no sense. Rendering: a new distinct embed-card component (avatar, name + @handle, full untruncated text, optional attached image, published-date-only timestamp, no like/retweet stats) replaces the plain article row wherever a tweet appears, on both the category-page list and the article detail page. Verified end-to-end against the real sample Nitter RSS feed (served locally): ingestion (all 100 items, tweet metadata correctly extracted, retweet/quote-tweet blockquotes correctly excluded from own-content text), publishing (bypasses clustering, tweet field threaded through to the published article), and rendering (embed card appears on the homepage feed and the article detail page, no duplicate title). Known follow-up: fxtwitter's JSON field names are based on public documentation, not a verified live response (that API is unreachable from this sandbox) — worth a real curl check before relying on the enrichment path in production; the RSS-only fallback path is what's actually been exercised here. --- backend/src/ingestion/adapters/base.ts | 3 + backend/src/ingestion/adapters/nitter.ts | 119 ++++++++++++++++++ backend/src/ingestion/poller.ts | 2 + backend/src/pipeline/publish.ts | 5 + backend/src/queue/priorityQueue.ts | 17 +-- backend/src/storage/db/articles.ts | 10 +- backend/src/storage/db/contentItems.ts | 6 +- backend/src/storage/db/index.ts | 10 +- backend/src/storage/db/types.ts | 6 +- frontend/src/lib/adminTypes.ts | 2 +- .../src/lib/components/ArticleListRow.svelte | 43 ++++--- frontend/src/lib/components/TweetCard.svelte | 100 +++++++++++++++ .../lib/components/admin/SourcesTab.svelte | 15 ++- frontend/src/lib/types.ts | 1 + frontend/src/routes/article/[id]/+page.svelte | 26 +++- 15 files changed, 327 insertions(+), 38 deletions(-) create mode 100644 backend/src/ingestion/adapters/nitter.ts create mode 100644 frontend/src/lib/components/TweetCard.svelte diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index cb79d61..723dfd7 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -9,6 +9,8 @@ export interface FetchedItem { videos: { url: string; provider?: string; embedHtml?: string }[]; link: string; publishedAt: string; + /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ + tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null }; raw: unknown; } @@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit>(); + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 10_000; + +/** + * Shape based on the publicly documented FixTweet/fxtwitter API + * (https://github.com/FixTweet/FxTwitter) — NOT verified against a live response in + * this environment (outbound access to api.fxtwitter.com is blocked here). Every field + * below is read with optional chaining and a fallback in fetchFxTwitter's caller, so a + * shape mismatch degrades gracefully to RSS-only data rather than breaking ingestion. + * Verify against a real `curl https://api.fxtwitter.com/2/status/` response and + * adjust the field paths here if they don't match. + */ +interface FxTweet { + text?: string; + created_timestamp?: number; + author?: { name?: string; screen_name?: string; avatar_url?: string }; + media?: { photos?: { url?: string }[] }; +} + +async function fetchFxTwitter(tweetId: string): Promise { + try { + const res = await fetch(`https://api.fxtwitter.com/2/status/${tweetId}`, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok) return null; + const json = (await res.json()) as { tweet?: FxTweet }; + return json?.tweet ?? null; + } catch (err) { + logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`); + return null; + } +} + +function extractTweetId(item: Parser.Item): string | null { + const guid = item.guid; + if (guid && /^\d+$/.test(guid)) return guid; + const fromLink = item.link?.match(/status\/(\d+)/)?.[1]; + return fromLink ?? null; +} + +/** + * A Nitter list/user RSS description is the item's own tweet content (a

plus + * optional ) followed, for retweets/quote-tweets, by a

wrapping the + * quoted tweet's own text/image/permalink. Only the part before that blockquote is this + * item's own content — nested quote-tweet rendering isn't part of the approved design. + */ +function ownContentHtml(descriptionHtml: string): string { + const cut = descriptionHtml.search(/|
]+src="([^"]+)"/i)?.[1] ?? null; +} + +export const nitterAdapter: SourceAdapter = { + async fetch(source: Source): Promise { + if (!source.url) return []; + + const feed = await parser.parseURL(source.url); + const items: FetchedItem[] = []; + + for (const item of feed.items) { + if (!item.link || !item.guid) continue; + const tweetId = extractTweetId(item); + if (!tweetId) { + logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`); + continue; + } + + // dc:creator is reliably the author of this item's own tweet text (rss-parser + // maps it to item.creator) — for a retweet, that's the original tweet's author, + // not whichever list member's retweet surfaced it in this feed. + const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown'; + const ownHtml = ownContentHtml(item.content ?? ''); + const rssImageUrl = extractImageUrl(ownHtml); + + const enrichment = await fetchFxTwitter(tweetId); + + const authorName = enrichment?.author?.name ?? handle; + const avatarUrl = enrichment?.author?.avatar_url ?? null; + const text = enrichment?.text ?? ownHtml; + const photoUrl = enrichment?.media?.photos?.[0]?.url ?? rssImageUrl; + const publishedAt = enrichment?.created_timestamp + ? new Date(enrichment.created_timestamp * 1000).toISOString() + : (item.isoDate ?? item.pubDate ?? new Date().toISOString()); + + items.push({ + title: item.title || text.slice(0, 100), + summary: text.slice(0, 500), + body: text, + images: photoUrl ? [{ url: photoUrl }] : [], + videos: [], + link: item.link, + publishedAt, + tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl }, + raw: { rss: item, fxtwitter: enrichment } + }); + } + + return items; + } +}; diff --git a/backend/src/ingestion/poller.ts b/backend/src/ingestion/poller.ts index 525d7ca..5a13c89 100644 --- a/backend/src/ingestion/poller.ts +++ b/backend/src/ingestion/poller.ts @@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js'; import { telegramAdapter } from './adapters/telegram.js'; import { apiAdapter } from './adapters/api.js'; import { youtubeAdapter } from './adapters/youtube.js'; +import { nitterAdapter } from './adapters/nitter.js'; import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js'; import { fetchFullArticle } from './articleFetcher.js'; import type { Source } from '../storage/db/types.js'; @@ -14,6 +15,7 @@ const adapters: Record = { telegram: telegramAdapter, api: apiAdapter, youtube: youtubeAdapter, + nitter: nitterAdapter, custom: apiAdapter }; diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 0b95f44..ad0bc98 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -85,12 +85,16 @@ export async function publishDirect(item: ContentItem): Promise { const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id } : null; + const tweet = item.tweet + ? { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl: item.tweet.avatarUrl, sourceItemId: item.id } + : null; const article = await articles.insertArticle({ title: item.title, body: item.body || item.summary, heroImage, video, + tweet, category, geo: item.geo, eventId: item.eventId, @@ -192,6 +196,7 @@ export async function publishCluster( body, heroImage, video, + tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass category, geo, eventId: opts.eventId ?? items[0]?.eventId ?? null, diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 181809e..12db2da 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -77,19 +77,22 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds); if (items.length === 0) return 0; - // YouTube videos never get LLM-merged with anything — each is always its own - // article (title/video/date/description), same shape whether the AI service is up - // or not. Route them straight to publishDirect, same as the no-AI passthrough path. - const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id)); - const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId)); + // YouTube videos and Nitter tweets never get LLM-merged with anything else — each + // is always its own article, same shape whether the AI service is up or not. Route + // them straight to publishDirect, same as the no-AI passthrough path. + const directPublishSourceIds = new Set( + sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id) + ); + const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); let publishedDirect = 0; - for (const item of youtubeItems) { + for (const item of directItems) { try { const article = await publishDirect(item); contentItemsDb.assignCluster([item.id], article.id); publishedDirect++; - logger.info('synthesis', `Published "${article.title}" directly (YouTube)`); + const source = sourcesDb.getSource(item.sourceId); + logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`); } catch (err) { logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`); } diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 1047542..17d5bbe 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -21,7 +21,8 @@ function rowToArticle(row: any): MergedArticle { threadId: row.thread_id, previousArticleId: row.previous_article_id, nextArticleId: row.next_article_id, - topStories: !!row.top_stories + topStories: !!row.top_stories, + tweet: row.tweet ? JSON.parse(row.tweet) : null }; } @@ -29,8 +30,8 @@ export function insertArticle(article: Omit): MergedArticle const id = `art-${randomUUID()}`; db.prepare( `INSERT INTO merged_articles - (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, article.title, @@ -49,7 +50,8 @@ export function insertArticle(article: Omit): MergedArticle article.threadId, article.previousArticleId, article.nextArticleId, - article.topStories ? 1 : 0 + article.topStories ? 1 : 0, + article.tweet ? JSON.stringify(article.tweet) : null ); if (article.previousArticleId) { db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId); diff --git a/backend/src/storage/db/contentItems.ts b/backend/src/storage/db/contentItems.ts index e33a35b..f4942b2 100644 --- a/backend/src/storage/db/contentItems.ts +++ b/backend/src/storage/db/contentItems.ts @@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem { embedding: row.embedding ? JSON.parse(row.embedding) : null, eventId: row.event_id, clusterId: row.cluster_id, + tweet: row.tweet ? JSON.parse(row.tweet) : null, raw: row.raw ? JSON.parse(row.raw) : null }; } @@ -28,8 +29,8 @@ export function insertContentItem(item: Omit): ContentItem { const id = `ci-${randomUUID()}`; db.prepare( `INSERT INTO content_items - (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, item.sourceId, @@ -47,6 +48,7 @@ export function insertContentItem(item: Omit): ContentItem { item.embedding ? JSON.stringify(item.embedding) : null, item.eventId, item.clusterId, + item.tweet ? JSON.stringify(item.tweet) : null, item.raw ? JSON.stringify(item.raw) : null ); return { ...item, id }; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index d14223b..850769f 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -58,6 +58,7 @@ export function migrate() { embedding TEXT, -- JSON float array event_id TEXT, cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis + tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only raw TEXT -- JSON, original payload ); CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id); @@ -82,7 +83,8 @@ export function migrate() { thread_id TEXT NOT NULL, previous_article_id TEXT, next_article_id TEXT, - top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?" + top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?" + tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only ); CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at); CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id); @@ -187,6 +189,12 @@ export function migrate() { if (!hasColumn('merged_articles', 'top_stories')) { db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('content_items', 'tweet')) { + db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT'); + } + if (!hasColumn('merged_articles', 'tweet')) { + db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT'); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index d62eb3c..778ff42 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -1,7 +1,7 @@ export interface Source { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; category: string[]; url: string | null; config: Record; @@ -31,6 +31,8 @@ export interface ContentItem { embedding: number[] | null; eventId: string | null; clusterId: string | null; + /** Nitter-sourced items only — null for everything else. */ + tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null } | null; raw: unknown; } @@ -47,6 +49,8 @@ export interface MergedArticle { body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; + /** Nitter-sourced articles only — the embed card's author info (see TweetCard.svelte). Never set alongside video. */ + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 9d0b128..5b7ba63 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -32,7 +32,7 @@ export interface AdminSettings { export interface AdminSource { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; category: string[]; url: string; config?: Record; diff --git a/frontend/src/lib/components/ArticleListRow.svelte b/frontend/src/lib/components/ArticleListRow.svelte index 1f37e95..d32248b 100644 --- a/frontend/src/lib/components/ArticleListRow.svelte +++ b/frontend/src/lib/components/ArticleListRow.svelte @@ -2,6 +2,7 @@ import type { MergedArticle } from '$lib/types'; import { timeAgo, exactTime, excerpt } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; + import TweetCard from './TweetCard.svelte'; let { article }: { article: MergedArticle } = $props(); @@ -12,28 +13,32 @@ ); - - {#if article.heroImage} - - {:else} -
- {/if} +{#if article.tweet} + +{:else} + + {#if article.heroImage} + + {:else} +
+ {/if} -
-
- {article.category[0] ?? ''} - · - {sourceLabel} - {#if article.video} +
+
+ {article.category[0] ?? ''} · - ▶ Video - {/if} + {sourceLabel} + {#if article.video} + · + ▶ Video + {/if} +
+
{article.title}
+
{excerpt(article.body)}
+
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
-
{article.title}
-
{excerpt(article.body)}
-
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
-
- + +{/if} diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index f3f3b11..bb83676 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -144,7 +144,17 @@ } const typeIcon = (type: string) => - type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•'; + type === 'rss' + ? '⟳' + : type === 'telegram' + ? '✈' + : type === 'youtube' + ? '▶' + : type === 'nitter' + ? '🐦' + : type === 'api' + ? '⇄' + : '•';
@@ -163,10 +173,13 @@ + {#if form.type === 'youtube'} + {:else if form.type === 'nitter'} + {:else} {/if} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 843cd8a..625b4f7 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -14,6 +14,7 @@ export interface MergedArticle { body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/routes/article/[id]/+page.svelte b/frontend/src/routes/article/[id]/+page.svelte index 3788cd0..418259d 100644 --- a/frontend/src/routes/article/[id]/+page.svelte +++ b/frontend/src/routes/article/[id]/+page.svelte @@ -2,6 +2,7 @@ import type { PageData } from './$types'; import { timeAgo, exactTime } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; + import TweetCard from '$lib/components/TweetCard.svelte'; let { data }: { data: PageData } = $props(); const a = $derived(data.article); @@ -22,9 +23,22 @@ {a.category.join(', ')}
-

{a.title}

+ {#if a.tweet} + +
+ Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} +
+ + + + {#if a.sources[0]} + View original tweet → + {/if} + {:else if a.video?.provider === 'youtube'} +

{a.title}

- {#if a.video?.provider === 'youtube'} @@ -46,6 +60,8 @@

{paragraph}

{/each} {:else} +

{a.title}

+
Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} {#if a.updatedAt !== a.publishedAt} @@ -180,6 +196,12 @@ height: 100%; border: none; } + .view-original { + display: inline-block; + font-size: 13px; + color: var(--text-accent); + margin-bottom: 20px; + } .tags { display: flex; gap: 8px; From bc6e75c1249d904c38209b05167249f461d47321 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 21:57:48 +0000 Subject: [PATCH 2/9] Fix fxtwitter endpoint URL and confirm response shape against a real call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Was calling /2/status/ (no username) based on the originally-given example; the actual working endpoint is //status/ (no version prefix), confirmed via a real curl response. text, created_timestamp, and author.name/avatar_url all match the assumed shape exactly — only media.photos[].url remains unverified (that test tweet had no photo), still guarded by the existing RSS-image fallback either way. --- backend/src/ingestion/adapters/nitter.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/src/ingestion/adapters/nitter.ts b/backend/src/ingestion/adapters/nitter.ts index c9f3cd0..3b3659c 100644 --- a/backend/src/ingestion/adapters/nitter.ts +++ b/backend/src/ingestion/adapters/nitter.ts @@ -17,13 +17,12 @@ const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS re const FETCH_TIMEOUT_MS = 10_000; /** - * Shape based on the publicly documented FixTweet/fxtwitter API - * (https://github.com/FixTweet/FxTwitter) — NOT verified against a live response in - * this environment (outbound access to api.fxtwitter.com is blocked here). Every field - * below is read with optional chaining and a fallback in fetchFxTwitter's caller, so a - * shape mismatch degrades gracefully to RSS-only data rather than breaking ingestion. - * Verify against a real `curl https://api.fxtwitter.com/2/status/` response and - * adjust the field paths here if they don't match. + * Shape confirmed against a real `curl https://api.fxtwitter.com//status/` + * response: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url` all + * verified exactly as read below. `media.photos[].url` is still unconfirmed — that + * response had no attached photo — but is read with optional chaining regardless, so a + * shape mismatch there just falls back to the RSS description's own (see + * fetch()'s photoUrl fallback) rather than breaking ingestion. */ interface FxTweet { text?: string; @@ -32,9 +31,10 @@ interface FxTweet { media?: { photos?: { url?: string }[] }; } -async function fetchFxTwitter(tweetId: string): Promise { +/** The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 — no API version prefix. */ +async function fetchFxTwitter(handle: string, tweetId: string): Promise { try { - const res = await fetch(`https://api.fxtwitter.com/2/status/${tweetId}`, { + const res = await fetch(`https://api.fxtwitter.com/${handle}/status/${tweetId}`, { headers: { 'User-Agent': USER_AGENT }, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); @@ -91,7 +91,7 @@ export const nitterAdapter: SourceAdapter = { const ownHtml = ownContentHtml(item.content ?? ''); const rssImageUrl = extractImageUrl(ownHtml); - const enrichment = await fetchFxTwitter(tweetId); + const enrichment = await fetchFxTwitter(handle, tweetId); const authorName = enrichment?.author?.name ?? handle; const avatarUrl = enrichment?.author?.avatar_url ?? null; From 8cc256f27d68076cd64b8857efef08e5a21280f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 22:17:44 +0000 Subject: [PATCH 3/9] Don't fall back to a favicon for image-less tweets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveHeroImage's favicon fallback exists so regular articles never look entirely bare, but for a tweet it meant an image-less tweet showed the Nitter instance's own favicon slapped on as if it were the tweet's photo. publishDirect now skips that fallback specifically for tweet items — TweetCard.svelte already renders cleanly with no image at all. --- backend/src/pipeline/publish.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index ad0bc98..e053072 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -35,11 +35,15 @@ function deriveTitle(body: string): string { /** * Resolves the hero image for an article: try the best candidate from the source * items, download and locally host it; if there isn't one, fall back to the site's - * favicon rather than leaving the article with no art at all. + * favicon rather than leaving the article with no art at all. That favicon fallback + * is skipped for tweets (allowFaviconFallback: false) — a Nitter instance's own + * favicon slapped onto an image-less tweet reads as a mistake, not a placeholder; + * TweetCard.svelte already handles no-image tweets gracefully with no image at all. */ async function resolveHeroImage( items: ContentItem[], - primaryLink: string + primaryLink: string, + allowFaviconFallback = true ): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> { const selected = selectBestImage(items); @@ -55,6 +59,10 @@ async function resolveHeroImage( return { heroImage: selected, storedMediaId: null }; } + if (!allowFaviconFallback) { + return { heroImage: null, storedMediaId: null }; + } + const favicon = faviconUrlFor(primaryLink); if (favicon) { const stored = await downloadAndStore(favicon, 'published', {}); @@ -81,7 +89,7 @@ async function resolveHeroImage( */ export async function publishDirect(item: ContentItem): Promise { const category = uniqueCategories([item]); - const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link); + const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link, !item.tweet); const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id } : null; From c0d90fc33b4c1c91c8886e4bea78de9c46a8e7ae Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 22:58:03 +0000 Subject: [PATCH 4/9] Fix multi-word category pages never matching their own articles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /category/x-news filtered by the raw URL slug ("x-news") instead of the real category name ("X News"), so it never matched merged_articles.category values for any multi-word category — only worked for the seeded defaults because they're all single words where the slug and name happen to be identical once lowercased. Now resolves the slug back to the actual category name via the site's own category list before filtering. --- frontend/src/routes/category/[name]/+page.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/frontend/src/routes/category/[name]/+page.ts b/frontend/src/routes/category/[name]/+page.ts index c83a4df..91366ea 100644 --- a/frontend/src/routes/category/[name]/+page.ts +++ b/frontend/src/routes/category/[name]/+page.ts @@ -1,14 +1,25 @@ import type { PageLoad } from './$types'; import { getFeed } from '$lib/api'; +import { slugify } from '$lib/format'; const PAGE_SIZE = 15; // Every category page (including "Local") filters by its category name — sources are // assigned categories directly via checkboxes in the admin Sources tab, so a source // tagged "Local" shows up here the same way one tagged "Tech" shows up on /category/tech. -export const load: PageLoad = async ({ params, fetch }) => { - const name = params.name; - const filters = { category: name }; +// +// The URL param is a slug (e.g. "x-news" from slugify("X News")), not the real category +// name — for single-word categories those happen to be the same lowercased, but for a +// multi-word name like "X News" the slug's hyphen never matches the stored "X News" +// (with a space) in a merged_articles.category LIKE match. Resolve the slug back to the +// real category name via the site's own category list (already loaded by the root +// layout) before filtering, rather than passing the raw slug straight through. +export const load: PageLoad = async ({ params, fetch, parent }) => { + const { categories } = await parent(); + const match = categories.find((c) => slugify(c.name) === params.name); + const categoryName = match?.name ?? params.name; + + const filters = { category: categoryName }; const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); - return { initial, filters, name, pageSize: PAGE_SIZE }; + return { initial, filters, name: categoryName, pageSize: PAGE_SIZE }; }; From c2b34623d1aa11a8ed787fe2f9c2249c6713d860 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 23:43:52 +0000 Subject: [PATCH 5/9] Add configurable tweet media hosting mode and fxtwitter base URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds nitterMediaMode (self-host/proxy/direct, default proxy) and fxtwitterBaseUrl to global settings with a new Retention tab panel. Tweet images and avatars now resolve through the chosen mode instead of always being downloaded — proxy mode streams media through a new SSRF-hardened /media/proxy route (hostname allowlist + DNS-rebinding defense) so the origin server's IP is never exposed to Twitter's CDN, direct hotlinks the original URL, and self-host keeps the prior always-download behavior. fxtwitterBaseUrl lets the enrichment call target a self-hosted FixTweet mirror instead of the public instance. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/api/mediaProxy.ts | 117 ++++++++++++++++++ backend/src/index.ts | 6 + backend/src/ingestion/adapters/nitter.ts | 11 +- backend/src/pipeline/publish.ts | 81 +++++++++--- backend/src/queue/priorityQueue.ts | 4 +- backend/src/storage/db/index.ts | 10 +- backend/src/storage/db/settings.ts | 5 + backend/src/storage/db/types.ts | 4 + frontend/src/lib/adminTypes.ts | 2 + frontend/src/lib/components/TweetCard.svelte | 2 +- .../lib/components/admin/RetentionTab.svelte | 59 +++++++++ 11 files changed, 277 insertions(+), 24 deletions(-) create mode 100644 backend/src/api/mediaProxy.ts diff --git a/backend/src/api/mediaProxy.ts b/backend/src/api/mediaProxy.ts new file mode 100644 index 0000000..f31970b --- /dev/null +++ b/backend/src/api/mediaProxy.ts @@ -0,0 +1,117 @@ +// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode): +// the visitor's browser requests media from this route instead of directly from +// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the +// remote host — the media itself is streamed straight through, never written to disk. +// +// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless +// tightly restricted: only twimg.com (Twitter's media CDN), the configured +// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter +// sources are allowed — and even an allowed hostname is rejected if it resolves to a +// private/loopback/link-local address (defends against DNS rebinding, not just a +// hostname string check). + +import type { FastifyInstance } from 'fastify'; +import dns from 'node:dns/promises'; +import { Readable } from 'node:stream'; +import * as sourcesDb from '../storage/db/sources.js'; +import { getSettings } from '../storage/db/settings.js'; +import { logger } from '../storage/db/logs.js'; + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 15_000; +const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i; + +function hostnameOf(rawUrl: string | null): string | null { + if (!rawUrl) return null; + try { + return new URL(rawUrl).hostname.toLowerCase(); + } catch { + return null; + } +} + +function isAllowedHost(hostname: string): boolean { + const lower = hostname.toLowerCase(); + if (TWITTER_MEDIA_HOST_RE.test(lower)) return true; + + const settings = getSettings(); + if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true; + + const nitterHosts = sourcesDb + .listSources() + .filter((s) => s.type === 'nitter') + .map((s) => hostnameOf(s.url)) + .filter((h): h is string => !!h); + return nitterHosts.includes(lower); +} + +function isPrivateOrReservedIp(ip: string, family: number): boolean { + if (family === 4) { + const [a, b] = ip.split('.').map(Number); + if (a === 10 || a === 127 || a === 0) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range + return false; + } + const lower = ip.toLowerCase(); + if (lower === '::1') return true; + if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7 + if (lower.startsWith('fe80')) return true; // link-local + if (lower.startsWith('::ffff:')) { + const v4 = lower.split(':').pop(); + if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4); + } + return false; +} + +export async function registerMediaProxy(app: FastifyInstance) { + app.get('/media/proxy', async (req, reply) => { + const { url } = req.query as { url?: string }; + if (!url) return reply.code(400).send({ error: 'url required' }); + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return reply.code(400).send({ error: 'invalid url' }); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return reply.code(400).send({ error: 'unsupported scheme' }); + } + + if (!isAllowedHost(parsed.hostname)) { + logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + let addresses: { address: string; family: number }[]; + try { + addresses = await dns.lookup(parsed.hostname, { all: true }); + } catch { + return reply.code(502).send({ error: 'DNS resolution failed' }); + } + if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) { + logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + try { + const res = await fetch(parsed.toString(), { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + return reply.code(502).send({ error: `upstream responded ${res.status}` }); + } + reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream'); + reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600'); + return reply.send(Readable.fromWeb(res.body as any)); + } catch (err) { + logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`); + return reply.code(502).send({ error: 'fetch failed' }); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 98746b5..1848fb3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -7,6 +7,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js'; import { registerAuth } from './api/auth.js'; import { registerPublicRoutes } from './api/public.js'; import { registerAdminRoutes } from './api/admin.js'; +import { registerMediaProxy } from './api/mediaProxy.js'; import { startScheduler } from './queue/scheduler.js'; import { logger } from './storage/db/logs.js'; @@ -79,6 +80,11 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); + // Static "/media/proxy" takes priority over the "/media/:filename" param route + // above regardless of registration order (find-my-way, Fastify's router, always + // prefers a static segment over a parametric one at the same depth). + await registerMediaProxy(app); + app.get('/health', async () => ({ ok: true })); await app.listen({ port: PORT, host: '0.0.0.0' }); diff --git a/backend/src/ingestion/adapters/nitter.ts b/backend/src/ingestion/adapters/nitter.ts index 3b3659c..fec2118 100644 --- a/backend/src/ingestion/adapters/nitter.ts +++ b/backend/src/ingestion/adapters/nitter.ts @@ -10,6 +10,7 @@ import Parser from 'rss-parser'; import type { Source } from '../../storage/db/types.js'; import type { SourceAdapter, FetchedItem } from './base.js'; import { logger } from '../../storage/db/logs.js'; +import { getSettings } from '../../storage/db/settings.js'; const parser = new Parser>(); @@ -31,10 +32,16 @@ interface FxTweet { media?: { photos?: { url?: string }[] }; } -/** The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 — no API version prefix. */ +/** + * The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 + * — no API version prefix. The base URL is admin-configurable (Retention tab) so a + * self-hosted FixTweet mirror (or another compatible public instance) can be used + * instead of the public api.fxtwitter.com default. + */ async function fetchFxTwitter(handle: string, tweetId: string): Promise { + const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, ''); try { - const res = await fetch(`https://api.fxtwitter.com/${handle}/status/${tweetId}`, { + const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, { headers: { 'User-Agent': USER_AGENT }, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index e053072..90b05b3 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -33,17 +33,18 @@ function deriveTitle(body: string): string { } /** - * Resolves the hero image for an article: try the best candidate from the source - * items, download and locally host it; if there isn't one, fall back to the site's - * favicon rather than leaving the article with no art at all. That favicon fallback - * is skipped for tweets (allowFaviconFallback: false) — a Nitter instance's own - * favicon slapped onto an image-less tweet reads as a mistake, not a placeholder; - * TweetCard.svelte already handles no-image tweets gracefully with no image at all. + * Resolves the hero image for a regular (non-tweet) article: try the best candidate + * from the source items, download and locally host it; if there isn't one, fall back + * to the site's favicon rather than leaving the article with no art at all. Tweets + * never reach this function — see resolveTweetMediaUrl below, which applies the + * admin-configured Nitter media mode instead of always downloading, and skips the + * favicon fallback entirely (a Nitter instance's own favicon slapped onto an + * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already + * handles no-image tweets gracefully with no image at all). */ async function resolveHeroImage( items: ContentItem[], - primaryLink: string, - allowFaviconFallback = true + primaryLink: string ): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> { const selected = selectBestImage(items); @@ -59,10 +60,6 @@ async function resolveHeroImage( return { heroImage: selected, storedMediaId: null }; } - if (!allowFaviconFallback) { - return { heroImage: null, storedMediaId: null }; - } - const favicon = faviconUrlFor(primaryLink); if (favicon) { const stored = await downloadAndStore(favicon, 'published', {}); @@ -77,6 +74,31 @@ async function resolveHeroImage( return { heroImage: null, storedMediaId: null }; } +/** + * Applies the admin-configured Nitter media mode (Retention tab) to a single + * externally-hosted tweet media URL — an attached photo or the author's avatar. + * 'self-host' downloads and serves it locally like any other article image; + * 'proxy' routes it through this server's own /media/proxy route so only this + * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the + * "anonymity" the admin asked for) without persisting anything to disk; 'direct' + * hotlinks the original URL unchanged, the cheapest option with no server involvement. + */ +async function resolveTweetMediaUrl( + url: string, + mode: GlobalSettings['nitterMediaMode'] +): Promise<{ url: string; storedMediaId: string | null }> { + if (mode === 'direct') return { url, storedMediaId: null }; + + if (mode === 'proxy') { + return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null }; + } + + const stored = await downloadAndStore(url, 'published', {}); + if (stored) return { url: stored.servedPath, storedMediaId: stored.id }; + // Download failed — fall back to hotlinking rather than losing the media entirely. + return { url, storedMediaId: null }; +} + /** * Publishes a single item as-is, with no AI calls at all — used when the AI service * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives @@ -87,15 +109,38 @@ async function resolveHeroImage( * tag-based thread detection — but these earlier articles aren't retroactively * rewritten or merged with anything after the fact. */ -export async function publishDirect(item: ContentItem): Promise { +export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise { const category = uniqueCategories([item]); - const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link, !item.tweet); + const storedMediaIds: string[] = []; + + let heroImage: MergedArticle['heroImage'] = null; + if (item.tweet) { + const selected = selectBestImage([item]); + if (selected) { + const resolved = await resolveTweetMediaUrl(selected.url, settings.nitterMediaMode); + heroImage = { url: resolved.url, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason }; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + } else { + const resolved = await resolveHeroImage([item], item.link); + heroImage = resolved.heroImage; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id } : null; - const tweet = item.tweet - ? { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl: item.tweet.avatarUrl, sourceItemId: item.id } - : null; + + let tweet: MergedArticle['tweet'] = null; + if (item.tweet) { + let avatarUrl: string | null = null; + if (item.tweet.avatarUrl) { + const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode); + avatarUrl = resolved.url; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + tweet = { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl, sourceItemId: item.id }; + } const article = await articles.insertArticle({ title: item.title, @@ -127,7 +172,7 @@ export async function publishDirect(item: ContentItem): Promise { topStories: anyPushesToTopStories([item]) }); - if (storedMediaId) promoteToPublished(storedMediaId, article.id); + for (const id of storedMediaIds) promoteToPublished(id, article.id); return article; } diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 12db2da..8c6a0b3 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise): GlobalSettings { merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, ai_service_host=?, ai_service_port=?, selected_models=?, + nitter_media_mode=?, fxtwitter_base_url=?, published_article_max_age_days=?, raw_item_max_age_days=?, storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=? WHERE id = 1` @@ -55,6 +58,8 @@ export function updateSettings(patch: Partial): GlobalSettings { merged.aiServiceHost, merged.aiServicePort, JSON.stringify(merged.selectedModels), + merged.nitterMediaMode, + merged.fxtwitterBaseUrl, merged.retention.publishedArticleMaxAgeDays, merged.retention.rawItemMaxAgeDays, merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 778ff42..ebf3829 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -119,6 +119,10 @@ export interface GlobalSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + /** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */ + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + /** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */ + fxtwitterBaseUrl: string; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 5b7ba63..66b5791 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -25,6 +25,8 @@ export interface AdminSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + fxtwitterBaseUrl: string; retention: RetentionSettings; categoryPriority: CategoryPriority[]; } diff --git a/frontend/src/lib/components/TweetCard.svelte b/frontend/src/lib/components/TweetCard.svelte index 52c80ee..271e217 100644 --- a/frontend/src/lib/components/TweetCard.svelte +++ b/frontend/src/lib/components/TweetCard.svelte @@ -18,7 +18,7 @@
{#if article.tweet?.avatarUrl} - + {:else}
{/if} diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index ad16ea6..2215d8d 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -11,6 +11,31 @@ let clearing = $state<'articles' | 'media' | null>(null); let clearResult = $state(null); + let nitterMediaMode = $state(settings.nitterMediaMode); + let fxtwitterBaseUrl = $state(settings.fxtwitterBaseUrl); + let nitterStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle'); + let nitterSaveTimer: ReturnType; + + function scheduleNitterSave() { + nitterStatus = 'saving'; + clearTimeout(nitterSaveTimer); + nitterSaveTimer = setTimeout(async () => { + try { + await updateSettings({ nitterMediaMode, fxtwitterBaseUrl }); + nitterStatus = 'saved'; + setTimeout(() => (nitterStatus = 'idle'), 1500); + } catch { + nitterStatus = 'error'; + } + }, 500); + } + + const mediaModes: { label: string; value: 'self-host' | 'proxy' | 'direct' }[] = [ + { label: 'Self-host', value: 'self-host' }, + { label: 'Proxy (recommended)', value: 'proxy' }, + { label: 'Direct', value: 'direct' } + ]; + async function handleClearArticles() { if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return; clearing = 'articles'; @@ -141,6 +166,40 @@
+
+
+ Nitter (tweet media) + +
+

+ How images and video attached to ingested tweets are served to visitors. Self-hosting + downloads and stores everything locally, same as regular article images. Proxying streams + each request through this server without persisting anything, so only this server's IP is + ever exposed to Twitter's CDN. Direct hotlinks the original URL straight from Twitter, with + no server involvement at all. +

+
+ {#each mediaModes as mode} + + {/each} +
+

+ Enrichment API used to fetch full tweet text, author info, and media — any fxtwitter/FixTweet- + compatible endpoint works. Defaults to the public fxtwitter.com instance; point this at a + self-hosted FixTweet mirror (or another public instance) instead if you'd rather not depend on it. +

+ +
+
Clear content

From d4ab69006134176d6d4fea141a878066d9274625 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 00:11:20 +0000 Subject: [PATCH 6/9] Render actual tweet video/multi-image media instead of a single thumbnail Tweets can carry up to 4 photos/videos/gifs; fxtwitter's media.all preserves their original order and, for videos, gives a real playable .mp4 plus a poster thumbnail. TweetCard now renders these as a 1/2/3/4-item grid (Twitter's own layout shapes) with fixed cell heights so a tall portrait image no longer dictates the whole card's height in the column view, and video/gif items play back with native controls instead of showing a static frame. Each item's url (and a video's thumbnail) still resolves through the configured Nitter media mode (self-host/proxy/direct) individually. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/ingestion/adapters/base.ts | 4 +- backend/src/ingestion/adapters/nitter.ts | 52 +++++++++++--- backend/src/pipeline/publish.ts | 52 +++++++++++--- backend/src/storage/db/types.ts | 16 ++++- frontend/src/lib/components/TweetCard.svelte | 73 ++++++++++++++++++-- frontend/src/lib/types.ts | 11 ++- 6 files changed, 177 insertions(+), 31 deletions(-) diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index 723dfd7..e767543 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,4 @@ -import type { Source, ContentItem } from '../../storage/db/types.js'; +import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -10,7 +10,7 @@ export interface FetchedItem { link: string; publishedAt: string; /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ - tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null }; + tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] }; raw: unknown; } diff --git a/backend/src/ingestion/adapters/nitter.ts b/backend/src/ingestion/adapters/nitter.ts index fec2118..f249b6b 100644 --- a/backend/src/ingestion/adapters/nitter.ts +++ b/backend/src/ingestion/adapters/nitter.ts @@ -7,29 +7,54 @@ // two outlets' coverage of the same news event does — same reasoning as YouTube. import Parser from 'rss-parser'; -import type { Source } from '../../storage/db/types.js'; +import type { Source, TweetMediaItem } from '../../storage/db/types.js'; import type { SourceAdapter, FetchedItem } from './base.js'; import { logger } from '../../storage/db/logs.js'; import { getSettings } from '../../storage/db/settings.js'; +/** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */ +const MAX_TWEET_MEDIA = 4; + const parser = new Parser>(); const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; const FETCH_TIMEOUT_MS = 10_000; /** - * Shape confirmed against a real `curl https://api.fxtwitter.com//status/` - * response: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url` all - * verified exactly as read below. `media.photos[].url` is still unconfirmed — that - * response had no attached photo — but is read with optional chaining regardless, so a - * shape mismatch there just falls back to the RSS description's own (see - * fetch()'s photoUrl fallback) rather than breaking ingestion. + * Shape confirmed against two real `curl https://api.fxtwitter.com//status/` + * responses: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url`, and + * (from a second, video-attached tweet) `media.all[]` — an ordered array covering both + * photos and videos, each with `type` ('photo' | 'video' | 'gif'), `url` (the direct + * playable/displayable URL — for video this is a real .mp4, not the .m3u8 playlist also + * present under `formats`), `thumbnail_url` (video/gif poster frame), and `width`/`height`. + * `media.all` is preferred over `media.photos`/`media.videos` since it's the only field + * that preserves the tweet's original media order. */ +interface FxTweetMedia { + type?: string; + url?: string; + thumbnail_url?: string; + width?: number; + height?: number; +} + interface FxTweet { text?: string; created_timestamp?: number; author?: { name?: string; screen_name?: string; avatar_url?: string }; - media?: { photos?: { url?: string }[] }; + media?: { all?: FxTweetMedia[]; photos?: FxTweetMedia[] }; +} + +/** Maps fxtwitter's media shape to our own, capped at the 4 items a tweet can carry. */ +function toTweetMedia(enrichment: FxTweet | null): TweetMediaItem[] { + const items = enrichment?.media?.all ?? enrichment?.media?.photos ?? []; + return items.slice(0, MAX_TWEET_MEDIA).map((m): TweetMediaItem => ({ + type: m.type === 'video' || m.type === 'gif' ? m.type : 'photo', + url: m.url ?? '', + thumbnailUrl: m.thumbnail_url ?? null, + width: m.width ?? null, + height: m.height ?? null + })).filter((m) => m.url); } /** @@ -103,7 +128,12 @@ export const nitterAdapter: SourceAdapter = { const authorName = enrichment?.author?.name ?? handle; const avatarUrl = enrichment?.author?.avatar_url ?? null; const text = enrichment?.text ?? ownHtml; - const photoUrl = enrichment?.media?.photos?.[0]?.url ?? rssImageUrl; + // Enrichment failed (or came back with no media) — fall back to the RSS + // description's own as a single photo, same as before multi-media support. + const media = toTweetMedia(enrichment); + if (media.length === 0 && rssImageUrl) { + media.push({ type: 'photo', url: rssImageUrl, thumbnailUrl: null, width: null, height: null }); + } const publishedAt = enrichment?.created_timestamp ? new Date(enrichment.created_timestamp * 1000).toISOString() : (item.isoDate ?? item.pubDate ?? new Date().toISOString()); @@ -112,11 +142,11 @@ export const nitterAdapter: SourceAdapter = { title: item.title || text.slice(0, 100), summary: text.slice(0, 500), body: text, - images: photoUrl ? [{ url: photoUrl }] : [], + images: [], videos: [], link: item.link, publishedAt, - tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl }, + tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media }, raw: { rss: item, fxtwitter: enrichment } }); } diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 90b05b3..2dd2880 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js'; import * as articles from '../storage/db/articles.js'; import * as tags from '../storage/db/tags.js'; import * as sources from '../storage/db/sources.js'; -import type { GlobalSettings, MergedArticle, ContentItem } from '../storage/db/types.js'; +import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js'; const FOLLOW_UP_LOOKBACK_DAYS = 3; @@ -99,6 +99,36 @@ async function resolveTweetMediaUrl( return { url, storedMediaId: null }; } +/** + * Resolves every attached photo/video/gif's url — and, for video/gif, its poster + * thumbnail — through the admin's chosen Nitter media mode. Order and item count are + * preserved; TweetCard.svelte renders this array directly, there's no separate + * "hero image" concept for tweets the way there is for regular articles. + */ +async function resolveTweetMedia( + media: TweetMediaItem[], + mode: GlobalSettings['nitterMediaMode'] +): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> { + const storedMediaIds: string[] = []; + const resolved: TweetMediaItem[] = []; + + for (const item of media) { + const url = await resolveTweetMediaUrl(item.url, mode); + if (url.storedMediaId) storedMediaIds.push(url.storedMediaId); + + let thumbnailUrl = item.thumbnailUrl; + if (thumbnailUrl) { + const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode); + thumbnailUrl = resolvedThumb.url; + if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId); + } + + resolved.push({ ...item, url: url.url, thumbnailUrl }); + } + + return { media: resolved, storedMediaIds }; +} + /** * Publishes a single item as-is, with no AI calls at all — used when the AI service * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives @@ -113,15 +143,9 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) const category = uniqueCategories([item]); const storedMediaIds: string[] = []; + // Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly. let heroImage: MergedArticle['heroImage'] = null; - if (item.tweet) { - const selected = selectBestImage([item]); - if (selected) { - const resolved = await resolveTweetMediaUrl(selected.url, settings.nitterMediaMode); - heroImage = { url: resolved.url, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason }; - if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); - } - } else { + if (!item.tweet) { const resolved = await resolveHeroImage([item], item.link); heroImage = resolved.heroImage; if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); @@ -139,7 +163,15 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) avatarUrl = resolved.url; if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); } - tweet = { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl, sourceItemId: item.id }; + const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode); + storedMediaIds.push(...resolvedMedia.storedMediaIds); + tweet = { + authorName: item.tweet.authorName, + authorHandle: item.tweet.authorHandle, + avatarUrl, + sourceItemId: item.id, + media: resolvedMedia.media + }; } const article = await articles.insertArticle({ diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index ebf3829..d267eea 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -14,6 +14,16 @@ export interface Source { createdAt: string; } +/** A single photo/video/gif attached to a tweet, in the tweet's own display order — fxtwitter caps this at 4. */ +export interface TweetMediaItem { + type: 'photo' | 'video' | 'gif'; + url: string; + /** Video/gif poster frame — null for photos. */ + thumbnailUrl: string | null; + width: number | null; + height: number | null; +} + export interface ContentItem { id: string; sourceId: string; @@ -32,7 +42,7 @@ export interface ContentItem { eventId: string | null; clusterId: string | null; /** Nitter-sourced items only — null for everything else. */ - tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null } | null; + tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null; raw: unknown; } @@ -49,8 +59,8 @@ export interface MergedArticle { body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; - /** Nitter-sourced articles only — the embed card's author info (see TweetCard.svelte). Never set alongside video. */ - tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null; + /** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */ + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/lib/components/TweetCard.svelte b/frontend/src/lib/components/TweetCard.svelte index 271e217..9d98156 100644 --- a/frontend/src/lib/components/TweetCard.svelte +++ b/frontend/src/lib/components/TweetCard.svelte @@ -6,6 +6,14 @@ let { article }: { article: MergedArticle } = $props(); const sourceLabel = $derived(article.sources[0]?.sourceName ?? 'Nitter'); + const media = $derived(article.tweet?.media.slice(0, 4) ?? []); + + // Native

{article.body}
- {#if article.heroImage} - + {#if media.length > 0} + {/if}
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
 @@ -86,10 +114,47 @@ white-space: pre-line; margin-bottom: 8px; } - .tweet-img { - width: 100%; + /* Fixed cell heights (rather than max-height on an auto-height image) so a tall + portrait photo or video is cropped to a sensible box instead of dictating the + whole card's height — this is what keeps a single vertical image from taking + over the column. Twitter's own 1/2/3/4-item grid shapes, at a size that fits + this app's narrower single-column feed. */ + .media-grid { + display: grid; + gap: 3px; border-radius: var(--radius); + overflow: hidden; margin-bottom: 8px; + background: var(--surface-1); + } + .media-grid[data-count='1'] { + grid-template-columns: 1fr; + height: 380px; + } + .media-grid[data-count='2'] { + grid-template-columns: 1fr 1fr; + height: 240px; + } + .media-grid[data-count='3'] { + grid-template-columns: 1fr 1fr; + grid-template-rows: 1fr 1fr; + height: 300px; + } + .media-grid[data-count='4'] { + grid-template-columns: 1fr 1fr; + grid-template-rows: 1fr 1fr; + height: 300px; + } + .media-cell { + overflow: hidden; + } + .media-cell.span-2 { + grid-row: 1 / 3; + } + .media-el { + width: 100%; + height: 100%; + object-fit: cover; display: block; background: var(--surface-1); } diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 625b4f7..fd889a1 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -8,13 +8,22 @@ export interface ArticleSource { publishedAt: string; } +/** A single photo/video/gif attached to a tweet, in the tweet's own display order — capped at 4. */ +export interface TweetMediaItem { + type: 'photo' | 'video' | 'gif'; + url: string; + thumbnailUrl: string | null; + width: number | null; + height: number | null; +} + export interface MergedArticle { id: string; title: string; body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; - tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null; + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; category: string[]; geo: string | null; eventId: string | null; From 78f59287dc164b48a35ca414f5fa035e56c91218 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 00:16:34 +0000 Subject: [PATCH 7/9] Fix crash rendering tweets published before tweet.media existed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit article.tweet?.media.slice(...) only guarded against a missing tweet object, not a missing media array — pre-existing published tweets in the DB predate that field and threw "Cannot read properties of undefined (reading 'slice')" on every category page containing one. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- frontend/src/lib/components/TweetCard.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/TweetCard.svelte b/frontend/src/lib/components/TweetCard.svelte index 9d98156..8ad62da 100644 --- a/frontend/src/lib/components/TweetCard.svelte +++ b/frontend/src/lib/components/TweetCard.svelte @@ -6,7 +6,9 @@ let { article }: { article: MergedArticle } = $props(); const sourceLabel = $derived(article.sources[0]?.sourceName ?? 'Nitter'); - const media = $derived(article.tweet?.media.slice(0, 4) ?? []); + // article.tweet.media is undefined for tweets published before this field existed — + // older rows in the DB weren't backfilled, so this can't assume it's always an array. + const media = $derived(article.tweet?.media?.slice(0, 4) ?? []); // Native {:else} From 7c44552ae0bfd2de57f80312e20aec9302eda675 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 00:39:30 +0000 Subject: [PATCH 9/9] Rework tweet card click targets: frame->tweet, photo->new tab, video->play MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole card previously linked to our own /article/[id] page. Per request, clicking the card frame now opens the original tweet (in a new tab) instead — there's no separate "full article" view for a tweet anyway. Clicking a photo opens that image by itself in a new tab (still resolved through the configured media mode, so proxy mode doesn't leak the browser's IP to Twitter when viewing the full image either). Clicking a video's native controls still just plays/pauses it rather than navigating anywhere. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- frontend/src/lib/components/TweetCard.svelte | 38 +++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/frontend/src/lib/components/TweetCard.svelte b/frontend/src/lib/components/TweetCard.svelte index 8cf1aa6..8305076 100644 --- a/frontend/src/lib/components/TweetCard.svelte +++ b/frontend/src/lib/components/TweetCard.svelte @@ -9,16 +9,26 @@ // article.tweet.media is undefined for tweets published before this field existed — // older rows in the DB weren't backfilled, so this can't assume it's always an array. const media = $derived(article.tweet?.media?.slice(0, 4) ?? []); + // The card's own href — clicking the frame goes to the original tweet, not our + // internal article page (there's no separate "full article" for a tweet anyway). + const tweetUrl = $derived(article.sources[0]?.link ?? `/article/${article.id}`); - // Native navigation (play/pause/scrub would otherwise just open the article instead). - // Plain images keep navigating as before — only the video element itself is exempted. - function handleMediaClick(e: MouseEvent) { - if ((e.target as HTMLElement).tagName === 'VIDEO') e.preventDefault(); + // A navigation + // (play/pause/scrub would otherwise just open the tweet in a new tab instead). + function stopVideoNav(e: MouseEvent) { + e.preventDefault(); + } + + // A photo click opens the image itself in a new tab, rather than the tweet link the + // rest of the card points to — also resolved through the configured media mode + // (proxy/self-host/direct) so opening the full image doesn't bypass proxy anonymity. + function openImage(e: MouseEvent, url: string) { + e.preventDefault(); + window.open(resolveMediaUrl(url), '_blank', 'noopener,noreferrer'); } - +
{article.category[0] ?? ''} · @@ -37,7 +47,7 @@
{article.body}
{#if media.length > 0} -