Merge remote-tracking branch 'origin/master' into development

# Conflicts:
#	backend/src/index.ts
#	backend/src/storage/db/index.ts

backend/src/api/mediaProxy.ts

@@ -0,0 +1,117 @@
+// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode):
+// the visitor's browser requests media from this route instead of directly from
+// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the
+// remote host — the media itself is streamed straight through, never written to disk.
+//
+// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless
+// tightly restricted: only twimg.com (Twitter's media CDN), the configured
+// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter
+// sources are allowed — and even an allowed hostname is rejected if it resolves to a
+// private/loopback/link-local address (defends against DNS rebinding, not just a
+// hostname string check).
+
+import type { FastifyInstance } from 'fastify';
+import dns from 'node:dns/promises';
+import { Readable } from 'node:stream';
+import * as sourcesDb from '../storage/db/sources.js';
+import { getSettings } from '../storage/db/settings.js';
+import { logger } from '../storage/db/logs.js';
+
+const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
+const FETCH_TIMEOUT_MS = 15_000;
+const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i;
+
+function hostnameOf(rawUrl: string | null): string | null {
+	if (!rawUrl) return null;
+	try {
+		return new URL(rawUrl).hostname.toLowerCase();
+	} catch {
+		return null;
+	}
+}
+
+function isAllowedHost(hostname: string): boolean {
+	const lower = hostname.toLowerCase();
+	if (TWITTER_MEDIA_HOST_RE.test(lower)) return true;
+
+	const settings = getSettings();
+	if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true;
+
+	const nitterHosts = sourcesDb
+		.listSources()
+		.filter((s) => s.type === 'nitter')
+		.map((s) => hostnameOf(s.url))
+		.filter((h): h is string => !!h);
+	return nitterHosts.includes(lower);
+}
+
+function isPrivateOrReservedIp(ip: string, family: number): boolean {
+	if (family === 4) {
+		const [a, b] = ip.split('.').map(Number);
+		if (a === 10 || a === 127 || a === 0) return true;
+		if (a === 169 && b === 254) return true;
+		if (a === 172 && b >= 16 && b <= 31) return true;
+		if (a === 192 && b === 168) return true;
+		if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range
+		return false;
+	}
+	const lower = ip.toLowerCase();
+	if (lower === '::1') return true;
+	if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7
+	if (lower.startsWith('fe80')) return true; // link-local
+	if (lower.startsWith('::ffff:')) {
+		const v4 = lower.split(':').pop();
+		if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4);
+	}
+	return false;
+}
+
+export async function registerMediaProxy(app: FastifyInstance) {
+	app.get('/media/proxy', async (req, reply) => {
+		const { url } = req.query as { url?: string };
+		if (!url) return reply.code(400).send({ error: 'url required' });
+
+		let parsed: URL;
+		try {
+			parsed = new URL(url);
+		} catch {
+			return reply.code(400).send({ error: 'invalid url' });
+		}
+
+		if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+			return reply.code(400).send({ error: 'unsupported scheme' });
+		}
+
+		if (!isAllowedHost(parsed.hostname)) {
+			logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`);
+			return reply.code(403).send({ error: 'host not allowed' });
+		}
+
+		let addresses: { address: string; family: number }[];
+		try {
+			addresses = await dns.lookup(parsed.hostname, { all: true });
+		} catch {
+			return reply.code(502).send({ error: 'DNS resolution failed' });
+		}
+		if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) {
+			logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`);
+			return reply.code(403).send({ error: 'host not allowed' });
+		}
+
+		try {
+			const res = await fetch(parsed.toString(), {
+				headers: { 'User-Agent': USER_AGENT },
+				signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
+			});
+			if (!res.ok || !res.body) {
+				return reply.code(502).send({ error: `upstream responded ${res.status}` });
+			}
+			reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream');
+			reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600');
+			return reply.send(Readable.fromWeb(res.body as any));
+		} catch (err) {
+			logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`);
+			return reply.code(502).send({ error: 'fetch failed' });
+		}
+	});
+}

backend/src/index.ts

@@ -8,6 +8,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js';
 import { registerAuth } from './api/auth.js';
 import { registerPublicRoutes } from './api/public.js';
 import { registerAdminRoutes } from './api/admin.js';
+import { registerMediaProxy } from './api/mediaProxy.js';
 import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js';
 import { startScheduler } from './queue/scheduler.js';
 import { logger } from './storage/db/logs.js';
@@ -88,6 +89,11 @@ async function main() {
 		return reply.send(fs.createReadStream(filePath));
 	});
 
+	// Static "/media/proxy" takes priority over the "/media/:filename" param route
+	// above regardless of registration order (find-my-way, Fastify's router, always
+	// prefers a static segment over a parametric one at the same depth).
+	await registerMediaProxy(app);
+
 	app.get('/health', async () => ({ ok: true }));
 
 	await app.listen({ port: PORT, host: '0.0.0.0' });

backend/src/ingestion/adapters/base.ts

@@ -1,4 +1,4 @@
-import type { Source, ContentItem } from '../../storage/db/types.js';
+import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js';
 import { cleanHtml, toSummary } from '../clean.js';
 
 export interface FetchedItem {
@@ -9,6 +9,8 @@ export interface FetchedItem {
 	videos: { url: string; provider?: string; embedHtml?: string }[];
 	link: string;
 	publishedAt: string;
+	/** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */
+	tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] };
 	raw: unknown;
 }
 
@@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit<ContentIt
 		embedding: null,
 		eventId: null,
 		clusterId: null,
+		tweet: item.tweet ? { ...item.tweet } : null,
 		raw: item.raw
 	};
 }

backend/src/ingestion/adapters/nitter.ts

@@ -0,0 +1,156 @@
+// Nitter is its own ingestion module, deliberately separate from the RSS adapter even
+// though the transport is RSS: a tweet is enriched with a second fetch to fxtwitter
+// (for author name/avatar and cleaner text) and always publishes as its own article —
+// see pipeline/publish.ts and queue/priorityQueue.ts, which route nitter-sourced items
+// straight to publishDirect rather than the LLM clustering/synthesis pipeline. Merging
+// two unrelated tweets into one AI-rewritten story would make no sense the way merging
+// two outlets' coverage of the same news event does — same reasoning as YouTube.
+
+import Parser from 'rss-parser';
+import type { Source, TweetMediaItem } from '../../storage/db/types.js';
+import type { SourceAdapter, FetchedItem } from './base.js';
+import { logger } from '../../storage/db/logs.js';
+import { getSettings } from '../../storage/db/settings.js';
+
+/** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */
+const MAX_TWEET_MEDIA = 4;
+
+const parser = new Parser<Record<string, unknown>>();
+
+const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
+const FETCH_TIMEOUT_MS = 10_000;
+
+/**
+ * Shape confirmed against two real `curl https://api.fxtwitter.com/<handle>/status/<id>`
+ * responses: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url`, and
+ * (from a second, video-attached tweet) `media.all[]` — an ordered array covering both
+ * photos and videos, each with `type` ('photo' | 'video' | 'gif'), `url` (the direct
+ * playable/displayable URL — for video this is a real .mp4, not the .m3u8 playlist also
+ * present under `formats`), `thumbnail_url` (video/gif poster frame), and `width`/`height`.
+ * `media.all` is preferred over `media.photos`/`media.videos` since it's the only field
+ * that preserves the tweet's original media order.
+ */
+interface FxTweetMedia {
+	type?: string;
+	url?: string;
+	thumbnail_url?: string;
+	width?: number;
+	height?: number;
+}
+
+interface FxTweet {
+	text?: string;
+	created_timestamp?: number;
+	author?: { name?: string; screen_name?: string; avatar_url?: string };
+	media?: { all?: FxTweetMedia[]; photos?: FxTweetMedia[] };
+}
+
+/** Maps fxtwitter's media shape to our own, capped at the 4 items a tweet can carry. */
+function toTweetMedia(enrichment: FxTweet | null): TweetMediaItem[] {
+	const items = enrichment?.media?.all ?? enrichment?.media?.photos ?? [];
+	return items.slice(0, MAX_TWEET_MEDIA).map((m): TweetMediaItem => ({
+		type: m.type === 'video' || m.type === 'gif' ? m.type : 'photo',
+		url: m.url ?? '',
+		thumbnailUrl: m.thumbnail_url ?? null,
+		width: m.width ?? null,
+		height: m.height ?? null
+	})).filter((m) => m.url);
+}
+
+/**
+ * The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123
+ * — no API version prefix. The base URL is admin-configurable (Retention tab) so a
+ * self-hosted FixTweet mirror (or another compatible public instance) can be used
+ * instead of the public api.fxtwitter.com default.
+ */
+async function fetchFxTwitter(handle: string, tweetId: string): Promise<FxTweet | null> {
+	const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, '');
+	try {
+		const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, {
+			headers: { 'User-Agent': USER_AGENT },
+			signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
+		});
+		if (!res.ok) return null;
+		const json = (await res.json()) as { tweet?: FxTweet };
+		return json?.tweet ?? null;
+	} catch (err) {
+		logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`);
+		return null;
+	}
+}
+
+function extractTweetId(item: Parser.Item): string | null {
+	const guid = item.guid;
+	if (guid && /^\d+$/.test(guid)) return guid;
+	const fromLink = item.link?.match(/status\/(\d+)/)?.[1];
+	return fromLink ?? null;
+}
+
+/**
+ * A Nitter list/user RSS description is the item's own tweet content (a <p> plus
+ * optional <img>) followed, for retweets/quote-tweets, by a <blockquote> wrapping the
+ * quoted tweet's own text/image/permalink. Only the part before that blockquote is this
+ * item's own content — nested quote-tweet rendering isn't part of the approved design.
+ */
+function ownContentHtml(descriptionHtml: string): string {
+	const cut = descriptionHtml.search(/<hr\s*\/?>|<blockquote/i);
+	return cut === -1 ? descriptionHtml : descriptionHtml.slice(0, cut);
+}
+
+function extractImageUrl(html: string): string | null {
+	return html.match(/<img[^>]+src="([^"]+)"/i)?.[1] ?? null;
+}
+
+export const nitterAdapter: SourceAdapter = {
+	async fetch(source: Source): Promise<FetchedItem[]> {
+		if (!source.url) return [];
+
+		const feed = await parser.parseURL(source.url);
+		const items: FetchedItem[] = [];
+
+		for (const item of feed.items) {
+			if (!item.link || !item.guid) continue;
+			const tweetId = extractTweetId(item);
+			if (!tweetId) {
+				logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`);
+				continue;
+			}
+
+			// dc:creator is reliably the author of this item's own tweet text (rss-parser
+			// maps it to item.creator) — for a retweet, that's the original tweet's author,
+			// not whichever list member's retweet surfaced it in this feed.
+			const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
+			const ownHtml = ownContentHtml(item.content ?? '');
+			const rssImageUrl = extractImageUrl(ownHtml);
+
+			const enrichment = await fetchFxTwitter(handle, tweetId);
+
+			const authorName = enrichment?.author?.name ?? handle;
+			const avatarUrl = enrichment?.author?.avatar_url ?? null;
+			const text = enrichment?.text ?? ownHtml;
+			// Enrichment failed (or came back with no media) — fall back to the RSS
+			// description's own <img> as a single photo, same as before multi-media support.
+			const media = toTweetMedia(enrichment);
+			if (media.length === 0 && rssImageUrl) {
+				media.push({ type: 'photo', url: rssImageUrl, thumbnailUrl: null, width: null, height: null });
+			}
+			const publishedAt = enrichment?.created_timestamp
+				? new Date(enrichment.created_timestamp * 1000).toISOString()
+				: (item.isoDate ?? item.pubDate ?? new Date().toISOString());
+
+			items.push({
+				title: item.title || text.slice(0, 100),
+				summary: text.slice(0, 500),
+				body: text,
+				images: [],
+				videos: [],
+				link: item.link,
+				publishedAt,
+				tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media },
+				raw: { rss: item, fxtwitter: enrichment }
+			});
+		}
+
+		return items;
+	}
+};

backend/src/ingestion/poller.ts

@@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js';
 import { telegramAdapter } from './adapters/telegram.js';
 import { apiAdapter } from './adapters/api.js';
 import { youtubeAdapter } from './adapters/youtube.js';
+import { nitterAdapter } from './adapters/nitter.js';
 import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
 import { fetchFullArticle } from './articleFetcher.js';
 import type { Source } from '../storage/db/types.js';
@@ -14,6 +15,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
 	telegram: telegramAdapter,
 	api: apiAdapter,
 	youtube: youtubeAdapter,
+	nitter: nitterAdapter,
 	custom: apiAdapter
 };
 

backend/src/pipeline/publish.ts

@@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js';
 import * as articles from '../storage/db/articles.js';
 import * as tags from '../storage/db/tags.js';
 import * as sources from '../storage/db/sources.js';
-import type { GlobalSettings, MergedArticle, ContentItem } from '../storage/db/types.js';
+import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js';
 
 const FOLLOW_UP_LOOKBACK_DAYS = 3;
 
@@ -33,9 +33,14 @@ function deriveTitle(body: string): string {
 }
 
 /**
- * Resolves the hero image for an article: try the best candidate from the source
- * items, download and locally host it; if there isn't one, fall back to the site's
- * favicon rather than leaving the article with no art at all.
+ * Resolves the hero image for a regular (non-tweet) article: try the best candidate
+ * from the source items, download and locally host it; if there isn't one, fall back
+ * to the site's favicon rather than leaving the article with no art at all. Tweets
+ * never reach this function — see resolveTweetMediaUrl below, which applies the
+ * admin-configured Nitter media mode instead of always downloading, and skips the
+ * favicon fallback entirely (a Nitter instance's own favicon slapped onto an
+ * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already
+ * handles no-image tweets gracefully with no image at all).
  */
 async function resolveHeroImage(
 	items: ContentItem[],
@@ -69,6 +74,61 @@ async function resolveHeroImage(
 	return { heroImage: null, storedMediaId: null };
 }
 
+/**
+ * Applies the admin-configured Nitter media mode (Retention tab) to a single
+ * externally-hosted tweet media URL — an attached photo or the author's avatar.
+ * 'self-host' downloads and serves it locally like any other article image;
+ * 'proxy' routes it through this server's own /media/proxy route so only this
+ * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the
+ * "anonymity" the admin asked for) without persisting anything to disk; 'direct'
+ * hotlinks the original URL unchanged, the cheapest option with no server involvement.
+ */
+async function resolveTweetMediaUrl(
+	url: string,
+	mode: GlobalSettings['nitterMediaMode']
+): Promise<{ url: string; storedMediaId: string | null }> {
+	if (mode === 'direct') return { url, storedMediaId: null };
+
+	if (mode === 'proxy') {
+		return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null };
+	}
+
+	const stored = await downloadAndStore(url, 'published', {});
+	if (stored) return { url: stored.servedPath, storedMediaId: stored.id };
+	// Download failed — fall back to hotlinking rather than losing the media entirely.
+	return { url, storedMediaId: null };
+}
+
+/**
+ * Resolves every attached photo/video/gif's url — and, for video/gif, its poster
+ * thumbnail — through the admin's chosen Nitter media mode. Order and item count are
+ * preserved; TweetCard.svelte renders this array directly, there's no separate
+ * "hero image" concept for tweets the way there is for regular articles.
+ */
+async function resolveTweetMedia(
+	media: TweetMediaItem[],
+	mode: GlobalSettings['nitterMediaMode']
+): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> {
+	const storedMediaIds: string[] = [];
+	const resolved: TweetMediaItem[] = [];
+
+	for (const item of media) {
+		const url = await resolveTweetMediaUrl(item.url, mode);
+		if (url.storedMediaId) storedMediaIds.push(url.storedMediaId);
+
+		let thumbnailUrl = item.thumbnailUrl;
+		if (thumbnailUrl) {
+			const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode);
+			thumbnailUrl = resolvedThumb.url;
+			if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId);
+		}
+
+		resolved.push({ ...item, url: url.url, thumbnailUrl });
+	}
+
+	return { media: resolved, storedMediaIds };
+}
+
 /**
  * Publishes a single item as-is, with no AI calls at all — used when the AI service
  * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
@@ -79,18 +139,47 @@ async function resolveHeroImage(
  * tag-based thread detection — but these earlier articles aren't retroactively
  * rewritten or merged with anything after the fact.
  */
-export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
+export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise<MergedArticle> {
 	const category = uniqueCategories([item]);
-	const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
+	const storedMediaIds: string[] = [];
+
+	// Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly.
+	let heroImage: MergedArticle['heroImage'] = null;
+	if (!item.tweet) {
+		const resolved = await resolveHeroImage([item], item.link);
+		heroImage = resolved.heroImage;
+		if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
+	}
+
 	const video = item.videos[0]
 		? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
 		: null;
 
+	let tweet: MergedArticle['tweet'] = null;
+	if (item.tweet) {
+		let avatarUrl: string | null = null;
+		if (item.tweet.avatarUrl) {
+			const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode);
+			avatarUrl = resolved.url;
+			if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
+		}
+		const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode);
+		storedMediaIds.push(...resolvedMedia.storedMediaIds);
+		tweet = {
+			authorName: item.tweet.authorName,
+			authorHandle: item.tweet.authorHandle,
+			avatarUrl,
+			sourceItemId: item.id,
+			media: resolvedMedia.media
+		};
+	}
+
 	const article = await articles.insertArticle({
 		title: item.title,
 		body: item.body || item.summary,
 		heroImage,
 		video,
+		tweet,
 		category,
 		geo: item.geo,
 		eventId: item.eventId,
@@ -115,7 +204,7 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
 		topStories: anyPushesToTopStories([item])
 	});
 
-	if (storedMediaId) promoteToPublished(storedMediaId, article.id);
+	for (const id of storedMediaIds) promoteToPublished(id, article.id);
 	return article;
 }
 
@@ -192,6 +281,7 @@ export async function publishCluster(
 		body,
 		heroImage,
 		video,
+		tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass
 		category,
 		geo,
 		eventId: opts.eventId ?? items[0]?.eventId ?? null,

backend/src/queue/priorityQueue.ts

@@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
 
 	for (const item of ranked) {
 		try {
-			const article = await publishDirect(item);
+			const article = await publishDirect(item, settings);
 			contentItemsDb.assignCluster([item.id], article.id);
 			published++;
 			logger.info('synthesis', `Published "${article.title}" directly (no AI available)`);
@@ -77,19 +77,22 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
 	const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
 	if (items.length === 0) return 0;
 
-	// YouTube videos never get LLM-merged with anything — each is always its own
-	// article (title/video/date/description), same shape whether the AI service is up
-	// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
-	const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
-	const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
+	// YouTube videos and Nitter tweets never get LLM-merged with anything else — each
+	// is always its own article, same shape whether the AI service is up or not. Route
+	// them straight to publishDirect, same as the no-AI passthrough path.
+	const directPublishSourceIds = new Set(
+		sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id)
+	);
+	const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
 
 	let publishedDirect = 0;
-	for (const item of youtubeItems) {
+	for (const item of directItems) {
 		try {
-			const article = await publishDirect(item);
+			const article = await publishDirect(item, settings);
 			contentItemsDb.assignCluster([item.id], article.id);
 			publishedDirect++;
-			logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
+			const source = sourcesDb.getSource(item.sourceId);
+			logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`);
 		} catch (err) {
 			logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
 		}

backend/src/storage/db/articles.ts

@@ -22,7 +22,8 @@ function rowToArticle(row: any): MergedArticle {
 		threadId: row.thread_id,
 		previousArticleId: row.previous_article_id,
 		nextArticleId: row.next_article_id,
-		topStories: !!row.top_stories
+		topStories: !!row.top_stories,
+		tweet: row.tweet ? JSON.parse(row.tweet) : null
 	};
 }
 
@@ -30,8 +31,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
 	const id = `art-${randomUUID()}`;
 	db.prepare(
 		`INSERT INTO merged_articles
-		 (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories)
-		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+		 (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	).run(
 		id,
 		article.title,
@@ -50,7 +51,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
 		article.threadId,
 		article.previousArticleId,
 		article.nextArticleId,
-		article.topStories ? 1 : 0
+		article.topStories ? 1 : 0,
+		article.tweet ? JSON.stringify(article.tweet) : null
 	);
 	if (article.previousArticleId) {
 		db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId);

backend/src/storage/db/contentItems.ts

@@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem {
 		embedding: row.embedding ? JSON.parse(row.embedding) : null,
 		eventId: row.event_id,
 		clusterId: row.cluster_id,
+		tweet: row.tweet ? JSON.parse(row.tweet) : null,
 		raw: row.raw ? JSON.parse(row.raw) : null
 	};
 }
@@ -28,8 +29,8 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
 	const id = `ci-${randomUUID()}`;
 	db.prepare(
 		`INSERT INTO content_items
-		 (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw)
-		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+		 (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	).run(
 		id,
 		item.sourceId,
@@ -47,6 +48,7 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
 		item.embedding ? JSON.stringify(item.embedding) : null,
 		item.eventId,
 		item.clusterId,
+		item.tweet ? JSON.stringify(item.tweet) : null,
 		item.raw ? JSON.stringify(item.raw) : null
 	);
 	return { ...item, id };

backend/src/storage/db/index.ts

@@ -58,6 +58,7 @@ export function migrate() {
 			embedding TEXT, -- JSON float array
 			event_id TEXT,
 			cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis
+			tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only
 			raw TEXT -- JSON, original payload
 		);
 		CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id);
@@ -82,7 +83,8 @@ export function migrate() {
 			thread_id TEXT NOT NULL,
 			previous_article_id TEXT,
 			next_article_id TEXT,
-			top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?"
+			top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?"
+			tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only
 		);
 		CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at);
 		CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id);
@@ -166,7 +168,9 @@ export function migrate() {
 			raw_item_max_age_days INTEGER DEFAULT 7,
 			storage_cap_enabled INTEGER NOT NULL DEFAULT 1,
 			storage_cap_value INTEGER NOT NULL DEFAULT 500,
-			storage_cap_unit TEXT NOT NULL DEFAULT 'GB'
+			storage_cap_unit TEXT NOT NULL DEFAULT 'GB',
+			nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct
+			fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'
 		);
 	`);
 
@@ -188,6 +192,18 @@ export function migrate() {
 	if (!hasColumn('merged_articles', 'top_stories')) {
 		db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0');
 	}
+	if (!hasColumn('content_items', 'tweet')) {
+		db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT');
+	}
+	if (!hasColumn('merged_articles', 'tweet')) {
+		db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT');
+	}
+	if (!hasColumn('global_settings', 'nitter_media_mode')) {
+		db.exec("ALTER TABLE global_settings ADD COLUMN nitter_media_mode TEXT NOT NULL DEFAULT 'proxy'");
+	}
+	if (!hasColumn('global_settings', 'fxtwitter_base_url')) {
+		db.exec("ALTER TABLE global_settings ADD COLUMN fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'");
+	}
 	if (!hasColumn('categories', 'is_private')) {
 		db.exec('ALTER TABLE categories ADD COLUMN is_private INTEGER NOT NULL DEFAULT 0');
 	}

backend/src/storage/db/settings.ts

@@ -13,6 +13,8 @@ function rowToSettings(row: any): GlobalSettings {
 		aiServiceHost: row.ai_service_host,
 		aiServicePort: row.ai_service_port,
 		selectedModels: JSON.parse(row.selected_models),
+		nitterMediaMode: row.nitter_media_mode,
+		fxtwitterBaseUrl: row.fxtwitter_base_url,
 		retention: {
 			publishedArticleMaxAgeDays: row.published_article_max_age_days,
 			rawItemMaxAgeDays: row.raw_item_max_age_days,
@@ -41,6 +43,7 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
 			merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?,
 			tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?,
 			ai_service_host=?, ai_service_port=?, selected_models=?,
+			nitter_media_mode=?, fxtwitter_base_url=?,
 			published_article_max_age_days=?, raw_item_max_age_days=?,
 			storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?
 		 WHERE id = 1`
@@ -55,6 +58,8 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
 		merged.aiServiceHost,
 		merged.aiServicePort,
 		JSON.stringify(merged.selectedModels),
+		merged.nitterMediaMode,
+		merged.fxtwitterBaseUrl,
 		merged.retention.publishedArticleMaxAgeDays,
 		merged.retention.rawItemMaxAgeDays,
 		merged.retention.storageCapEnabled ? 1 : 0,

backend/src/storage/db/types.ts

@@ -1,7 +1,7 @@
 export interface Source {
 	id: string;
 	name: string;
-	type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
+	type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
 	category: string[];
 	url: string | null;
 	config: Record<string, unknown>;
@@ -14,6 +14,16 @@ export interface Source {
 	createdAt: string;
 }
 
+/** A single photo/video/gif attached to a tweet, in the tweet's own display order — fxtwitter caps this at 4. */
+export interface TweetMediaItem {
+	type: 'photo' | 'video' | 'gif';
+	url: string;
+	/** Video/gif poster frame — null for photos. */
+	thumbnailUrl: string | null;
+	width: number | null;
+	height: number | null;
+}
+
 export interface ContentItem {
 	id: string;
 	sourceId: string;
@@ -31,6 +41,8 @@ export interface ContentItem {
 	embedding: number[] | null;
 	eventId: string | null;
 	clusterId: string | null;
+	/** Nitter-sourced items only — null for everything else. */
+	tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null;
 	raw: unknown;
 }
 
@@ -47,6 +59,8 @@ export interface MergedArticle {
 	body: string;
 	heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
 	video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
+	/** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */
+	tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
 	category: string[];
 	geo: string | null;
 	eventId: string | null;
@@ -117,6 +131,10 @@ export interface GlobalSettings {
 	aiServiceHost: string;
 	aiServicePort: number;
 	selectedModels: { embedding: string; image: string; synthesis: string };
+	/** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */
+	nitterMediaMode: 'self-host' | 'proxy' | 'direct';
+	/** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */
+	fxtwitterBaseUrl: string;
 	retention: {
 		publishedArticleMaxAgeDays: number | null;
 		rawItemMaxAgeDays: number | null;