Add configurable tweet media hosting mode and fxtwitter base URL

Adds nitterMediaMode (self-host/proxy/direct, default proxy) and
fxtwitterBaseUrl to global settings with a new Retention tab panel.
Tweet images and avatars now resolve through the chosen mode instead
of always being downloaded — proxy mode streams media through a new
SSRF-hardened /media/proxy route (hostname allowlist + DNS-rebinding
defense) so the origin server's IP is never exposed to Twitter's CDN,
direct hotlinks the original URL, and self-host keeps the prior
always-download behavior. fxtwitterBaseUrl lets the enrichment call
target a self-hosted FixTweet mirror instead of the public instance.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8

backend/src/api/mediaProxy.ts

@@ -0,0 +1,117 @@
+// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode):
+// the visitor's browser requests media from this route instead of directly from
+// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the
+// remote host — the media itself is streamed straight through, never written to disk.
+//
+// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless
+// tightly restricted: only twimg.com (Twitter's media CDN), the configured
+// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter
+// sources are allowed — and even an allowed hostname is rejected if it resolves to a
+// private/loopback/link-local address (defends against DNS rebinding, not just a
+// hostname string check).
+
+import type { FastifyInstance } from 'fastify';
+import dns from 'node:dns/promises';
+import { Readable } from 'node:stream';
+import * as sourcesDb from '../storage/db/sources.js';
+import { getSettings } from '../storage/db/settings.js';
+import { logger } from '../storage/db/logs.js';
+
+const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
+const FETCH_TIMEOUT_MS = 15_000;
+const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i;
+
+function hostnameOf(rawUrl: string | null): string | null {
+	if (!rawUrl) return null;
+	try {
+		return new URL(rawUrl).hostname.toLowerCase();
+	} catch {
+		return null;
+	}
+}
+
+function isAllowedHost(hostname: string): boolean {
+	const lower = hostname.toLowerCase();
+	if (TWITTER_MEDIA_HOST_RE.test(lower)) return true;
+
+	const settings = getSettings();
+	if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true;
+
+	const nitterHosts = sourcesDb
+		.listSources()
+		.filter((s) => s.type === 'nitter')
+		.map((s) => hostnameOf(s.url))
+		.filter((h): h is string => !!h);
+	return nitterHosts.includes(lower);
+}
+
+function isPrivateOrReservedIp(ip: string, family: number): boolean {
+	if (family === 4) {
+		const [a, b] = ip.split('.').map(Number);
+		if (a === 10 || a === 127 || a === 0) return true;
+		if (a === 169 && b === 254) return true;
+		if (a === 172 && b >= 16 && b <= 31) return true;
+		if (a === 192 && b === 168) return true;
+		if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range
+		return false;
+	}
+	const lower = ip.toLowerCase();
+	if (lower === '::1') return true;
+	if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7
+	if (lower.startsWith('fe80')) return true; // link-local
+	if (lower.startsWith('::ffff:')) {
+		const v4 = lower.split(':').pop();
+		if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4);
+	}
+	return false;
+}
+
+export async function registerMediaProxy(app: FastifyInstance) {
+	app.get('/media/proxy', async (req, reply) => {
+		const { url } = req.query as { url?: string };
+		if (!url) return reply.code(400).send({ error: 'url required' });
+
+		let parsed: URL;
+		try {
+			parsed = new URL(url);
+		} catch {
+			return reply.code(400).send({ error: 'invalid url' });
+		}
+
+		if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+			return reply.code(400).send({ error: 'unsupported scheme' });
+		}
+
+		if (!isAllowedHost(parsed.hostname)) {
+			logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`);
+			return reply.code(403).send({ error: 'host not allowed' });
+		}
+
+		let addresses: { address: string; family: number }[];
+		try {
+			addresses = await dns.lookup(parsed.hostname, { all: true });
+		} catch {
+			return reply.code(502).send({ error: 'DNS resolution failed' });
+		}
+		if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) {
+			logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`);
+			return reply.code(403).send({ error: 'host not allowed' });
+		}
+
+		try {
+			const res = await fetch(parsed.toString(), {
+				headers: { 'User-Agent': USER_AGENT },
+				signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
+			});
+			if (!res.ok || !res.body) {
+				return reply.code(502).send({ error: `upstream responded ${res.status}` });
+			}
+			reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream');
+			reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600');
+			return reply.send(Readable.fromWeb(res.body as any));
+		} catch (err) {
+			logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`);
+			return reply.code(502).send({ error: 'fetch failed' });
+		}
+	});
+}

backend/src/index.ts

@@ -7,6 +7,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js';
 import { registerAuth } from './api/auth.js';
 import { registerPublicRoutes } from './api/public.js';
 import { registerAdminRoutes } from './api/admin.js';
+import { registerMediaProxy } from './api/mediaProxy.js';
 import { startScheduler } from './queue/scheduler.js';
 import { logger } from './storage/db/logs.js';
 
@@ -79,6 +80,11 @@ async function main() {
 		return reply.send(fs.createReadStream(filePath));
 	});
 
+	// Static "/media/proxy" takes priority over the "/media/:filename" param route
+	// above regardless of registration order (find-my-way, Fastify's router, always
+	// prefers a static segment over a parametric one at the same depth).
+	await registerMediaProxy(app);
+
 	app.get('/health', async () => ({ ok: true }));
 
 	await app.listen({ port: PORT, host: '0.0.0.0' });

backend/src/ingestion/adapters/nitter.ts

@@ -10,6 +10,7 @@ import Parser from 'rss-parser';
 import type { Source } from '../../storage/db/types.js';
 import type { SourceAdapter, FetchedItem } from './base.js';
 import { logger } from '../../storage/db/logs.js';
+import { getSettings } from '../../storage/db/settings.js';
 
 const parser = new Parser<Record<string, unknown>>();
 
@@ -31,10 +32,16 @@ interface FxTweet {
 	media?: { photos?: { url?: string }[] };
 }
 
-/** The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 — no API version prefix. */
+/**
+ * The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123
+ * — no API version prefix. The base URL is admin-configurable (Retention tab) so a
+ * self-hosted FixTweet mirror (or another compatible public instance) can be used
+ * instead of the public api.fxtwitter.com default.
+ */
 async function fetchFxTwitter(handle: string, tweetId: string): Promise<FxTweet | null> {
+	const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, '');
 	try {
-		const res = await fetch(`https://api.fxtwitter.com/${handle}/status/${tweetId}`, {
+		const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, {
 			headers: { 'User-Agent': USER_AGENT },
 			signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
 		});

backend/src/pipeline/publish.ts

@@ -33,17 +33,18 @@ function deriveTitle(body: string): string {
 }
 
 /**
- * Resolves the hero image for an article: try the best candidate from the source
- * items, download and locally host it; if there isn't one, fall back to the site's
- * favicon rather than leaving the article with no art at all. That favicon fallback
- * is skipped for tweets (allowFaviconFallback: false) — a Nitter instance's own
- * favicon slapped onto an image-less tweet reads as a mistake, not a placeholder;
- * TweetCard.svelte already handles no-image tweets gracefully with no image at all.
+ * Resolves the hero image for a regular (non-tweet) article: try the best candidate
+ * from the source items, download and locally host it; if there isn't one, fall back
+ * to the site's favicon rather than leaving the article with no art at all. Tweets
+ * never reach this function — see resolveTweetMediaUrl below, which applies the
+ * admin-configured Nitter media mode instead of always downloading, and skips the
+ * favicon fallback entirely (a Nitter instance's own favicon slapped onto an
+ * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already
+ * handles no-image tweets gracefully with no image at all).
  */
 async function resolveHeroImage(
 	items: ContentItem[],
-	primaryLink: string,
-	allowFaviconFallback = true
+	primaryLink: string
 ): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> {
 	const selected = selectBestImage(items);
 
@@ -59,10 +60,6 @@ async function resolveHeroImage(
 		return { heroImage: selected, storedMediaId: null };
 	}
 
-	if (!allowFaviconFallback) {
-		return { heroImage: null, storedMediaId: null };
-	}
-
 	const favicon = faviconUrlFor(primaryLink);
 	if (favicon) {
 		const stored = await downloadAndStore(favicon, 'published', {});
@@ -77,6 +74,31 @@ async function resolveHeroImage(
 	return { heroImage: null, storedMediaId: null };
 }
 
+/**
+ * Applies the admin-configured Nitter media mode (Retention tab) to a single
+ * externally-hosted tweet media URL — an attached photo or the author's avatar.
+ * 'self-host' downloads and serves it locally like any other article image;
+ * 'proxy' routes it through this server's own /media/proxy route so only this
+ * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the
+ * "anonymity" the admin asked for) without persisting anything to disk; 'direct'
+ * hotlinks the original URL unchanged, the cheapest option with no server involvement.
+ */
+async function resolveTweetMediaUrl(
+	url: string,
+	mode: GlobalSettings['nitterMediaMode']
+): Promise<{ url: string; storedMediaId: string | null }> {
+	if (mode === 'direct') return { url, storedMediaId: null };
+
+	if (mode === 'proxy') {
+		return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null };
+	}
+
+	const stored = await downloadAndStore(url, 'published', {});
+	if (stored) return { url: stored.servedPath, storedMediaId: stored.id };
+	// Download failed — fall back to hotlinking rather than losing the media entirely.
+	return { url, storedMediaId: null };
+}
+
 /**
  * Publishes a single item as-is, with no AI calls at all — used when the AI service
  * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
@@ -87,15 +109,38 @@ async function resolveHeroImage(
  * tag-based thread detection — but these earlier articles aren't retroactively
  * rewritten or merged with anything after the fact.
  */
-export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
+export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise<MergedArticle> {
 	const category = uniqueCategories([item]);
-	const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link, !item.tweet);
+	const storedMediaIds: string[] = [];
+
+	let heroImage: MergedArticle['heroImage'] = null;
+	if (item.tweet) {
+		const selected = selectBestImage([item]);
+		if (selected) {
+			const resolved = await resolveTweetMediaUrl(selected.url, settings.nitterMediaMode);
+			heroImage = { url: resolved.url, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason };
+			if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
+		}
+	} else {
+		const resolved = await resolveHeroImage([item], item.link);
+		heroImage = resolved.heroImage;
+		if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
+	}
+
 	const video = item.videos[0]
 		? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
 		: null;
-	const tweet = item.tweet
-		? { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl: item.tweet.avatarUrl, sourceItemId: item.id }
-		: null;
+
+	let tweet: MergedArticle['tweet'] = null;
+	if (item.tweet) {
+		let avatarUrl: string | null = null;
+		if (item.tweet.avatarUrl) {
+			const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode);
+			avatarUrl = resolved.url;
+			if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
+		}
+		tweet = { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl, sourceItemId: item.id };
+	}
 
 	const article = await articles.insertArticle({
 		title: item.title,
@@ -127,7 +172,7 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
 		topStories: anyPushesToTopStories([item])
 	});
 
-	if (storedMediaId) promoteToPublished(storedMediaId, article.id);
+	for (const id of storedMediaIds) promoteToPublished(id, article.id);
 	return article;
 }
 

backend/src/queue/priorityQueue.ts

@@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
 
 	for (const item of ranked) {
 		try {
-			const article = await publishDirect(item);
+			const article = await publishDirect(item, settings);
 			contentItemsDb.assignCluster([item.id], article.id);
 			published++;
 			logger.info('synthesis', `Published "${article.title}" directly (no AI available)`);
@@ -88,7 +88,7 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
 	let publishedDirect = 0;
 	for (const item of directItems) {
 		try {
-			const article = await publishDirect(item);
+			const article = await publishDirect(item, settings);
 			contentItemsDb.assignCluster([item.id], article.id);
 			publishedDirect++;
 			const source = sourcesDb.getSource(item.sourceId);

backend/src/storage/db/index.ts

@@ -167,7 +167,9 @@ export function migrate() {
 			raw_item_max_age_days INTEGER DEFAULT 7,
 			storage_cap_enabled INTEGER NOT NULL DEFAULT 1,
 			storage_cap_value INTEGER NOT NULL DEFAULT 500,
-			storage_cap_unit TEXT NOT NULL DEFAULT 'GB'
+			storage_cap_unit TEXT NOT NULL DEFAULT 'GB',
+			nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct
+			fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'
 		);
 	`);
 
@@ -195,6 +197,12 @@ export function migrate() {
 	if (!hasColumn('merged_articles', 'tweet')) {
 		db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT');
 	}
+	if (!hasColumn('global_settings', 'nitter_media_mode')) {
+		db.exec("ALTER TABLE global_settings ADD COLUMN nitter_media_mode TEXT NOT NULL DEFAULT 'proxy'");
+	}
+	if (!hasColumn('global_settings', 'fxtwitter_base_url')) {
+		db.exec("ALTER TABLE global_settings ADD COLUMN fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'");
+	}
 
 	// Seed default categories if none exist yet. "News" sits right under "Top stories" —
 	// general news sources belong here, not on "Top stories" itself, which isn't a real

backend/src/storage/db/settings.ts

@@ -13,6 +13,8 @@ function rowToSettings(row: any): GlobalSettings {
 		aiServiceHost: row.ai_service_host,
 		aiServicePort: row.ai_service_port,
 		selectedModels: JSON.parse(row.selected_models),
+		nitterMediaMode: row.nitter_media_mode,
+		fxtwitterBaseUrl: row.fxtwitter_base_url,
 		retention: {
 			publishedArticleMaxAgeDays: row.published_article_max_age_days,
 			rawItemMaxAgeDays: row.raw_item_max_age_days,
@@ -41,6 +43,7 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
 			merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?,
 			tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?,
 			ai_service_host=?, ai_service_port=?, selected_models=?,
+			nitter_media_mode=?, fxtwitter_base_url=?,
 			published_article_max_age_days=?, raw_item_max_age_days=?,
 			storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?
 		 WHERE id = 1`
@@ -55,6 +58,8 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
 		merged.aiServiceHost,
 		merged.aiServicePort,
 		JSON.stringify(merged.selectedModels),
+		merged.nitterMediaMode,
+		merged.fxtwitterBaseUrl,
 		merged.retention.publishedArticleMaxAgeDays,
 		merged.retention.rawItemMaxAgeDays,
 		merged.retention.storageCapEnabled ? 1 : 0,

backend/src/storage/db/types.ts

@@ -119,6 +119,10 @@ export interface GlobalSettings {
 	aiServiceHost: string;
 	aiServicePort: number;
 	selectedModels: { embedding: string; image: string; synthesis: string };
+	/** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */
+	nitterMediaMode: 'self-host' | 'proxy' | 'direct';
+	/** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */
+	fxtwitterBaseUrl: string;
 	retention: {
 		publishedArticleMaxAgeDays: number | null;
 		rawItemMaxAgeDays: number | null;