diff --git a/backend/src/api/mediaProxy.ts b/backend/src/api/mediaProxy.ts new file mode 100644 index 0000000..f31970b --- /dev/null +++ b/backend/src/api/mediaProxy.ts @@ -0,0 +1,117 @@ +// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode): +// the visitor's browser requests media from this route instead of directly from +// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the +// remote host — the media itself is streamed straight through, never written to disk. +// +// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless +// tightly restricted: only twimg.com (Twitter's media CDN), the configured +// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter +// sources are allowed — and even an allowed hostname is rejected if it resolves to a +// private/loopback/link-local address (defends against DNS rebinding, not just a +// hostname string check). + +import type { FastifyInstance } from 'fastify'; +import dns from 'node:dns/promises'; +import { Readable } from 'node:stream'; +import * as sourcesDb from '../storage/db/sources.js'; +import { getSettings } from '../storage/db/settings.js'; +import { logger } from '../storage/db/logs.js'; + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 15_000; +const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i; + +function hostnameOf(rawUrl: string | null): string | null { + if (!rawUrl) return null; + try { + return new URL(rawUrl).hostname.toLowerCase(); + } catch { + return null; + } +} + +function isAllowedHost(hostname: string): boolean { + const lower = hostname.toLowerCase(); + if (TWITTER_MEDIA_HOST_RE.test(lower)) return true; + + const settings = getSettings(); + if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true; + + const nitterHosts = sourcesDb + .listSources() + .filter((s) => s.type === 'nitter') + .map((s) => hostnameOf(s.url)) + .filter((h): h is string => !!h); + return nitterHosts.includes(lower); +} + +function isPrivateOrReservedIp(ip: string, family: number): boolean { + if (family === 4) { + const [a, b] = ip.split('.').map(Number); + if (a === 10 || a === 127 || a === 0) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range + return false; + } + const lower = ip.toLowerCase(); + if (lower === '::1') return true; + if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7 + if (lower.startsWith('fe80')) return true; // link-local + if (lower.startsWith('::ffff:')) { + const v4 = lower.split(':').pop(); + if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4); + } + return false; +} + +export async function registerMediaProxy(app: FastifyInstance) { + app.get('/media/proxy', async (req, reply) => { + const { url } = req.query as { url?: string }; + if (!url) return reply.code(400).send({ error: 'url required' }); + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return reply.code(400).send({ error: 'invalid url' }); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return reply.code(400).send({ error: 'unsupported scheme' }); + } + + if (!isAllowedHost(parsed.hostname)) { + logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + let addresses: { address: string; family: number }[]; + try { + addresses = await dns.lookup(parsed.hostname, { all: true }); + } catch { + return reply.code(502).send({ error: 'DNS resolution failed' }); + } + if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) { + logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + try { + const res = await fetch(parsed.toString(), { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + return reply.code(502).send({ error: `upstream responded ${res.status}` }); + } + reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream'); + reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600'); + return reply.send(Readable.fromWeb(res.body as any)); + } catch (err) { + logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`); + return reply.code(502).send({ error: 'fetch failed' }); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 98746b5..1848fb3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -7,6 +7,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js'; import { registerAuth } from './api/auth.js'; import { registerPublicRoutes } from './api/public.js'; import { registerAdminRoutes } from './api/admin.js'; +import { registerMediaProxy } from './api/mediaProxy.js'; import { startScheduler } from './queue/scheduler.js'; import { logger } from './storage/db/logs.js'; @@ -79,6 +80,11 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); + // Static "/media/proxy" takes priority over the "/media/:filename" param route + // above regardless of registration order (find-my-way, Fastify's router, always + // prefers a static segment over a parametric one at the same depth). + await registerMediaProxy(app); + app.get('/health', async () => ({ ok: true })); await app.listen({ port: PORT, host: '0.0.0.0' }); diff --git a/backend/src/ingestion/adapters/nitter.ts b/backend/src/ingestion/adapters/nitter.ts index 3b3659c..fec2118 100644 --- a/backend/src/ingestion/adapters/nitter.ts +++ b/backend/src/ingestion/adapters/nitter.ts @@ -10,6 +10,7 @@ import Parser from 'rss-parser'; import type { Source } from '../../storage/db/types.js'; import type { SourceAdapter, FetchedItem } from './base.js'; import { logger } from '../../storage/db/logs.js'; +import { getSettings } from '../../storage/db/settings.js'; const parser = new Parser>(); @@ -31,10 +32,16 @@ interface FxTweet { media?: { photos?: { url?: string }[] }; } -/** The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 — no API version prefix. */ +/** + * The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 + * — no API version prefix. The base URL is admin-configurable (Retention tab) so a + * self-hosted FixTweet mirror (or another compatible public instance) can be used + * instead of the public api.fxtwitter.com default. + */ async function fetchFxTwitter(handle: string, tweetId: string): Promise { + const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, ''); try { - const res = await fetch(`https://api.fxtwitter.com/${handle}/status/${tweetId}`, { + const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, { headers: { 'User-Agent': USER_AGENT }, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index e053072..90b05b3 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -33,17 +33,18 @@ function deriveTitle(body: string): string { } /** - * Resolves the hero image for an article: try the best candidate from the source - * items, download and locally host it; if there isn't one, fall back to the site's - * favicon rather than leaving the article with no art at all. That favicon fallback - * is skipped for tweets (allowFaviconFallback: false) — a Nitter instance's own - * favicon slapped onto an image-less tweet reads as a mistake, not a placeholder; - * TweetCard.svelte already handles no-image tweets gracefully with no image at all. + * Resolves the hero image for a regular (non-tweet) article: try the best candidate + * from the source items, download and locally host it; if there isn't one, fall back + * to the site's favicon rather than leaving the article with no art at all. Tweets + * never reach this function — see resolveTweetMediaUrl below, which applies the + * admin-configured Nitter media mode instead of always downloading, and skips the + * favicon fallback entirely (a Nitter instance's own favicon slapped onto an + * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already + * handles no-image tweets gracefully with no image at all). */ async function resolveHeroImage( items: ContentItem[], - primaryLink: string, - allowFaviconFallback = true + primaryLink: string ): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> { const selected = selectBestImage(items); @@ -59,10 +60,6 @@ async function resolveHeroImage( return { heroImage: selected, storedMediaId: null }; } - if (!allowFaviconFallback) { - return { heroImage: null, storedMediaId: null }; - } - const favicon = faviconUrlFor(primaryLink); if (favicon) { const stored = await downloadAndStore(favicon, 'published', {}); @@ -77,6 +74,31 @@ async function resolveHeroImage( return { heroImage: null, storedMediaId: null }; } +/** + * Applies the admin-configured Nitter media mode (Retention tab) to a single + * externally-hosted tweet media URL — an attached photo or the author's avatar. + * 'self-host' downloads and serves it locally like any other article image; + * 'proxy' routes it through this server's own /media/proxy route so only this + * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the + * "anonymity" the admin asked for) without persisting anything to disk; 'direct' + * hotlinks the original URL unchanged, the cheapest option with no server involvement. + */ +async function resolveTweetMediaUrl( + url: string, + mode: GlobalSettings['nitterMediaMode'] +): Promise<{ url: string; storedMediaId: string | null }> { + if (mode === 'direct') return { url, storedMediaId: null }; + + if (mode === 'proxy') { + return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null }; + } + + const stored = await downloadAndStore(url, 'published', {}); + if (stored) return { url: stored.servedPath, storedMediaId: stored.id }; + // Download failed — fall back to hotlinking rather than losing the media entirely. + return { url, storedMediaId: null }; +} + /** * Publishes a single item as-is, with no AI calls at all — used when the AI service * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives @@ -87,15 +109,38 @@ async function resolveHeroImage( * tag-based thread detection — but these earlier articles aren't retroactively * rewritten or merged with anything after the fact. */ -export async function publishDirect(item: ContentItem): Promise { +export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise { const category = uniqueCategories([item]); - const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link, !item.tweet); + const storedMediaIds: string[] = []; + + let heroImage: MergedArticle['heroImage'] = null; + if (item.tweet) { + const selected = selectBestImage([item]); + if (selected) { + const resolved = await resolveTweetMediaUrl(selected.url, settings.nitterMediaMode); + heroImage = { url: resolved.url, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason }; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + } else { + const resolved = await resolveHeroImage([item], item.link); + heroImage = resolved.heroImage; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id } : null; - const tweet = item.tweet - ? { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl: item.tweet.avatarUrl, sourceItemId: item.id } - : null; + + let tweet: MergedArticle['tweet'] = null; + if (item.tweet) { + let avatarUrl: string | null = null; + if (item.tweet.avatarUrl) { + const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode); + avatarUrl = resolved.url; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + tweet = { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl, sourceItemId: item.id }; + } const article = await articles.insertArticle({ title: item.title, @@ -127,7 +172,7 @@ export async function publishDirect(item: ContentItem): Promise { topStories: anyPushesToTopStories([item]) }); - if (storedMediaId) promoteToPublished(storedMediaId, article.id); + for (const id of storedMediaIds) promoteToPublished(id, article.id); return article; } diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 12db2da..8c6a0b3 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise): GlobalSettings { merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, ai_service_host=?, ai_service_port=?, selected_models=?, + nitter_media_mode=?, fxtwitter_base_url=?, published_article_max_age_days=?, raw_item_max_age_days=?, storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=? WHERE id = 1` @@ -55,6 +58,8 @@ export function updateSettings(patch: Partial): GlobalSettings { merged.aiServiceHost, merged.aiServicePort, JSON.stringify(merged.selectedModels), + merged.nitterMediaMode, + merged.fxtwitterBaseUrl, merged.retention.publishedArticleMaxAgeDays, merged.retention.rawItemMaxAgeDays, merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 778ff42..ebf3829 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -119,6 +119,10 @@ export interface GlobalSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + /** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */ + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + /** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */ + fxtwitterBaseUrl: string; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 5b7ba63..66b5791 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -25,6 +25,8 @@ export interface AdminSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + fxtwitterBaseUrl: string; retention: RetentionSettings; categoryPriority: CategoryPriority[]; } diff --git a/frontend/src/lib/components/TweetCard.svelte b/frontend/src/lib/components/TweetCard.svelte index 52c80ee..271e217 100644 --- a/frontend/src/lib/components/TweetCard.svelte +++ b/frontend/src/lib/components/TweetCard.svelte @@ -18,7 +18,7 @@
{#if article.tweet?.avatarUrl} - + {:else}
{/if} diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index ad16ea6..2215d8d 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -11,6 +11,31 @@ let clearing = $state<'articles' | 'media' | null>(null); let clearResult = $state(null); + let nitterMediaMode = $state(settings.nitterMediaMode); + let fxtwitterBaseUrl = $state(settings.fxtwitterBaseUrl); + let nitterStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle'); + let nitterSaveTimer: ReturnType; + + function scheduleNitterSave() { + nitterStatus = 'saving'; + clearTimeout(nitterSaveTimer); + nitterSaveTimer = setTimeout(async () => { + try { + await updateSettings({ nitterMediaMode, fxtwitterBaseUrl }); + nitterStatus = 'saved'; + setTimeout(() => (nitterStatus = 'idle'), 1500); + } catch { + nitterStatus = 'error'; + } + }, 500); + } + + const mediaModes: { label: string; value: 'self-host' | 'proxy' | 'direct' }[] = [ + { label: 'Self-host', value: 'self-host' }, + { label: 'Proxy (recommended)', value: 'proxy' }, + { label: 'Direct', value: 'direct' } + ]; + async function handleClearArticles() { if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return; clearing = 'articles'; @@ -141,6 +166,40 @@
+
+
+ Nitter (tweet media) + +
+

+ How images and video attached to ingested tweets are served to visitors. Self-hosting + downloads and stores everything locally, same as regular article images. Proxying streams + each request through this server without persisting anything, so only this server's IP is + ever exposed to Twitter's CDN. Direct hotlinks the original URL straight from Twitter, with + no server involvement at all. +

+
+ {#each mediaModes as mode} + + {/each} +
+

+ Enrichment API used to fetch full tweet text, author info, and media — any fxtwitter/FixTweet- + compatible endpoint works. Defaults to the public fxtwitter.com instance; point this at a + self-hosted FixTweet mirror (or another public instance) instead if you'd rather not depend on it. +

+ +
+
Clear content