diff --git a/backend/src/api/mediaProxy.ts b/backend/src/api/mediaProxy.ts new file mode 100644 index 0000000..f31970b --- /dev/null +++ b/backend/src/api/mediaProxy.ts @@ -0,0 +1,117 @@ +// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode): +// the visitor's browser requests media from this route instead of directly from +// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the +// remote host — the media itself is streamed straight through, never written to disk. +// +// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless +// tightly restricted: only twimg.com (Twitter's media CDN), the configured +// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter +// sources are allowed — and even an allowed hostname is rejected if it resolves to a +// private/loopback/link-local address (defends against DNS rebinding, not just a +// hostname string check). + +import type { FastifyInstance } from 'fastify'; +import dns from 'node:dns/promises'; +import { Readable } from 'node:stream'; +import * as sourcesDb from '../storage/db/sources.js'; +import { getSettings } from '../storage/db/settings.js'; +import { logger } from '../storage/db/logs.js'; + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 15_000; +const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i; + +function hostnameOf(rawUrl: string | null): string | null { + if (!rawUrl) return null; + try { + return new URL(rawUrl).hostname.toLowerCase(); + } catch { + return null; + } +} + +function isAllowedHost(hostname: string): boolean { + const lower = hostname.toLowerCase(); + if (TWITTER_MEDIA_HOST_RE.test(lower)) return true; + + const settings = getSettings(); + if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true; + + const nitterHosts = sourcesDb + .listSources() + .filter((s) => s.type === 'nitter') + .map((s) => hostnameOf(s.url)) + .filter((h): h is string => !!h); + return nitterHosts.includes(lower); +} + +function isPrivateOrReservedIp(ip: string, family: number): boolean { + if (family === 4) { + const [a, b] = ip.split('.').map(Number); + if (a === 10 || a === 127 || a === 0) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range + return false; + } + const lower = ip.toLowerCase(); + if (lower === '::1') return true; + if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7 + if (lower.startsWith('fe80')) return true; // link-local + if (lower.startsWith('::ffff:')) { + const v4 = lower.split(':').pop(); + if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4); + } + return false; +} + +export async function registerMediaProxy(app: FastifyInstance) { + app.get('/media/proxy', async (req, reply) => { + const { url } = req.query as { url?: string }; + if (!url) return reply.code(400).send({ error: 'url required' }); + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return reply.code(400).send({ error: 'invalid url' }); + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return reply.code(400).send({ error: 'unsupported scheme' }); + } + + if (!isAllowedHost(parsed.hostname)) { + logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + let addresses: { address: string; family: number }[]; + try { + addresses = await dns.lookup(parsed.hostname, { all: true }); + } catch { + return reply.code(502).send({ error: 'DNS resolution failed' }); + } + if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) { + logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`); + return reply.code(403).send({ error: 'host not allowed' }); + } + + try { + const res = await fetch(parsed.toString(), { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + return reply.code(502).send({ error: `upstream responded ${res.status}` }); + } + reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream'); + reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600'); + return reply.send(Readable.fromWeb(res.body as any)); + } catch (err) { + logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`); + return reply.code(502).send({ error: 'fetch failed' }); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 1e71e82..0e4c485 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -8,6 +8,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js'; import { registerAuth } from './api/auth.js'; import { registerPublicRoutes } from './api/public.js'; import { registerAdminRoutes } from './api/admin.js'; +import { registerMediaProxy } from './api/mediaProxy.js'; import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js'; import { startScheduler } from './queue/scheduler.js'; import { logger } from './storage/db/logs.js'; @@ -88,6 +89,11 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); + // Static "/media/proxy" takes priority over the "/media/:filename" param route + // above regardless of registration order (find-my-way, Fastify's router, always + // prefers a static segment over a parametric one at the same depth). + await registerMediaProxy(app); + app.get('/health', async () => ({ ok: true })); await app.listen({ port: PORT, host: '0.0.0.0' }); diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index cb79d61..e767543 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,4 @@ -import type { Source, ContentItem } from '../../storage/db/types.js'; +import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -9,6 +9,8 @@ export interface FetchedItem { videos: { url: string; provider?: string; embedHtml?: string }[]; link: string; publishedAt: string; + /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ + tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] }; raw: unknown; } @@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit>(); + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 10_000; + +/** + * Shape confirmed against two real `curl https://api.fxtwitter.com//status/` + * responses: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url`, and + * (from a second, video-attached tweet) `media.all[]` — an ordered array covering both + * photos and videos, each with `type` ('photo' | 'video' | 'gif'), `url` (the direct + * playable/displayable URL — for video this is a real .mp4, not the .m3u8 playlist also + * present under `formats`), `thumbnail_url` (video/gif poster frame), and `width`/`height`. + * `media.all` is preferred over `media.photos`/`media.videos` since it's the only field + * that preserves the tweet's original media order. + */ +interface FxTweetMedia { + type?: string; + url?: string; + thumbnail_url?: string; + width?: number; + height?: number; +} + +interface FxTweet { + text?: string; + created_timestamp?: number; + author?: { name?: string; screen_name?: string; avatar_url?: string }; + media?: { all?: FxTweetMedia[]; photos?: FxTweetMedia[] }; +} + +/** Maps fxtwitter's media shape to our own, capped at the 4 items a tweet can carry. */ +function toTweetMedia(enrichment: FxTweet | null): TweetMediaItem[] { + const items = enrichment?.media?.all ?? enrichment?.media?.photos ?? []; + return items.slice(0, MAX_TWEET_MEDIA).map((m): TweetMediaItem => ({ + type: m.type === 'video' || m.type === 'gif' ? m.type : 'photo', + url: m.url ?? '', + thumbnailUrl: m.thumbnail_url ?? null, + width: m.width ?? null, + height: m.height ?? null + })).filter((m) => m.url); +} + +/** + * The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123 + * — no API version prefix. The base URL is admin-configurable (Retention tab) so a + * self-hosted FixTweet mirror (or another compatible public instance) can be used + * instead of the public api.fxtwitter.com default. + */ +async function fetchFxTwitter(handle: string, tweetId: string): Promise { + const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, ''); + try { + const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok) return null; + const json = (await res.json()) as { tweet?: FxTweet }; + return json?.tweet ?? null; + } catch (err) { + logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`); + return null; + } +} + +function extractTweetId(item: Parser.Item): string | null { + const guid = item.guid; + if (guid && /^\d+$/.test(guid)) return guid; + const fromLink = item.link?.match(/status\/(\d+)/)?.[1]; + return fromLink ?? null; +} + +/** + * A Nitter list/user RSS description is the item's own tweet content (a

plus + * optional ) followed, for retweets/quote-tweets, by a

wrapping the + * quoted tweet's own text/image/permalink. Only the part before that blockquote is this + * item's own content — nested quote-tweet rendering isn't part of the approved design. + */ +function ownContentHtml(descriptionHtml: string): string { + const cut = descriptionHtml.search(/|
]+src="([^"]+)"/i)?.[1] ?? null; +} + +export const nitterAdapter: SourceAdapter = { + async fetch(source: Source): Promise { + if (!source.url) return []; + + const feed = await parser.parseURL(source.url); + const items: FetchedItem[] = []; + + for (const item of feed.items) { + if (!item.link || !item.guid) continue; + const tweetId = extractTweetId(item); + if (!tweetId) { + logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`); + continue; + } + + // dc:creator is reliably the author of this item's own tweet text (rss-parser + // maps it to item.creator) — for a retweet, that's the original tweet's author, + // not whichever list member's retweet surfaced it in this feed. + const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown'; + const ownHtml = ownContentHtml(item.content ?? ''); + const rssImageUrl = extractImageUrl(ownHtml); + + const enrichment = await fetchFxTwitter(handle, tweetId); + + const authorName = enrichment?.author?.name ?? handle; + const avatarUrl = enrichment?.author?.avatar_url ?? null; + const text = enrichment?.text ?? ownHtml; + // Enrichment failed (or came back with no media) — fall back to the RSS + // description's own as a single photo, same as before multi-media support. + const media = toTweetMedia(enrichment); + if (media.length === 0 && rssImageUrl) { + media.push({ type: 'photo', url: rssImageUrl, thumbnailUrl: null, width: null, height: null }); + } + const publishedAt = enrichment?.created_timestamp + ? new Date(enrichment.created_timestamp * 1000).toISOString() + : (item.isoDate ?? item.pubDate ?? new Date().toISOString()); + + items.push({ + title: item.title || text.slice(0, 100), + summary: text.slice(0, 500), + body: text, + images: [], + videos: [], + link: item.link, + publishedAt, + tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media }, + raw: { rss: item, fxtwitter: enrichment } + }); + } + + return items; + } +}; diff --git a/backend/src/ingestion/poller.ts b/backend/src/ingestion/poller.ts index 525d7ca..5a13c89 100644 --- a/backend/src/ingestion/poller.ts +++ b/backend/src/ingestion/poller.ts @@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js'; import { telegramAdapter } from './adapters/telegram.js'; import { apiAdapter } from './adapters/api.js'; import { youtubeAdapter } from './adapters/youtube.js'; +import { nitterAdapter } from './adapters/nitter.js'; import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js'; import { fetchFullArticle } from './articleFetcher.js'; import type { Source } from '../storage/db/types.js'; @@ -14,6 +15,7 @@ const adapters: Record = { telegram: telegramAdapter, api: apiAdapter, youtube: youtubeAdapter, + nitter: nitterAdapter, custom: apiAdapter }; diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 0b95f44..2dd2880 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js'; import * as articles from '../storage/db/articles.js'; import * as tags from '../storage/db/tags.js'; import * as sources from '../storage/db/sources.js'; -import type { GlobalSettings, MergedArticle, ContentItem } from '../storage/db/types.js'; +import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js'; const FOLLOW_UP_LOOKBACK_DAYS = 3; @@ -33,9 +33,14 @@ function deriveTitle(body: string): string { } /** - * Resolves the hero image for an article: try the best candidate from the source - * items, download and locally host it; if there isn't one, fall back to the site's - * favicon rather than leaving the article with no art at all. + * Resolves the hero image for a regular (non-tweet) article: try the best candidate + * from the source items, download and locally host it; if there isn't one, fall back + * to the site's favicon rather than leaving the article with no art at all. Tweets + * never reach this function — see resolveTweetMediaUrl below, which applies the + * admin-configured Nitter media mode instead of always downloading, and skips the + * favicon fallback entirely (a Nitter instance's own favicon slapped onto an + * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already + * handles no-image tweets gracefully with no image at all). */ async function resolveHeroImage( items: ContentItem[], @@ -69,6 +74,61 @@ async function resolveHeroImage( return { heroImage: null, storedMediaId: null }; } +/** + * Applies the admin-configured Nitter media mode (Retention tab) to a single + * externally-hosted tweet media URL — an attached photo or the author's avatar. + * 'self-host' downloads and serves it locally like any other article image; + * 'proxy' routes it through this server's own /media/proxy route so only this + * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the + * "anonymity" the admin asked for) without persisting anything to disk; 'direct' + * hotlinks the original URL unchanged, the cheapest option with no server involvement. + */ +async function resolveTweetMediaUrl( + url: string, + mode: GlobalSettings['nitterMediaMode'] +): Promise<{ url: string; storedMediaId: string | null }> { + if (mode === 'direct') return { url, storedMediaId: null }; + + if (mode === 'proxy') { + return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null }; + } + + const stored = await downloadAndStore(url, 'published', {}); + if (stored) return { url: stored.servedPath, storedMediaId: stored.id }; + // Download failed — fall back to hotlinking rather than losing the media entirely. + return { url, storedMediaId: null }; +} + +/** + * Resolves every attached photo/video/gif's url — and, for video/gif, its poster + * thumbnail — through the admin's chosen Nitter media mode. Order and item count are + * preserved; TweetCard.svelte renders this array directly, there's no separate + * "hero image" concept for tweets the way there is for regular articles. + */ +async function resolveTweetMedia( + media: TweetMediaItem[], + mode: GlobalSettings['nitterMediaMode'] +): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> { + const storedMediaIds: string[] = []; + const resolved: TweetMediaItem[] = []; + + for (const item of media) { + const url = await resolveTweetMediaUrl(item.url, mode); + if (url.storedMediaId) storedMediaIds.push(url.storedMediaId); + + let thumbnailUrl = item.thumbnailUrl; + if (thumbnailUrl) { + const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode); + thumbnailUrl = resolvedThumb.url; + if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId); + } + + resolved.push({ ...item, url: url.url, thumbnailUrl }); + } + + return { media: resolved, storedMediaIds }; +} + /** * Publishes a single item as-is, with no AI calls at all — used when the AI service * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives @@ -79,18 +139,47 @@ async function resolveHeroImage( * tag-based thread detection — but these earlier articles aren't retroactively * rewritten or merged with anything after the fact. */ -export async function publishDirect(item: ContentItem): Promise { +export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise { const category = uniqueCategories([item]); - const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link); + const storedMediaIds: string[] = []; + + // Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly. + let heroImage: MergedArticle['heroImage'] = null; + if (!item.tweet) { + const resolved = await resolveHeroImage([item], item.link); + heroImage = resolved.heroImage; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id } : null; + let tweet: MergedArticle['tweet'] = null; + if (item.tweet) { + let avatarUrl: string | null = null; + if (item.tweet.avatarUrl) { + const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode); + avatarUrl = resolved.url; + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode); + storedMediaIds.push(...resolvedMedia.storedMediaIds); + tweet = { + authorName: item.tweet.authorName, + authorHandle: item.tweet.authorHandle, + avatarUrl, + sourceItemId: item.id, + media: resolvedMedia.media + }; + } + const article = await articles.insertArticle({ title: item.title, body: item.body || item.summary, heroImage, video, + tweet, category, geo: item.geo, eventId: item.eventId, @@ -115,7 +204,7 @@ export async function publishDirect(item: ContentItem): Promise { topStories: anyPushesToTopStories([item]) }); - if (storedMediaId) promoteToPublished(storedMediaId, article.id); + for (const id of storedMediaIds) promoteToPublished(id, article.id); return article; } @@ -192,6 +281,7 @@ export async function publishCluster( body, heroImage, video, + tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass category, geo, eventId: opts.eventId ?? items[0]?.eventId ?? null, diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 181809e..8c6a0b3 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise s.type === 'youtube').map((s) => s.id)); - const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId)); + // YouTube videos and Nitter tweets never get LLM-merged with anything else — each + // is always its own article, same shape whether the AI service is up or not. Route + // them straight to publishDirect, same as the no-AI passthrough path. + const directPublishSourceIds = new Set( + sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id) + ); + const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); let publishedDirect = 0; - for (const item of youtubeItems) { + for (const item of directItems) { try { - const article = await publishDirect(item); + const article = await publishDirect(item, settings); contentItemsDb.assignCluster([item.id], article.id); publishedDirect++; - logger.info('synthesis', `Published "${article.title}" directly (YouTube)`); + const source = sourcesDb.getSource(item.sourceId); + logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`); } catch (err) { logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`); } diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 4328722..8c08ff3 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -22,7 +22,8 @@ function rowToArticle(row: any): MergedArticle { threadId: row.thread_id, previousArticleId: row.previous_article_id, nextArticleId: row.next_article_id, - topStories: !!row.top_stories + topStories: !!row.top_stories, + tweet: row.tweet ? JSON.parse(row.tweet) : null }; } @@ -30,8 +31,8 @@ export function insertArticle(article: Omit): MergedArticle const id = `art-${randomUUID()}`; db.prepare( `INSERT INTO merged_articles - (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, article.title, @@ -50,7 +51,8 @@ export function insertArticle(article: Omit): MergedArticle article.threadId, article.previousArticleId, article.nextArticleId, - article.topStories ? 1 : 0 + article.topStories ? 1 : 0, + article.tweet ? JSON.stringify(article.tweet) : null ); if (article.previousArticleId) { db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId); diff --git a/backend/src/storage/db/contentItems.ts b/backend/src/storage/db/contentItems.ts index e33a35b..f4942b2 100644 --- a/backend/src/storage/db/contentItems.ts +++ b/backend/src/storage/db/contentItems.ts @@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem { embedding: row.embedding ? JSON.parse(row.embedding) : null, eventId: row.event_id, clusterId: row.cluster_id, + tweet: row.tweet ? JSON.parse(row.tweet) : null, raw: row.raw ? JSON.parse(row.raw) : null }; } @@ -28,8 +29,8 @@ export function insertContentItem(item: Omit): ContentItem { const id = `ci-${randomUUID()}`; db.prepare( `INSERT INTO content_items - (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, item.sourceId, @@ -47,6 +48,7 @@ export function insertContentItem(item: Omit): ContentItem { item.embedding ? JSON.stringify(item.embedding) : null, item.eventId, item.clusterId, + item.tweet ? JSON.stringify(item.tweet) : null, item.raw ? JSON.stringify(item.raw) : null ); return { ...item, id }; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index d66b76a..220dcaa 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -58,6 +58,7 @@ export function migrate() { embedding TEXT, -- JSON float array event_id TEXT, cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis + tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only raw TEXT -- JSON, original payload ); CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id); @@ -82,7 +83,8 @@ export function migrate() { thread_id TEXT NOT NULL, previous_article_id TEXT, next_article_id TEXT, - top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?" + top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?" + tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only ); CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at); CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id); @@ -166,7 +168,9 @@ export function migrate() { raw_item_max_age_days INTEGER DEFAULT 7, storage_cap_enabled INTEGER NOT NULL DEFAULT 1, storage_cap_value INTEGER NOT NULL DEFAULT 500, - storage_cap_unit TEXT NOT NULL DEFAULT 'GB' + storage_cap_unit TEXT NOT NULL DEFAULT 'GB', + nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct + fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com' ); `); @@ -188,6 +192,18 @@ export function migrate() { if (!hasColumn('merged_articles', 'top_stories')) { db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('content_items', 'tweet')) { + db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT'); + } + if (!hasColumn('merged_articles', 'tweet')) { + db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT'); + } + if (!hasColumn('global_settings', 'nitter_media_mode')) { + db.exec("ALTER TABLE global_settings ADD COLUMN nitter_media_mode TEXT NOT NULL DEFAULT 'proxy'"); + } + if (!hasColumn('global_settings', 'fxtwitter_base_url')) { + db.exec("ALTER TABLE global_settings ADD COLUMN fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'"); + } if (!hasColumn('categories', 'is_private')) { db.exec('ALTER TABLE categories ADD COLUMN is_private INTEGER NOT NULL DEFAULT 0'); } diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 35a97f5..f9fd366 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -13,6 +13,8 @@ function rowToSettings(row: any): GlobalSettings { aiServiceHost: row.ai_service_host, aiServicePort: row.ai_service_port, selectedModels: JSON.parse(row.selected_models), + nitterMediaMode: row.nitter_media_mode, + fxtwitterBaseUrl: row.fxtwitter_base_url, retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days, @@ -41,6 +43,7 @@ export function updateSettings(patch: Partial): GlobalSettings { merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, ai_service_host=?, ai_service_port=?, selected_models=?, + nitter_media_mode=?, fxtwitter_base_url=?, published_article_max_age_days=?, raw_item_max_age_days=?, storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=? WHERE id = 1` @@ -55,6 +58,8 @@ export function updateSettings(patch: Partial): GlobalSettings { merged.aiServiceHost, merged.aiServicePort, JSON.stringify(merged.selectedModels), + merged.nitterMediaMode, + merged.fxtwitterBaseUrl, merged.retention.publishedArticleMaxAgeDays, merged.retention.rawItemMaxAgeDays, merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 2e5b00c..a71712e 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -1,7 +1,7 @@ export interface Source { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; category: string[]; url: string | null; config: Record; @@ -14,6 +14,16 @@ export interface Source { createdAt: string; } +/** A single photo/video/gif attached to a tweet, in the tweet's own display order — fxtwitter caps this at 4. */ +export interface TweetMediaItem { + type: 'photo' | 'video' | 'gif'; + url: string; + /** Video/gif poster frame — null for photos. */ + thumbnailUrl: string | null; + width: number | null; + height: number | null; +} + export interface ContentItem { id: string; sourceId: string; @@ -31,6 +41,8 @@ export interface ContentItem { embedding: number[] | null; eventId: string | null; clusterId: string | null; + /** Nitter-sourced items only — null for everything else. */ + tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null; raw: unknown; } @@ -47,6 +59,8 @@ export interface MergedArticle { body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; + /** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */ + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; category: string[]; geo: string | null; eventId: string | null; @@ -117,6 +131,10 @@ export interface GlobalSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + /** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */ + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + /** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */ + fxtwitterBaseUrl: string; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index c1891c8..f5357e2 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -26,6 +26,8 @@ export interface AdminSettings { aiServiceHost: string; aiServicePort: number; selectedModels: { embedding: string; image: string; synthesis: string }; + nitterMediaMode: 'self-host' | 'proxy' | 'direct'; + fxtwitterBaseUrl: string; retention: RetentionSettings; categoryPriority: CategoryPriority[]; } @@ -33,7 +35,7 @@ export interface AdminSettings { export interface AdminSource { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; category: string[]; url: string; config?: Record; diff --git a/frontend/src/lib/components/ArticleListRow.svelte b/frontend/src/lib/components/ArticleListRow.svelte index 1f37e95..d32248b 100644 --- a/frontend/src/lib/components/ArticleListRow.svelte +++ b/frontend/src/lib/components/ArticleListRow.svelte @@ -2,6 +2,7 @@ import type { MergedArticle } from '$lib/types'; import { timeAgo, exactTime, excerpt } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; + import TweetCard from './TweetCard.svelte'; let { article }: { article: MergedArticle } = $props(); @@ -12,28 +13,32 @@ ); - - {#if article.heroImage} - - {:else} -
- {/if} +{#if article.tweet} + +{:else} + + {#if article.heroImage} + + {:else} +
+ {/if} -
-
- {article.category[0] ?? ''} - · - {sourceLabel} - {#if article.video} +
+
+ {article.category[0] ?? ''} · - ▶ Video - {/if} + {sourceLabel} + {#if article.video} + · + ▶ Video + {/if} +
+
{article.title}
+
{excerpt(article.body)}
+
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
-
{article.title}
-
{excerpt(article.body)}
-
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
-
- + +{/if} diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index ad16ea6..2215d8d 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -11,6 +11,31 @@ let clearing = $state<'articles' | 'media' | null>(null); let clearResult = $state(null); + let nitterMediaMode = $state(settings.nitterMediaMode); + let fxtwitterBaseUrl = $state(settings.fxtwitterBaseUrl); + let nitterStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle'); + let nitterSaveTimer: ReturnType; + + function scheduleNitterSave() { + nitterStatus = 'saving'; + clearTimeout(nitterSaveTimer); + nitterSaveTimer = setTimeout(async () => { + try { + await updateSettings({ nitterMediaMode, fxtwitterBaseUrl }); + nitterStatus = 'saved'; + setTimeout(() => (nitterStatus = 'idle'), 1500); + } catch { + nitterStatus = 'error'; + } + }, 500); + } + + const mediaModes: { label: string; value: 'self-host' | 'proxy' | 'direct' }[] = [ + { label: 'Self-host', value: 'self-host' }, + { label: 'Proxy (recommended)', value: 'proxy' }, + { label: 'Direct', value: 'direct' } + ]; + async function handleClearArticles() { if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return; clearing = 'articles'; @@ -141,6 +166,40 @@
+
+
+ Nitter (tweet media) + +
+

+ How images and video attached to ingested tweets are served to visitors. Self-hosting + downloads and stores everything locally, same as regular article images. Proxying streams + each request through this server without persisting anything, so only this server's IP is + ever exposed to Twitter's CDN. Direct hotlinks the original URL straight from Twitter, with + no server involvement at all. +

+
+ {#each mediaModes as mode} + + {/each} +
+

+ Enrichment API used to fetch full tweet text, author info, and media — any fxtwitter/FixTweet- + compatible endpoint works. Defaults to the public fxtwitter.com instance; point this at a + self-hosted FixTweet mirror (or another public instance) instead if you'd rather not depend on it. +

+ +
+
Clear content

diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index f3f3b11..bb83676 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -144,7 +144,17 @@ } const typeIcon = (type: string) => - type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•'; + type === 'rss' + ? '⟳' + : type === 'telegram' + ? '✈' + : type === 'youtube' + ? '▶' + : type === 'nitter' + ? '🐦' + : type === 'api' + ? '⇄' + : '•';

@@ -163,10 +173,13 @@ + {#if form.type === 'youtube'} + {:else if form.type === 'nitter'} + {:else} {/if} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 830b8e2..08ec8ab 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -8,12 +8,22 @@ export interface ArticleSource { publishedAt: string; } +/** A single photo/video/gif attached to a tweet, in the tweet's own display order — capped at 4. */ +export interface TweetMediaItem { + type: 'photo' | 'video' | 'gif'; + url: string; + thumbnailUrl: string | null; + width: number | null; + height: number | null; +} + export interface MergedArticle { id: string; title: string; body: string; heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; + tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/routes/article/[id]/+page.svelte b/frontend/src/routes/article/[id]/+page.svelte index 3788cd0..418259d 100644 --- a/frontend/src/routes/article/[id]/+page.svelte +++ b/frontend/src/routes/article/[id]/+page.svelte @@ -2,6 +2,7 @@ import type { PageData } from './$types'; import { timeAgo, exactTime } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; + import TweetCard from '$lib/components/TweetCard.svelte'; let { data }: { data: PageData } = $props(); const a = $derived(data.article); @@ -22,9 +23,22 @@ {a.category.join(', ')}
-

{a.title}

+ {#if a.tweet} + +
+ Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} +
+ + + + {#if a.sources[0]} + View original tweet → + {/if} + {:else if a.video?.provider === 'youtube'} +

{a.title}

- {#if a.video?.provider === 'youtube'} @@ -46,6 +60,8 @@

{paragraph}

{/each} {:else} +

{a.title}

+
Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} {#if a.updatedAt !== a.publishedAt} @@ -180,6 +196,12 @@ height: 100%; border: none; } + .view-original { + display: inline-block; + font-size: 13px; + color: var(--text-accent); + margin-bottom: 20px; + } .tags { display: flex; gap: 8px; diff --git a/frontend/src/routes/category/[name]/+page.ts b/frontend/src/routes/category/[name]/+page.ts index c83a4df..91366ea 100644 --- a/frontend/src/routes/category/[name]/+page.ts +++ b/frontend/src/routes/category/[name]/+page.ts @@ -1,14 +1,25 @@ import type { PageLoad } from './$types'; import { getFeed } from '$lib/api'; +import { slugify } from '$lib/format'; const PAGE_SIZE = 15; // Every category page (including "Local") filters by its category name — sources are // assigned categories directly via checkboxes in the admin Sources tab, so a source // tagged "Local" shows up here the same way one tagged "Tech" shows up on /category/tech. -export const load: PageLoad = async ({ params, fetch }) => { - const name = params.name; - const filters = { category: name }; +// +// The URL param is a slug (e.g. "x-news" from slugify("X News")), not the real category +// name — for single-word categories those happen to be the same lowercased, but for a +// multi-word name like "X News" the slug's hyphen never matches the stored "X News" +// (with a space) in a merged_articles.category LIKE match. Resolve the slug back to the +// real category name via the site's own category list (already loaded by the root +// layout) before filtering, rather than passing the raw slug straight through. +export const load: PageLoad = async ({ params, fetch, parent }) => { + const { categories } = await parent(); + const match = categories.find((c) => slugify(c.name) === params.name); + const categoryName = match?.name ?? params.name; + + const filters = { category: categoryName }; const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); - return { initial, filters, name, pageSize: PAGE_SIZE }; + return { initial, filters, name: categoryName, pageSize: PAGE_SIZE }; };