Merge remote-tracking branch 'origin/master' into development

# Conflicts:
#	backend/src/index.ts
#	backend/src/storage/db/index.ts
This commit is contained in:
Claude
2026-07-23 03:02:10 +00:00
20 changed files with 784 additions and 53 deletions
+117
View File
@@ -0,0 +1,117 @@
// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode):
// the visitor's browser requests media from this route instead of directly from
// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the
// remote host — the media itself is streamed straight through, never written to disk.
//
// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless
// tightly restricted: only twimg.com (Twitter's media CDN), the configured
// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter
// sources are allowed — and even an allowed hostname is rejected if it resolves to a
// private/loopback/link-local address (defends against DNS rebinding, not just a
// hostname string check).
import type { FastifyInstance } from 'fastify';
import dns from 'node:dns/promises';
import { Readable } from 'node:stream';
import * as sourcesDb from '../storage/db/sources.js';
import { getSettings } from '../storage/db/settings.js';
import { logger } from '../storage/db/logs.js';
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
const FETCH_TIMEOUT_MS = 15_000;
const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i;
function hostnameOf(rawUrl: string | null): string | null {
if (!rawUrl) return null;
try {
return new URL(rawUrl).hostname.toLowerCase();
} catch {
return null;
}
}
function isAllowedHost(hostname: string): boolean {
const lower = hostname.toLowerCase();
if (TWITTER_MEDIA_HOST_RE.test(lower)) return true;
const settings = getSettings();
if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true;
const nitterHosts = sourcesDb
.listSources()
.filter((s) => s.type === 'nitter')
.map((s) => hostnameOf(s.url))
.filter((h): h is string => !!h);
return nitterHosts.includes(lower);
}
function isPrivateOrReservedIp(ip: string, family: number): boolean {
if (family === 4) {
const [a, b] = ip.split('.').map(Number);
if (a === 10 || a === 127 || a === 0) return true;
if (a === 169 && b === 254) return true;
if (a === 172 && b >= 16 && b <= 31) return true;
if (a === 192 && b === 168) return true;
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range
return false;
}
const lower = ip.toLowerCase();
if (lower === '::1') return true;
if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7
if (lower.startsWith('fe80')) return true; // link-local
if (lower.startsWith('::ffff:')) {
const v4 = lower.split(':').pop();
if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4);
}
return false;
}
export async function registerMediaProxy(app: FastifyInstance) {
app.get('/media/proxy', async (req, reply) => {
const { url } = req.query as { url?: string };
if (!url) return reply.code(400).send({ error: 'url required' });
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return reply.code(400).send({ error: 'invalid url' });
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
return reply.code(400).send({ error: 'unsupported scheme' });
}
if (!isAllowedHost(parsed.hostname)) {
logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`);
return reply.code(403).send({ error: 'host not allowed' });
}
let addresses: { address: string; family: number }[];
try {
addresses = await dns.lookup(parsed.hostname, { all: true });
} catch {
return reply.code(502).send({ error: 'DNS resolution failed' });
}
if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) {
logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`);
return reply.code(403).send({ error: 'host not allowed' });
}
try {
const res = await fetch(parsed.toString(), {
headers: { 'User-Agent': USER_AGENT },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
});
if (!res.ok || !res.body) {
return reply.code(502).send({ error: `upstream responded ${res.status}` });
}
reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream');
reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600');
return reply.send(Readable.fromWeb(res.body as any));
} catch (err) {
logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`);
return reply.code(502).send({ error: 'fetch failed' });
}
});
}
+6
View File
@@ -8,6 +8,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js';
import { registerAuth } from './api/auth.js';
import { registerPublicRoutes } from './api/public.js';
import { registerAdminRoutes } from './api/admin.js';
import { registerMediaProxy } from './api/mediaProxy.js';
import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js';
import { startScheduler } from './queue/scheduler.js';
import { logger } from './storage/db/logs.js';
@@ -88,6 +89,11 @@ async function main() {
return reply.send(fs.createReadStream(filePath));
});
// Static "/media/proxy" takes priority over the "/media/:filename" param route
// above regardless of registration order (find-my-way, Fastify's router, always
// prefers a static segment over a parametric one at the same depth).
await registerMediaProxy(app);
app.get('/health', async () => ({ ok: true }));
await app.listen({ port: PORT, host: '0.0.0.0' });
+4 -1
View File
@@ -1,4 +1,4 @@
import type { Source, ContentItem } from '../../storage/db/types.js';
import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js';
import { cleanHtml, toSummary } from '../clean.js';
export interface FetchedItem {
@@ -9,6 +9,8 @@ export interface FetchedItem {
videos: { url: string; provider?: string; embedHtml?: string }[];
link: string;
publishedAt: string;
/** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */
tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] };
raw: unknown;
}
@@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit<ContentIt
embedding: null,
eventId: null,
clusterId: null,
tweet: item.tweet ? { ...item.tweet } : null,
raw: item.raw
};
}
+156
View File
@@ -0,0 +1,156 @@
// Nitter is its own ingestion module, deliberately separate from the RSS adapter even
// though the transport is RSS: a tweet is enriched with a second fetch to fxtwitter
// (for author name/avatar and cleaner text) and always publishes as its own article —
// see pipeline/publish.ts and queue/priorityQueue.ts, which route nitter-sourced items
// straight to publishDirect rather than the LLM clustering/synthesis pipeline. Merging
// two unrelated tweets into one AI-rewritten story would make no sense the way merging
// two outlets' coverage of the same news event does — same reasoning as YouTube.
import Parser from 'rss-parser';
import type { Source, TweetMediaItem } from '../../storage/db/types.js';
import type { SourceAdapter, FetchedItem } from './base.js';
import { logger } from '../../storage/db/logs.js';
import { getSettings } from '../../storage/db/settings.js';
/** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */
const MAX_TWEET_MEDIA = 4;
const parser = new Parser<Record<string, unknown>>();
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
const FETCH_TIMEOUT_MS = 10_000;
/**
* Shape confirmed against two real `curl https://api.fxtwitter.com/<handle>/status/<id>`
* responses: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url`, and
* (from a second, video-attached tweet) `media.all[]` — an ordered array covering both
* photos and videos, each with `type` ('photo' | 'video' | 'gif'), `url` (the direct
* playable/displayable URL — for video this is a real .mp4, not the .m3u8 playlist also
* present under `formats`), `thumbnail_url` (video/gif poster frame), and `width`/`height`.
* `media.all` is preferred over `media.photos`/`media.videos` since it's the only field
* that preserves the tweet's original media order.
*/
interface FxTweetMedia {
type?: string;
url?: string;
thumbnail_url?: string;
width?: number;
height?: number;
}
interface FxTweet {
text?: string;
created_timestamp?: number;
author?: { name?: string; screen_name?: string; avatar_url?: string };
media?: { all?: FxTweetMedia[]; photos?: FxTweetMedia[] };
}
/** Maps fxtwitter's media shape to our own, capped at the 4 items a tweet can carry. */
function toTweetMedia(enrichment: FxTweet | null): TweetMediaItem[] {
const items = enrichment?.media?.all ?? enrichment?.media?.photos ?? [];
return items.slice(0, MAX_TWEET_MEDIA).map((m): TweetMediaItem => ({
type: m.type === 'video' || m.type === 'gif' ? m.type : 'photo',
url: m.url ?? '',
thumbnailUrl: m.thumbnail_url ?? null,
width: m.width ?? null,
height: m.height ?? null
})).filter((m) => m.url);
}
/**
* The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123
* — no API version prefix. The base URL is admin-configurable (Retention tab) so a
* self-hosted FixTweet mirror (or another compatible public instance) can be used
* instead of the public api.fxtwitter.com default.
*/
async function fetchFxTwitter(handle: string, tweetId: string): Promise<FxTweet | null> {
const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, '');
try {
const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, {
headers: { 'User-Agent': USER_AGENT },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
});
if (!res.ok) return null;
const json = (await res.json()) as { tweet?: FxTweet };
return json?.tweet ?? null;
} catch (err) {
logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`);
return null;
}
}
function extractTweetId(item: Parser.Item): string | null {
const guid = item.guid;
if (guid && /^\d+$/.test(guid)) return guid;
const fromLink = item.link?.match(/status\/(\d+)/)?.[1];
return fromLink ?? null;
}
/**
* A Nitter list/user RSS description is the item's own tweet content (a <p> plus
* optional <img>) followed, for retweets/quote-tweets, by a <blockquote> wrapping the
* quoted tweet's own text/image/permalink. Only the part before that blockquote is this
* item's own content — nested quote-tweet rendering isn't part of the approved design.
*/
function ownContentHtml(descriptionHtml: string): string {
const cut = descriptionHtml.search(/<hr\s*\/?>|<blockquote/i);
return cut === -1 ? descriptionHtml : descriptionHtml.slice(0, cut);
}
function extractImageUrl(html: string): string | null {
return html.match(/<img[^>]+src="([^"]+)"/i)?.[1] ?? null;
}
export const nitterAdapter: SourceAdapter = {
async fetch(source: Source): Promise<FetchedItem[]> {
if (!source.url) return [];
const feed = await parser.parseURL(source.url);
const items: FetchedItem[] = [];
for (const item of feed.items) {
if (!item.link || !item.guid) continue;
const tweetId = extractTweetId(item);
if (!tweetId) {
logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`);
continue;
}
// dc:creator is reliably the author of this item's own tweet text (rss-parser
// maps it to item.creator) — for a retweet, that's the original tweet's author,
// not whichever list member's retweet surfaced it in this feed.
const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
const ownHtml = ownContentHtml(item.content ?? '');
const rssImageUrl = extractImageUrl(ownHtml);
const enrichment = await fetchFxTwitter(handle, tweetId);
const authorName = enrichment?.author?.name ?? handle;
const avatarUrl = enrichment?.author?.avatar_url ?? null;
const text = enrichment?.text ?? ownHtml;
// Enrichment failed (or came back with no media) — fall back to the RSS
// description's own <img> as a single photo, same as before multi-media support.
const media = toTweetMedia(enrichment);
if (media.length === 0 && rssImageUrl) {
media.push({ type: 'photo', url: rssImageUrl, thumbnailUrl: null, width: null, height: null });
}
const publishedAt = enrichment?.created_timestamp
? new Date(enrichment.created_timestamp * 1000).toISOString()
: (item.isoDate ?? item.pubDate ?? new Date().toISOString());
items.push({
title: item.title || text.slice(0, 100),
summary: text.slice(0, 500),
body: text,
images: [],
videos: [],
link: item.link,
publishedAt,
tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media },
raw: { rss: item, fxtwitter: enrichment }
});
}
return items;
}
};
+2
View File
@@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js';
import { telegramAdapter } from './adapters/telegram.js';
import { apiAdapter } from './adapters/api.js';
import { youtubeAdapter } from './adapters/youtube.js';
import { nitterAdapter } from './adapters/nitter.js';
import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
import { fetchFullArticle } from './articleFetcher.js';
import type { Source } from '../storage/db/types.js';
@@ -14,6 +15,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
telegram: telegramAdapter,
api: apiAdapter,
youtube: youtubeAdapter,
nitter: nitterAdapter,
custom: apiAdapter
};
+97 -7
View File
@@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js';
import * as articles from '../storage/db/articles.js';
import * as tags from '../storage/db/tags.js';
import * as sources from '../storage/db/sources.js';
import type { GlobalSettings, MergedArticle, ContentItem } from '../storage/db/types.js';
import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js';
const FOLLOW_UP_LOOKBACK_DAYS = 3;
@@ -33,9 +33,14 @@ function deriveTitle(body: string): string {
}
/**
* Resolves the hero image for an article: try the best candidate from the source
* items, download and locally host it; if there isn't one, fall back to the site's
* favicon rather than leaving the article with no art at all.
* Resolves the hero image for a regular (non-tweet) article: try the best candidate
* from the source items, download and locally host it; if there isn't one, fall back
* to the site's favicon rather than leaving the article with no art at all. Tweets
* never reach this function — see resolveTweetMediaUrl below, which applies the
* admin-configured Nitter media mode instead of always downloading, and skips the
* favicon fallback entirely (a Nitter instance's own favicon slapped onto an
* image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already
* handles no-image tweets gracefully with no image at all).
*/
async function resolveHeroImage(
items: ContentItem[],
@@ -69,6 +74,61 @@ async function resolveHeroImage(
return { heroImage: null, storedMediaId: null };
}
/**
* Applies the admin-configured Nitter media mode (Retention tab) to a single
* externally-hosted tweet media URL — an attached photo or the author's avatar.
* 'self-host' downloads and serves it locally like any other article image;
* 'proxy' routes it through this server's own /media/proxy route so only this
* server's IP is ever exposed to Twitter/the Nitter instance's CDN (the
* "anonymity" the admin asked for) without persisting anything to disk; 'direct'
* hotlinks the original URL unchanged, the cheapest option with no server involvement.
*/
async function resolveTweetMediaUrl(
url: string,
mode: GlobalSettings['nitterMediaMode']
): Promise<{ url: string; storedMediaId: string | null }> {
if (mode === 'direct') return { url, storedMediaId: null };
if (mode === 'proxy') {
return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null };
}
const stored = await downloadAndStore(url, 'published', {});
if (stored) return { url: stored.servedPath, storedMediaId: stored.id };
// Download failed — fall back to hotlinking rather than losing the media entirely.
return { url, storedMediaId: null };
}
/**
* Resolves every attached photo/video/gif's url — and, for video/gif, its poster
* thumbnail — through the admin's chosen Nitter media mode. Order and item count are
* preserved; TweetCard.svelte renders this array directly, there's no separate
* "hero image" concept for tweets the way there is for regular articles.
*/
async function resolveTweetMedia(
media: TweetMediaItem[],
mode: GlobalSettings['nitterMediaMode']
): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> {
const storedMediaIds: string[] = [];
const resolved: TweetMediaItem[] = [];
for (const item of media) {
const url = await resolveTweetMediaUrl(item.url, mode);
if (url.storedMediaId) storedMediaIds.push(url.storedMediaId);
let thumbnailUrl = item.thumbnailUrl;
if (thumbnailUrl) {
const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode);
thumbnailUrl = resolvedThumb.url;
if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId);
}
resolved.push({ ...item, url: url.url, thumbnailUrl });
}
return { media: resolved, storedMediaIds };
}
/**
* Publishes a single item as-is, with no AI calls at all — used when the AI service
* isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
@@ -79,18 +139,47 @@ async function resolveHeroImage(
* tag-based thread detection — but these earlier articles aren't retroactively
* rewritten or merged with anything after the fact.
*/
export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise<MergedArticle> {
const category = uniqueCategories([item]);
const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
const storedMediaIds: string[] = [];
// Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly.
let heroImage: MergedArticle['heroImage'] = null;
if (!item.tweet) {
const resolved = await resolveHeroImage([item], item.link);
heroImage = resolved.heroImage;
if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
}
const video = item.videos[0]
? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
: null;
let tweet: MergedArticle['tweet'] = null;
if (item.tweet) {
let avatarUrl: string | null = null;
if (item.tweet.avatarUrl) {
const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode);
avatarUrl = resolved.url;
if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
}
const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode);
storedMediaIds.push(...resolvedMedia.storedMediaIds);
tweet = {
authorName: item.tweet.authorName,
authorHandle: item.tweet.authorHandle,
avatarUrl,
sourceItemId: item.id,
media: resolvedMedia.media
};
}
const article = await articles.insertArticle({
title: item.title,
body: item.body || item.summary,
heroImage,
video,
tweet,
category,
geo: item.geo,
eventId: item.eventId,
@@ -115,7 +204,7 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
topStories: anyPushesToTopStories([item])
});
if (storedMediaId) promoteToPublished(storedMediaId, article.id);
for (const id of storedMediaIds) promoteToPublished(id, article.id);
return article;
}
@@ -192,6 +281,7 @@ export async function publishCluster(
body,
heroImage,
video,
tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass
category,
geo,
eventId: opts.eventId ?? items[0]?.eventId ?? null,
+12 -9
View File
@@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
for (const item of ranked) {
try {
const article = await publishDirect(item);
const article = await publishDirect(item, settings);
contentItemsDb.assignCluster([item.id], article.id);
published++;
logger.info('synthesis', `Published "${article.title}" directly (no AI available)`);
@@ -77,19 +77,22 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
if (items.length === 0) return 0;
// YouTube videos never get LLM-merged with anything — each is always its own
// article (title/video/date/description), same shape whether the AI service is up
// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
// YouTube videos and Nitter tweets never get LLM-merged with anything else — each
// is always its own article, same shape whether the AI service is up or not. Route
// them straight to publishDirect, same as the no-AI passthrough path.
const directPublishSourceIds = new Set(
sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id)
);
const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
let publishedDirect = 0;
for (const item of youtubeItems) {
for (const item of directItems) {
try {
const article = await publishDirect(item);
const article = await publishDirect(item, settings);
contentItemsDb.assignCluster([item.id], article.id);
publishedDirect++;
logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
const source = sourcesDb.getSource(item.sourceId);
logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`);
} catch (err) {
logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
}
+6 -4
View File
@@ -22,7 +22,8 @@ function rowToArticle(row: any): MergedArticle {
threadId: row.thread_id,
previousArticleId: row.previous_article_id,
nextArticleId: row.next_article_id,
topStories: !!row.top_stories
topStories: !!row.top_stories,
tweet: row.tweet ? JSON.parse(row.tweet) : null
};
}
@@ -30,8 +31,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
const id = `art-${randomUUID()}`;
db.prepare(
`INSERT INTO merged_articles
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
id,
article.title,
@@ -50,7 +51,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
article.threadId,
article.previousArticleId,
article.nextArticleId,
article.topStories ? 1 : 0
article.topStories ? 1 : 0,
article.tweet ? JSON.stringify(article.tweet) : null
);
if (article.previousArticleId) {
db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId);
+4 -2
View File
@@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem {
embedding: row.embedding ? JSON.parse(row.embedding) : null,
eventId: row.event_id,
clusterId: row.cluster_id,
tweet: row.tweet ? JSON.parse(row.tweet) : null,
raw: row.raw ? JSON.parse(row.raw) : null
};
}
@@ -28,8 +29,8 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
const id = `ci-${randomUUID()}`;
db.prepare(
`INSERT INTO content_items
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
id,
item.sourceId,
@@ -47,6 +48,7 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
item.embedding ? JSON.stringify(item.embedding) : null,
item.eventId,
item.clusterId,
item.tweet ? JSON.stringify(item.tweet) : null,
item.raw ? JSON.stringify(item.raw) : null
);
return { ...item, id };
+18 -2
View File
@@ -58,6 +58,7 @@ export function migrate() {
embedding TEXT, -- JSON float array
event_id TEXT,
cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis
tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only
raw TEXT -- JSON, original payload
);
CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id);
@@ -82,7 +83,8 @@ export function migrate() {
thread_id TEXT NOT NULL,
previous_article_id TEXT,
next_article_id TEXT,
top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?"
top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?"
tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only
);
CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at);
CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id);
@@ -166,7 +168,9 @@ export function migrate() {
raw_item_max_age_days INTEGER DEFAULT 7,
storage_cap_enabled INTEGER NOT NULL DEFAULT 1,
storage_cap_value INTEGER NOT NULL DEFAULT 500,
storage_cap_unit TEXT NOT NULL DEFAULT 'GB'
storage_cap_unit TEXT NOT NULL DEFAULT 'GB',
nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct
fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'
);
`);
@@ -188,6 +192,18 @@ export function migrate() {
if (!hasColumn('merged_articles', 'top_stories')) {
db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0');
}
if (!hasColumn('content_items', 'tweet')) {
db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT');
}
if (!hasColumn('merged_articles', 'tweet')) {
db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT');
}
if (!hasColumn('global_settings', 'nitter_media_mode')) {
db.exec("ALTER TABLE global_settings ADD COLUMN nitter_media_mode TEXT NOT NULL DEFAULT 'proxy'");
}
if (!hasColumn('global_settings', 'fxtwitter_base_url')) {
db.exec("ALTER TABLE global_settings ADD COLUMN fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'");
}
if (!hasColumn('categories', 'is_private')) {
db.exec('ALTER TABLE categories ADD COLUMN is_private INTEGER NOT NULL DEFAULT 0');
}
+5
View File
@@ -13,6 +13,8 @@ function rowToSettings(row: any): GlobalSettings {
aiServiceHost: row.ai_service_host,
aiServicePort: row.ai_service_port,
selectedModels: JSON.parse(row.selected_models),
nitterMediaMode: row.nitter_media_mode,
fxtwitterBaseUrl: row.fxtwitter_base_url,
retention: {
publishedArticleMaxAgeDays: row.published_article_max_age_days,
rawItemMaxAgeDays: row.raw_item_max_age_days,
@@ -41,6 +43,7 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?,
tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?,
ai_service_host=?, ai_service_port=?, selected_models=?,
nitter_media_mode=?, fxtwitter_base_url=?,
published_article_max_age_days=?, raw_item_max_age_days=?,
storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?
WHERE id = 1`
@@ -55,6 +58,8 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
merged.aiServiceHost,
merged.aiServicePort,
JSON.stringify(merged.selectedModels),
merged.nitterMediaMode,
merged.fxtwitterBaseUrl,
merged.retention.publishedArticleMaxAgeDays,
merged.retention.rawItemMaxAgeDays,
merged.retention.storageCapEnabled ? 1 : 0,
+19 -1
View File
@@ -1,7 +1,7 @@
export interface Source {
id: string;
name: string;
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
category: string[];
url: string | null;
config: Record<string, unknown>;
@@ -14,6 +14,16 @@ export interface Source {
createdAt: string;
}
/** A single photo/video/gif attached to a tweet, in the tweet's own display order — fxtwitter caps this at 4. */
export interface TweetMediaItem {
type: 'photo' | 'video' | 'gif';
url: string;
/** Video/gif poster frame — null for photos. */
thumbnailUrl: string | null;
width: number | null;
height: number | null;
}
export interface ContentItem {
id: string;
sourceId: string;
@@ -31,6 +41,8 @@ export interface ContentItem {
embedding: number[] | null;
eventId: string | null;
clusterId: string | null;
/** Nitter-sourced items only — null for everything else. */
tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null;
raw: unknown;
}
@@ -47,6 +59,8 @@ export interface MergedArticle {
body: string;
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
/** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
category: string[];
geo: string | null;
eventId: string | null;
@@ -117,6 +131,10 @@ export interface GlobalSettings {
aiServiceHost: string;
aiServicePort: number;
selectedModels: { embedding: string; image: string; synthesis: string };
/** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */
nitterMediaMode: 'self-host' | 'proxy' | 'direct';
/** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */
fxtwitterBaseUrl: string;
retention: {
publishedArticleMaxAgeDays: number | null;
rawItemMaxAgeDays: number | null;
+3 -1
View File
@@ -26,6 +26,8 @@ export interface AdminSettings {
aiServiceHost: string;
aiServicePort: number;
selectedModels: { embedding: string; image: string; synthesis: string };
nitterMediaMode: 'self-host' | 'proxy' | 'direct';
fxtwitterBaseUrl: string;
retention: RetentionSettings;
categoryPriority: CategoryPriority[];
}
@@ -33,7 +35,7 @@ export interface AdminSettings {
export interface AdminSource {
id: string;
name: string;
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
category: string[];
url: string;
config?: Record<string, unknown>;
@@ -2,6 +2,7 @@
import type { MergedArticle } from '$lib/types';
import { timeAgo, exactTime, excerpt } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
import TweetCard from './TweetCard.svelte';
let { article }: { article: MergedArticle } = $props();
@@ -12,28 +13,32 @@
);
</script>
<a class="row" href={`/article/${article.id}`}>
{#if article.heroImage}
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
{:else}
<div class="thumb placeholder"></div>
{/if}
{#if article.tweet}
<TweetCard {article} />
{:else}
<a class="row" href={`/article/${article.id}`}>
{#if article.heroImage}
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
{:else}
<div class="thumb placeholder"></div>
{/if}
<div class="content">
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>{sourceLabel}</span>
{#if article.video}
<div class="content">
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>▶ Video</span>
{/if}
<span>{sourceLabel}</span>
{#if article.video}
<span>&middot;</span>
<span>▶ Video</span>
{/if}
</div>
<div class="title">{article.title}</div>
<div class="excerpt">{excerpt(article.body)}</div>
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</div>
<div class="title">{article.title}</div>
<div class="excerpt">{excerpt(article.body)}</div>
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</div>
</a>
</a>
{/if}
<style>
.row {
@@ -0,0 +1,189 @@
<script lang="ts">
import type { MergedArticle } from '$lib/types';
import { timeAgo, exactTime } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
let { article }: { article: MergedArticle } = $props();
const sourceLabel = $derived(article.sources[0]?.sourceName ?? 'Nitter');
// article.tweet.media is undefined for tweets published before this field existed —
// older rows in the DB weren't backfilled, so this can't assume it's always an array.
const media = $derived(article.tweet?.media?.slice(0, 4) ?? []);
// The card's own href — clicking the frame goes to the original tweet, not our
// internal article page (there's no separate "full article" for a tweet anyway).
const tweetUrl = $derived(article.sources[0]?.link ?? `/article/${article.id}`);
// A <video controls> click must not fall through to the card's own <a> navigation
// (play/pause/scrub would otherwise just open the tweet in a new tab instead).
function stopVideoNav(e: MouseEvent) {
e.preventDefault();
}
// A photo click opens the image itself in a new tab, rather than the tweet link the
// rest of the card points to — also resolved through the configured media mode
// (proxy/self-host/direct) so opening the full image doesn't bypass proxy anonymity.
function openImage(e: MouseEvent, url: string) {
e.preventDefault();
window.open(resolveMediaUrl(url), '_blank', 'noopener,noreferrer');
}
</script>
<a class="tweet-card" href={tweetUrl} target="_blank" rel="noreferrer">
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>{sourceLabel}</span>
<span>&middot;</span>
<span>🐦 Tweet</span>
</div>
<div class="author-row">
{#if article.tweet?.avatarUrl}
<img class="avatar" src={resolveMediaUrl(article.tweet.avatarUrl)} alt="" loading="lazy" />
{:else}
<div class="avatar placeholder"></div>
{/if}
<span class="name">{article.tweet?.authorName}</span>
<span class="handle">@{article.tweet?.authorHandle}</span>
</div>
<div class="text">{article.body}</div>
{#if media.length > 0}
<div class="media-grid" data-count={media.length}>
{#each media as item, i (item.url)}
<div class="media-cell" class:span-2={media.length === 3 && i === 0}>
{#if item.type === 'video' || item.type === 'gif'}
<video
class="media-el"
controls
preload="metadata"
playsinline
loop={item.type === 'gif'}
muted={item.type === 'gif'}
poster={item.thumbnailUrl ? resolveMediaUrl(item.thumbnailUrl) : undefined}
onclick={stopVideoNav}
>
<source src={resolveMediaUrl(item.url)} type="video/mp4" />
</video>
{:else}
<button type="button" class="media-btn" onclick={(e) => openImage(e, item.url)}>
<img class="media-el" src={resolveMediaUrl(item.url)} alt="" loading="lazy" />
</button>
{/if}
</div>
{/each}
</div>
{/if}
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</a>
<style>
.tweet-card {
display: block;
border: 0.5px solid var(--border);
border-radius: 12px;
padding: 14px;
margin-bottom: 12px;
background: var(--surface-2);
color: inherit;
}
.tweet-card:hover {
text-decoration: none;
border-color: var(--border-accent);
}
.meta {
font-size: 11px;
color: var(--text-accent);
display: flex;
gap: 6px;
margin-bottom: 10px;
}
.author-row {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 8px;
}
.avatar {
width: 36px;
height: 36px;
border-radius: 50%;
object-fit: cover;
flex-shrink: 0;
background: var(--surface-1);
}
.avatar.placeholder {
display: block;
}
.name {
font-size: 14px;
font-weight: 600;
color: var(--text-primary);
}
.handle {
font-size: 13px;
color: var(--text-muted);
}
.text {
font-size: 14px;
line-height: 1.5;
color: var(--text-primary);
white-space: pre-line;
margin-bottom: 8px;
}
/* Fixed cell heights (rather than max-height on an auto-height image) so a tall
portrait photo or video is cropped to a sensible box instead of dictating the
whole card's height — this is what keeps a single vertical image from taking
over the column. Twitter's own 1/2/3/4-item grid shapes, at a size that fits
this app's narrower single-column feed. */
.media-grid {
display: grid;
gap: 3px;
border-radius: var(--radius);
overflow: hidden;
margin-bottom: 8px;
background: var(--surface-1);
}
.media-grid[data-count='1'] {
grid-template-columns: 1fr;
height: 380px;
}
.media-grid[data-count='2'] {
grid-template-columns: 1fr 1fr;
height: 240px;
}
.media-grid[data-count='3'] {
grid-template-columns: 1fr 1fr;
grid-template-rows: 1fr 1fr;
height: 300px;
}
.media-grid[data-count='4'] {
grid-template-columns: 1fr 1fr;
grid-template-rows: 1fr 1fr;
height: 300px;
}
.media-cell {
overflow: hidden;
}
.media-cell.span-2 {
grid-row: 1 / 3;
}
.media-btn {
display: block;
width: 100%;
height: 100%;
padding: 0;
border: none;
background: none;
cursor: pointer;
}
.media-el {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
background: var(--surface-1);
}
.time {
font-size: 11px;
color: var(--text-muted);
}
</style>
@@ -11,6 +11,31 @@
let clearing = $state<'articles' | 'media' | null>(null);
let clearResult = $state<string | null>(null);
let nitterMediaMode = $state(settings.nitterMediaMode);
let fxtwitterBaseUrl = $state(settings.fxtwitterBaseUrl);
let nitterStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle');
let nitterSaveTimer: ReturnType<typeof setTimeout>;
function scheduleNitterSave() {
nitterStatus = 'saving';
clearTimeout(nitterSaveTimer);
nitterSaveTimer = setTimeout(async () => {
try {
await updateSettings({ nitterMediaMode, fxtwitterBaseUrl });
nitterStatus = 'saved';
setTimeout(() => (nitterStatus = 'idle'), 1500);
} catch {
nitterStatus = 'error';
}
}, 500);
}
const mediaModes: { label: string; value: 'self-host' | 'proxy' | 'direct' }[] = [
{ label: 'Self-host', value: 'self-host' },
{ label: 'Proxy (recommended)', value: 'proxy' },
{ label: 'Direct', value: 'direct' }
];
async function handleClearArticles() {
if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return;
clearing = 'articles';
@@ -141,6 +166,40 @@
</div>
</div>
<div class="panel">
<div class="head">
<span class="panel-title">Nitter (tweet media)</span>
<SaveStatus status={nitterStatus} />
</div>
<p class="hint">
How images and video attached to ingested tweets are served to visitors. Self-hosting
downloads and stores everything locally, same as regular article images. Proxying streams
each request through this server without persisting anything, so only this server's IP is
ever exposed to Twitter's CDN. Direct hotlinks the original URL straight from Twitter, with
no server involvement at all.
</p>
<div class="pill-row">
{#each mediaModes as mode}
<button
class="pill"
class:active={nitterMediaMode === mode.value}
onclick={() => {
nitterMediaMode = mode.value;
scheduleNitterSave();
}}
>
{mode.label}
</button>
{/each}
</div>
<p class="hint" style="margin-top: 14px; margin-bottom: 6px;">
Enrichment API used to fetch full tweet text, author info, and media — any fxtwitter/FixTweet-
compatible endpoint works. Defaults to the public fxtwitter.com instance; point this at a
self-hosted FixTweet mirror (or another public instance) instead if you'd rather not depend on it.
</p>
<input type="text" bind:value={fxtwitterBaseUrl} oninput={scheduleNitterSave} placeholder="https://api.fxtwitter.com" style="width: 100%" />
</div>
<div class="panel">
<span class="panel-title">Clear content</span>
<p class="hint">
@@ -144,7 +144,17 @@
}
const typeIcon = (type: string) =>
type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•';
type === 'rss'
? '⟳'
: type === 'telegram'
? '✈'
: type === 'youtube'
? '▶'
: type === 'nitter'
? '🐦'
: type === 'api'
? '⇄'
: '•';
</script>
<div class="toolbar">
@@ -163,10 +173,13 @@
<option value="api">API</option>
<option value="telegram">Telegram</option>
<option value="youtube">YouTube</option>
<option value="nitter">Nitter</option>
<option value="custom">Custom</option>
</select>
{#if form.type === 'youtube'}
<input placeholder="Channel URL (@handle or /channel/UC…), or channel ID" bind:value={form.channelId} />
{:else if form.type === 'nitter'}
<input placeholder="Nitter list/user RSS feed URL" bind:value={form.url} />
{:else}
<input placeholder="URL or channel" bind:value={form.url} />
{/if}
+10
View File
@@ -8,12 +8,22 @@ export interface ArticleSource {
publishedAt: string;
}
/** A single photo/video/gif attached to a tweet, in the tweet's own display order — capped at 4. */
export interface TweetMediaItem {
type: 'photo' | 'video' | 'gif';
url: string;
thumbnailUrl: string | null;
width: number | null;
height: number | null;
}
export interface MergedArticle {
id: string;
title: string;
body: string;
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
category: string[];
geo: string | null;
eventId: string | null;
+24 -2
View File
@@ -2,6 +2,7 @@
import type { PageData } from './$types';
import { timeAgo, exactTime } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
import TweetCard from '$lib/components/TweetCard.svelte';
let { data }: { data: PageData } = $props();
const a = $derived(data.article);
@@ -22,9 +23,22 @@
<span>{a.category.join(', ')}</span>
</div>
<h1>{a.title}</h1>
{#if a.tweet}
<!-- Tweets have no separate "full article" concept — the embed card itself is the
whole story, same as the list view. No <h1> (it would just duplicate the tweet
text already shown in the card) and no hero-image/body paragraphs either. -->
<div class="dates">
<span>Published {timeAgo(a.publishedAt)} &middot; {exactTime(a.publishedAt)}</span>
</div>
<TweetCard article={a} />
{#if a.sources[0]}
<a class="view-original" href={a.sources[0].link} target="_blank" rel="noreferrer">View original tweet →</a>
{/if}
{:else if a.video?.provider === 'youtube'}
<h1>{a.title}</h1>
{#if a.video?.provider === 'youtube'}
<!-- YouTube's own layout: title, then the video itself, then when it was published
and its description — no hero image (the embed already shows the thumbnail) and
no cross-source merging (see backend priorityQueue.ts — YouTube items never merge). -->
@@ -46,6 +60,8 @@
<p>{paragraph}</p>
{/each}
{:else}
<h1>{a.title}</h1>
<div class="dates">
<span>Published {timeAgo(a.publishedAt)} &middot; {exactTime(a.publishedAt)}</span>
{#if a.updatedAt !== a.publishedAt}
@@ -180,6 +196,12 @@
height: 100%;
border: none;
}
.view-original {
display: inline-block;
font-size: 13px;
color: var(--text-accent);
margin-bottom: 20px;
}
.tags {
display: flex;
gap: 8px;
+15 -4
View File
@@ -1,14 +1,25 @@
import type { PageLoad } from './$types';
import { getFeed } from '$lib/api';
import { slugify } from '$lib/format';
const PAGE_SIZE = 15;
// Every category page (including "Local") filters by its category name — sources are
// assigned categories directly via checkboxes in the admin Sources tab, so a source
// tagged "Local" shows up here the same way one tagged "Tech" shows up on /category/tech.
export const load: PageLoad = async ({ params, fetch }) => {
const name = params.name;
const filters = { category: name };
//
// The URL param is a slug (e.g. "x-news" from slugify("X News")), not the real category
// name — for single-word categories those happen to be the same lowercased, but for a
// multi-word name like "X News" the slug's hyphen never matches the stored "X News"
// (with a space) in a merged_articles.category LIKE match. Resolve the slug back to the
// real category name via the site's own category list (already loaded by the root
// layout) before filtering, rather than passing the raw slug straight through.
export const load: PageLoad = async ({ params, fetch, parent }) => {
const { categories } = await parent();
const match = categories.find((c) => slugify(c.name) === params.name);
const categoryName = match?.name ?? params.name;
const filters = { category: categoryName };
const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch);
return { initial, filters, name, pageSize: PAGE_SIZE };
return { initial, filters, name: categoryName, pageSize: PAGE_SIZE };
};