Merge pull request #10 from Salastil/claude/nitter-rss-tweets
Add Nitter (Twitter) source support with dedicated tweet rendering
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
// Backs the "proxy" Nitter media mode (see Retention tab / GlobalSettings.nitterMediaMode):
|
||||
// the visitor's browser requests media from this route instead of directly from
|
||||
// Twitter/the Nitter instance's CDN, so only this server's IP is ever exposed to the
|
||||
// remote host — the media itself is streamed straight through, never written to disk.
|
||||
//
|
||||
// Since this route fetches whatever URL it's given, it's a textbook SSRF vector unless
|
||||
// tightly restricted: only twimg.com (Twitter's media CDN), the configured
|
||||
// fxtwitterBaseUrl's host, and the hostnames of the admin's own configured Nitter
|
||||
// sources are allowed — and even an allowed hostname is rejected if it resolves to a
|
||||
// private/loopback/link-local address (defends against DNS rebinding, not just a
|
||||
// hostname string check).
|
||||
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import dns from 'node:dns/promises';
|
||||
import { Readable } from 'node:stream';
|
||||
import * as sourcesDb from '../storage/db/sources.js';
|
||||
import { getSettings } from '../storage/db/settings.js';
|
||||
import { logger } from '../storage/db/logs.js';
|
||||
|
||||
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
|
||||
const FETCH_TIMEOUT_MS = 15_000;
|
||||
const TWITTER_MEDIA_HOST_RE = /(^|\.)twimg\.com$/i;
|
||||
|
||||
function hostnameOf(rawUrl: string | null): string | null {
|
||||
if (!rawUrl) return null;
|
||||
try {
|
||||
return new URL(rawUrl).hostname.toLowerCase();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isAllowedHost(hostname: string): boolean {
|
||||
const lower = hostname.toLowerCase();
|
||||
if (TWITTER_MEDIA_HOST_RE.test(lower)) return true;
|
||||
|
||||
const settings = getSettings();
|
||||
if (hostnameOf(settings.fxtwitterBaseUrl) === lower) return true;
|
||||
|
||||
const nitterHosts = sourcesDb
|
||||
.listSources()
|
||||
.filter((s) => s.type === 'nitter')
|
||||
.map((s) => hostnameOf(s.url))
|
||||
.filter((h): h is string => !!h);
|
||||
return nitterHosts.includes(lower);
|
||||
}
|
||||
|
||||
function isPrivateOrReservedIp(ip: string, family: number): boolean {
|
||||
if (family === 4) {
|
||||
const [a, b] = ip.split('.').map(Number);
|
||||
if (a === 10 || a === 127 || a === 0) return true;
|
||||
if (a === 169 && b === 254) return true;
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
if (a === 192 && b === 168) return true;
|
||||
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT range
|
||||
return false;
|
||||
}
|
||||
const lower = ip.toLowerCase();
|
||||
if (lower === '::1') return true;
|
||||
if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // unique local fc00::/7
|
||||
if (lower.startsWith('fe80')) return true; // link-local
|
||||
if (lower.startsWith('::ffff:')) {
|
||||
const v4 = lower.split(':').pop();
|
||||
if (v4?.includes('.')) return isPrivateOrReservedIp(v4, 4);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function registerMediaProxy(app: FastifyInstance) {
|
||||
app.get('/media/proxy', async (req, reply) => {
|
||||
const { url } = req.query as { url?: string };
|
||||
if (!url) return reply.code(400).send({ error: 'url required' });
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return reply.code(400).send({ error: 'invalid url' });
|
||||
}
|
||||
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
return reply.code(400).send({ error: 'unsupported scheme' });
|
||||
}
|
||||
|
||||
if (!isAllowedHost(parsed.hostname)) {
|
||||
logger.warn('media-proxy', `Blocked proxy request to disallowed host: ${parsed.hostname}`);
|
||||
return reply.code(403).send({ error: 'host not allowed' });
|
||||
}
|
||||
|
||||
let addresses: { address: string; family: number }[];
|
||||
try {
|
||||
addresses = await dns.lookup(parsed.hostname, { all: true });
|
||||
} catch {
|
||||
return reply.code(502).send({ error: 'DNS resolution failed' });
|
||||
}
|
||||
if (addresses.some((a) => isPrivateOrReservedIp(a.address, a.family))) {
|
||||
logger.warn('media-proxy', `Blocked proxy request resolving to a private/reserved address: ${parsed.hostname}`);
|
||||
return reply.code(403).send({ error: 'host not allowed' });
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(parsed.toString(), {
|
||||
headers: { 'User-Agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
|
||||
});
|
||||
if (!res.ok || !res.body) {
|
||||
return reply.code(502).send({ error: `upstream responded ${res.status}` });
|
||||
}
|
||||
reply.header('content-type', res.headers.get('content-type') ?? 'application/octet-stream');
|
||||
reply.header('cache-control', res.headers.get('cache-control') ?? 'public, max-age=3600');
|
||||
return reply.send(Readable.fromWeb(res.body as any));
|
||||
} catch (err) {
|
||||
logger.warn('media-proxy', `Proxy fetch failed for ${parsed.toString()}: ${(err as Error).message}`);
|
||||
return reply.code(502).send({ error: 'fetch failed' });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { ADMIN_API_KEY } from './api/apiKey.js';
|
||||
import { registerAuth } from './api/auth.js';
|
||||
import { registerPublicRoutes } from './api/public.js';
|
||||
import { registerAdminRoutes } from './api/admin.js';
|
||||
import { registerMediaProxy } from './api/mediaProxy.js';
|
||||
import { startScheduler } from './queue/scheduler.js';
|
||||
import { logger } from './storage/db/logs.js';
|
||||
|
||||
@@ -79,6 +80,11 @@ async function main() {
|
||||
return reply.send(fs.createReadStream(filePath));
|
||||
});
|
||||
|
||||
// Static "/media/proxy" takes priority over the "/media/:filename" param route
|
||||
// above regardless of registration order (find-my-way, Fastify's router, always
|
||||
// prefers a static segment over a parametric one at the same depth).
|
||||
await registerMediaProxy(app);
|
||||
|
||||
app.get('/health', async () => ({ ok: true }));
|
||||
|
||||
await app.listen({ port: PORT, host: '0.0.0.0' });
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { Source, ContentItem } from '../../storage/db/types.js';
|
||||
import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js';
|
||||
import { cleanHtml, toSummary } from '../clean.js';
|
||||
|
||||
export interface FetchedItem {
|
||||
@@ -9,6 +9,8 @@ export interface FetchedItem {
|
||||
videos: { url: string; provider?: string; embedHtml?: string }[];
|
||||
link: string;
|
||||
publishedAt: string;
|
||||
/** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */
|
||||
tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] };
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
@@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit<ContentIt
|
||||
embedding: null,
|
||||
eventId: null,
|
||||
clusterId: null,
|
||||
tweet: item.tweet ? { ...item.tweet } : null,
|
||||
raw: item.raw
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
// Nitter is its own ingestion module, deliberately separate from the RSS adapter even
|
||||
// though the transport is RSS: a tweet is enriched with a second fetch to fxtwitter
|
||||
// (for author name/avatar and cleaner text) and always publishes as its own article —
|
||||
// see pipeline/publish.ts and queue/priorityQueue.ts, which route nitter-sourced items
|
||||
// straight to publishDirect rather than the LLM clustering/synthesis pipeline. Merging
|
||||
// two unrelated tweets into one AI-rewritten story would make no sense the way merging
|
||||
// two outlets' coverage of the same news event does — same reasoning as YouTube.
|
||||
|
||||
import Parser from 'rss-parser';
|
||||
import type { Source, TweetMediaItem } from '../../storage/db/types.js';
|
||||
import type { SourceAdapter, FetchedItem } from './base.js';
|
||||
import { logger } from '../../storage/db/logs.js';
|
||||
import { getSettings } from '../../storage/db/settings.js';
|
||||
|
||||
/** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */
|
||||
const MAX_TWEET_MEDIA = 4;
|
||||
|
||||
const parser = new Parser<Record<string, unknown>>();
|
||||
|
||||
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
|
||||
const FETCH_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* Shape confirmed against two real `curl https://api.fxtwitter.com/<handle>/status/<id>`
|
||||
* responses: `text`, `created_timestamp` (unix seconds), `author.name`/`avatar_url`, and
|
||||
* (from a second, video-attached tweet) `media.all[]` — an ordered array covering both
|
||||
* photos and videos, each with `type` ('photo' | 'video' | 'gif'), `url` (the direct
|
||||
* playable/displayable URL — for video this is a real .mp4, not the .m3u8 playlist also
|
||||
* present under `formats`), `thumbnail_url` (video/gif poster frame), and `width`/`height`.
|
||||
* `media.all` is preferred over `media.photos`/`media.videos` since it's the only field
|
||||
* that preserves the tweet's original media order.
|
||||
*/
|
||||
interface FxTweetMedia {
|
||||
type?: string;
|
||||
url?: string;
|
||||
thumbnail_url?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
interface FxTweet {
|
||||
text?: string;
|
||||
created_timestamp?: number;
|
||||
author?: { name?: string; screen_name?: string; avatar_url?: string };
|
||||
media?: { all?: FxTweetMedia[]; photos?: FxTweetMedia[] };
|
||||
}
|
||||
|
||||
/** Maps fxtwitter's media shape to our own, capped at the 4 items a tweet can carry. */
|
||||
function toTweetMedia(enrichment: FxTweet | null): TweetMediaItem[] {
|
||||
const items = enrichment?.media?.all ?? enrichment?.media?.photos ?? [];
|
||||
return items.slice(0, MAX_TWEET_MEDIA).map((m): TweetMediaItem => ({
|
||||
type: m.type === 'video' || m.type === 'gif' ? m.type : 'photo',
|
||||
url: m.url ?? '',
|
||||
thumbnailUrl: m.thumbnail_url ?? null,
|
||||
width: m.width ?? null,
|
||||
height: m.height ?? null
|
||||
})).filter((m) => m.url);
|
||||
}
|
||||
|
||||
/**
|
||||
* The endpoint is keyed by handle + status ID, e.g. https://api.fxtwitter.com/zerohedge/status/123
|
||||
* — no API version prefix. The base URL is admin-configurable (Retention tab) so a
|
||||
* self-hosted FixTweet mirror (or another compatible public instance) can be used
|
||||
* instead of the public api.fxtwitter.com default.
|
||||
*/
|
||||
async function fetchFxTwitter(handle: string, tweetId: string): Promise<FxTweet | null> {
|
||||
const baseUrl = getSettings().fxtwitterBaseUrl.replace(/\/+$/, '');
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/${handle}/status/${tweetId}`, {
|
||||
headers: { 'User-Agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const json = (await res.json()) as { tweet?: FxTweet };
|
||||
return json?.tweet ?? null;
|
||||
} catch (err) {
|
||||
logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractTweetId(item: Parser.Item): string | null {
|
||||
const guid = item.guid;
|
||||
if (guid && /^\d+$/.test(guid)) return guid;
|
||||
const fromLink = item.link?.match(/status\/(\d+)/)?.[1];
|
||||
return fromLink ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A Nitter list/user RSS description is the item's own tweet content (a <p> plus
|
||||
* optional <img>) followed, for retweets/quote-tweets, by a <blockquote> wrapping the
|
||||
* quoted tweet's own text/image/permalink. Only the part before that blockquote is this
|
||||
* item's own content — nested quote-tweet rendering isn't part of the approved design.
|
||||
*/
|
||||
function ownContentHtml(descriptionHtml: string): string {
|
||||
const cut = descriptionHtml.search(/<hr\s*\/?>|<blockquote/i);
|
||||
return cut === -1 ? descriptionHtml : descriptionHtml.slice(0, cut);
|
||||
}
|
||||
|
||||
function extractImageUrl(html: string): string | null {
|
||||
return html.match(/<img[^>]+src="([^"]+)"/i)?.[1] ?? null;
|
||||
}
|
||||
|
||||
export const nitterAdapter: SourceAdapter = {
|
||||
async fetch(source: Source): Promise<FetchedItem[]> {
|
||||
if (!source.url) return [];
|
||||
|
||||
const feed = await parser.parseURL(source.url);
|
||||
const items: FetchedItem[] = [];
|
||||
|
||||
for (const item of feed.items) {
|
||||
if (!item.link || !item.guid) continue;
|
||||
const tweetId = extractTweetId(item);
|
||||
if (!tweetId) {
|
||||
logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// dc:creator is reliably the author of this item's own tweet text (rss-parser
|
||||
// maps it to item.creator) — for a retweet, that's the original tweet's author,
|
||||
// not whichever list member's retweet surfaced it in this feed.
|
||||
const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
|
||||
const ownHtml = ownContentHtml(item.content ?? '');
|
||||
const rssImageUrl = extractImageUrl(ownHtml);
|
||||
|
||||
const enrichment = await fetchFxTwitter(handle, tweetId);
|
||||
|
||||
const authorName = enrichment?.author?.name ?? handle;
|
||||
const avatarUrl = enrichment?.author?.avatar_url ?? null;
|
||||
const text = enrichment?.text ?? ownHtml;
|
||||
// Enrichment failed (or came back with no media) — fall back to the RSS
|
||||
// description's own <img> as a single photo, same as before multi-media support.
|
||||
const media = toTweetMedia(enrichment);
|
||||
if (media.length === 0 && rssImageUrl) {
|
||||
media.push({ type: 'photo', url: rssImageUrl, thumbnailUrl: null, width: null, height: null });
|
||||
}
|
||||
const publishedAt = enrichment?.created_timestamp
|
||||
? new Date(enrichment.created_timestamp * 1000).toISOString()
|
||||
: (item.isoDate ?? item.pubDate ?? new Date().toISOString());
|
||||
|
||||
items.push({
|
||||
title: item.title || text.slice(0, 100),
|
||||
summary: text.slice(0, 500),
|
||||
body: text,
|
||||
images: [],
|
||||
videos: [],
|
||||
link: item.link,
|
||||
publishedAt,
|
||||
tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media },
|
||||
raw: { rss: item, fxtwitter: enrichment }
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
};
|
||||
@@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js';
|
||||
import { telegramAdapter } from './adapters/telegram.js';
|
||||
import { apiAdapter } from './adapters/api.js';
|
||||
import { youtubeAdapter } from './adapters/youtube.js';
|
||||
import { nitterAdapter } from './adapters/nitter.js';
|
||||
import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
|
||||
import { fetchFullArticle } from './articleFetcher.js';
|
||||
import type { Source } from '../storage/db/types.js';
|
||||
@@ -14,6 +15,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
|
||||
telegram: telegramAdapter,
|
||||
api: apiAdapter,
|
||||
youtube: youtubeAdapter,
|
||||
nitter: nitterAdapter,
|
||||
custom: apiAdapter
|
||||
};
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js';
|
||||
import * as articles from '../storage/db/articles.js';
|
||||
import * as tags from '../storage/db/tags.js';
|
||||
import * as sources from '../storage/db/sources.js';
|
||||
import type { GlobalSettings, MergedArticle, ContentItem } from '../storage/db/types.js';
|
||||
import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js';
|
||||
|
||||
const FOLLOW_UP_LOOKBACK_DAYS = 3;
|
||||
|
||||
@@ -33,9 +33,14 @@ function deriveTitle(body: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the hero image for an article: try the best candidate from the source
|
||||
* items, download and locally host it; if there isn't one, fall back to the site's
|
||||
* favicon rather than leaving the article with no art at all.
|
||||
* Resolves the hero image for a regular (non-tweet) article: try the best candidate
|
||||
* from the source items, download and locally host it; if there isn't one, fall back
|
||||
* to the site's favicon rather than leaving the article with no art at all. Tweets
|
||||
* never reach this function — see resolveTweetMediaUrl below, which applies the
|
||||
* admin-configured Nitter media mode instead of always downloading, and skips the
|
||||
* favicon fallback entirely (a Nitter instance's own favicon slapped onto an
|
||||
* image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already
|
||||
* handles no-image tweets gracefully with no image at all).
|
||||
*/
|
||||
async function resolveHeroImage(
|
||||
items: ContentItem[],
|
||||
@@ -69,6 +74,61 @@ async function resolveHeroImage(
|
||||
return { heroImage: null, storedMediaId: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the admin-configured Nitter media mode (Retention tab) to a single
|
||||
* externally-hosted tweet media URL — an attached photo or the author's avatar.
|
||||
* 'self-host' downloads and serves it locally like any other article image;
|
||||
* 'proxy' routes it through this server's own /media/proxy route so only this
|
||||
* server's IP is ever exposed to Twitter/the Nitter instance's CDN (the
|
||||
* "anonymity" the admin asked for) without persisting anything to disk; 'direct'
|
||||
* hotlinks the original URL unchanged, the cheapest option with no server involvement.
|
||||
*/
|
||||
async function resolveTweetMediaUrl(
|
||||
url: string,
|
||||
mode: GlobalSettings['nitterMediaMode']
|
||||
): Promise<{ url: string; storedMediaId: string | null }> {
|
||||
if (mode === 'direct') return { url, storedMediaId: null };
|
||||
|
||||
if (mode === 'proxy') {
|
||||
return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null };
|
||||
}
|
||||
|
||||
const stored = await downloadAndStore(url, 'published', {});
|
||||
if (stored) return { url: stored.servedPath, storedMediaId: stored.id };
|
||||
// Download failed — fall back to hotlinking rather than losing the media entirely.
|
||||
return { url, storedMediaId: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves every attached photo/video/gif's url — and, for video/gif, its poster
|
||||
* thumbnail — through the admin's chosen Nitter media mode. Order and item count are
|
||||
* preserved; TweetCard.svelte renders this array directly, there's no separate
|
||||
* "hero image" concept for tweets the way there is for regular articles.
|
||||
*/
|
||||
async function resolveTweetMedia(
|
||||
media: TweetMediaItem[],
|
||||
mode: GlobalSettings['nitterMediaMode']
|
||||
): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> {
|
||||
const storedMediaIds: string[] = [];
|
||||
const resolved: TweetMediaItem[] = [];
|
||||
|
||||
for (const item of media) {
|
||||
const url = await resolveTweetMediaUrl(item.url, mode);
|
||||
if (url.storedMediaId) storedMediaIds.push(url.storedMediaId);
|
||||
|
||||
let thumbnailUrl = item.thumbnailUrl;
|
||||
if (thumbnailUrl) {
|
||||
const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode);
|
||||
thumbnailUrl = resolvedThumb.url;
|
||||
if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId);
|
||||
}
|
||||
|
||||
resolved.push({ ...item, url: url.url, thumbnailUrl });
|
||||
}
|
||||
|
||||
return { media: resolved, storedMediaIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes a single item as-is, with no AI calls at all — used when the AI service
|
||||
* isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
|
||||
@@ -79,18 +139,47 @@ async function resolveHeroImage(
|
||||
* tag-based thread detection — but these earlier articles aren't retroactively
|
||||
* rewritten or merged with anything after the fact.
|
||||
*/
|
||||
export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
|
||||
export async function publishDirect(item: ContentItem, settings: GlobalSettings): Promise<MergedArticle> {
|
||||
const category = uniqueCategories([item]);
|
||||
const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
|
||||
const storedMediaIds: string[] = [];
|
||||
|
||||
// Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly.
|
||||
let heroImage: MergedArticle['heroImage'] = null;
|
||||
if (!item.tweet) {
|
||||
const resolved = await resolveHeroImage([item], item.link);
|
||||
heroImage = resolved.heroImage;
|
||||
if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
|
||||
}
|
||||
|
||||
const video = item.videos[0]
|
||||
? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
|
||||
: null;
|
||||
|
||||
let tweet: MergedArticle['tweet'] = null;
|
||||
if (item.tweet) {
|
||||
let avatarUrl: string | null = null;
|
||||
if (item.tweet.avatarUrl) {
|
||||
const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode);
|
||||
avatarUrl = resolved.url;
|
||||
if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
|
||||
}
|
||||
const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode);
|
||||
storedMediaIds.push(...resolvedMedia.storedMediaIds);
|
||||
tweet = {
|
||||
authorName: item.tweet.authorName,
|
||||
authorHandle: item.tweet.authorHandle,
|
||||
avatarUrl,
|
||||
sourceItemId: item.id,
|
||||
media: resolvedMedia.media
|
||||
};
|
||||
}
|
||||
|
||||
const article = await articles.insertArticle({
|
||||
title: item.title,
|
||||
body: item.body || item.summary,
|
||||
heroImage,
|
||||
video,
|
||||
tweet,
|
||||
category,
|
||||
geo: item.geo,
|
||||
eventId: item.eventId,
|
||||
@@ -115,7 +204,7 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
|
||||
topStories: anyPushesToTopStories([item])
|
||||
});
|
||||
|
||||
if (storedMediaId) promoteToPublished(storedMediaId, article.id);
|
||||
for (const id of storedMediaIds) promoteToPublished(id, article.id);
|
||||
return article;
|
||||
}
|
||||
|
||||
@@ -192,6 +281,7 @@ export async function publishCluster(
|
||||
body,
|
||||
heroImage,
|
||||
video,
|
||||
tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass
|
||||
category,
|
||||
geo,
|
||||
eventId: opts.eventId ?? items[0]?.eventId ?? null,
|
||||
|
||||
@@ -54,7 +54,7 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
|
||||
|
||||
for (const item of ranked) {
|
||||
try {
|
||||
const article = await publishDirect(item);
|
||||
const article = await publishDirect(item, settings);
|
||||
contentItemsDb.assignCluster([item.id], article.id);
|
||||
published++;
|
||||
logger.info('synthesis', `Published "${article.title}" directly (no AI available)`);
|
||||
@@ -77,19 +77,22 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
|
||||
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
|
||||
if (items.length === 0) return 0;
|
||||
|
||||
// YouTube videos never get LLM-merged with anything — each is always its own
|
||||
// article (title/video/date/description), same shape whether the AI service is up
|
||||
// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
|
||||
const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
|
||||
const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
|
||||
// YouTube videos and Nitter tweets never get LLM-merged with anything else — each
|
||||
// is always its own article, same shape whether the AI service is up or not. Route
|
||||
// them straight to publishDirect, same as the no-AI passthrough path.
|
||||
const directPublishSourceIds = new Set(
|
||||
sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id)
|
||||
);
|
||||
const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
|
||||
|
||||
let publishedDirect = 0;
|
||||
for (const item of youtubeItems) {
|
||||
for (const item of directItems) {
|
||||
try {
|
||||
const article = await publishDirect(item);
|
||||
const article = await publishDirect(item, settings);
|
||||
contentItemsDb.assignCluster([item.id], article.id);
|
||||
publishedDirect++;
|
||||
logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
|
||||
const source = sourcesDb.getSource(item.sourceId);
|
||||
logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`);
|
||||
} catch (err) {
|
||||
logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
@@ -21,7 +21,8 @@ function rowToArticle(row: any): MergedArticle {
|
||||
threadId: row.thread_id,
|
||||
previousArticleId: row.previous_article_id,
|
||||
nextArticleId: row.next_article_id,
|
||||
topStories: !!row.top_stories
|
||||
topStories: !!row.top_stories,
|
||||
tweet: row.tweet ? JSON.parse(row.tweet) : null
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,8 +30,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
|
||||
const id = `art-${randomUUID()}`;
|
||||
db.prepare(
|
||||
`INSERT INTO merged_articles
|
||||
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
id,
|
||||
article.title,
|
||||
@@ -49,7 +50,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
|
||||
article.threadId,
|
||||
article.previousArticleId,
|
||||
article.nextArticleId,
|
||||
article.topStories ? 1 : 0
|
||||
article.topStories ? 1 : 0,
|
||||
article.tweet ? JSON.stringify(article.tweet) : null
|
||||
);
|
||||
if (article.previousArticleId) {
|
||||
db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId);
|
||||
|
||||
@@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem {
|
||||
embedding: row.embedding ? JSON.parse(row.embedding) : null,
|
||||
eventId: row.event_id,
|
||||
clusterId: row.cluster_id,
|
||||
tweet: row.tweet ? JSON.parse(row.tweet) : null,
|
||||
raw: row.raw ? JSON.parse(row.raw) : null
|
||||
};
|
||||
}
|
||||
@@ -28,8 +29,8 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
|
||||
const id = `ci-${randomUUID()}`;
|
||||
db.prepare(
|
||||
`INSERT INTO content_items
|
||||
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
id,
|
||||
item.sourceId,
|
||||
@@ -47,6 +48,7 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
|
||||
item.embedding ? JSON.stringify(item.embedding) : null,
|
||||
item.eventId,
|
||||
item.clusterId,
|
||||
item.tweet ? JSON.stringify(item.tweet) : null,
|
||||
item.raw ? JSON.stringify(item.raw) : null
|
||||
);
|
||||
return { ...item, id };
|
||||
|
||||
@@ -58,6 +58,7 @@ export function migrate() {
|
||||
embedding TEXT, -- JSON float array
|
||||
event_id TEXT,
|
||||
cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis
|
||||
tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only
|
||||
raw TEXT -- JSON, original payload
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id);
|
||||
@@ -82,7 +83,8 @@ export function migrate() {
|
||||
thread_id TEXT NOT NULL,
|
||||
previous_article_id TEXT,
|
||||
next_article_id TEXT,
|
||||
top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?"
|
||||
top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?"
|
||||
tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id);
|
||||
@@ -165,7 +167,9 @@ export function migrate() {
|
||||
raw_item_max_age_days INTEGER DEFAULT 7,
|
||||
storage_cap_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
storage_cap_value INTEGER NOT NULL DEFAULT 500,
|
||||
storage_cap_unit TEXT NOT NULL DEFAULT 'GB'
|
||||
storage_cap_unit TEXT NOT NULL DEFAULT 'GB',
|
||||
nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct
|
||||
fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'
|
||||
);
|
||||
`);
|
||||
|
||||
@@ -187,6 +191,18 @@ export function migrate() {
|
||||
if (!hasColumn('merged_articles', 'top_stories')) {
|
||||
db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
if (!hasColumn('content_items', 'tweet')) {
|
||||
db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT');
|
||||
}
|
||||
if (!hasColumn('merged_articles', 'tweet')) {
|
||||
db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT');
|
||||
}
|
||||
if (!hasColumn('global_settings', 'nitter_media_mode')) {
|
||||
db.exec("ALTER TABLE global_settings ADD COLUMN nitter_media_mode TEXT NOT NULL DEFAULT 'proxy'");
|
||||
}
|
||||
if (!hasColumn('global_settings', 'fxtwitter_base_url')) {
|
||||
db.exec("ALTER TABLE global_settings ADD COLUMN fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com'");
|
||||
}
|
||||
|
||||
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
|
||||
// general news sources belong here, not on "Top stories" itself, which isn't a real
|
||||
|
||||
@@ -13,6 +13,8 @@ function rowToSettings(row: any): GlobalSettings {
|
||||
aiServiceHost: row.ai_service_host,
|
||||
aiServicePort: row.ai_service_port,
|
||||
selectedModels: JSON.parse(row.selected_models),
|
||||
nitterMediaMode: row.nitter_media_mode,
|
||||
fxtwitterBaseUrl: row.fxtwitter_base_url,
|
||||
retention: {
|
||||
publishedArticleMaxAgeDays: row.published_article_max_age_days,
|
||||
rawItemMaxAgeDays: row.raw_item_max_age_days,
|
||||
@@ -41,6 +43,7 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
|
||||
merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?,
|
||||
tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?,
|
||||
ai_service_host=?, ai_service_port=?, selected_models=?,
|
||||
nitter_media_mode=?, fxtwitter_base_url=?,
|
||||
published_article_max_age_days=?, raw_item_max_age_days=?,
|
||||
storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?
|
||||
WHERE id = 1`
|
||||
@@ -55,6 +58,8 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
|
||||
merged.aiServiceHost,
|
||||
merged.aiServicePort,
|
||||
JSON.stringify(merged.selectedModels),
|
||||
merged.nitterMediaMode,
|
||||
merged.fxtwitterBaseUrl,
|
||||
merged.retention.publishedArticleMaxAgeDays,
|
||||
merged.retention.rawItemMaxAgeDays,
|
||||
merged.retention.storageCapEnabled ? 1 : 0,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
|
||||
category: string[];
|
||||
url: string | null;
|
||||
config: Record<string, unknown>;
|
||||
@@ -14,6 +14,16 @@ export interface Source {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** A single photo/video/gif attached to a tweet, in the tweet's own display order — fxtwitter caps this at 4. */
|
||||
export interface TweetMediaItem {
|
||||
type: 'photo' | 'video' | 'gif';
|
||||
url: string;
|
||||
/** Video/gif poster frame — null for photos. */
|
||||
thumbnailUrl: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
}
|
||||
|
||||
export interface ContentItem {
|
||||
id: string;
|
||||
sourceId: string;
|
||||
@@ -31,6 +41,8 @@ export interface ContentItem {
|
||||
embedding: number[] | null;
|
||||
eventId: string | null;
|
||||
clusterId: string | null;
|
||||
/** Nitter-sourced items only — null for everything else. */
|
||||
tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
@@ -47,6 +59,8 @@ export interface MergedArticle {
|
||||
body: string;
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
/** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */
|
||||
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
@@ -115,6 +129,10 @@ export interface GlobalSettings {
|
||||
aiServiceHost: string;
|
||||
aiServicePort: number;
|
||||
selectedModels: { embedding: string; image: string; synthesis: string };
|
||||
/** How tweet media (attached photos, avatars) is served — see pipeline/publish.ts's resolveTweetMedia. */
|
||||
nitterMediaMode: 'self-host' | 'proxy' | 'direct';
|
||||
/** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */
|
||||
fxtwitterBaseUrl: string;
|
||||
retention: {
|
||||
publishedArticleMaxAgeDays: number | null;
|
||||
rawItemMaxAgeDays: number | null;
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface AdminSettings {
|
||||
aiServiceHost: string;
|
||||
aiServicePort: number;
|
||||
selectedModels: { embedding: string; image: string; synthesis: string };
|
||||
nitterMediaMode: 'self-host' | 'proxy' | 'direct';
|
||||
fxtwitterBaseUrl: string;
|
||||
retention: RetentionSettings;
|
||||
categoryPriority: CategoryPriority[];
|
||||
}
|
||||
@@ -32,7 +34,7 @@ export interface AdminSettings {
|
||||
export interface AdminSource {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
|
||||
category: string[];
|
||||
url: string;
|
||||
config?: Record<string, unknown>;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import type { MergedArticle } from '$lib/types';
|
||||
import { timeAgo, exactTime, excerpt } from '$lib/format';
|
||||
import { resolveMediaUrl } from '$lib/config';
|
||||
import TweetCard from './TweetCard.svelte';
|
||||
|
||||
let { article }: { article: MergedArticle } = $props();
|
||||
|
||||
@@ -12,28 +13,32 @@
|
||||
);
|
||||
</script>
|
||||
|
||||
<a class="row" href={`/article/${article.id}`}>
|
||||
{#if article.heroImage}
|
||||
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
|
||||
{:else}
|
||||
<div class="thumb placeholder"></div>
|
||||
{/if}
|
||||
{#if article.tweet}
|
||||
<TweetCard {article} />
|
||||
{:else}
|
||||
<a class="row" href={`/article/${article.id}`}>
|
||||
{#if article.heroImage}
|
||||
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
|
||||
{:else}
|
||||
<div class="thumb placeholder"></div>
|
||||
{/if}
|
||||
|
||||
<div class="content">
|
||||
<div class="meta">
|
||||
<span>{article.category[0] ?? ''}</span>
|
||||
<span>·</span>
|
||||
<span>{sourceLabel}</span>
|
||||
{#if article.video}
|
||||
<div class="content">
|
||||
<div class="meta">
|
||||
<span>{article.category[0] ?? ''}</span>
|
||||
<span>·</span>
|
||||
<span>▶ Video</span>
|
||||
{/if}
|
||||
<span>{sourceLabel}</span>
|
||||
{#if article.video}
|
||||
<span>·</span>
|
||||
<span>▶ Video</span>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="title">{article.title}</div>
|
||||
<div class="excerpt">{excerpt(article.body)}</div>
|
||||
<div class="time">{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}</div>
|
||||
</div>
|
||||
<div class="title">{article.title}</div>
|
||||
<div class="excerpt">{excerpt(article.body)}</div>
|
||||
<div class="time">{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}</div>
|
||||
</div>
|
||||
</a>
|
||||
</a>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.row {
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
<script lang="ts">
|
||||
import type { MergedArticle } from '$lib/types';
|
||||
import { timeAgo, exactTime } from '$lib/format';
|
||||
import { resolveMediaUrl } from '$lib/config';
|
||||
|
||||
let { article }: { article: MergedArticle } = $props();
|
||||
|
||||
const sourceLabel = $derived(article.sources[0]?.sourceName ?? 'Nitter');
|
||||
// article.tweet.media is undefined for tweets published before this field existed —
|
||||
// older rows in the DB weren't backfilled, so this can't assume it's always an array.
|
||||
const media = $derived(article.tweet?.media?.slice(0, 4) ?? []);
|
||||
// The card's own href — clicking the frame goes to the original tweet, not our
|
||||
// internal article page (there's no separate "full article" for a tweet anyway).
|
||||
const tweetUrl = $derived(article.sources[0]?.link ?? `/article/${article.id}`);
|
||||
|
||||
// A <video controls> click must not fall through to the card's own <a> navigation
|
||||
// (play/pause/scrub would otherwise just open the tweet in a new tab instead).
|
||||
function stopVideoNav(e: MouseEvent) {
|
||||
e.preventDefault();
|
||||
}
|
||||
|
||||
// A photo click opens the image itself in a new tab, rather than the tweet link the
|
||||
// rest of the card points to — also resolved through the configured media mode
|
||||
// (proxy/self-host/direct) so opening the full image doesn't bypass proxy anonymity.
|
||||
function openImage(e: MouseEvent, url: string) {
|
||||
e.preventDefault();
|
||||
window.open(resolveMediaUrl(url), '_blank', 'noopener,noreferrer');
|
||||
}
|
||||
</script>
|
||||
|
||||
<a class="tweet-card" href={tweetUrl} target="_blank" rel="noreferrer">
|
||||
<div class="meta">
|
||||
<span>{article.category[0] ?? ''}</span>
|
||||
<span>·</span>
|
||||
<span>{sourceLabel}</span>
|
||||
<span>·</span>
|
||||
<span>🐦 Tweet</span>
|
||||
</div>
|
||||
<div class="author-row">
|
||||
{#if article.tweet?.avatarUrl}
|
||||
<img class="avatar" src={resolveMediaUrl(article.tweet.avatarUrl)} alt="" loading="lazy" />
|
||||
{:else}
|
||||
<div class="avatar placeholder"></div>
|
||||
{/if}
|
||||
<span class="name">{article.tweet?.authorName}</span>
|
||||
<span class="handle">@{article.tweet?.authorHandle}</span>
|
||||
</div>
|
||||
<div class="text">{article.body}</div>
|
||||
{#if media.length > 0}
|
||||
<div class="media-grid" data-count={media.length}>
|
||||
{#each media as item, i (item.url)}
|
||||
<div class="media-cell" class:span-2={media.length === 3 && i === 0}>
|
||||
{#if item.type === 'video' || item.type === 'gif'}
|
||||
<video
|
||||
class="media-el"
|
||||
controls
|
||||
preload="metadata"
|
||||
playsinline
|
||||
loop={item.type === 'gif'}
|
||||
muted={item.type === 'gif'}
|
||||
poster={item.thumbnailUrl ? resolveMediaUrl(item.thumbnailUrl) : undefined}
|
||||
onclick={stopVideoNav}
|
||||
>
|
||||
<source src={resolveMediaUrl(item.url)} type="video/mp4" />
|
||||
</video>
|
||||
{:else}
|
||||
<button type="button" class="media-btn" onclick={(e) => openImage(e, item.url)}>
|
||||
<img class="media-el" src={resolveMediaUrl(item.url)} alt="" loading="lazy" />
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
<div class="time">{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}</div>
|
||||
</a>
|
||||
|
||||
<style>
|
||||
.tweet-card {
|
||||
display: block;
|
||||
border: 0.5px solid var(--border);
|
||||
border-radius: 12px;
|
||||
padding: 14px;
|
||||
margin-bottom: 12px;
|
||||
background: var(--surface-2);
|
||||
color: inherit;
|
||||
}
|
||||
.tweet-card:hover {
|
||||
text-decoration: none;
|
||||
border-color: var(--border-accent);
|
||||
}
|
||||
.meta {
|
||||
font-size: 11px;
|
||||
color: var(--text-accent);
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.author-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.avatar {
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
flex-shrink: 0;
|
||||
background: var(--surface-1);
|
||||
}
|
||||
.avatar.placeholder {
|
||||
display: block;
|
||||
}
|
||||
.name {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
.handle {
|
||||
font-size: 13px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.text {
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
color: var(--text-primary);
|
||||
white-space: pre-line;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
/* Fixed cell heights (rather than max-height on an auto-height image) so a tall
|
||||
portrait photo or video is cropped to a sensible box instead of dictating the
|
||||
whole card's height — this is what keeps a single vertical image from taking
|
||||
over the column. Twitter's own 1/2/3/4-item grid shapes, at a size that fits
|
||||
this app's narrower single-column feed. */
|
||||
.media-grid {
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
border-radius: var(--radius);
|
||||
overflow: hidden;
|
||||
margin-bottom: 8px;
|
||||
background: var(--surface-1);
|
||||
}
|
||||
.media-grid[data-count='1'] {
|
||||
grid-template-columns: 1fr;
|
||||
height: 380px;
|
||||
}
|
||||
.media-grid[data-count='2'] {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
height: 240px;
|
||||
}
|
||||
.media-grid[data-count='3'] {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
grid-template-rows: 1fr 1fr;
|
||||
height: 300px;
|
||||
}
|
||||
.media-grid[data-count='4'] {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
grid-template-rows: 1fr 1fr;
|
||||
height: 300px;
|
||||
}
|
||||
.media-cell {
|
||||
overflow: hidden;
|
||||
}
|
||||
.media-cell.span-2 {
|
||||
grid-row: 1 / 3;
|
||||
}
|
||||
.media-btn {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
.media-el {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
background: var(--surface-1);
|
||||
}
|
||||
.time {
|
||||
font-size: 11px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
</style>
|
||||
@@ -11,6 +11,31 @@
|
||||
let clearing = $state<'articles' | 'media' | null>(null);
|
||||
let clearResult = $state<string | null>(null);
|
||||
|
||||
let nitterMediaMode = $state(settings.nitterMediaMode);
|
||||
let fxtwitterBaseUrl = $state(settings.fxtwitterBaseUrl);
|
||||
let nitterStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle');
|
||||
let nitterSaveTimer: ReturnType<typeof setTimeout>;
|
||||
|
||||
function scheduleNitterSave() {
|
||||
nitterStatus = 'saving';
|
||||
clearTimeout(nitterSaveTimer);
|
||||
nitterSaveTimer = setTimeout(async () => {
|
||||
try {
|
||||
await updateSettings({ nitterMediaMode, fxtwitterBaseUrl });
|
||||
nitterStatus = 'saved';
|
||||
setTimeout(() => (nitterStatus = 'idle'), 1500);
|
||||
} catch {
|
||||
nitterStatus = 'error';
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
|
||||
const mediaModes: { label: string; value: 'self-host' | 'proxy' | 'direct' }[] = [
|
||||
{ label: 'Self-host', value: 'self-host' },
|
||||
{ label: 'Proxy (recommended)', value: 'proxy' },
|
||||
{ label: 'Direct', value: 'direct' }
|
||||
];
|
||||
|
||||
async function handleClearArticles() {
|
||||
if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return;
|
||||
clearing = 'articles';
|
||||
@@ -141,6 +166,40 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="head">
|
||||
<span class="panel-title">Nitter (tweet media)</span>
|
||||
<SaveStatus status={nitterStatus} />
|
||||
</div>
|
||||
<p class="hint">
|
||||
How images and video attached to ingested tweets are served to visitors. Self-hosting
|
||||
downloads and stores everything locally, same as regular article images. Proxying streams
|
||||
each request through this server without persisting anything, so only this server's IP is
|
||||
ever exposed to Twitter's CDN. Direct hotlinks the original URL straight from Twitter, with
|
||||
no server involvement at all.
|
||||
</p>
|
||||
<div class="pill-row">
|
||||
{#each mediaModes as mode}
|
||||
<button
|
||||
class="pill"
|
||||
class:active={nitterMediaMode === mode.value}
|
||||
onclick={() => {
|
||||
nitterMediaMode = mode.value;
|
||||
scheduleNitterSave();
|
||||
}}
|
||||
>
|
||||
{mode.label}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
<p class="hint" style="margin-top: 14px; margin-bottom: 6px;">
|
||||
Enrichment API used to fetch full tweet text, author info, and media — any fxtwitter/FixTweet-
|
||||
compatible endpoint works. Defaults to the public fxtwitter.com instance; point this at a
|
||||
self-hosted FixTweet mirror (or another public instance) instead if you'd rather not depend on it.
|
||||
</p>
|
||||
<input type="text" bind:value={fxtwitterBaseUrl} oninput={scheduleNitterSave} placeholder="https://api.fxtwitter.com" style="width: 100%" />
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<span class="panel-title">Clear content</span>
|
||||
<p class="hint">
|
||||
|
||||
@@ -144,7 +144,17 @@
|
||||
}
|
||||
|
||||
const typeIcon = (type: string) =>
|
||||
type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•';
|
||||
type === 'rss'
|
||||
? '⟳'
|
||||
: type === 'telegram'
|
||||
? '✈'
|
||||
: type === 'youtube'
|
||||
? '▶'
|
||||
: type === 'nitter'
|
||||
? '🐦'
|
||||
: type === 'api'
|
||||
? '⇄'
|
||||
: '•';
|
||||
</script>
|
||||
|
||||
<div class="toolbar">
|
||||
@@ -163,10 +173,13 @@
|
||||
<option value="api">API</option>
|
||||
<option value="telegram">Telegram</option>
|
||||
<option value="youtube">YouTube</option>
|
||||
<option value="nitter">Nitter</option>
|
||||
<option value="custom">Custom</option>
|
||||
</select>
|
||||
{#if form.type === 'youtube'}
|
||||
<input placeholder="Channel URL (@handle or /channel/UC…), or channel ID" bind:value={form.channelId} />
|
||||
{:else if form.type === 'nitter'}
|
||||
<input placeholder="Nitter list/user RSS feed URL" bind:value={form.url} />
|
||||
{:else}
|
||||
<input placeholder="URL or channel" bind:value={form.url} />
|
||||
{/if}
|
||||
|
||||
@@ -8,12 +8,22 @@ export interface ArticleSource {
|
||||
publishedAt: string;
|
||||
}
|
||||
|
||||
/** A single photo/video/gif attached to a tweet, in the tweet's own display order — capped at 4. */
|
||||
export interface TweetMediaItem {
|
||||
type: 'photo' | 'video' | 'gif';
|
||||
url: string;
|
||||
thumbnailUrl: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
}
|
||||
|
||||
export interface MergedArticle {
|
||||
id: string;
|
||||
title: string;
|
||||
body: string;
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import type { PageData } from './$types';
|
||||
import { timeAgo, exactTime } from '$lib/format';
|
||||
import { resolveMediaUrl } from '$lib/config';
|
||||
import TweetCard from '$lib/components/TweetCard.svelte';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
const a = $derived(data.article);
|
||||
@@ -22,9 +23,22 @@
|
||||
<span>{a.category.join(', ')}</span>
|
||||
</div>
|
||||
|
||||
<h1>{a.title}</h1>
|
||||
{#if a.tweet}
|
||||
<!-- Tweets have no separate "full article" concept — the embed card itself is the
|
||||
whole story, same as the list view. No <h1> (it would just duplicate the tweet
|
||||
text already shown in the card) and no hero-image/body paragraphs either. -->
|
||||
<div class="dates">
|
||||
<span>Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)}</span>
|
||||
</div>
|
||||
|
||||
<TweetCard article={a} />
|
||||
|
||||
{#if a.sources[0]}
|
||||
<a class="view-original" href={a.sources[0].link} target="_blank" rel="noreferrer">View original tweet →</a>
|
||||
{/if}
|
||||
{:else if a.video?.provider === 'youtube'}
|
||||
<h1>{a.title}</h1>
|
||||
|
||||
{#if a.video?.provider === 'youtube'}
|
||||
<!-- YouTube's own layout: title, then the video itself, then when it was published
|
||||
and its description — no hero image (the embed already shows the thumbnail) and
|
||||
no cross-source merging (see backend priorityQueue.ts — YouTube items never merge). -->
|
||||
@@ -46,6 +60,8 @@
|
||||
<p>{paragraph}</p>
|
||||
{/each}
|
||||
{:else}
|
||||
<h1>{a.title}</h1>
|
||||
|
||||
<div class="dates">
|
||||
<span>Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)}</span>
|
||||
{#if a.updatedAt !== a.publishedAt}
|
||||
@@ -180,6 +196,12 @@
|
||||
height: 100%;
|
||||
border: none;
|
||||
}
|
||||
.view-original {
|
||||
display: inline-block;
|
||||
font-size: 13px;
|
||||
color: var(--text-accent);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.tags {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
|
||||
@@ -1,14 +1,25 @@
|
||||
import type { PageLoad } from './$types';
|
||||
import { getFeed } from '$lib/api';
|
||||
import { slugify } from '$lib/format';
|
||||
|
||||
const PAGE_SIZE = 15;
|
||||
|
||||
// Every category page (including "Local") filters by its category name — sources are
|
||||
// assigned categories directly via checkboxes in the admin Sources tab, so a source
|
||||
// tagged "Local" shows up here the same way one tagged "Tech" shows up on /category/tech.
|
||||
export const load: PageLoad = async ({ params, fetch }) => {
|
||||
const name = params.name;
|
||||
const filters = { category: name };
|
||||
//
|
||||
// The URL param is a slug (e.g. "x-news" from slugify("X News")), not the real category
|
||||
// name — for single-word categories those happen to be the same lowercased, but for a
|
||||
// multi-word name like "X News" the slug's hyphen never matches the stored "X News"
|
||||
// (with a space) in a merged_articles.category LIKE match. Resolve the slug back to the
|
||||
// real category name via the site's own category list (already loaded by the root
|
||||
// layout) before filtering, rather than passing the raw slug straight through.
|
||||
export const load: PageLoad = async ({ params, fetch, parent }) => {
|
||||
const { categories } = await parent();
|
||||
const match = categories.find((c) => slugify(c.name) === params.name);
|
||||
const categoryName = match?.name ?? params.name;
|
||||
|
||||
const filters = { category: categoryName };
|
||||
const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch);
|
||||
return { initial, filters, name, pageSize: PAGE_SIZE };
|
||||
return { initial, filters, name: categoryName, pageSize: PAGE_SIZE };
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user