Add "Nitter" source type: tweets rendered as a distinct embed card

Nitter list/user RSS feeds are ingested as their own source type, enriched
via fxtwitter (author name/handle/avatar, cleaner text, attached photo) with
a graceful RSS-only fallback when that enrichment fails. Tweets always
publish directly, one per article, and never enter the LLM
clustering/synthesis pipeline — the same bypass already used for YouTube,
since merging unrelated tweets together makes no sense.

Rendering: a new distinct embed-card component (avatar, name + @handle,
full untruncated text, optional attached image, published-date-only
timestamp, no like/retweet stats) replaces the plain article row wherever a
tweet appears, on both the category-page list and the article detail page.

Verified end-to-end against the real sample Nitter RSS feed (served
locally): ingestion (all 100 items, tweet metadata correctly extracted,
retweet/quote-tweet blockquotes correctly excluded from own-content text),
publishing (bypasses clustering, tweet field threaded through to the
published article), and rendering (embed card appears on the homepage feed
and the article detail page, no duplicate title).

Known follow-up: fxtwitter's JSON field names are based on public
documentation, not a verified live response (that API is unreachable from
this sandbox) — worth a real curl check before relying on the enrichment
path in production; the RSS-only fallback path is what's actually been
exercised here.
This commit is contained in:
Claude
2026-07-22 21:54:37 +00:00
parent 30e3576206
commit 5cb9e6e4cd
15 changed files with 327 additions and 38 deletions
+3
View File
@@ -9,6 +9,8 @@ export interface FetchedItem {
videos: { url: string; provider?: string; embedHtml?: string }[];
link: string;
publishedAt: string;
/** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */
tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null };
raw: unknown;
}
@@ -40,6 +42,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit<ContentIt
embedding: null,
eventId: null,
clusterId: null,
tweet: item.tweet ? { ...item.tweet } : null,
raw: item.raw
};
}
+119
View File
@@ -0,0 +1,119 @@
// Nitter is its own ingestion module, deliberately separate from the RSS adapter even
// though the transport is RSS: a tweet is enriched with a second fetch to fxtwitter
// (for author name/avatar and cleaner text) and always publishes as its own article —
// see pipeline/publish.ts and queue/priorityQueue.ts, which route nitter-sourced items
// straight to publishDirect rather than the LLM clustering/synthesis pipeline. Merging
// two unrelated tweets into one AI-rewritten story would make no sense the way merging
// two outlets' coverage of the same news event does — same reasoning as YouTube.
import Parser from 'rss-parser';
import type { Source } from '../../storage/db/types.js';
import type { SourceAdapter, FetchedItem } from './base.js';
import { logger } from '../../storage/db/logs.js';
const parser = new Parser<Record<string, unknown>>();
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
const FETCH_TIMEOUT_MS = 10_000;
/**
* Shape based on the publicly documented FixTweet/fxtwitter API
* (https://github.com/FixTweet/FxTwitter) — NOT verified against a live response in
* this environment (outbound access to api.fxtwitter.com is blocked here). Every field
* below is read with optional chaining and a fallback in fetchFxTwitter's caller, so a
* shape mismatch degrades gracefully to RSS-only data rather than breaking ingestion.
* Verify against a real `curl https://api.fxtwitter.com/2/status/<id>` response and
* adjust the field paths here if they don't match.
*/
interface FxTweet {
text?: string;
created_timestamp?: number;
author?: { name?: string; screen_name?: string; avatar_url?: string };
media?: { photos?: { url?: string }[] };
}
async function fetchFxTwitter(tweetId: string): Promise<FxTweet | null> {
try {
const res = await fetch(`https://api.fxtwitter.com/2/status/${tweetId}`, {
headers: { 'User-Agent': USER_AGENT },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
});
if (!res.ok) return null;
const json = (await res.json()) as { tweet?: FxTweet };
return json?.tweet ?? null;
} catch (err) {
logger.warn('nitter', `fxtwitter enrichment failed for tweet ${tweetId}: ${(err as Error).message}`);
return null;
}
}
function extractTweetId(item: Parser.Item): string | null {
const guid = item.guid;
if (guid && /^\d+$/.test(guid)) return guid;
const fromLink = item.link?.match(/status\/(\d+)/)?.[1];
return fromLink ?? null;
}
/**
* A Nitter list/user RSS description is the item's own tweet content (a <p> plus
* optional <img>) followed, for retweets/quote-tweets, by a <blockquote> wrapping the
* quoted tweet's own text/image/permalink. Only the part before that blockquote is this
* item's own content — nested quote-tweet rendering isn't part of the approved design.
*/
function ownContentHtml(descriptionHtml: string): string {
const cut = descriptionHtml.search(/<hr\s*\/?>|<blockquote/i);
return cut === -1 ? descriptionHtml : descriptionHtml.slice(0, cut);
}
function extractImageUrl(html: string): string | null {
return html.match(/<img[^>]+src="([^"]+)"/i)?.[1] ?? null;
}
export const nitterAdapter: SourceAdapter = {
async fetch(source: Source): Promise<FetchedItem[]> {
if (!source.url) return [];
const feed = await parser.parseURL(source.url);
const items: FetchedItem[] = [];
for (const item of feed.items) {
if (!item.link || !item.guid) continue;
const tweetId = extractTweetId(item);
if (!tweetId) {
logger.warn('nitter', `Couldn't extract a tweet ID from "${item.link}" — skipping`);
continue;
}
// dc:creator is reliably the author of this item's own tweet text (rss-parser
// maps it to item.creator) — for a retweet, that's the original tweet's author,
// not whichever list member's retweet surfaced it in this feed.
const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
const ownHtml = ownContentHtml(item.content ?? '');
const rssImageUrl = extractImageUrl(ownHtml);
const enrichment = await fetchFxTwitter(tweetId);
const authorName = enrichment?.author?.name ?? handle;
const avatarUrl = enrichment?.author?.avatar_url ?? null;
const text = enrichment?.text ?? ownHtml;
const photoUrl = enrichment?.media?.photos?.[0]?.url ?? rssImageUrl;
const publishedAt = enrichment?.created_timestamp
? new Date(enrichment.created_timestamp * 1000).toISOString()
: (item.isoDate ?? item.pubDate ?? new Date().toISOString());
items.push({
title: item.title || text.slice(0, 100),
summary: text.slice(0, 500),
body: text,
images: photoUrl ? [{ url: photoUrl }] : [],
videos: [],
link: item.link,
publishedAt,
tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl },
raw: { rss: item, fxtwitter: enrichment }
});
}
return items;
}
};
+2
View File
@@ -5,6 +5,7 @@ import { rssAdapter } from './adapters/rss.js';
import { telegramAdapter } from './adapters/telegram.js';
import { apiAdapter } from './adapters/api.js';
import { youtubeAdapter } from './adapters/youtube.js';
import { nitterAdapter } from './adapters/nitter.js';
import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
import { fetchFullArticle } from './articleFetcher.js';
import type { Source } from '../storage/db/types.js';
@@ -14,6 +15,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
telegram: telegramAdapter,
api: apiAdapter,
youtube: youtubeAdapter,
nitter: nitterAdapter,
custom: apiAdapter
};
+5
View File
@@ -85,12 +85,16 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
const video = item.videos[0]
? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
: null;
const tweet = item.tweet
? { authorName: item.tweet.authorName, authorHandle: item.tweet.authorHandle, avatarUrl: item.tweet.avatarUrl, sourceItemId: item.id }
: null;
const article = await articles.insertArticle({
title: item.title,
body: item.body || item.summary,
heroImage,
video,
tweet,
category,
geo: item.geo,
eventId: item.eventId,
@@ -192,6 +196,7 @@ export async function publishCluster(
body,
heroImage,
video,
tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass
category,
geo,
eventId: opts.eventId ?? items[0]?.eventId ?? null,
+10 -7
View File
@@ -77,19 +77,22 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
if (items.length === 0) return 0;
// YouTube videos never get LLM-merged with anything — each is always its own
// article (title/video/date/description), same shape whether the AI service is up
// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
// YouTube videos and Nitter tweets never get LLM-merged with anything else — each
// is always its own article, same shape whether the AI service is up or not. Route
// them straight to publishDirect, same as the no-AI passthrough path.
const directPublishSourceIds = new Set(
sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id)
);
const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
let publishedDirect = 0;
for (const item of youtubeItems) {
for (const item of directItems) {
try {
const article = await publishDirect(item);
contentItemsDb.assignCluster([item.id], article.id);
publishedDirect++;
logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
const source = sourcesDb.getSource(item.sourceId);
logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`);
} catch (err) {
logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
}
+6 -4
View File
@@ -21,7 +21,8 @@ function rowToArticle(row: any): MergedArticle {
threadId: row.thread_id,
previousArticleId: row.previous_article_id,
nextArticleId: row.next_article_id,
topStories: !!row.top_stories
topStories: !!row.top_stories,
tweet: row.tweet ? JSON.parse(row.tweet) : null
};
}
@@ -29,8 +30,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
const id = `art-${randomUUID()}`;
db.prepare(
`INSERT INTO merged_articles
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
id,
article.title,
@@ -49,7 +50,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
article.threadId,
article.previousArticleId,
article.nextArticleId,
article.topStories ? 1 : 0
article.topStories ? 1 : 0,
article.tweet ? JSON.stringify(article.tweet) : null
);
if (article.previousArticleId) {
db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId);
+4 -2
View File
@@ -20,6 +20,7 @@ function rowToItem(row: any): ContentItem {
embedding: row.embedding ? JSON.parse(row.embedding) : null,
eventId: row.event_id,
clusterId: row.cluster_id,
tweet: row.tweet ? JSON.parse(row.tweet) : null,
raw: row.raw ? JSON.parse(row.raw) : null
};
}
@@ -28,8 +29,8 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
const id = `ci-${randomUUID()}`;
db.prepare(
`INSERT INTO content_items
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
id,
item.sourceId,
@@ -47,6 +48,7 @@ export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
item.embedding ? JSON.stringify(item.embedding) : null,
item.eventId,
item.clusterId,
item.tweet ? JSON.stringify(item.tweet) : null,
item.raw ? JSON.stringify(item.raw) : null
);
return { ...item, id };
+9 -1
View File
@@ -58,6 +58,7 @@ export function migrate() {
embedding TEXT, -- JSON float array
event_id TEXT,
cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis
tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only
raw TEXT -- JSON, original payload
);
CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id);
@@ -82,7 +83,8 @@ export function migrate() {
thread_id TEXT NOT NULL,
previous_article_id TEXT,
next_article_id TEXT,
top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?"
top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?"
tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only
);
CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at);
CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id);
@@ -187,6 +189,12 @@ export function migrate() {
if (!hasColumn('merged_articles', 'top_stories')) {
db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0');
}
if (!hasColumn('content_items', 'tweet')) {
db.exec('ALTER TABLE content_items ADD COLUMN tweet TEXT');
}
if (!hasColumn('merged_articles', 'tweet')) {
db.exec('ALTER TABLE merged_articles ADD COLUMN tweet TEXT');
}
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
// general news sources belong here, not on "Top stories" itself, which isn't a real
+5 -1
View File
@@ -1,7 +1,7 @@
export interface Source {
id: string;
name: string;
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
category: string[];
url: string | null;
config: Record<string, unknown>;
@@ -31,6 +31,8 @@ export interface ContentItem {
embedding: number[] | null;
eventId: string | null;
clusterId: string | null;
/** Nitter-sourced items only — null for everything else. */
tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null } | null;
raw: unknown;
}
@@ -47,6 +49,8 @@ export interface MergedArticle {
body: string;
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
/** Nitter-sourced articles only — the embed card's author info (see TweetCard.svelte). Never set alongside video. */
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null;
category: string[];
geo: string | null;
eventId: string | null;
+1 -1
View File
@@ -32,7 +32,7 @@ export interface AdminSettings {
export interface AdminSource {
id: string;
name: string;
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
category: string[];
url: string;
config?: Record<string, unknown>;
@@ -2,6 +2,7 @@
import type { MergedArticle } from '$lib/types';
import { timeAgo, exactTime, excerpt } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
import TweetCard from './TweetCard.svelte';
let { article }: { article: MergedArticle } = $props();
@@ -12,28 +13,32 @@
);
</script>
<a class="row" href={`/article/${article.id}`}>
{#if article.heroImage}
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
{:else}
<div class="thumb placeholder"></div>
{/if}
{#if article.tweet}
<TweetCard {article} />
{:else}
<a class="row" href={`/article/${article.id}`}>
{#if article.heroImage}
<img class="thumb" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
{:else}
<div class="thumb placeholder"></div>
{/if}
<div class="content">
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>{sourceLabel}</span>
{#if article.video}
<div class="content">
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>▶ Video</span>
{/if}
<span>{sourceLabel}</span>
{#if article.video}
<span>&middot;</span>
<span>▶ Video</span>
{/if}
</div>
<div class="title">{article.title}</div>
<div class="excerpt">{excerpt(article.body)}</div>
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</div>
<div class="title">{article.title}</div>
<div class="excerpt">{excerpt(article.body)}</div>
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</div>
</a>
</a>
{/if}
<style>
.row {
@@ -0,0 +1,100 @@
<script lang="ts">
import type { MergedArticle } from '$lib/types';
import { timeAgo, exactTime } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
let { article }: { article: MergedArticle } = $props();
const sourceLabel = $derived(article.sources[0]?.sourceName ?? 'Nitter');
</script>
<a class="tweet-card" href={`/article/${article.id}`}>
<div class="meta">
<span>{article.category[0] ?? ''}</span>
<span>&middot;</span>
<span>{sourceLabel}</span>
<span>&middot;</span>
<span>🐦 Tweet</span>
</div>
<div class="author-row">
{#if article.tweet?.avatarUrl}
<img class="avatar" src={article.tweet.avatarUrl} alt="" loading="lazy" />
{:else}
<div class="avatar placeholder"></div>
{/if}
<span class="name">{article.tweet?.authorName}</span>
<span class="handle">@{article.tweet?.authorHandle}</span>
</div>
<div class="text">{article.body}</div>
{#if article.heroImage}
<img class="tweet-img" src={resolveMediaUrl(article.heroImage.url)} alt="" loading="lazy" />
{/if}
<div class="time">{timeAgo(article.publishedAt)} &middot; {exactTime(article.publishedAt)}</div>
</a>
<style>
.tweet-card {
display: block;
border: 0.5px solid var(--border);
border-radius: 12px;
padding: 14px;
margin-bottom: 12px;
background: var(--surface-2);
color: inherit;
}
.tweet-card:hover {
text-decoration: none;
border-color: var(--border-accent);
}
.meta {
font-size: 11px;
color: var(--text-accent);
display: flex;
gap: 6px;
margin-bottom: 10px;
}
.author-row {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 8px;
}
.avatar {
width: 36px;
height: 36px;
border-radius: 50%;
object-fit: cover;
flex-shrink: 0;
background: var(--surface-1);
}
.avatar.placeholder {
display: block;
}
.name {
font-size: 14px;
font-weight: 600;
color: var(--text-primary);
}
.handle {
font-size: 13px;
color: var(--text-muted);
}
.text {
font-size: 14px;
line-height: 1.5;
color: var(--text-primary);
white-space: pre-line;
margin-bottom: 8px;
}
.tweet-img {
width: 100%;
border-radius: var(--radius);
margin-bottom: 8px;
display: block;
background: var(--surface-1);
}
.time {
font-size: 11px;
color: var(--text-muted);
}
</style>
@@ -144,7 +144,17 @@
}
const typeIcon = (type: string) =>
type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•';
type === 'rss'
? '⟳'
: type === 'telegram'
? '✈'
: type === 'youtube'
? '▶'
: type === 'nitter'
? '🐦'
: type === 'api'
? '⇄'
: '•';
</script>
<div class="toolbar">
@@ -163,10 +173,13 @@
<option value="api">API</option>
<option value="telegram">Telegram</option>
<option value="youtube">YouTube</option>
<option value="nitter">Nitter</option>
<option value="custom">Custom</option>
</select>
{#if form.type === 'youtube'}
<input placeholder="Channel URL (@handle or /channel/UC…), or channel ID" bind:value={form.channelId} />
{:else if form.type === 'nitter'}
<input placeholder="Nitter list/user RSS feed URL" bind:value={form.url} />
{:else}
<input placeholder="URL or channel" bind:value={form.url} />
{/if}
+1
View File
@@ -14,6 +14,7 @@ export interface MergedArticle {
body: string;
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string } | null;
category: string[];
geo: string | null;
eventId: string | null;
+24 -2
View File
@@ -2,6 +2,7 @@
import type { PageData } from './$types';
import { timeAgo, exactTime } from '$lib/format';
import { resolveMediaUrl } from '$lib/config';
import TweetCard from '$lib/components/TweetCard.svelte';
let { data }: { data: PageData } = $props();
const a = $derived(data.article);
@@ -22,9 +23,22 @@
<span>{a.category.join(', ')}</span>
</div>
<h1>{a.title}</h1>
{#if a.tweet}
<!-- Tweets have no separate "full article" concept — the embed card itself is the
whole story, same as the list view. No <h1> (it would just duplicate the tweet
text already shown in the card) and no hero-image/body paragraphs either. -->
<div class="dates">
<span>Published {timeAgo(a.publishedAt)} &middot; {exactTime(a.publishedAt)}</span>
</div>
<TweetCard article={a} />
{#if a.sources[0]}
<a class="view-original" href={a.sources[0].link} target="_blank" rel="noreferrer">View original tweet →</a>
{/if}
{:else if a.video?.provider === 'youtube'}
<h1>{a.title}</h1>
{#if a.video?.provider === 'youtube'}
<!-- YouTube's own layout: title, then the video itself, then when it was published
and its description — no hero image (the embed already shows the thumbnail) and
no cross-source merging (see backend priorityQueue.ts — YouTube items never merge). -->
@@ -46,6 +60,8 @@
<p>{paragraph}</p>
{/each}
{:else}
<h1>{a.title}</h1>
<div class="dates">
<span>Published {timeAgo(a.publishedAt)} &middot; {exactTime(a.publishedAt)}</span>
{#if a.updatedAt !== a.publishedAt}
@@ -180,6 +196,12 @@
height: 100%;
border: none;
}
.view-original {
display: inline-block;
font-size: 13px;
color: var(--text-accent);
margin-bottom: 20px;
}
.tags {
display: flex;
gap: 8px;