Merge pull request #13 from Salastil/claude/nitter-rss-tweets
Claude/nitter rss tweets
This commit is contained in:
Salastil
@@ -1,4 +1,4 @@
|
||||
import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js';
|
||||
import type { Source, ContentItem, TweetMediaItem, QuotedTweet } from '../../storage/db/types.js';
|
||||
import { cleanHtml, toSummary } from '../clean.js';
|
||||
|
||||
export interface FetchedItem {
|
||||
@@ -10,7 +10,15 @@ export interface FetchedItem {
|
||||
link: string;
|
||||
publishedAt: string;
|
||||
/** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */
|
||||
tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] };
|
||||
tweet?: {
|
||||
id: string;
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
avatarUrl: string | null;
|
||||
media: TweetMediaItem[];
|
||||
repostedByHandle: string | null;
|
||||
quotedTweet: QuotedTweet | null;
|
||||
};
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,10 +7,11 @@
|
||||
// two outlets' coverage of the same news event does — same reasoning as YouTube.
|
||||
|
||||
import Parser from 'rss-parser';
|
||||
import type { Source, TweetMediaItem } from '../../storage/db/types.js';
|
||||
import type { Source, TweetMediaItem, QuotedTweet } from '../../storage/db/types.js';
|
||||
import type { SourceAdapter, FetchedItem } from './base.js';
|
||||
import { logger } from '../../storage/db/logs.js';
|
||||
import { getSettings } from '../../storage/db/settings.js';
|
||||
import { cleanHtml } from '../clean.js';
|
||||
|
||||
/** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */
|
||||
const MAX_TWEET_MEDIA = 4;
|
||||
@@ -101,6 +102,46 @@ function extractImageUrl(html: string): string | null {
|
||||
return html.match(/<img[^>]+src="([^"]+)"/i)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A bare retweet's RSS title is prefixed "RT by @handle: ..." — confirmed against a real
|
||||
* sample (e.g. `<title>RT by @rawsalerts: BREAKING: MyPillow CEO...</title>` with
|
||||
* `<dc:creator>@Polymarket</dc:creator>`, i.e. dc:creator is already the original author;
|
||||
* only the retweeter's handle is missing from anywhere else in the feed). No blockquote
|
||||
* accompanies a bare retweet — the description is the original tweet's own content directly.
|
||||
*/
|
||||
function extractRepostedByHandle(title: string | undefined): string | null {
|
||||
return title?.match(/^RT by @(\w+):/i)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A quote-tweet's RSS description is the quoter's own commentary followed by an `<hr/>`
|
||||
* and a `<blockquote>` wrapping the quoted tweet — confirmed against a real sample:
|
||||
* `<blockquote><b>zerohedge (@zerohedge)</b><p>...quoted text...<img .../></p>
|
||||
* <footer>— <cite><a href="https://.../status/...">...</a></cite></footer></blockquote>`.
|
||||
* Nitter carries the quoted tweet's author, text, one image, and permalink fully inline —
|
||||
* no extra fxtwitter call needed to render it as a nested preview.
|
||||
*/
|
||||
function extractQuotedTweet(descriptionHtml: string): QuotedTweet | null {
|
||||
const blockquoteMatch = descriptionHtml.match(/<blockquote>([\s\S]*?)<\/blockquote>/i);
|
||||
if (!blockquoteMatch) return null;
|
||||
const inner = blockquoteMatch[1];
|
||||
|
||||
const authorMatch = inner.match(/<b>\s*([^<(]+?)\s*\(@([^)]+)\)\s*<\/b>/i);
|
||||
const linkMatch = inner.match(/<footer>[\s\S]*?<a href="([^"]+)"/i);
|
||||
if (!authorMatch || !linkMatch) return null;
|
||||
|
||||
const afterAuthor = inner.slice((authorMatch.index ?? 0) + authorMatch[0].length);
|
||||
const beforeFooter = afterAuthor.replace(/<footer>[\s\S]*$/i, '');
|
||||
|
||||
return {
|
||||
authorName: authorMatch[1].trim(),
|
||||
authorHandle: authorMatch[2].trim(),
|
||||
text: cleanHtml(beforeFooter),
|
||||
imageUrl: extractImageUrl(inner),
|
||||
link: linkMatch[1]
|
||||
};
|
||||
}
|
||||
|
||||
export const nitterAdapter: SourceAdapter = {
|
||||
async fetch(source: Source): Promise<FetchedItem[]> {
|
||||
if (!source.url) return [];
|
||||
@@ -116,16 +157,22 @@ export const nitterAdapter: SourceAdapter = {
|
||||
continue;
|
||||
}
|
||||
|
||||
// dc:creator is reliably the author of this item's own tweet text (rss-parser
|
||||
// maps it to item.creator) — for a retweet, that's the original tweet's author,
|
||||
// not whichever list member's retweet surfaced it in this feed.
|
||||
const handle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
|
||||
// dc:creator (rss-parser maps it to item.creator) is reliably the author of the
|
||||
// tweet actually being shown — confirmed against a real "RT by @X: ..." sample,
|
||||
// where dc:creator was the ORIGINAL author, not the retweeter (the retweeter's
|
||||
// handle only ever appears in the title's "RT by @X:" prefix, extracted below).
|
||||
const rssHandle = (item.creator ?? '').replace(/^@/, '') || 'unknown';
|
||||
const ownHtml = ownContentHtml(item.content ?? '');
|
||||
const rssImageUrl = extractImageUrl(ownHtml);
|
||||
const repostedByHandle = extractRepostedByHandle(item.title);
|
||||
// A bare retweet never carries a blockquote (its description IS the original
|
||||
// tweet directly), so this is naturally null whenever repostedByHandle is set.
|
||||
const quotedTweet = extractQuotedTweet(item.content ?? '');
|
||||
|
||||
const enrichment = await fetchFxTwitter(handle, tweetId);
|
||||
const enrichment = await fetchFxTwitter(rssHandle, tweetId);
|
||||
|
||||
const authorName = enrichment?.author?.name ?? handle;
|
||||
const authorName = enrichment?.author?.name ?? rssHandle;
|
||||
const handle = enrichment?.author?.screen_name ?? rssHandle;
|
||||
const avatarUrl = enrichment?.author?.avatar_url ?? null;
|
||||
const text = enrichment?.text ?? ownHtml;
|
||||
// Enrichment failed (or came back with no media) — fall back to the RSS
|
||||
@@ -146,7 +193,7 @@ export const nitterAdapter: SourceAdapter = {
|
||||
videos: [],
|
||||
link: item.link,
|
||||
publishedAt,
|
||||
tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media },
|
||||
tweet: { id: tweetId, authorName, authorHandle: handle, avatarUrl, media, repostedByHandle, quotedTweet },
|
||||
raw: { rss: item, fxtwitter: enrichment }
|
||||
});
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { logger } from '../storage/db/logs.js';
|
||||
import * as articles from '../storage/db/articles.js';
|
||||
import * as tags from '../storage/db/tags.js';
|
||||
import * as sources from '../storage/db/sources.js';
|
||||
import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js';
|
||||
import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem, QuotedTweet } from '../storage/db/types.js';
|
||||
|
||||
const FOLLOW_UP_LOOKBACK_DAYS = 3;
|
||||
|
||||
@@ -129,6 +129,16 @@ async function resolveTweetMedia(
|
||||
return { media: resolved, storedMediaIds };
|
||||
}
|
||||
|
||||
/** Resolves a quote-tweet's embedded image (if any) through the Nitter media mode, same as any other tweet media. */
|
||||
async function resolveQuotedTweet(
|
||||
quoted: QuotedTweet,
|
||||
mode: GlobalSettings['nitterMediaMode']
|
||||
): Promise<{ quotedTweet: QuotedTweet; storedMediaId: string | null }> {
|
||||
if (!quoted.imageUrl) return { quotedTweet: quoted, storedMediaId: null };
|
||||
const resolved = await resolveTweetMediaUrl(quoted.imageUrl, mode);
|
||||
return { quotedTweet: { ...quoted, imageUrl: resolved.url }, storedMediaId: resolved.storedMediaId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes a single item as-is, with no AI calls at all — used when the AI service
|
||||
* isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
|
||||
@@ -165,12 +175,22 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings)
|
||||
}
|
||||
const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode);
|
||||
storedMediaIds.push(...resolvedMedia.storedMediaIds);
|
||||
|
||||
let quotedTweet: QuotedTweet | null = null;
|
||||
if (item.tweet.quotedTweet) {
|
||||
const resolvedQuoted = await resolveQuotedTweet(item.tweet.quotedTweet, settings.nitterMediaMode);
|
||||
quotedTweet = resolvedQuoted.quotedTweet;
|
||||
if (resolvedQuoted.storedMediaId) storedMediaIds.push(resolvedQuoted.storedMediaId);
|
||||
}
|
||||
|
||||
tweet = {
|
||||
authorName: item.tweet.authorName,
|
||||
authorHandle: item.tweet.authorHandle,
|
||||
avatarUrl,
|
||||
sourceItemId: item.id,
|
||||
media: resolvedMedia.media
|
||||
media: resolvedMedia.media,
|
||||
repostedByHandle: item.tweet.repostedByHandle,
|
||||
quotedTweet
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,19 @@ export interface TweetMediaItem {
|
||||
height: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The tweet embedded inside a quote-tweet's own <blockquote> — Nitter's RSS carries this
|
||||
* fully inline (author, text, one image, permalink), so no extra fxtwitter call is needed
|
||||
* to render it. Rendered as a smaller frame nested inside the quoting tweet's card.
|
||||
*/
|
||||
export interface QuotedTweet {
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
text: string;
|
||||
imageUrl: string | null;
|
||||
link: string;
|
||||
}
|
||||
|
||||
export interface ContentItem {
|
||||
id: string;
|
||||
sourceId: string;
|
||||
@@ -42,7 +55,17 @@ export interface ContentItem {
|
||||
eventId: string | null;
|
||||
clusterId: string | null;
|
||||
/** Nitter-sourced items only — null for everything else. */
|
||||
tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null;
|
||||
tweet: {
|
||||
id: string;
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
avatarUrl: string | null;
|
||||
media: TweetMediaItem[];
|
||||
/** Set when this item is a bare retweet — the retweeter's handle, e.g. from RSS "RT by @X:". */
|
||||
repostedByHandle: string | null;
|
||||
/** Set when this item is a quote-tweet — the tweet embedded in its <blockquote>. */
|
||||
quotedTweet: QuotedTweet | null;
|
||||
} | null;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
@@ -60,7 +83,15 @@ export interface MergedArticle {
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
/** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */
|
||||
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
|
||||
tweet: {
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
avatarUrl: string | null;
|
||||
sourceItemId: string;
|
||||
media: TweetMediaItem[];
|
||||
repostedByHandle: string | null;
|
||||
quotedTweet: QuotedTweet | null;
|
||||
} | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
|
||||
@@ -26,9 +26,19 @@
|
||||
e.preventDefault();
|
||||
window.open(resolveMediaUrl(url), '_blank', 'noopener,noreferrer');
|
||||
}
|
||||
|
||||
// The nested quoted-tweet frame opens its own permalink — a Nitter URL already, not
|
||||
// backend-hosted media, so no resolveMediaUrl needed here.
|
||||
function openQuoted(e: MouseEvent, link: string) {
|
||||
e.preventDefault();
|
||||
window.open(link, '_blank', 'noopener,noreferrer');
|
||||
}
|
||||
</script>
|
||||
|
||||
<a class="tweet-card" href={tweetUrl} target="_blank" rel="noreferrer">
|
||||
{#if article.tweet?.repostedByHandle}
|
||||
<div class="repost-line">🔁 Reposted by @{article.tweet.repostedByHandle}</div>
|
||||
{/if}
|
||||
<div class="meta">
|
||||
<span>{article.category[0] ?? ''}</span>
|
||||
<span>·</span>
|
||||
@@ -72,6 +82,20 @@
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
{#if article.tweet?.quotedTweet}
|
||||
{@const quoted = article.tweet.quotedTweet}
|
||||
<div class="quote-label">↩️ Replying to @{quoted.authorHandle}</div>
|
||||
<button type="button" class="quoted-card" onclick={(e) => openQuoted(e, quoted.link)}>
|
||||
<div class="quoted-author-row">
|
||||
<span class="quoted-name">{quoted.authorName}</span>
|
||||
<span class="quoted-handle">@{quoted.authorHandle}</span>
|
||||
</div>
|
||||
<div class="quoted-text">{quoted.text}</div>
|
||||
{#if quoted.imageUrl}
|
||||
<img class="quoted-img" src={resolveMediaUrl(quoted.imageUrl)} alt="" loading="lazy" />
|
||||
{/if}
|
||||
</button>
|
||||
{/if}
|
||||
<div class="time">{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}</div>
|
||||
</a>
|
||||
|
||||
@@ -89,6 +113,12 @@
|
||||
text-decoration: none;
|
||||
border-color: var(--border-accent);
|
||||
}
|
||||
.repost-line {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
color: var(--text-secondary);
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.meta {
|
||||
font-size: 11px;
|
||||
color: var(--text-accent);
|
||||
@@ -182,6 +212,57 @@
|
||||
display: block;
|
||||
background: var(--surface-1);
|
||||
}
|
||||
/* Nested inside the same outer frame — deliberately lighter-weight than the card
|
||||
itself (no independent hover border, tighter radius, tinted background instead of
|
||||
its own border-forward "card" look) so it reads as inset content, not a second card. */
|
||||
.quote-label {
|
||||
font-size: 11.5px;
|
||||
font-weight: 600;
|
||||
color: var(--text-secondary);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.quoted-card {
|
||||
display: block;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
padding: 10px;
|
||||
margin-bottom: 8px;
|
||||
background: var(--surface-1);
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
.quoted-author-row {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
gap: 6px;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
.quoted-name {
|
||||
font-size: 12.5px;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
.quoted-handle {
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.quoted-text {
|
||||
font-size: 12.5px;
|
||||
line-height: 1.45;
|
||||
color: var(--text-secondary);
|
||||
white-space: pre-line;
|
||||
}
|
||||
.quoted-img {
|
||||
width: 100%;
|
||||
max-height: 140px;
|
||||
object-fit: cover;
|
||||
border-radius: 6px;
|
||||
margin-top: 6px;
|
||||
display: block;
|
||||
background: var(--surface-2);
|
||||
}
|
||||
.time {
|
||||
font-size: 11px;
|
||||
color: var(--text-muted);
|
||||
|
||||
@@ -17,13 +17,30 @@ export interface TweetMediaItem {
|
||||
height: number | null;
|
||||
}
|
||||
|
||||
/** The tweet embedded in a quote-tweet's own preview — rendered as a smaller nested frame. */
|
||||
export interface QuotedTweet {
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
text: string;
|
||||
imageUrl: string | null;
|
||||
link: string;
|
||||
}
|
||||
|
||||
export interface MergedArticle {
|
||||
id: string;
|
||||
title: string;
|
||||
body: string;
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null;
|
||||
tweet: {
|
||||
authorName: string;
|
||||
authorHandle: string;
|
||||
avatarUrl: string | null;
|
||||
sourceItemId: string;
|
||||
media: TweetMediaItem[];
|
||||
repostedByHandle: string | null;
|
||||
quotedTweet: QuotedTweet | null;
|
||||
} | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
|
||||
Reference in New Issue
Block a user