diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index e767543..3002913 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,4 @@ -import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js'; +import type { Source, ContentItem, TweetMediaItem, QuotedTweet } from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -10,7 +10,15 @@ export interface FetchedItem { link: string; publishedAt: string; /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ - tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] }; + tweet?: { + id: string; + authorName: string; + authorHandle: string; + avatarUrl: string | null; + media: TweetMediaItem[]; + repostedByHandle: string | null; + quotedTweet: QuotedTweet | null; + }; raw: unknown; } diff --git a/backend/src/ingestion/adapters/nitter.ts b/backend/src/ingestion/adapters/nitter.ts index 8e1060f..a13b3d1 100644 --- a/backend/src/ingestion/adapters/nitter.ts +++ b/backend/src/ingestion/adapters/nitter.ts @@ -7,10 +7,11 @@ // two outlets' coverage of the same news event does — same reasoning as YouTube. import Parser from 'rss-parser'; -import type { Source, TweetMediaItem } from '../../storage/db/types.js'; +import type { Source, TweetMediaItem, QuotedTweet } from '../../storage/db/types.js'; import type { SourceAdapter, FetchedItem } from './base.js'; import { logger } from '../../storage/db/logs.js'; import { getSettings } from '../../storage/db/settings.js'; +import { cleanHtml, toSummary } from '../clean.js'; /** fxtwitter caps a tweet at 4 attached photos/videos/gifs, in tweet display order. */ const MAX_TWEET_MEDIA = 4; @@ -101,6 +102,46 @@ function extractImageUrl(html: string): string | null { return html.match(/]+src="([^"]+)"/i)?.[1] ?? null; } +/** + * A bare retweet's RSS title is prefixed "RT by @handle: ..." — confirmed against a real + * sample (e.g. `RT by @rawsalerts: BREAKING: MyPillow CEO...` with + * `@Polymarket`, i.e. dc:creator is already the original author; + * only the retweeter's handle is missing from anywhere else in the feed). No blockquote + * accompanies a bare retweet — the description is the original tweet's own content directly. + */ +function extractRepostedByHandle(title: string | undefined): string | null { + return title?.match(/^RT by @(\w+):/i)?.[1] ?? null; +} + +/** + * A quote-tweet's RSS description is the quoter's own commentary followed by an `
` + * and a `
` wrapping the quoted tweet — confirmed against a real sample: + * `
zerohedge (@zerohedge)

...quoted text...

+ *
`. + * Nitter carries the quoted tweet's author, text, one image, and permalink fully inline — + * no extra fxtwitter call needed to render it as a nested preview. + */ +function extractQuotedTweet(descriptionHtml: string): QuotedTweet | null { + const blockquoteMatch = descriptionHtml.match(/
([\s\S]*?)<\/blockquote>/i); + if (!blockquoteMatch) return null; + const inner = blockquoteMatch[1]; + + const authorMatch = inner.match(/\s*([^<(]+?)\s*\(@([^)]+)\)\s*<\/b>/i); + const linkMatch = inner.match(/