5cb9e6e4cd
Nitter list/user RSS feeds are ingested as their own source type, enriched via fxtwitter (author name/handle/avatar, cleaner text, attached photo) with a graceful RSS-only fallback when that enrichment fails. Tweets always publish directly, one per article, and never enter the LLM clustering/synthesis pipeline — the same bypass already used for YouTube, since merging unrelated tweets together makes no sense. Rendering: a new distinct embed-card component (avatar, name + @handle, full untruncated text, optional attached image, published-date-only timestamp, no like/retweet stats) replaces the plain article row wherever a tweet appears, on both the category-page list and the article detail page. Verified end-to-end against the real sample Nitter RSS feed (served locally): ingestion (all 100 items, tweet metadata correctly extracted, retweet/quote-tweet blockquotes correctly excluded from own-content text), publishing (bypasses clustering, tweet field threaded through to the published article), and rendering (embed card appears on the homepage feed and the article detail page, no duplicate title). Known follow-up: fxtwitter's JSON field names are based on public documentation, not a verified live response (that API is unreachable from this sandbox) — worth a real curl check before relying on the enrichment path in production; the RSS-only fallback path is what's actually been exercised here.
116 lines
3.8 KiB
TypeScript
116 lines
3.8 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { db } from './index.js';
|
|
import type { ContentItem } from './types.js';
|
|
|
|
function rowToItem(row: any): ContentItem {
|
|
return {
|
|
id: row.id,
|
|
sourceId: row.source_id,
|
|
type: row.type,
|
|
title: row.title,
|
|
summary: row.summary,
|
|
body: row.body,
|
|
images: JSON.parse(row.images),
|
|
videos: JSON.parse(row.videos),
|
|
link: row.link,
|
|
publishedAt: row.published_at,
|
|
fetchedAt: row.fetched_at,
|
|
tags: JSON.parse(row.tags),
|
|
geo: row.geo,
|
|
embedding: row.embedding ? JSON.parse(row.embedding) : null,
|
|
eventId: row.event_id,
|
|
clusterId: row.cluster_id,
|
|
tweet: row.tweet ? JSON.parse(row.tweet) : null,
|
|
raw: row.raw ? JSON.parse(row.raw) : null
|
|
};
|
|
}
|
|
|
|
export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
|
|
const id = `ci-${randomUUID()}`;
|
|
db.prepare(
|
|
`INSERT INTO content_items
|
|
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
).run(
|
|
id,
|
|
item.sourceId,
|
|
item.type,
|
|
item.title,
|
|
item.summary,
|
|
item.body,
|
|
JSON.stringify(item.images),
|
|
JSON.stringify(item.videos),
|
|
item.link,
|
|
item.publishedAt,
|
|
item.fetchedAt,
|
|
JSON.stringify(item.tags),
|
|
item.geo,
|
|
item.embedding ? JSON.stringify(item.embedding) : null,
|
|
item.eventId,
|
|
item.clusterId,
|
|
item.tweet ? JSON.stringify(item.tweet) : null,
|
|
item.raw ? JSON.stringify(item.raw) : null
|
|
);
|
|
return { ...item, id };
|
|
}
|
|
|
|
/** Avoids re-ingesting the same story twice on repeated polls. */
|
|
export function existsByLink(link: string): boolean {
|
|
const row = db.prepare('SELECT id FROM content_items WHERE link = ?').get(link);
|
|
return !!row;
|
|
}
|
|
|
|
export function unclusteredItemsExcludingSources(excludeSourceIds: string[]): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id IS NULL').all();
|
|
const items = rows.map(rowToItem);
|
|
if (excludeSourceIds.length === 0) return items;
|
|
return items.filter((i) => !excludeSourceIds.includes(i.sourceId));
|
|
}
|
|
|
|
export function unclusteredItemsForSources(sourceIds: string[], sinceISO: string): ContentItem[] {
|
|
if (sourceIds.length === 0) return [];
|
|
const placeholders = sourceIds.map(() => '?').join(',');
|
|
const rows = db
|
|
.prepare(`SELECT * FROM content_items WHERE cluster_id IS NULL AND source_id IN (${placeholders}) AND fetched_at > ?`)
|
|
.all(...sourceIds, sinceISO);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function setEmbedding(id: string, embedding: number[]) {
|
|
db.prepare('UPDATE content_items SET embedding = ? WHERE id = ?').run(JSON.stringify(embedding), id);
|
|
}
|
|
|
|
export function assignCluster(ids: string[], clusterId: string) {
|
|
const stmt = db.prepare('UPDATE content_items SET cluster_id = ? WHERE id = ?');
|
|
for (const id of ids) stmt.run(clusterId, id);
|
|
}
|
|
|
|
export function itemsByCluster(clusterId: string): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id = ?').all(clusterId);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function itemsOlderThan(days: number): ContentItem[] {
|
|
const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE fetched_at < ?').all(cutoff);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function deleteContentItems(ids: string[]) {
|
|
const stmt = db.prepare('DELETE FROM content_items WHERE id = ?');
|
|
for (const id of ids) stmt.run(id);
|
|
}
|
|
|
|
export function itemsForSource(sourceId: string): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE source_id = ?').all(sourceId);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function deleteContentItemsForSource(sourceId: string) {
|
|
db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId);
|
|
}
|
|
|
|
export function deleteAllContentItems() {
|
|
db.prepare('DELETE FROM content_items').run();
|
|
}
|