Claude
e204c70e00
- Fix "Body cannot be empty" error on DELETE by making the JSON content-type parser tolerate empty bodies, and by only sending Content-Type from the frontend when a request actually has one. - Deleting a source now cascades: raw content items and any article composed entirely from that source are removed too, plus their media. - Add admin endpoints/UI to clear all articles, all media, or a single source's content without deleting the source, so things can be repopulated fresh. - Sources can now be assigned multiple categories via checkboxes (instead of free text) and edited in place, not just added/deleted. - Add a "News" default category (seeded fresh, backfilled on existing DBs) so general news sources have a real home instead of the pseudo-category "Top stories", which is just the homepage's all-categories chronological view. - Widen the site's content column 15% (1080px -> 1242px). - Add YouTube as its own source type/ingestion module: pulls a channel's public Atom feed, and each video always publishes directly as its own article (title, embedded video, publish date, description) rather than going through the cross-source clustering/synthesis pipeline.
114 lines
3.7 KiB
TypeScript
114 lines
3.7 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { db } from './index.js';
|
|
import type { ContentItem } from './types.js';
|
|
|
|
function rowToItem(row: any): ContentItem {
|
|
return {
|
|
id: row.id,
|
|
sourceId: row.source_id,
|
|
type: row.type,
|
|
title: row.title,
|
|
summary: row.summary,
|
|
body: row.body,
|
|
images: JSON.parse(row.images),
|
|
videos: JSON.parse(row.videos),
|
|
link: row.link,
|
|
publishedAt: row.published_at,
|
|
fetchedAt: row.fetched_at,
|
|
tags: JSON.parse(row.tags),
|
|
geo: row.geo,
|
|
embedding: row.embedding ? JSON.parse(row.embedding) : null,
|
|
eventId: row.event_id,
|
|
clusterId: row.cluster_id,
|
|
raw: row.raw ? JSON.parse(row.raw) : null
|
|
};
|
|
}
|
|
|
|
export function insertContentItem(item: Omit<ContentItem, 'id'>): ContentItem {
|
|
const id = `ci-${randomUUID()}`;
|
|
db.prepare(
|
|
`INSERT INTO content_items
|
|
(id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, raw)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
).run(
|
|
id,
|
|
item.sourceId,
|
|
item.type,
|
|
item.title,
|
|
item.summary,
|
|
item.body,
|
|
JSON.stringify(item.images),
|
|
JSON.stringify(item.videos),
|
|
item.link,
|
|
item.publishedAt,
|
|
item.fetchedAt,
|
|
JSON.stringify(item.tags),
|
|
item.geo,
|
|
item.embedding ? JSON.stringify(item.embedding) : null,
|
|
item.eventId,
|
|
item.clusterId,
|
|
item.raw ? JSON.stringify(item.raw) : null
|
|
);
|
|
return { ...item, id };
|
|
}
|
|
|
|
/** Avoids re-ingesting the same story twice on repeated polls. */
|
|
export function existsByLink(link: string): boolean {
|
|
const row = db.prepare('SELECT id FROM content_items WHERE link = ?').get(link);
|
|
return !!row;
|
|
}
|
|
|
|
export function unclusteredItemsExcludingSources(excludeSourceIds: string[]): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id IS NULL').all();
|
|
const items = rows.map(rowToItem);
|
|
if (excludeSourceIds.length === 0) return items;
|
|
return items.filter((i) => !excludeSourceIds.includes(i.sourceId));
|
|
}
|
|
|
|
export function unclusteredItemsForSources(sourceIds: string[], sinceISO: string): ContentItem[] {
|
|
if (sourceIds.length === 0) return [];
|
|
const placeholders = sourceIds.map(() => '?').join(',');
|
|
const rows = db
|
|
.prepare(`SELECT * FROM content_items WHERE cluster_id IS NULL AND source_id IN (${placeholders}) AND fetched_at > ?`)
|
|
.all(...sourceIds, sinceISO);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function setEmbedding(id: string, embedding: number[]) {
|
|
db.prepare('UPDATE content_items SET embedding = ? WHERE id = ?').run(JSON.stringify(embedding), id);
|
|
}
|
|
|
|
export function assignCluster(ids: string[], clusterId: string) {
|
|
const stmt = db.prepare('UPDATE content_items SET cluster_id = ? WHERE id = ?');
|
|
for (const id of ids) stmt.run(clusterId, id);
|
|
}
|
|
|
|
export function itemsByCluster(clusterId: string): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id = ?').all(clusterId);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function itemsOlderThan(days: number): ContentItem[] {
|
|
const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE fetched_at < ?').all(cutoff);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function deleteContentItems(ids: string[]) {
|
|
const stmt = db.prepare('DELETE FROM content_items WHERE id = ?');
|
|
for (const id of ids) stmt.run(id);
|
|
}
|
|
|
|
export function itemsForSource(sourceId: string): ContentItem[] {
|
|
const rows = db.prepare('SELECT * FROM content_items WHERE source_id = ?').all(sourceId);
|
|
return rows.map(rowToItem);
|
|
}
|
|
|
|
export function deleteContentItemsForSource(sourceId: string) {
|
|
db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId);
|
|
}
|
|
|
|
export function deleteAllContentItems() {
|
|
db.prepare('DELETE FROM content_items').run();
|
|
}
|