Fix source deletion, add content clearing, multi-category/editable sources, News category, wider layout, and a YouTube source module

- Fix "Body cannot be empty" error on DELETE by making the JSON content-type
  parser tolerate empty bodies, and by only sending Content-Type from the
  frontend when a request actually has one.
- Deleting a source now cascades: raw content items and any article composed
  entirely from that source are removed too, plus their media.
- Add admin endpoints/UI to clear all articles, all media, or a single
  source's content without deleting the source, so things can be repopulated
  fresh.
- Sources can now be assigned multiple categories via checkboxes (instead of
  free text) and edited in place, not just added/deleted.
- Add a "News" default category (seeded fresh, backfilled on existing DBs) so
  general news sources have a real home instead of the pseudo-category "Top
  stories", which is just the homepage's all-categories chronological view.
- Widen the site's content column 15% (1080px -> 1242px).
- Add YouTube as its own source type/ingestion module: pulls a channel's
  public Atom feed, and each video always publishes directly as its own
  article (title, embedded video, publish date, description) rather than
  going through the cross-source clustering/synthesis pipeline.

backend/src/api/admin.ts

@@ -3,6 +3,7 @@ import * as settingsDb from '../storage/db/settings.js';
 import * as sourcesDb from '../storage/db/sources.js';
 import * as eventsDb from '../storage/db/events.js';
 import * as categoriesDb from '../storage/db/categories.js';
+import { clearSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
 import { OllamaProvider } from '../inference/ollama-provider.js';
 import { pollSourceNow } from '../ingestion/poller.js';
 import { logger, listLogs } from '../storage/db/logs.js';
@@ -58,10 +59,31 @@ export async function registerAdminRoutes(app: FastifyInstance) {
 
 	app.delete('/api/admin/sources/:id', async (req, reply) => {
 		const { id } = req.params as { id: string };
+		// Deleting a source deletes its raw content and any article composed entirely
+		// from it too — otherwise stale articles from a source the admin just removed
+		// keep showing up on the site pointing at nothing.
+		clearSourceContent(id);
 		sourcesDb.deleteSource(id);
 		return reply.code(204).send();
 	});
 
+	// --- Content clearing (re-populate a source, or the whole site, from scratch) ---
+	app.delete('/api/admin/content/sources/:id', async (req, reply) => {
+		const { id } = req.params as { id: string };
+		const result = clearSourceContent(id);
+		return reply.code(200).send(result);
+	});
+
+	app.delete('/api/admin/content/articles', async (_req, reply) => {
+		const deleted = clearAllArticles();
+		return reply.code(200).send({ deleted });
+	});
+
+	app.delete('/api/admin/content/media', async (_req, reply) => {
+		const deleted = clearAllMedia();
+		return reply.code(200).send({ deleted });
+	});
+
 	// Manual "poll now" — the refresh icon on each source in the admin panel.
 	app.post('/api/admin/sources/:id/poll', async (req, reply) => {
 		const { id } = req.params as { id: string };

backend/src/index.ts

@@ -37,6 +37,20 @@ async function main() {
 	});
 	await app.register(cookie);
 
+	// Overrides Fastify's default JSON body parser, which throws "Body cannot be empty
+	// when content-type is set to 'application/json'" for any bodyless request (DELETE,
+	// or POST with no payload) that still carries a Content-Type header — exactly what
+	// browsers' fetch() does when a client sets that header unconditionally. An empty
+	// body is just as valid as `{}` for routes that don't read req.body at all.
+	app.addContentTypeParser('application/json', { parseAs: 'string' }, (_req, body, done) => {
+		if (typeof body !== 'string' || body.trim() === '') return done(null, {});
+		try {
+			done(null, JSON.parse(body));
+		} catch (err) {
+			done(err as Error, undefined);
+		}
+	});
+
 	await registerAuth(app);
 	await registerPublicRoutes(app);
 	await registerAdminRoutes(app);

backend/src/ingestion/adapters/youtube.ts

@@ -0,0 +1,90 @@
+// YouTube is its own ingestion module, deliberately separate from the RSS/Telegram
+// adapters: a channel's public Atom feed (no API key needed) is fetched directly, and
+// each entry becomes its own single-video article — see pipeline/publish.ts and
+// queue/priorityQueue.ts, which route youtube-sourced items straight to publishDirect
+// rather than through the LLM clustering/synthesis pipeline. Merging two unrelated
+// videos into one AI-rewritten story would make no sense the way merging two outlets'
+// coverage of the same news event does.
+
+import Parser from 'rss-parser';
+import type { Source } from '../../storage/db/types.js';
+import type { SourceAdapter, FetchedItem } from './base.js';
+import { logger } from '../../storage/db/logs.js';
+
+type YoutubeEntry = Parser.Item & {
+	'media:group'?: {
+		'media:description'?: string[];
+		'media:thumbnail'?: { $?: { url?: string } }[];
+	}[];
+};
+
+const parser = new Parser<Record<string, unknown>, YoutubeEntry>({
+	customFields: {
+		item: [['media:group', 'media:group']]
+	}
+});
+
+/** Builds the channel's Atom feed URL — YouTube publishes these publicly with no API key required. */
+function feedUrl(source: Source): string | null {
+	if (source.url) return source.url;
+	const channelId = source.config?.channelId as string | undefined;
+	if (channelId) return `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
+	const playlistId = source.config?.playlistId as string | undefined;
+	if (playlistId) return `https://www.youtube.com/feeds/videos.xml?playlist_id=${encodeURIComponent(playlistId)}`;
+	return null;
+}
+
+function extractVideoId(url: string): string | null {
+	const match =
+		url.match(/[?&]v=([\w-]{6,})/) || url.match(/youtu\.be\/([\w-]{6,})/) || url.match(/\/embed\/([\w-]{6,})/);
+	return match ? match[1] : null;
+}
+
+function extractDescription(item: YoutubeEntry): string {
+	const group = item['media:group']?.[0];
+	return group?.['media:description']?.[0] ?? '';
+}
+
+function extractThumbnail(item: YoutubeEntry): string | null {
+	const group = item['media:group']?.[0];
+	return group?.['media:thumbnail']?.[0]?.$?.url ?? null;
+}
+
+export const youtubeAdapter: SourceAdapter = {
+	async fetch(source: Source): Promise<FetchedItem[]> {
+		const url = feedUrl(source);
+		if (!url) {
+			logger.warn('youtube', `Source "${source.name}" has no url, channelId, or playlistId configured — skipping`);
+			return [];
+		}
+
+		const feed = await parser.parseURL(url);
+		const items: FetchedItem[] = [];
+
+		for (const item of feed.items) {
+			if (!item.link || !item.title) continue;
+			const videoId = extractVideoId(item.link);
+			const description = extractDescription(item);
+			const thumbnail = extractThumbnail(item);
+
+			items.push({
+				title: item.title,
+				summary: description.slice(0, 500),
+				body: description || null,
+				images: thumbnail ? [{ url: thumbnail }] : [],
+				videos: [
+					{
+						url: item.link,
+						provider: 'youtube',
+						embedHtml: videoId ? `https://www.youtube.com/embed/${videoId}` : undefined
+					}
+				],
+				link: item.link,
+				publishedAt: item.isoDate ?? item.pubDate ?? new Date().toISOString(),
+				raw: item
+			});
+		}
+
+		return items;
+	}
+};

backend/src/ingestion/poller.ts

@@ -4,6 +4,7 @@ import { logger } from '../storage/db/logs.js';
 import { rssAdapter } from './adapters/rss.js';
 import { telegramAdapter } from './adapters/telegram.js';
 import { apiAdapter } from './adapters/api.js';
+import { youtubeAdapter } from './adapters/youtube.js';
 import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
 import { fetchFullArticle } from './articleFetcher.js';
 import type { Source } from '../storage/db/types.js';
@@ -12,6 +13,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
 	rss: rssAdapter,
 	telegram: telegramAdapter,
 	api: apiAdapter,
+	youtube: youtubeAdapter,
 	custom: apiAdapter
 };
 

backend/src/pipeline/publish.ts

@@ -77,7 +77,9 @@ async function resolveHeroImage(
 export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
 	const category = uniqueCategories([item]);
 	const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
-	const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, sourceItemId: item.id } : null;
+	const video = item.videos[0]
+		? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
+		: null;
 
 	const article = await articles.insertArticle({
 		title: item.title,
@@ -140,7 +142,12 @@ export async function publishCluster(
 	const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? '');
 	const videoItem = items.find((i) => i.videos.length > 0);
 	const video = videoItem
-		? { url: videoItem.videos[0].url, provider: videoItem.videos[0].provider, sourceItemId: videoItem.id }
+		? {
+				url: videoItem.videos[0].url,
+				provider: videoItem.videos[0].provider,
+				embedUrl: videoItem.videos[0].embedHtml,
+				sourceItemId: videoItem.id
+			}
 		: null;
 
 	const category = uniqueCategories(items);

backend/src/queue/priorityQueue.ts

@@ -9,6 +9,13 @@ import { publishCluster, publishDirect } from '../pipeline/publish.js';
 import { logger } from '../storage/db/logs.js';
 import type { GlobalSettings, ContentItem } from '../storage/db/types.js';
 
+function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
+	const matches: T[] = [];
+	const rest: T[] = [];
+	for (const item of items) (predicate(item) ? matches : rest).push(item);
+	return [matches, rest];
+}
+
 function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>): number {
 	const source = sourcesDb.getSource(item.sourceId);
 	const cats = source?.category ?? [];
@@ -70,10 +77,28 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
 	const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
 	if (items.length === 0) return 0;
 
+	// YouTube videos never get LLM-merged with anything — each is always its own
+	// article (title/video/date/description), same shape whether the AI service is up
+	// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
+	const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
+	const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
+
+	let publishedDirect = 0;
+	for (const item of youtubeItems) {
+		try {
+			const article = await publishDirect(item);
+			contentItemsDb.assignCluster([item.id], article.id);
+			publishedDirect++;
+			logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
+		} catch (err) {
+			logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
+		}
+	}
+
 	const categories = categoriesDb.listCategories();
 	const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
 
-	const ranked = items
+	const ranked = mergeableItems
 		.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) }))
 		.sort((a, b) => a.rank - b.rank)
 		.map((r) => r.item);
@@ -119,5 +144,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
 		);
 	}
 
-	return published;
+	return published + publishedDirect;
 }

backend/src/storage/contentCascade.ts

@@ -0,0 +1,61 @@
+// Deleting or "clearing" a source shouldn't leave orphaned merged articles behind
+// pointing at raw items that no longer exist — this is the one place that coordinates
+// content_items, merged_articles, and media_assets together, since none of those three
+// tables have a single FK chain connecting them all (see storage/db/index.ts schema
+// comments: sources are copied into merged_articles.sources at publish time, not
+// referenced live).
+
+import * as contentItemsDb from './db/contentItems.js';
+import * as articlesDb from './db/articles.js';
+import { deleteMediaByArticleId, deleteMediaByContentItemIds, deleteAllMedia } from './media/index.js';
+import { logger } from './db/logs.js';
+
+export interface ClearResult {
+	itemsDeleted: number;
+	articlesDeleted: number;
+}
+
+/**
+ * Removes every raw content item ingested from a source, plus any merged article that
+ * was composed entirely from that source's items (so it doesn't linger on the site
+ * pointing at deleted raw data). Articles that merged this source's coverage together
+ * with other sources' are left alone — stripping just this source's contribution back
+ * out of an already-published multi-source article isn't something the merge pipeline
+ * supports undoing.
+ */
+export function clearSourceContent(sourceId: string): ClearResult {
+	const items = contentItemsDb.itemsForSource(sourceId);
+	const itemIds = new Set(items.map((i) => i.id));
+
+	let articlesDeleted = 0;
+	if (itemIds.size > 0) {
+		for (const article of articlesDb.allArticlesNewestFirst()) {
+			if (article.sources.length > 0 && article.sources.every((s) => itemIds.has(s.itemId))) {
+				deleteMediaByArticleId(article.id);
+				articlesDb.deleteArticle(article.id);
+				articlesDeleted++;
+			}
+		}
+		deleteMediaByContentItemIds([...itemIds]);
+	}
+
+	contentItemsDb.deleteContentItemsForSource(sourceId);
+	logger.info('admin', `Cleared content for source ${sourceId}: ${itemIds.size} item(s), ${articlesDeleted} article(s)`);
+	return { itemsDeleted: itemIds.size, articlesDeleted };
+}
+
+/** Wipes every published article and its media, keeping raw ingested items intact so they can be re-synthesized fresh. */
+export function clearAllArticles(): number {
+	const articles = articlesDb.allArticlesNewestFirst();
+	for (const article of articles) deleteMediaByArticleId(article.id);
+	articlesDb.deleteAllArticles();
+	logger.info('admin', `Cleared all articles: ${articles.length} removed`);
+	return articles.length;
+}
+
+/** Wipes every locally-stored media file (both candidate and published tiers). */
+export function clearAllMedia(): number {
+	const count = deleteAllMedia();
+	logger.info('admin', `Cleared all media: ${count} file(s) removed`);
+	return count;
+}

backend/src/storage/db/articles.ts

@@ -131,3 +131,7 @@ export function findRecentArticleByTags(tagIds: string[], sinceDays: number): Me
 export function deleteArticle(id: string) {
 	db.prepare('DELETE FROM merged_articles WHERE id = ?').run(id);
 }
+
+export function deleteAllArticles() {
+	db.prepare('DELETE FROM merged_articles').run();
+}

backend/src/storage/db/contentItems.ts

@@ -98,3 +98,16 @@ export function deleteContentItems(ids: string[]) {
 	const stmt = db.prepare('DELETE FROM content_items WHERE id = ?');
 	for (const id of ids) stmt.run(id);
 }
+
+export function itemsForSource(sourceId: string): ContentItem[] {
+	const rows = db.prepare('SELECT * FROM content_items WHERE source_id = ?').all(sourceId);
+	return rows.map(rowToItem);
+}
+
+export function deleteContentItemsForSource(sourceId: string) {
+	db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId);
+}
+
+export function deleteAllContentItems() {
+	db.prepare('DELETE FROM content_items').run();
+}

backend/src/storage/db/index.ts

@@ -182,15 +182,29 @@ export function migrate() {
 		).run();
 	}
 
-	// Seed default categories if none exist yet.
+	// Seed default categories if none exist yet. "News" sits right under "Top stories" —
+	// general news sources belong here, not on "Top stories" itself, which isn't a real
+	// filterable tag: it's the homepage's all-categories-chronological view (see
+	// +layout.svelte's nav mapping and /api/feed's no-category-filter default).
 	const catCount = db.prepare('SELECT COUNT(*) as c FROM categories').get() as { c: number };
 	if (catCount.c === 0) {
-		const defaults = ['Top stories', 'Local', 'World', 'Business', 'Tech', 'Culture'];
+		const defaults = ['Top stories', 'News', 'Local', 'World', 'Business', 'Tech', 'Culture'];
 		const stmt = db.prepare(
 			'INSERT INTO categories (id, name, priority_rank, is_default) VALUES (?, ?, ?, 1)'
 		);
 		defaults.forEach((name, i) => {
 			stmt.run(`cat-${name.toLowerCase().replace(/\s+/g, '-')}`, name, i + 1);
 		});
+	} else {
+		// Backfill for installs seeded before "News" existed.
+		const hasNews = db.prepare("SELECT id FROM categories WHERE lower(name) = 'news'").get();
+		if (!hasNews) {
+			const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number };
+			db.prepare('INSERT INTO categories (id, name, priority_rank, is_default) VALUES (?, ?, ?, 1)').run(
+				'cat-news',
+				'News',
+				maxRank.m + 1
+			);
+		}
 	}
 }

backend/src/storage/db/types.ts

@@ -1,7 +1,7 @@
 export interface Source {
 	id: string;
 	name: string;
-	type: 'rss' | 'api' | 'telegram' | 'custom';
+	type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
 	category: string[];
 	url: string | null;
 	config: Record<string, unknown>;
@@ -44,7 +44,7 @@ export interface MergedArticle {
 	title: string;
 	body: string;
 	heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
-	video: { url: string; provider?: string; sourceItemId: string } | null;
+	video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
 	category: string[];
 	geo: string | null;
 	eventId: string | null;

backend/src/storage/media/index.ts

@@ -70,6 +70,39 @@ export function deleteCandidateMediaOlderThan(days: number): number {
 	return rows.length;
 }
 
+function deleteRows(rows: { id: string; local_path: string }[]) {
+	for (const row of rows) fs.rm(row.local_path, () => {});
+	return rows.length;
+}
+
+export function deleteMediaByArticleId(articleId: string): number {
+	const rows = db.prepare('SELECT id, local_path FROM media_assets WHERE article_id = ?').all(articleId) as {
+		id: string;
+		local_path: string;
+	}[];
+	const count = deleteRows(rows);
+	db.prepare('DELETE FROM media_assets WHERE article_id = ?').run(articleId);
+	return count;
+}
+
+export function deleteMediaByContentItemIds(contentItemIds: string[]): number {
+	if (contentItemIds.length === 0) return 0;
+	const placeholders = contentItemIds.map(() => '?').join(',');
+	const rows = db
+		.prepare(`SELECT id, local_path FROM media_assets WHERE content_item_id IN (${placeholders})`)
+		.all(...contentItemIds) as { id: string; local_path: string }[];
+	const count = deleteRows(rows);
+	db.prepare(`DELETE FROM media_assets WHERE content_item_id IN (${placeholders})`).run(...contentItemIds);
+	return count;
+}
+
+export function deleteAllMedia(): number {
+	const rows = db.prepare('SELECT id, local_path FROM media_assets').all() as { id: string; local_path: string }[];
+	const count = deleteRows(rows);
+	db.prepare('DELETE FROM media_assets').run();
+	return count;
+}
+
 function guessExtension(contentType: string, url: string): string {
 	if (contentType.includes('jpeg')) return '.jpg';
 	if (contentType.includes('png')) return '.png';