Merge pull request #3 from Salastil/claude/source-management-content-1w2yno

Add a per-source "Push to Top Stories?" opt-in, off by default

backend/src/pipeline/publish.ts

@@ -21,6 +21,11 @@ function uniqueCategories(items: ContentItem[]): string[] {
 	return [...cats];
 }
 
+/** An article shows on the homepage if any of its contributing sources opted into "Push to Top Stories?". */
+function anyPushesToTopStories(items: ContentItem[]): boolean {
+	return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false);
+}
+
 /** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */
 function deriveTitle(body: string): string {
 	const firstLine = body.split('\n')[0];
@@ -106,7 +111,8 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
 		tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement
 		threadId: randomUUID(),
 		previousArticleId: null,
-		nextArticleId: null
+		nextArticleId: null,
+		topStories: anyPushesToTopStories([item])
 	});
 
 	if (storedMediaId) promoteToPublished(storedMediaId, article.id);
@@ -197,7 +203,8 @@ export async function publishCluster(
 		tags: tagIds,
 		threadId,
 		previousArticleId,
-		nextArticleId: null
+		nextArticleId: null,
+		topStories: anyPushesToTopStories(items)
 	});
 
 	if (storedMediaId) {

backend/src/storage/db/articles.ts

@@ -20,7 +20,8 @@ function rowToArticle(row: any): MergedArticle {
 		tags: JSON.parse(row.tags),
 		threadId: row.thread_id,
 		previousArticleId: row.previous_article_id,
-		nextArticleId: row.next_article_id
+		nextArticleId: row.next_article_id,
+		topStories: !!row.top_stories
 	};
 }
 
@@ -28,8 +29,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
 	const id = `art-${randomUUID()}`;
 	db.prepare(
 		`INSERT INTO merged_articles
-		 (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id)
+		 (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories)
-		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	).run(
 		id,
 		article.title,
@@ -47,7 +48,8 @@ export function insertArticle(article: Omit<MergedArticle, 'id'>): MergedArticle
 		JSON.stringify(article.tags),
 		article.threadId,
 		article.previousArticleId,
-		article.nextArticleId
+		article.nextArticleId,
+		article.topStories ? 1 : 0
 	);
 	if (article.previousArticleId) {
 		db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId);
@@ -76,6 +78,17 @@ export function queryFeed(filters: {
 }): MergedArticle[] {
 	let sql = 'SELECT * FROM merged_articles WHERE 1=1';
 	const params: unknown[] = [];
+
+	// The bare feed (no category/geo/eventId/tag — i.e. the homepage/"Top stories") only
+	// shows articles whose contributing source(s) opted into "Push to Top Stories?" —
+	// otherwise every ingested article from every source would flood the homepage.
+	// Any explicit filter (a real category page, Local's geo filter, a tag or event page)
+	// is unaffected — those show everything matching, regardless of this flag.
+	const isHomepage = !filters.category && !filters.geo && !filters.eventId && !filters.tag;
+	if (isHomepage) {
+		sql += ' AND top_stories = 1';
+	}
+
 	if (filters.category) {
 		sql += ' AND category LIKE ?';
 		params.push(`%"${filters.category}"%`);

backend/src/storage/db/index.ts

@@ -28,6 +28,7 @@ export function migrate() {
 			config TEXT NOT NULL DEFAULT '{}', -- JSON: apiKey, telegramChannelId, authHeaders
 			poll_interval_minutes INTEGER NOT NULL DEFAULT 15,
 			enabled INTEGER NOT NULL DEFAULT 1,
+			push_to_top_stories INTEGER NOT NULL DEFAULT 0, -- opt-in: keeps the homepage from being flooded by every ingested source
 			last_polled_at TEXT,
 			last_error TEXT,
 			created_at TEXT NOT NULL
@@ -73,7 +74,8 @@ export function migrate() {
 			tags TEXT NOT NULL DEFAULT '[]', -- JSON tag ids
 			thread_id TEXT NOT NULL,
 			previous_article_id TEXT,
-			next_article_id TEXT
+			next_article_id TEXT,
+			top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?"
 		);
 		CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at);
 		CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id);
@@ -182,10 +184,22 @@ export function migrate() {
 		).run();
 	}
 
+	// Backfill new columns for installs seeded before they existed — node:sqlite's
+	// CREATE TABLE IF NOT EXISTS doesn't add columns to an already-existing table.
+	const hasColumn = (table: string, column: string) =>
+		(db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[]).some((c) => c.name === column);
+	if (!hasColumn('sources', 'push_to_top_stories')) {
+		db.exec('ALTER TABLE sources ADD COLUMN push_to_top_stories INTEGER NOT NULL DEFAULT 0');
+	}
+	if (!hasColumn('merged_articles', 'top_stories')) {
+		db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0');
+	}
+
 	// Seed default categories if none exist yet. "News" sits right under "Top stories" —
 	// general news sources belong here, not on "Top stories" itself, which isn't a real
-	// filterable tag: it's the homepage's all-categories-chronological view (see
+	// filterable tag: it's the homepage view, now scoped to only the articles whose
-	// +layout.svelte's nav mapping and /api/feed's no-category-filter default).
+	// sources opted into "Push to Top Stories?" (see sources.push_to_top_stories and
+	// articles.queryFeed's isHomepage gate) rather than every ingested article.
 	const catCount = db.prepare('SELECT COUNT(*) as c FROM categories').get() as { c: number };
 	if (catCount.c === 0) {
 		const defaults = ['Top stories', 'News', 'Local', 'World', 'Business', 'Tech', 'Culture'];

backend/src/storage/db/sources.ts

@@ -12,6 +12,7 @@ function rowToSource(row: any): Source {
 		config: JSON.parse(row.config),
 		pollIntervalMinutes: row.poll_interval_minutes,
 		enabled: !!row.enabled,
+		pushToTopStories: !!row.push_to_top_stories,
 		lastPolledAt: row.last_polled_at,
 		lastError: row.last_error,
 		createdAt: row.created_at
@@ -37,8 +38,8 @@ export function createSource(input: Partial<Source>): Source {
 	const id = `src-${randomUUID()}`;
 	const now = new Date().toISOString();
 	db.prepare(
-		`INSERT INTO sources (id, name, type, category, url, config, poll_interval_minutes, enabled, last_polled_at, last_error, created_at)
+		`INSERT INTO sources (id, name, type, category, url, config, poll_interval_minutes, enabled, push_to_top_stories, last_polled_at, last_error, created_at)
-		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?)`
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?)`
 	).run(
 		id,
 		input.name ?? 'Untitled source',
@@ -48,6 +49,7 @@ export function createSource(input: Partial<Source>): Source {
 		JSON.stringify(input.config ?? {}),
 		input.pollIntervalMinutes ?? 15,
 		input.enabled === false ? 0 : 1,
+		input.pushToTopStories ? 1 : 0,
 		now
 	);
 	return getSource(id)!;
@@ -58,7 +60,7 @@ export function updateSource(id: string, patch: Partial<Source>): Source | null
 	if (!existing) return null;
 	const merged = { ...existing, ...patch };
 	db.prepare(
-		`UPDATE sources SET name=?, type=?, category=?, url=?, config=?, poll_interval_minutes=?, enabled=?, last_polled_at=?, last_error=? WHERE id=?`
+		`UPDATE sources SET name=?, type=?, category=?, url=?, config=?, poll_interval_minutes=?, enabled=?, push_to_top_stories=?, last_polled_at=?, last_error=? WHERE id=?`
 	).run(
 		merged.name,
 		merged.type,
@@ -67,6 +69,7 @@ export function updateSource(id: string, patch: Partial<Source>): Source | null
 		JSON.stringify(merged.config),
 		merged.pollIntervalMinutes,
 		merged.enabled ? 1 : 0,
+		merged.pushToTopStories ? 1 : 0,
 		merged.lastPolledAt,
 		merged.lastError,
 		id

backend/src/storage/db/types.ts

@@ -7,6 +7,8 @@ export interface Source {
 	config: Record<string, unknown>;
 	pollIntervalMinutes: number;
 	enabled: boolean;
+	/** Opt-in — default false, so the homepage ("Top stories") isn't flooded by every ingested source. */
+	pushToTopStories: boolean;
 	lastPolledAt: string | null;
 	lastError: string | null;
 	createdAt: string;
@@ -57,6 +59,8 @@ export interface MergedArticle {
 	threadId: string;
 	previousArticleId: string | null;
 	nextArticleId: string | null;
+	/** True if any contributing source opted into "Push to Top Stories?" — gates the homepage feed, see articles.queryFeed. */
+	topStories: boolean;
 }
 
 export interface Tag {

frontend/src/lib/adminTypes.ts

@@ -38,6 +38,7 @@ export interface AdminSource {
 	config?: Record<string, unknown>;
 	pollIntervalMinutes: number;
 	enabled: boolean;
+	pushToTopStories: boolean;
 	lastPolledAt: string | null;
 	lastError: string | null;
 }

frontend/src/lib/components/admin/SourcesTab.svelte

@@ -24,7 +24,8 @@
 			url: '',
 			channelId: '',
 			categorySet: new Set<string>(),
-			pollIntervalMinutes: 15
+			pollIntervalMinutes: 15,
+			pushToTopStories: false
 		};
 	}
 
@@ -43,7 +44,8 @@
 			url: source.type === 'youtube' ? '' : source.url,
 			channelId: source.type === 'youtube' ? (source.url || (source.config?.channelId as string) || '') : '',
 			categorySet: new Set(source.category),
-			pollIntervalMinutes: source.pollIntervalMinutes
+			pollIntervalMinutes: source.pollIntervalMinutes,
+			pushToTopStories: source.pushToTopStories
 		};
 		editingId = source.id;
 		showAdd = true;
@@ -69,7 +71,8 @@
 				type: form.type,
 				url: form.channelId,
 				category,
-				pollIntervalMinutes: form.pollIntervalMinutes
+				pollIntervalMinutes: form.pollIntervalMinutes,
+				pushToTopStories: form.pushToTopStories
 			};
 		}
 		return {
@@ -77,7 +80,8 @@
 			type: form.type,
 			url: form.url,
 			category,
-			pollIntervalMinutes: form.pollIntervalMinutes
+			pollIntervalMinutes: form.pollIntervalMinutes,
+			pushToTopStories: form.pushToTopStories
 		};
 	}
 
@@ -119,6 +123,11 @@
 		sources = sources.map((s) => (s.id === source.id ? updated : s));
 	}
 
+	async function toggleTopStories(source: AdminSource) {
+		const updated = await updateSource(source.id, { pushToTopStories: !source.pushToTopStories });
+		sources = sources.map((s) => (s.id === source.id ? updated : s));
+	}
+
 	async function pollNow(source: AdminSource) {
 		pollingId = source.id;
 		justPolled = null;
@@ -176,6 +185,11 @@
 				</label>
 			{/each}
 		</div>
+		<label class="top-stories-check">
+			<input type="checkbox" bind:checked={form.pushToTopStories} />
+			Push to Top Stories?
+			<span class="hint">Off by default — keeps the homepage from being flooded by every source.</span>
+		</label>
 		<div class="add-actions">
 			<button onclick={cancelForm}>Cancel</button>
 			<button class="primary" onclick={handleSubmit}>{editingId ? 'Save' : 'Add'}</button>
@@ -226,6 +240,14 @@
 			<span class="cat">{source.category.join(', ')}</span>
 			<span class="cat">{source.pollIntervalMinutes} min</span>
 			<div class="actions">
+				<button
+					class="icon-btn"
+					class:starred={source.pushToTopStories}
+					onclick={() => toggleTopStories(source)}
+					title={source.pushToTopStories ? 'Pushing to Top Stories — click to stop' : 'Not pushed to Top Stories — click to enable'}
+				>
+					{source.pushToTopStories ? '★' : '☆'}
+				</button>
 				<button class="icon-btn" onclick={() => startEdit(source)} title="Edit">✎</button>
 				<button class="icon-btn" onclick={() => toggleEnabled(source)} title={source.enabled ? 'Disable' : 'Enable'}>
 					{source.enabled ? '⏸' : '▶'}
@@ -292,6 +314,22 @@
 	.category-check input {
 		width: auto;
 	}
+	.top-stories-check {
+		display: flex;
+		align-items: center;
+		flex-wrap: wrap;
+		gap: 6px;
+		font-size: 12px;
+		color: var(--text-secondary);
+		margin-bottom: 12px;
+	}
+	.top-stories-check input {
+		width: auto;
+	}
+	.top-stories-check .hint {
+		font-size: 11px;
+		color: var(--text-muted);
+	}
 	.add-actions {
 		display: flex;
 		gap: 8px;
@@ -308,7 +346,7 @@
 	}
 	.row {
 		display: grid;
-		grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 96px;
+		grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 120px;
 		gap: 10px;
 		padding: 10px;
 		align-items: center;
@@ -386,4 +424,7 @@
 	.icon-btn.danger:hover {
 		color: var(--text-danger);
 	}
+	.icon-btn.starred {
+		color: var(--text-accent);
+	}
 </style>