From 64ef671c5e5447cc7d7c1636510aec1f5025a29b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 19:13:36 +0000 Subject: [PATCH] Add a per-source "Push to Top Stories?" opt-in, off by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ingested article used to show up on the homepage regardless of its source, which meant a handful of high-volume feeds could flood "Top Stories." Sources now default to not appearing there; a source has to explicitly opt in via a new checkbox (also toggleable inline with a star icon) for its articles to show up on the homepage feed. An article shows there if any of its contributing sources opted in — merged/clustered stories aren't held to requiring all sources to agree. Category pages, Local, tags, and events are unaffected; this only gates the bare, no-filter homepage query. Schema: sources.push_to_top_stories and merged_articles.top_stories, both backfilled for existing databases via ALTER TABLE. --- backend/src/pipeline/publish.ts | 11 +++- backend/src/storage/db/articles.ts | 21 ++++++-- backend/src/storage/db/index.ts | 20 ++++++-- backend/src/storage/db/sources.ts | 9 ++-- backend/src/storage/db/types.ts | 4 ++ frontend/src/lib/adminTypes.ts | 1 + .../lib/components/admin/SourcesTab.svelte | 51 +++++++++++++++++-- 7 files changed, 100 insertions(+), 17 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index c2ffd90..0b95f44 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -21,6 +21,11 @@ function uniqueCategories(items: ContentItem[]): string[] { return [...cats]; } +/** An article shows on the homepage if any of its contributing sources opted into "Push to Top Stories?". */ +function anyPushesToTopStories(items: ContentItem[]): boolean { + return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false); +} + /** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */ function deriveTitle(body: string): string { const firstLine = body.split('\n')[0]; @@ -106,7 +111,8 @@ export async function publishDirect(item: ContentItem): Promise { tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement threadId: randomUUID(), previousArticleId: null, - nextArticleId: null + nextArticleId: null, + topStories: anyPushesToTopStories([item]) }); if (storedMediaId) promoteToPublished(storedMediaId, article.id); @@ -197,7 +203,8 @@ export async function publishCluster( tags: tagIds, threadId, previousArticleId, - nextArticleId: null + nextArticleId: null, + topStories: anyPushesToTopStories(items) }); if (storedMediaId) { diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 286ce50..1047542 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -20,7 +20,8 @@ function rowToArticle(row: any): MergedArticle { tags: JSON.parse(row.tags), threadId: row.thread_id, previousArticleId: row.previous_article_id, - nextArticleId: row.next_article_id + nextArticleId: row.next_article_id, + topStories: !!row.top_stories }; } @@ -28,8 +29,8 @@ export function insertArticle(article: Omit): MergedArticle const id = `art-${randomUUID()}`; db.prepare( `INSERT INTO merged_articles - (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, article.title, @@ -47,7 +48,8 @@ export function insertArticle(article: Omit): MergedArticle JSON.stringify(article.tags), article.threadId, article.previousArticleId, - article.nextArticleId + article.nextArticleId, + article.topStories ? 1 : 0 ); if (article.previousArticleId) { db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId); @@ -76,6 +78,17 @@ export function queryFeed(filters: { }): MergedArticle[] { let sql = 'SELECT * FROM merged_articles WHERE 1=1'; const params: unknown[] = []; + + // The bare feed (no category/geo/eventId/tag — i.e. the homepage/"Top stories") only + // shows articles whose contributing source(s) opted into "Push to Top Stories?" — + // otherwise every ingested article from every source would flood the homepage. + // Any explicit filter (a real category page, Local's geo filter, a tag or event page) + // is unaffected — those show everything matching, regardless of this flag. + const isHomepage = !filters.category && !filters.geo && !filters.eventId && !filters.tag; + if (isHomepage) { + sql += ' AND top_stories = 1'; + } + if (filters.category) { sql += ' AND category LIKE ?'; params.push(`%"${filters.category}"%`); diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 8baf1a1..1cb89af 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -28,6 +28,7 @@ export function migrate() { config TEXT NOT NULL DEFAULT '{}', -- JSON: apiKey, telegramChannelId, authHeaders poll_interval_minutes INTEGER NOT NULL DEFAULT 15, enabled INTEGER NOT NULL DEFAULT 1, + push_to_top_stories INTEGER NOT NULL DEFAULT 0, -- opt-in: keeps the homepage from being flooded by every ingested source last_polled_at TEXT, last_error TEXT, created_at TEXT NOT NULL @@ -73,7 +74,8 @@ export function migrate() { tags TEXT NOT NULL DEFAULT '[]', -- JSON tag ids thread_id TEXT NOT NULL, previous_article_id TEXT, - next_article_id TEXT + next_article_id TEXT, + top_stories INTEGER NOT NULL DEFAULT 0 -- true if any contributing source opted into "Push to Top Stories?" ); CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at); CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id); @@ -182,10 +184,22 @@ export function migrate() { ).run(); } + // Backfill new columns for installs seeded before they existed — node:sqlite's + // CREATE TABLE IF NOT EXISTS doesn't add columns to an already-existing table. + const hasColumn = (table: string, column: string) => + (db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[]).some((c) => c.name === column); + if (!hasColumn('sources', 'push_to_top_stories')) { + db.exec('ALTER TABLE sources ADD COLUMN push_to_top_stories INTEGER NOT NULL DEFAULT 0'); + } + if (!hasColumn('merged_articles', 'top_stories')) { + db.exec('ALTER TABLE merged_articles ADD COLUMN top_stories INTEGER NOT NULL DEFAULT 0'); + } + // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real - // filterable tag: it's the homepage's all-categories-chronological view (see - // +layout.svelte's nav mapping and /api/feed's no-category-filter default). + // filterable tag: it's the homepage view, now scoped to only the articles whose + // sources opted into "Push to Top Stories?" (see sources.push_to_top_stories and + // articles.queryFeed's isHomepage gate) rather than every ingested article. const catCount = db.prepare('SELECT COUNT(*) as c FROM categories').get() as { c: number }; if (catCount.c === 0) { const defaults = ['Top stories', 'News', 'Local', 'World', 'Business', 'Tech', 'Culture']; diff --git a/backend/src/storage/db/sources.ts b/backend/src/storage/db/sources.ts index e050428..ea6992e 100644 --- a/backend/src/storage/db/sources.ts +++ b/backend/src/storage/db/sources.ts @@ -12,6 +12,7 @@ function rowToSource(row: any): Source { config: JSON.parse(row.config), pollIntervalMinutes: row.poll_interval_minutes, enabled: !!row.enabled, + pushToTopStories: !!row.push_to_top_stories, lastPolledAt: row.last_polled_at, lastError: row.last_error, createdAt: row.created_at @@ -37,8 +38,8 @@ export function createSource(input: Partial): Source { const id = `src-${randomUUID()}`; const now = new Date().toISOString(); db.prepare( - `INSERT INTO sources (id, name, type, category, url, config, poll_interval_minutes, enabled, last_polled_at, last_error, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?)` + `INSERT INTO sources (id, name, type, category, url, config, poll_interval_minutes, enabled, push_to_top_stories, last_polled_at, last_error, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?)` ).run( id, input.name ?? 'Untitled source', @@ -48,6 +49,7 @@ export function createSource(input: Partial): Source { JSON.stringify(input.config ?? {}), input.pollIntervalMinutes ?? 15, input.enabled === false ? 0 : 1, + input.pushToTopStories ? 1 : 0, now ); return getSource(id)!; @@ -58,7 +60,7 @@ export function updateSource(id: string, patch: Partial): Source | null if (!existing) return null; const merged = { ...existing, ...patch }; db.prepare( - `UPDATE sources SET name=?, type=?, category=?, url=?, config=?, poll_interval_minutes=?, enabled=?, last_polled_at=?, last_error=? WHERE id=?` + `UPDATE sources SET name=?, type=?, category=?, url=?, config=?, poll_interval_minutes=?, enabled=?, push_to_top_stories=?, last_polled_at=?, last_error=? WHERE id=?` ).run( merged.name, merged.type, @@ -67,6 +69,7 @@ export function updateSource(id: string, patch: Partial): Source | null JSON.stringify(merged.config), merged.pollIntervalMinutes, merged.enabled ? 1 : 0, + merged.pushToTopStories ? 1 : 0, merged.lastPolledAt, merged.lastError, id diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 5da4f9f..d62eb3c 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -7,6 +7,8 @@ export interface Source { config: Record; pollIntervalMinutes: number; enabled: boolean; + /** Opt-in — default false, so the homepage ("Top stories") isn't flooded by every ingested source. */ + pushToTopStories: boolean; lastPolledAt: string | null; lastError: string | null; createdAt: string; @@ -57,6 +59,8 @@ export interface MergedArticle { threadId: string; previousArticleId: string | null; nextArticleId: string | null; + /** True if any contributing source opted into "Push to Top Stories?" — gates the homepage feed, see articles.queryFeed. */ + topStories: boolean; } export interface Tag { diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index c367191..9d0b128 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -38,6 +38,7 @@ export interface AdminSource { config?: Record; pollIntervalMinutes: number; enabled: boolean; + pushToTopStories: boolean; lastPolledAt: string | null; lastError: string | null; } diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index 506585d..0726238 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -24,7 +24,8 @@ url: '', channelId: '', categorySet: new Set(), - pollIntervalMinutes: 15 + pollIntervalMinutes: 15, + pushToTopStories: false }; } @@ -43,7 +44,8 @@ url: source.type === 'youtube' ? '' : source.url, channelId: source.type === 'youtube' ? (source.url || (source.config?.channelId as string) || '') : '', categorySet: new Set(source.category), - pollIntervalMinutes: source.pollIntervalMinutes + pollIntervalMinutes: source.pollIntervalMinutes, + pushToTopStories: source.pushToTopStories }; editingId = source.id; showAdd = true; @@ -69,7 +71,8 @@ type: form.type, url: form.channelId, category, - pollIntervalMinutes: form.pollIntervalMinutes + pollIntervalMinutes: form.pollIntervalMinutes, + pushToTopStories: form.pushToTopStories }; } return { @@ -77,7 +80,8 @@ type: form.type, url: form.url, category, - pollIntervalMinutes: form.pollIntervalMinutes + pollIntervalMinutes: form.pollIntervalMinutes, + pushToTopStories: form.pushToTopStories }; } @@ -119,6 +123,11 @@ sources = sources.map((s) => (s.id === source.id ? updated : s)); } + async function toggleTopStories(source: AdminSource) { + const updated = await updateSource(source.id, { pushToTopStories: !source.pushToTopStories }); + sources = sources.map((s) => (s.id === source.id ? updated : s)); + } + async function pollNow(source: AdminSource) { pollingId = source.id; justPolled = null; @@ -176,6 +185,11 @@ {/each} +
@@ -226,6 +240,14 @@ {source.category.join(', ')} {source.pollIntervalMinutes} min
+