diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 8792ef5..cf2b1f3 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -38,12 +38,28 @@ export async function registerAdminRoutes(app: FastifyInstance) { categoriesDb.setCategoryOrder(body.categoryPriority); delete body.categoryPriority; } + const before = settingsDb.getSettings(); const settings = withStorageUsed(settingsDb.updateSettings(body)); if (body.weather) { // Poll immediately rather than waiting for the next scheduler tick (up to 45 // minutes) — the admin just changed the location/unit and expects to see it reflected. pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); } + if (body.widgets) { + // Re-enabling a widget (see the Widgets tab) should show fresh data right away + // instead of waiting out its normal cadence (up to 45m/15m/1h) — scheduler.ts + // skips polling entirely while a widget is disabled, so there's nothing recent + // to fall back on otherwise. + if (body.widgets.weather && !before.widgets.weather) { + pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); + } + if (body.widgets.stocks && !before.widgets.stocks) { + pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); + } + if (body.widgets.poe2 && !before.widgets.poe2) { + pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`)); + } + } return { ...settings, categoryPriority: categoriesDb.listCategories() }; }); diff --git a/backend/src/api/public.ts b/backend/src/api/public.ts index 9765fb4..e2a0898 100644 --- a/backend/src/api/public.ts +++ b/backend/src/api/public.ts @@ -46,8 +46,10 @@ export async function registerPublicRoutes(app: FastifyInstance) { }); app.get('/api/events', async () => { - // Public fields only — sourceIds, cadenceTime etc. stay admin-only. - return eventsDb.listEvents().map((e) => ({ id: e.id, name: e.name, active: e.active, cadence: e.cadence })); + // Public fields only — sourceIds, keywords etc. stay admin-only. + return eventsDb + .listEvents() + .map((e) => ({ id: e.id, name: e.name, active: e.active, recapIntervalHours: e.recapIntervalHours, isSpillover: e.isSpillover })); }); // Drives the site nav — admin-editable (add/remove/reorder) via /api/admin/categories, @@ -60,6 +62,14 @@ export async function registerPublicRoutes(app: FastifyInstance) { return categories.filter((c) => !c.isPrivate); }); + // Per-widget enable flags + display order — see the admin panel's consolidated + // "Widgets" tab. Weather/Stocks/PoE2's backend pollers are also gated on these + // flags (see scheduler.ts); Sidebar.svelte renders in exactly this order. + app.get('/api/widgets', async () => { + const { widgets, widgetOrder } = settingsDb.getSettings(); + return { ...widgets, order: widgetOrder }; + }); + // Sidebar widgets — see WeatherTab/StocksTab/BookmarksTab in the admin panel. app.get('/api/weather', async () => settingsDb.getSettings().weather); diff --git a/backend/src/ingestion/poller.ts b/backend/src/ingestion/poller.ts index 5a13c89..f751b32 100644 --- a/backend/src/ingestion/poller.ts +++ b/backend/src/ingestion/poller.ts @@ -15,16 +15,15 @@ const adapters: Record = { telegram: telegramAdapter, api: apiAdapter, youtube: youtubeAdapter, - nitter: nitterAdapter, - custom: apiAdapter + nitter: nitterAdapter }; // Which source types point at a real webpage worth following for the full article, // as opposed to Telegram where the message itself *is* the content. const FOLLOWS_LINK_FOR_FULL_ARTICLE: Source['type'][] = ['rss', 'api']; -export async function pollDueSources(defaultIntervalMinutes: number): Promise { - const due = sourcesDb.sourcesDueForPoll(defaultIntervalMinutes); +export async function pollDueSources(): Promise { + const due = sourcesDb.sourcesDueForPoll(); let ingested = 0; for (const source of due) { ingested += await pollOne(source); diff --git a/backend/src/queue/eventsRecap.ts b/backend/src/queue/eventsRecap.ts index 3541e37..f66bc08 100644 --- a/backend/src/queue/eventsRecap.ts +++ b/backend/src/queue/eventsRecap.ts @@ -5,29 +5,14 @@ import { publishEventRecap } from '../pipeline/publish.js'; import type { GlobalSettings } from '../storage/db/types.js'; import { logger } from '../storage/db/logs.js'; +// null means recaps are off for this item — e.g. one just organizing a commit or +// torrent RSS feed under its own nav entry, with nothing that needs periodically +// summarizing. Individual items still publish immediately regardless (see +// priorityQueue.ts); this only gates the periodic AI wrap-up below. function isDue(event: ReturnType[number]): boolean { - const now = new Date(); + if (event.recapIntervalHours === null) return false; const last = event.lastRecapAt ? new Date(event.lastRecapAt) : null; - - // "Continuous" no longer means "recap every tick" — individual items matching this - // event now publish immediately regardless of cadence (see priorityQueue.ts), so the - // recap job's only remaining purpose is the periodic AI wrap-up. Treated the same as - // hourly so an ongoing event still gets occasional recaps without spamming a - // near-duplicate one on every synthesis tick. - if (event.cadence === 'continuous' || event.cadence === 'hourly') { - return !last || now.getTime() - last.getTime() >= 3600_000; - } - - if (event.cadence === 'daily') { - if (!event.cadenceTime) return false; - const [h, m] = event.cadenceTime.split(':').map(Number); - const scheduledToday = new Date(now); - scheduledToday.setHours(h, m, 0, 0); - const alreadyRecappedToday = last && last.toDateString() === now.toDateString(); - return now >= scheduledToday && !alreadyRecappedToday; - } - - return false; + return !last || Date.now() - last.getTime() >= event.recapIntervalHours * 3600_000; } /** diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index c6234fa..1eded36 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -7,7 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js'; import { clusterItems } from '../pipeline/clustering.js'; import { publishCluster, publishDirect } from '../pipeline/publish.js'; import { logger } from '../storage/db/logs.js'; -import type { GlobalSettings, ContentItem, TrackedEvent } from '../storage/db/types.js'; +import type { GlobalSettings, ContentItem, TrackedEvent, Source } from '../storage/db/types.js'; function partition(items: T[], predicate: (item: T) => boolean): [T[], T[]] { const matches: T[] = []; @@ -31,8 +31,8 @@ function claimedEventId(item: ContentItem, events: TrackedEvent[]): string | nul return match?.id ?? null; } -function primaryCategoryRank(item: ContentItem, rankByName: Map): number { - const source = sourcesDb.getSource(item.sourceId); +function primaryCategoryRank(item: ContentItem, rankByName: Map, sourcesById: Map): number { + const source = sourcesById.get(item.sourceId); const cats = source?.category ?? []; let best = Infinity; for (const cat of cats) { @@ -45,6 +45,33 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map) return best; } +/** + * Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same + * publish-then-tag-then-log/error shape, differing only in how the success/failure + * message describes why the item skipped merging. + */ +async function publishItemsDirect( + items: ContentItem[], + settings: GlobalSettings, + activeEvents: TrackedEvent[], + describeSuccess: (item: ContentItem) => string, + failureLabel: string +): Promise { + let published = 0; + for (const item of items) { + try { + const eventId = claimedEventId(item, activeEvents) ?? undefined; + const article = await publishDirect(item, settings, { eventId }); + contentItemsDb.assignCluster([item.id], article.id); + published++; + logger.info('synthesis', `Published "${article.title}" directly (${describeSuccess(item)})`); + } catch (err) { + logger.error('synthesis', `${failureLabel} for "${item.title}": ${(err as Error).message}`); + } + } + return published; +} + /** * Fallback for when the AI service isn't reachable yet — publishes every eligible item * immediately rather than leaving pages empty until Ollama is set up. Unlike the AI @@ -58,28 +85,15 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise [s.id, s])); const categories = categoriesDb.listCategories(); const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); const ranked = items - .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) })) + .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) .sort((a, b) => a.rank - b.rank) .map((r) => r.item); - let published = 0; - - for (const item of ranked) { - try { - const eventId = claimedEventId(item, activeEvents) ?? undefined; - const article = await publishDirect(item, settings, { eventId }); - contentItemsDb.assignCluster([item.id], article.id); - published++; - logger.info('synthesis', `Published "${article.title}" directly (no AI available)`); - } catch (err) { - logger.error('synthesis', `Passthrough publish failed for "${item.title}": ${(err as Error).message}`); - } - } - - return published; + return publishItemsDirect(ranked, settings, activeEvents, () => 'no AI available', 'Passthrough publish failed'); } /** @@ -96,36 +110,32 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const items = contentItemsDb.unclusteredItemsExcludingSources([]); if (items.length === 0) return 0; + // One fetch of the full source list per cycle, reused below for both the + // direct-publish partition and each item's category/type lookups — avoids a + // separate sourcesDb.getSource() round-trip per item. + const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with // anything else — each is always its own article, same shape whether the AI service // is up or not. Route them straight to publishDirect, same as the no-AI passthrough path. const directPublishSourceIds = new Set( - sourcesDb - .listSources() - .filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram') - .map((s) => s.id) + [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - let publishedDirect = 0; - for (const item of directItems) { - try { - const eventId = claimedEventId(item, activeEvents) ?? undefined; - const article = await publishDirect(item, settings, { eventId }); - contentItemsDb.assignCluster([item.id], article.id); - publishedDirect++; - const source = sourcesDb.getSource(item.sourceId); - logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`); - } catch (err) { - logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`); - } - } + const publishedDirect = await publishItemsDirect( + directItems, + settings, + activeEvents, + (item) => sourcesById.get(item.sourceId)?.type ?? 'unknown', + 'Direct publish failed' + ); const categories = categoriesDb.listCategories(); const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); const ranked = mergeableItems - .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) })) + .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) .sort((a, b) => a.rank - b.rank) .map((r) => r.item); diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index b9d958c..76dd754 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -24,8 +24,7 @@ export function startScheduler() { setInterval(async () => { try { - const settings = settingsDb.getSettings(); - const ingested = await pollDueSources(settings.defaultPollIntervalMinutes); + const ingested = await pollDueSources(); if (ingested > 0) logger.info('scheduler', `Poll tick: ingested ${ingested} new item(s)`); } catch (err) { logger.error('scheduler', `Poll tick failed: ${(err as Error).message}`); @@ -70,19 +69,32 @@ export function startScheduler() { // Immediate first call for all three — unlike RSS sources (whose "due" check makes a // brand-new source eligible on the very next 1-minute tick), weather/stocks/poe2 have // no such shortcut; without this the sidebar is empty for up to 45/15/60 minutes after - // every restart. - pollWeatherNow().catch((err) => logger.error('weather', `Initial poll failed: ${err.message}`)); + // every restart. Each is also gated on its Widgets-tab enabled flag (see + // admin/settings' consolidated Widgets tab) — disabling a widget stops these external + // calls entirely rather than just hiding the sidebar box, so there's no pointless + // polling for something nobody's looking at. Re-enabling it triggers an immediate + // poll instead (see admin.ts's PATCH /api/admin/settings), same as this initial call. + if (settingsDb.getSettings().widgets.weather) { + pollWeatherNow().catch((err) => logger.error('weather', `Initial poll failed: ${err.message}`)); + } setInterval(() => { + if (!settingsDb.getSettings().widgets.weather) return; pollWeatherNow().catch((err) => logger.error('weather', `Poll tick failed: ${err.message}`)); }, WEATHER_TICK_MS); - pollStocksNow().catch((err) => logger.error('stocks', `Initial poll failed: ${err.message}`)); + if (settingsDb.getSettings().widgets.stocks) { + pollStocksNow().catch((err) => logger.error('stocks', `Initial poll failed: ${err.message}`)); + } setInterval(() => { + if (!settingsDb.getSettings().widgets.stocks) return; pollStocksNow().catch((err) => logger.error('stocks', `Poll tick failed: ${err.message}`)); }, STOCKS_TICK_MS); - pollPoe2Now().catch((err) => logger.error('poe2', `Initial poll failed: ${err.message}`)); + if (settingsDb.getSettings().widgets.poe2) { + pollPoe2Now().catch((err) => logger.error('poe2', `Initial poll failed: ${err.message}`)); + } setInterval(() => { + if (!settingsDb.getSettings().widgets.poe2) return; pollPoe2Now().catch((err) => logger.error('poe2', `Poll tick failed: ${err.message}`)); }, POE2_TICK_MS); diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 85ea331..30f0e47 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -143,13 +143,6 @@ export function articlesForEventSince(eventId: string, since: string): MergedArt return rows.map(rowToArticle); } -export function latestArticleInThread(threadId: string): MergedArticle | null { - const row = db - .prepare('SELECT * FROM merged_articles WHERE thread_id = ? ORDER BY published_at DESC LIMIT 1') - .get(threadId); - return row ? rowToArticle(row) : null; -} - export function articlesOlderThan(days: number): MergedArticle[] { const cutoff = new Date(Date.now() - days * 86_400_000).toISOString(); const rows = db.prepare('SELECT * FROM merged_articles WHERE published_at < ?').all(cutoff); diff --git a/backend/src/storage/db/contentItems.ts b/backend/src/storage/db/contentItems.ts index 057df87..3356053 100644 --- a/backend/src/storage/db/contentItems.ts +++ b/backend/src/storage/db/contentItems.ts @@ -69,15 +69,6 @@ export function unclusteredItemsExcludingSources(excludeSourceIds: string[]): Co return items.filter((i) => !excludeSourceIds.includes(i.sourceId)); } -export function unclusteredItemsForSources(sourceIds: string[], sinceISO: string): ContentItem[] { - if (sourceIds.length === 0) return []; - const placeholders = sourceIds.map(() => '?').join(','); - const rows = db - .prepare(`SELECT * FROM content_items WHERE cluster_id IS NULL AND source_id IN (${placeholders}) AND fetched_at > ?`) - .all(...sourceIds, sinceISO); - return rows.map(rowToItem); -} - export function setEmbedding(id: string, embedding: number[]) { db.prepare('UPDATE content_items SET embedding = ? WHERE id = ?').run(JSON.stringify(embedding), id); } @@ -93,11 +84,6 @@ export function resetClusterForItems(ids: string[]) { for (const id of ids) stmt.run(id); } -export function itemsByCluster(clusterId: string): ContentItem[] { - const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id = ?').all(clusterId); - return rows.map(rowToItem); -} - export function itemsOlderThan(days: number): ContentItem[] { const cutoff = new Date(Date.now() - days * 86_400_000).toISOString(); const rows = db.prepare('SELECT * FROM content_items WHERE fetched_at < ?').all(cutoff); @@ -117,7 +103,3 @@ export function itemsForSource(sourceId: string): ContentItem[] { export function deleteContentItemsForSource(sourceId: string) { db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId); } - -export function deleteAllContentItems() { - db.prepare('DELETE FROM content_items').run(); -} diff --git a/backend/src/storage/db/events.ts b/backend/src/storage/db/events.ts index 70ab9f4..e2b12ab 100644 --- a/backend/src/storage/db/events.ts +++ b/backend/src/storage/db/events.ts @@ -9,9 +9,9 @@ function rowToEvent(row: any): TrackedEvent { description: row.description, sourceIds: JSON.parse(row.source_ids), keywords: JSON.parse(row.keywords), - cadence: row.cadence, - cadenceTime: row.cadence_time, + recapIntervalHours: row.recap_interval_hours, active: !!row.active, + isSpillover: !!row.is_spillover, retentionOverrideDays: row.retention_override_days, lastRecapAt: row.last_recap_at, createdAt: row.created_at @@ -52,7 +52,7 @@ export function createEvent(input: Partial): TrackedEvent { const id = `evt-${randomUUID()}`; const now = new Date().toISOString(); db.prepare( - `INSERT INTO tracked_events (id, name, description, source_ids, keywords, cadence, cadence_time, active, retention_override_days, last_recap_at, created_at) + `INSERT INTO tracked_events (id, name, description, source_ids, keywords, recap_interval_hours, active, is_spillover, retention_override_days, last_recap_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)` ).run( id, @@ -60,9 +60,9 @@ export function createEvent(input: Partial): TrackedEvent { input.description ?? '', JSON.stringify(input.sourceIds ?? []), JSON.stringify(input.keywords ?? []), - input.cadence ?? 'continuous', - input.cadenceTime ?? null, + input.recapIntervalHours ?? null, input.active === false ? 0 : 1, + input.isSpillover ? 1 : 0, input.retentionOverrideDays ?? null, now ); @@ -74,15 +74,15 @@ export function updateEvent(id: string, patch: Partial): TrackedEv if (!existing) return null; const merged = { ...existing, ...patch }; db.prepare( - `UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, cadence=?, cadence_time=?, active=?, retention_override_days=?, last_recap_at=? WHERE id=?` + `UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, recap_interval_hours=?, active=?, is_spillover=?, retention_override_days=?, last_recap_at=? WHERE id=?` ).run( merged.name, merged.description, JSON.stringify(merged.sourceIds), JSON.stringify(merged.keywords), - merged.cadence, - merged.cadenceTime, + merged.recapIntervalHours, merged.active ? 1 : 0, + merged.isSpillover ? 1 : 0, merged.retentionOverrideDays, merged.lastRecapAt, id diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 0697e94..7be16c5 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -37,7 +37,7 @@ export function migrate() { CREATE TABLE IF NOT EXISTS sources ( id TEXT PRIMARY KEY, name TEXT NOT NULL, - type TEXT NOT NULL, -- rss | api | telegram | custom + type TEXT NOT NULL, -- rss | api | telegram | youtube | nitter category TEXT NOT NULL DEFAULT '[]', -- JSON array url TEXT, config TEXT NOT NULL DEFAULT '{}', -- JSON: apiKey, telegramChannelId, authHeaders @@ -130,9 +130,9 @@ export function migrate() { description TEXT NOT NULL DEFAULT '', source_ids TEXT NOT NULL DEFAULT '[]', -- JSON keywords TEXT NOT NULL DEFAULT '[]', -- JSON string array — empty means "match everything from source_ids" - cadence TEXT NOT NULL DEFAULT 'continuous', - cadence_time TEXT, + recap_interval_hours INTEGER, -- hours between AI recaps; NULL = recaps off for this item active INTEGER NOT NULL DEFAULT 1, + is_spillover INTEGER NOT NULL DEFAULT 0, retention_override_days INTEGER, last_recap_at TEXT, created_at TEXT NOT NULL @@ -168,7 +168,6 @@ export function migrate() { CREATE TABLE IF NOT EXISTS global_settings ( id INTEGER PRIMARY KEY CHECK (id = 1), -- singleton row merge_strictness INTEGER NOT NULL DEFAULT 3, - default_poll_interval_minutes INTEGER NOT NULL DEFAULT 15, hold_before_publish_minutes INTEGER NOT NULL DEFAULT 30, tag_dedup_threshold REAL NOT NULL DEFAULT 0.82, tag_expiry_days INTEGER NOT NULL DEFAULT 21, @@ -185,6 +184,11 @@ export function migrate() { nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com', telegram_media_mode TEXT NOT NULL DEFAULT 'self-host', -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL) + widget_weather_enabled INTEGER NOT NULL DEFAULT 1, + widget_stocks_enabled INTEGER NOT NULL DEFAULT 1, + widget_bookmarks_enabled INTEGER NOT NULL DEFAULT 1, + widget_poe2_enabled INTEGER NOT NULL DEFAULT 1, + widget_order TEXT NOT NULL DEFAULT '["weather","stocks","poe2","bookmarks"]', -- JSON array, admin-sortable via the Widgets tab weather_location_name TEXT, weather_latitude REAL, weather_longitude REAL, @@ -200,7 +204,6 @@ export function migrate() { weather_updated_at TEXT, -- ISO timestamp, NULL pre-first-poll poe2_league_id TEXT, poe2_league_name TEXT, - poe2_primary_currency_name TEXT, -- unused since the watchlist moved to arbitrary currency pairs (no single "quoted in" currency anymore) — column kept rather than dropped, SQLite ALTER TABLE can't drop columns without a full table rebuild poe2_updated_at TEXT ); @@ -326,6 +329,12 @@ export function migrate() { if (!hasColumn('tracked_events', 'keywords')) { db.exec("ALTER TABLE tracked_events ADD COLUMN keywords TEXT NOT NULL DEFAULT '[]'"); } + if (!hasColumn('tracked_events', 'is_spillover')) { + db.exec('ALTER TABLE tracked_events ADD COLUMN is_spillover INTEGER NOT NULL DEFAULT 0'); + } + if (!hasColumn('tracked_events', 'recap_interval_hours')) { + db.exec('ALTER TABLE tracked_events ADD COLUMN recap_interval_hours INTEGER'); + } if (!hasColumn('merged_articles', 'is_recap')) { db.exec('ALTER TABLE merged_articles ADD COLUMN is_recap INTEGER NOT NULL DEFAULT 0'); } @@ -347,9 +356,19 @@ export function migrate() { if (!hasColumn('global_settings', 'poe2_league_id')) { db.exec('ALTER TABLE global_settings ADD COLUMN poe2_league_id TEXT'); db.exec('ALTER TABLE global_settings ADD COLUMN poe2_league_name TEXT'); - db.exec('ALTER TABLE global_settings ADD COLUMN poe2_primary_currency_name TEXT'); db.exec('ALTER TABLE global_settings ADD COLUMN poe2_updated_at TEXT'); } + if (!hasColumn('global_settings', 'widget_weather_enabled')) { + db.exec('ALTER TABLE global_settings ADD COLUMN widget_weather_enabled INTEGER NOT NULL DEFAULT 1'); + db.exec('ALTER TABLE global_settings ADD COLUMN widget_stocks_enabled INTEGER NOT NULL DEFAULT 1'); + db.exec('ALTER TABLE global_settings ADD COLUMN widget_bookmarks_enabled INTEGER NOT NULL DEFAULT 1'); + db.exec('ALTER TABLE global_settings ADD COLUMN widget_poe2_enabled INTEGER NOT NULL DEFAULT 1'); + } + if (!hasColumn('global_settings', 'widget_order')) { + db.exec( + `ALTER TABLE global_settings ADD COLUMN widget_order TEXT NOT NULL DEFAULT '["weather","stocks","poe2","bookmarks"]'` + ); + } // Seed a handful of sensible default tickers so the Stocks widget isn't empty on a // fresh install — the admin can remove/replace any of them via the Stocks tab. @@ -368,19 +387,6 @@ export function migrate() { }); } - // Stocks switched data providers from Stooq (walled off behind a proof-of-work - // challenge) to Yahoo Finance, which uses different symbol syntax — rewrites only - // rows still holding exactly one of the three old Stooq-format default symbols we - // ourselves seeded, never touching a symbol the admin typed in themselves. - const stooqToYahooSymbols: [string, string][] = [ - ['^dji', '^DJI'], - ['^spx', '^GSPC'], - ['btcusd', 'BTC-USD'] - ]; - for (const [oldSymbol, newSymbol] of stooqToYahooSymbols) { - db.prepare('UPDATE stock_tickers SET symbol = ? WHERE symbol = ?').run(newSymbol, oldSymbol); - } - // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real // filterable tag: it's the homepage view, now scoped to only the articles whose diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 35723a3..9b0469c 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -4,7 +4,6 @@ import type { GlobalSettings } from './types.js'; function rowToSettings(row: any): GlobalSettings { return { mergeStrictness: row.merge_strictness, - defaultPollIntervalMinutes: row.default_poll_interval_minutes, holdBeforePublishMinutes: row.hold_before_publish_minutes, tagDedupThreshold: row.tag_dedup_threshold, tagExpiryDays: row.tag_expiry_days, @@ -16,6 +15,13 @@ function rowToSettings(row: any): GlobalSettings { nitterMediaMode: row.nitter_media_mode, fxtwitterBaseUrl: row.fxtwitter_base_url, telegramMediaMode: row.telegram_media_mode, + widgets: { + weather: !!row.widget_weather_enabled, + stocks: !!row.widget_stocks_enabled, + bookmarks: !!row.widget_bookmarks_enabled, + poe2: !!row.widget_poe2_enabled + }, + widgetOrder: JSON.parse(row.widget_order), retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days, @@ -58,54 +64,68 @@ export function updateSettings(patch: Partial): GlobalSettings { retention: { ...current.retention, ...(patch.retention ?? {}) }, selectedModels: { ...current.selectedModels, ...(patch.selectedModels ?? {}) }, weather: { ...current.weather, ...(patch.weather ?? {}) }, - poe2: { ...current.poe2, ...(patch.poe2 ?? {}) } + poe2: { ...current.poe2, ...(patch.poe2 ?? {}) }, + widgets: { ...current.widgets, ...(patch.widgets ?? {}) } }; + // Named params (rather than positional `?`) so this list can be reordered or + // extended without the column list and the bound-values list silently drifting + // out of sync — node:sqlite binds each by its `$name` key, not position. db.prepare( `UPDATE global_settings SET - merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, - tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, - ai_service_host=?, ai_service_port=?, selected_models=?, - nitter_media_mode=?, fxtwitter_base_url=?, telegram_media_mode=?, - published_article_max_age_days=?, raw_item_max_age_days=?, - storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?, - weather_location_name=?, weather_latitude=?, weather_longitude=?, weather_unit=?, - weather_wind_unit=?, weather_pressure_unit=?, - weather_current=?, weather_hourly=?, weather_daily=?, weather_alerts=?, weather_updated_at=?, - poe2_league_id=?, poe2_league_name=?, poe2_updated_at=? + merge_strictness=$merge_strictness, + hold_before_publish_minutes=$hold_before_publish_minutes, + tag_dedup_threshold=$tag_dedup_threshold, tag_expiry_days=$tag_expiry_days, + follow_up_min_hours_since_last=$follow_up_min_hours_since_last, follow_up_min_new_sources=$follow_up_min_new_sources, + ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models, + nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, telegram_media_mode=$telegram_media_mode, + widget_weather_enabled=$widget_weather_enabled, widget_stocks_enabled=$widget_stocks_enabled, + widget_bookmarks_enabled=$widget_bookmarks_enabled, widget_poe2_enabled=$widget_poe2_enabled, + widget_order=$widget_order, + published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days, + storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit, + weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude, + weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit, + weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily, + weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at, + poe2_league_id=$poe2_league_id, poe2_league_name=$poe2_league_name, poe2_updated_at=$poe2_updated_at WHERE id = 1` - ).run( - merged.mergeStrictness, - merged.defaultPollIntervalMinutes, - merged.holdBeforePublishMinutes, - merged.tagDedupThreshold, - merged.tagExpiryDays, - merged.followUpMinHoursSinceLast, - merged.followUpMinNewSources, - merged.aiServiceHost, - merged.aiServicePort, - JSON.stringify(merged.selectedModels), - merged.nitterMediaMode, - merged.fxtwitterBaseUrl, - merged.telegramMediaMode, - merged.retention.publishedArticleMaxAgeDays, - merged.retention.rawItemMaxAgeDays, - merged.retention.storageCapEnabled ? 1 : 0, - merged.retention.storageCapValue, - merged.retention.storageCapUnit, - merged.weather.locationName, - merged.weather.latitude, - merged.weather.longitude, - merged.weather.unit, - merged.weather.windUnit, - merged.weather.pressureUnit, - merged.weather.current ? JSON.stringify(merged.weather.current) : null, - JSON.stringify(merged.weather.hourly), - JSON.stringify(merged.weather.daily), - JSON.stringify(merged.weather.alerts), - merged.weather.updatedAt, - merged.poe2.leagueId, - merged.poe2.leagueName, - merged.poe2.updatedAt - ); + ).run({ + $merge_strictness: merged.mergeStrictness, + $hold_before_publish_minutes: merged.holdBeforePublishMinutes, + $tag_dedup_threshold: merged.tagDedupThreshold, + $tag_expiry_days: merged.tagExpiryDays, + $follow_up_min_hours_since_last: merged.followUpMinHoursSinceLast, + $follow_up_min_new_sources: merged.followUpMinNewSources, + $ai_service_host: merged.aiServiceHost, + $ai_service_port: merged.aiServicePort, + $selected_models: JSON.stringify(merged.selectedModels), + $nitter_media_mode: merged.nitterMediaMode, + $fxtwitter_base_url: merged.fxtwitterBaseUrl, + $telegram_media_mode: merged.telegramMediaMode, + $widget_weather_enabled: merged.widgets.weather ? 1 : 0, + $widget_stocks_enabled: merged.widgets.stocks ? 1 : 0, + $widget_bookmarks_enabled: merged.widgets.bookmarks ? 1 : 0, + $widget_poe2_enabled: merged.widgets.poe2 ? 1 : 0, + $widget_order: JSON.stringify(merged.widgetOrder), + $published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays, + $raw_item_max_age_days: merged.retention.rawItemMaxAgeDays, + $storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0, + $storage_cap_value: merged.retention.storageCapValue, + $storage_cap_unit: merged.retention.storageCapUnit, + $weather_location_name: merged.weather.locationName, + $weather_latitude: merged.weather.latitude, + $weather_longitude: merged.weather.longitude, + $weather_unit: merged.weather.unit, + $weather_wind_unit: merged.weather.windUnit, + $weather_pressure_unit: merged.weather.pressureUnit, + $weather_current: merged.weather.current ? JSON.stringify(merged.weather.current) : null, + $weather_hourly: JSON.stringify(merged.weather.hourly), + $weather_daily: JSON.stringify(merged.weather.daily), + $weather_alerts: JSON.stringify(merged.weather.alerts), + $weather_updated_at: merged.weather.updatedAt, + $poe2_league_id: merged.poe2.leagueId, + $poe2_league_name: merged.poe2.leagueName, + $poe2_updated_at: merged.poe2.updatedAt + }); return getSettings(); } diff --git a/backend/src/storage/db/sources.ts b/backend/src/storage/db/sources.ts index ea6992e..8d89a8e 100644 --- a/backend/src/storage/db/sources.ts +++ b/backend/src/storage/db/sources.ts @@ -89,12 +89,11 @@ export function markPolled(id: string, error: string | null) { ); } -/** Sources due for polling right now, based on their own interval (or the global default). */ -export function sourcesDueForPoll(defaultIntervalMinutes: number): Source[] { +/** Sources due for polling right now, based on their own interval. */ +export function sourcesDueForPoll(): Source[] { const now = Date.now(); return listEnabledSources().filter((s) => { if (!s.lastPolledAt) return true; - const interval = (s.pollIntervalMinutes || defaultIntervalMinutes) * 60_000; - return now - new Date(s.lastPolledAt).getTime() >= interval; + return now - new Date(s.lastPolledAt).getTime() >= s.pollIntervalMinutes * 60_000; }); } diff --git a/backend/src/storage/db/tags.ts b/backend/src/storage/db/tags.ts index e2ea9f6..455f47a 100644 --- a/backend/src/storage/db/tags.ts +++ b/backend/src/storage/db/tags.ts @@ -28,13 +28,6 @@ export function listActiveTags(): Tag[] { return rows.map(rowToTag); } -export function getTagsByIds(ids: string[]): Tag[] { - if (ids.length === 0) return []; - const placeholders = ids.map(() => '?').join(','); - const rows = db.prepare(`SELECT * FROM tags WHERE id IN (${placeholders})`).all(...ids); - return rows.map(rowToTag); -} - function cosineSimilarity(a: number[], b: number[]): number { if (a.length === 0 || b.length === 0 || a.length !== b.length) return 0; let dot = 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 45b759b..406db07 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -1,7 +1,7 @@ export interface Source { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter'; category: string[]; url: string | null; config: Record; @@ -178,9 +178,11 @@ export interface TrackedEvent { sourceIds: string[]; /** Only items whose title/summary/body contain at least one of these (case-insensitive) qualify for this event — empty means "match everything from sourceIds", the original behavior. */ keywords: string[]; - cadence: 'continuous' | 'daily' | 'hourly' | 'custom'; - cadenceTime: string | null; + /** Hours between AI recaps, or null to turn recaps off entirely — e.g. an item that's just organizing a commit or torrent RSS feed under one nav entry, with nothing that needs periodically summarizing. Individual articles still publish immediately either way (see priorityQueue.ts); this only gates eventsRecap.ts's periodic wrap-up. */ + recapIntervalHours: 1 | 3 | 6 | 12 | 24 | null; active: boolean; + /** Collapses into the "More »" nav tab instead of getting its own top-level tab — same idea as Category.isSpillover. */ + isSpillover: boolean; retentionOverrideDays: number | null; lastRecapAt: string | null; createdAt: string; @@ -265,7 +267,6 @@ export interface Bookmark { export interface GlobalSettings { mergeStrictness: 1 | 2 | 3 | 4 | 5; - defaultPollIntervalMinutes: number; holdBeforePublishMinutes: number; tagDedupThreshold: number; tagExpiryDays: number; @@ -280,6 +281,15 @@ export interface GlobalSettings { fxtwitterBaseUrl: string; /** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */ telegramMediaMode: 'self-host' | 'proxy'; + /** Per-widget enable flags — see admin/settings' consolidated "Widgets" tab. Weather/Stocks/PoE2's backend pollers (scheduler.ts) are gated on these too, not just sidebar visibility; Bookmarks has no poller so its flag only affects the sidebar. */ + widgets: { + weather: boolean; + stocks: boolean; + bookmarks: boolean; + poe2: boolean; + }; + /** Sidebar widget display order, admin-sortable via the Widgets tab's up/down arrows — mirrored exactly by Sidebar.svelte. */ + widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/docs/review-2026-07.md b/docs/review-2026-07.md new file mode 100644 index 0000000..008a633 --- /dev/null +++ b/docs/review-2026-07.md @@ -0,0 +1,230 @@ +# Homefeed program review — July 2026 + +Scope: dead code, inefficiency, and UI/system consistency across the whole app +(backend + frontend, public site + admin panel), as requested after a long run of +incremental feature work. Findings below are left as originally written (historical +record); the "Resolved since this report" section tracks what's since been acted on. + +## Resolved since this report + +- **1.1** `ArticleCard.svelte` — deleted. +- **1.2** All five zero-call-site exports (`getTagsByIds`, `unclusteredItemsForSources`, + `itemsByCluster`, `deleteAllContentItems`, `latestArticleInThread`) — removed. +- **1.3** `poe2_primary_currency_name` — dropped from the schema entirely (CREATE TABLE + literal + the ALTER TABLE migration line), not just left as an inert column. +- **Inefficiencies** — the unconditional `stooqToYahooSymbols` startup rewrite + (long-since a no-op) was deleted outright; `settings.ts:updateSettings` now binds + named `$column` parameters instead of 33 positional `?`s, removing the + reorder-and-silently-corrupt risk; `runPassthroughCycle`/`runSynthesisCycle`'s + direct-publish logic now share one `publishItemsDirect` helper; the per-item + `sourcesDb.getSource()` N+1 in the synthesis tick was replaced with a single + `sourcesDb.listSources()` map reused for both the direct-publish partition and each + item's category-rank lookup. (`settings.ts`'s remaining structural size and + `migrate()`'s 410-line mixed-idiom growth were left as-is — real fixes but a larger + refactor than this pass warranted.) +- **`'custom'` source type** — removed entirely: dropped from the `Source['type']` + union (backend + frontend), the poller's adapter map, and the admin add-source + dropdown. It had no adapter of its own (silently aliased to the API adapter) and no + admin-facing purpose distinct from `api`. + +Everything above was verified via `tsc --noEmit`/`svelte-check` (0 errors on both +sides), a full build of both packages, and a live smoke test against a fresh DB +(migrate() + a settings roundtrip through the new named-param query, confirming the +column is gone and unrelated fields are untouched). + +See the **staleness/refresh recommendation** delivered separately in conversation — +that item was discussed, not changed, per the request to advise rather than implement. + +--- + +## 1. Dead code + +### 1.1 Frontend +- `frontend/src/lib/components/ArticleCard.svelte` — zero references anywhere under + `frontend/src` (confirmed via `grep -rn "ArticleCard" frontend/src`, zero matches + outside its own file). Superseded by `ArticleListRow.svelte`. Safe to delete. + +### 1.2 Backend +Exported functions with no call sites found anywhere in `backend/src`: +- `backend/src/storage/db/tags.ts:31` — `getTagsByIds` +- `backend/src/storage/db/contentItems.ts:72` — `unclusteredItemsForSources` +- `backend/src/storage/db/contentItems.ts:96` — `itemsByCluster` +- `backend/src/storage/db/contentItems.ts:121` — `deleteAllContentItems` +- `backend/src/storage/db/articles.ts:146` — `latestArticleInThread` + +### 1.3 Dead settings field +- `global_settings.poe2_primary_currency_name` — left over from before PoE2 moved to + per-pair tracking (base/quote currencies with a directly-computed rate, no longer + "everything quoted in one reference currency"). The column is harmless — dropping it + would mean a SQLite table rebuild for one nullable text column, not worth doing on + its own. `GlobalSettings.poe2` on the TS side no longer exposes it, so it's already + fully inert; only the raw DB column remains as an artifact. + +### 1.4 Over-exported (not dead, but worth a look) +These are only ever called from within their own module, so exporting them invites +cross-module coupling that hasn't happened yet but could: +- `backend/src/storage/db/sources.ts:27` — `listEnabledSources` +- `backend/src/storage/db/events.ts:46` — `getEvent` +- `backend/src/clustering.ts:9` — `strictnessToThreshold` + +--- + +## 2. Inefficiencies + +- **`backend/src/storage/db/index.ts:375-382`** — the `stooqToYahooSymbols` stock + symbol rewrite runs unconditionally on every server start, forever, with no + skip-guard once it's already been applied to a given database. Same category of + issue as the unconditional `DROP TABLE IF EXISTS admin_users` / `admin_sessions` at + lines 25-26 — both re-run one-time work on every boot instead of gating it behind a + "have I already done this" check. +- **`backend/src/storage/db/settings.ts:updateSettings` (lines 77-109)** — 33 + positional `?` SQL parameters matched to 33 arguments passed to `.run()` in the same + order, with nothing enforcing that the two lists stay in sync if either is edited. + Not currently broken, but a single misordered edit here would silently write the + wrong value into the wrong column with no type error to catch it. +- Synthesis's 60-second tick calls `sourcesDb.listSources()` on every iteration, which + is redundant if the source list hasn't changed since the last tick — a small, + low-priority optimization. +- `runPassthroughCycle` and `runSynthesisCycle`'s "direct publish" halves are + near-duplicate copy-pasted logic rather than a shared helper. +- The `migrate()` function (410 lines) mixes four different guarded/unguarded + backfill idioms with no schema-version tracking, which makes it hard to tell at a + glance whether a given block still needs to run or is a no-op on any DB that's + already current. + +--- + +## 3. UI / layout consistency audit (public site) + +- **`/weather` is not reachable from the top nav** — it's only linkable via the + sidebar widget, unlike every other route (home, categories, events, `/more`), which + are all in top nav. +- **Sidebar widgets don't share a component.** Weather, Stocks, and PoE2 each + hand-roll their own `.widget` shell but do converge on the same `.head` + + interval-tag pattern (`today`, `24h`, etc.). `BookmarksWidget` is the outlier: it + lacks that `.head`/interval wrapper the other three use, so it looks structurally + different from its neighbors in the same sidebar. +- Confirmed: the PoE2 icon-removal and 24h-only simplification from earlier this + session are fully complete on the frontend (verified via grep — no remaining icon + references). + +--- + +## 4. UI / layout consistency audit (admin panel) + +- **Near-duplicate CRUD tabs.** `StocksTab.svelte` and `BookmarksTab.svelte` are + close to byte-identical in structure (add row, list, delete button). `Poe2Tab.svelte` + is a variant of the same pattern; `WeatherTab.svelte` uses a different + settings-form pattern entirely. None of the three share a common component, so a fix + to one (e.g. the null/`—` display bug fixed in Poe2Tab this session) doesn't + propagate to the others even where the same bug class could exist. +- **Admin tab list has an implicit, unlabeled grouping** — Sources & Content / + Sidebar widgets (Weather, Stocks, PoE2, Bookmarks) / Integrations (Telegram) / + System (Logs, Settings) — but the tab bar renders them as one flat list with no + visual separation or heading, so the grouping only exists in the code's mental + model, not on screen. +- **`/admin` itself is a dead redirect-only render** — it exists only to bounce to + `/admin/settings` or another tab, with no content of its own. + +--- + +## 5. Cross-system / type consistency findings + +### 5.1 Duplicated types +- `WeatherHourEntry`, `WeatherDayEntry`, `WeatherCurrentConditions`, and + `WeatherAlert` are byte-identical between `frontend/src/lib/types.ts` and + `adminTypes.ts` — genuine duplication, not just similar shapes. +- `Category` and `CategoryPriority` are same-shape-but-differently-named — worth + confirming whether that's intentional (different semantic roles) or just drift from + copy-paste. +- Backend-side: `Tweet`/Telegram-message inline shapes are duplicated between + `ContentItem` and `MergedArticle` in `backend/src/storage/db/types.ts` (lines + 78-108 vs. 125-143) rather than being extracted into a shared named interface. + +### 5.2 CSS token drift +- A `12px` "card radius" value is hardcoded across roughly 20 files instead of using + the existing `var(--radius)` token (which is 8px) — meaning cards and the + token-driven `--radius` elements don't actually share one visual language despite + looking like they should. +- `LogsTab.svelte:99-101,157` hardcodes `#a8710f`/`#fff` for the "warn" log-level + color rather than using a token, unlike "err" which correctly uses `--text-danger`. + There's no `--text-warning` token defined to match. + +### 5.3 Ingestion pipeline (traced end-to-end) +Flow: adapter → poller → priority queue → clustering/synthesis (or direct-publish +bypass) → articles table → public feed. The telegram/nitter/youtube direct-publish +bypass (skipping clustering/synthesis entirely for those source types) is coherent +and already explicitly commented as intentional — not an oversight, despite looking +unusual at first glance. + +One asymmetry does look like a genuine oversight rather than a documented design +choice: the `'custom'` source type is excluded from both +`FOLLOWS_LINK_FOR_FULL_ARTICLE` (`backend/src/poller.ts:24`, which only lists +`['rss', 'api']`) and from `directPublishSourceIds` in +`backend/src/queue/priorityQueue.ts` — with no comment anywhere explaining why +`custom` is treated differently from `rss`, which otherwise behaves the same way a +`custom` source presumably would. + +### 5.4 Frontend staleness beyond the sidebar +This session added a 5-minute `invalidate('app:sidebar')` timer +(`frontend/src/routes/+layout.svelte` / `+layout.ts`) so the sidebar widgets stay +fresh in a tab left open. That is the **only** refresh mechanism anywhere in the +frontend. Every other route — home (`/`), `/category/[name]`, `/event/[id]`, +`/article/[id]`, `/more`, and notably **`/admin/settings`** — is a pure one-shot +`load()` with no refresh at all. This is fine for content pages (a stale article list +is a minor nuisance, fixed by navigating), but is a real staleness risk specifically +for `/admin/settings`: an admin who leaves that page open to watch Logs, AI-status, +or Telegram connection status will see indefinitely stale state with no signal that +it's stale. + +### 5.5 Public API field leak (most concrete/actionable finding) +`backend/src/api/public.ts` — the categories, bookmarks, and events routes all +explicitly filter their response shape down to public-safe fields: + +```ts +// lines ~48-51, 57-61, 68-72 — filtered +app.get('/api/events', async () => eventsDb.listEvents().filter(e => e.active).map(({ id, name, ... }) => ({ id, name, ... }))); +``` + +But stocks and PoE2 do not: + +```ts +// line 66 +app.get('/api/stocks', async () => stocksDb.listStockTickers()); + +// lines 74-77 +app.get('/api/poe2', async () => { + const { leagueName, updatedAt } = settingsDb.getSettings().poe2; + return { leagueName, updatedAt, entries: poe2WatchlistDb.listWatchlist() }; +}); +``` + +`listStockTickers()` and `listWatchlist()` return the full DB row, including +`lastError` and `lastPolledAt` — internal poller-diagnostic fields with no reason to +be visible to an unauthenticated visitor. This is inconsistent with how every other +public route in the same file handles the same concern, and is a straightforward fix: +map each entry down to its public-facing fields the same way categories/bookmarks/ +events already do. + +--- + +## Appendix: surveyed and found fine + +Recorded here so these don't get re-litigated in a future review: + +- **Logging** — a single shared `logger`, consistent lowercase tags, and only two + `console.log` calls outside of it, both explicitly commented as intentional + exceptions (in `index.ts`). +- **API route organization** — `admin.ts` (329 lines / 38 routes) is cleanly + sectioned into 11 logical groups; `public.ts` (78 lines / 9 routes) has no + duplicated logic. 52 routes total, no redundant endpoints found. +- **Scheduler cadences** — weather (45m), stocks (15m), and PoE2 (1h) all poll + immediately on start by design; the ingestion poll/synthesis loop (60s) and + retention sweep (1h) deliberately don't, also by design. No redundant external + fetches were found across the whole scheduler table. +- **Module organization pattern** — the `client.ts` (raw external API) + + `poller.ts` (orchestration) split is followed consistently for weather, stocks, and + PoE2. Telegram is the one deliberate exception (a stateful GramJS client/credentials + singleton, ingested through the generic per-source `ingestion/poller.ts` rather than + `queue/scheduler.ts`) — this is explicitly commented in the code as an intentional + divergence, not an inconsistency to fix. diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 2cc08d6..d0a1063 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -118,9 +118,15 @@ export interface AdminPoe2Settings { updatedAt: string | null; } +export interface AdminWidgetsEnabled { + weather: boolean; + stocks: boolean; + bookmarks: boolean; + poe2: boolean; +} + export interface AdminSettings { mergeStrictness: 1 | 2 | 3 | 4 | 5; - defaultPollIntervalMinutes: number; holdBeforePublishMinutes: number; tagDedupThreshold: number; tagExpiryDays: number; @@ -132,6 +138,8 @@ export interface AdminSettings { nitterMediaMode: 'self-host' | 'proxy' | 'direct'; fxtwitterBaseUrl: string; telegramMediaMode: 'self-host' | 'proxy'; + widgets: AdminWidgetsEnabled; + widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; retention: RetentionSettings; categoryPriority: CategoryPriority[]; weather: AdminWeatherSettings; @@ -141,7 +149,7 @@ export interface AdminSettings { export interface AdminSource { id: string; name: string; - type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom'; + type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter'; category: string[]; url: string; config?: Record; @@ -159,9 +167,10 @@ export interface AdminTrackedEvent { sourceIds: string[]; /** Only items whose title/summary/body contain at least one of these (case-insensitive) qualify for this event — empty means "match everything from sourceIds". */ keywords: string[]; - cadence: 'continuous' | 'daily' | 'hourly' | 'custom'; - cadenceTime: string | null; + /** Hours between AI recaps, or null to turn recaps off entirely for this item. */ + recapIntervalHours: 1 | 3 | 6 | 12 | 24 | null; active: boolean; + isSpillover: boolean; retentionOverrideDays: number | null; } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index e0afe7d..dba92e5 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,5 +1,5 @@ import { getBackendUrl } from './config'; -import type { MergedArticle, Tag, TrackedEventPublic, Category, Weather, StockTicker, Bookmark, Poe2Data } from './types'; +import type { MergedArticle, Tag, TrackedEventPublic, Category, Weather, StockTicker, Bookmark, Poe2Data, WidgetsEnabled } from './types'; async function get(path: string, fetchFn: typeof fetch = fetch): Promise { // credentials: 'include' so the private-access cookie (see lib/privateAccess.ts) @@ -57,3 +57,7 @@ export function getBookmarks(fetchFn?: typeof fetch): Promise { export function getPoe2(fetchFn?: typeof fetch): Promise { return get('/api/poe2', fetchFn); } + +export function getWidgetsEnabled(fetchFn?: typeof fetch): Promise { + return get('/api/widgets', fetchFn); +} diff --git a/frontend/src/lib/components/ArticleCard.svelte b/frontend/src/lib/components/ArticleCard.svelte deleted file mode 100644 index 009c574..0000000 --- a/frontend/src/lib/components/ArticleCard.svelte +++ /dev/null @@ -1,76 +0,0 @@ - - - - {#if article.heroImage} - - {:else} -
- {/if} -
- {#if article.video} - ▶ Video - {:else if article.sourceCount > 1} - ⇄ {sourceLabel} - {:else} - {sourceLabel} - {/if} -
-
{article.title}
-
{timeAgo(article.publishedAt)}
- - - diff --git a/frontend/src/lib/components/admin/CollapsibleSection.svelte b/frontend/src/lib/components/admin/CollapsibleSection.svelte new file mode 100644 index 0000000..364e858 --- /dev/null +++ b/frontend/src/lib/components/admin/CollapsibleSection.svelte @@ -0,0 +1,69 @@ + + +
+ + {#if expanded} +
+ {@render children()} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/admin/EventsTab.svelte b/frontend/src/lib/components/admin/EventsTab.svelte index db7c18d..fd9a0e3 100644 --- a/frontend/src/lib/components/admin/EventsTab.svelte +++ b/frontend/src/lib/components/admin/EventsTab.svelte @@ -5,7 +5,9 @@ let { events: initial, sources }: { events: AdminTrackedEvent[]; sources: AdminSource[] } = $props(); let events = $state([...initial]); let showAdd = $state(false); - let newEvent = $state({ name: '', cadence: 'daily' as AdminTrackedEvent['cadence'], cadenceTime: '18:00' }); + // Off by default — plenty of items (a commit feed, a torrent feed) exist just to + // organize sources under one nav entry and never need an AI recap. + let newEvent = $state({ name: '', recapIntervalHours: null as AdminTrackedEvent['recapIntervalHours'] }); let editingId = $state(null); function emptyEditForm() { @@ -14,17 +16,13 @@ description: '', sourceIdSet: new Set(), keywordsText: '', - cadence: 'daily' as AdminTrackedEvent['cadence'], - cadenceTime: '18:00', + recapIntervalHours: null as AdminTrackedEvent['recapIntervalHours'], + isSpillover: false, retentionOverrideDays: null as number | null }; } let editForm = $state(emptyEditForm()); - function sourceNames(ids: string[]) { - return ids.map((id) => sources.find((s) => s.id === id)?.name).filter(Boolean).join(', ') || 'No sources assigned'; - } - async function handleAdd() { if (!newEvent.name) return; const created = await addEvent({ @@ -32,12 +30,12 @@ description: '', sourceIds: [], keywords: [], - cadence: newEvent.cadence, - cadenceTime: newEvent.cadence === 'daily' ? newEvent.cadenceTime : null, + recapIntervalHours: newEvent.recapIntervalHours, + isSpillover: false, retentionOverrideDays: null }); events = [...events, created]; - newEvent = { name: '', cadence: 'daily', cadenceTime: '18:00' }; + newEvent = { name: '', recapIntervalHours: null }; showAdd = false; } @@ -58,8 +56,8 @@ description: event.description, sourceIdSet: new Set(event.sourceIds), keywordsText: event.keywords.join(', '), - cadence: event.cadence, - cadenceTime: event.cadenceTime ?? '18:00', + recapIntervalHours: event.recapIntervalHours, + isSpillover: event.isSpillover, retentionOverrideDays: event.retentionOverrideDays }; } @@ -86,8 +84,8 @@ description: editForm.description, sourceIds: [...editForm.sourceIdSet], keywords, - cadence: editForm.cadence, - cadenceTime: editForm.cadence === 'daily' ? editForm.cadenceTime : null, + recapIntervalHours: editForm.recapIntervalHours, + isSpillover: editForm.isSpillover, retentionOverrideDays: editForm.retentionOverrideDays }); events = events.map((e) => (e.id === editingId ? updated : e)); @@ -96,22 +94,31 @@
- {events.length} tracked events - + {events.length} tracked items +
{#if showAdd}
- - +
+
+
Recap cadence
+ - {#if newEvent.cadence === 'daily'} - - {/if} +

+ How often an AI recap is written summarizing this item's coverage, on top of its + individual articles (which publish immediately either way). Off by default — leave it + off for something you're just organizing under its own nav entry (a commit feed, a + torrent feed) with nothing that needs summarizing. +

@@ -126,15 +133,7 @@ {#if editingId === event.id}
- - - {#if editForm.cadence === 'daily'} - - {/if} +
@@ -156,10 +155,33 @@

Comma-separated words, phrases, or emoji — only items from the sources above whose - title/summary/body contain at least one qualify for this event's recap. Leave blank to + title/summary/body contain at least one qualify for this item's recap. Leave blank to include everything from the assigned sources.

+
+
Recap cadence
+ +

+ How often an AI recap is written summarizing this item's coverage, on top of its + individual articles (which publish immediately either way). Off by default — leave + it off for something you're just organizing under its own nav entry (a commit feed, + a torrent feed) with nothing that needs summarizing. +

+
+ + +
@@ -170,12 +192,7 @@
{event.name}
- {sourceNames(event.sourceIds)} - {#if event.keywords.length > 0} - · matching {event.keywords.map((k) => `"${k}"`).join(', ')} - {/if} - · - {event.cadence === 'daily' ? `daily recap at ${event.cadenceTime}` : event.cadence} + {event.sourceIds.length} source{event.sourceIds.length === 1 ? '' : 's'} active
toggleActive(event)} role="button" tabindex="0"> @@ -314,4 +331,27 @@ .edit-panel .hint { margin: 6px 0 10px; } + .cadence-block { + margin-top: 12px; + padding-top: 12px; + border-top: 0.5px solid var(--border); + } + .cadence-block .hint { + margin-top: 6px; + } + .edit-spillover { + display: flex; + margin-top: 12px; + } + .spillover-toggle { + display: flex; + align-items: center; + gap: 5px; + font-size: 11px; + color: var(--text-secondary); + white-space: nowrap; + } + .spillover-toggle input { + width: auto; + } diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index be0c028..8e0ced1 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -1,9 +1,11 @@ -
-
- Merge strictness - -
-

How similar articles must be before they're combined into one story.

-
- Loose - - Strict - {local.mergeStrictness} -
-
- -
-
- Poll interval -

How often each source is checked for new items.

- -
-
- Hold before publish -

Wait window to gather more sources before finalizing a story.

- -
-
- -
- Follow-up articles -

- Instead of editing a published article, a distinct follow-up is created once enough new - corroborating sources arrive after enough time has passed. -

-
-
- - -
-
- - -
-
-
- -
- Category priority +

Synthesis queue processes higher-ranked categories first. Nothing is dropped — lower categories just wait longer when the queue is busy. This list also drives the site's nav — @@ -225,6 +155,69 @@ {addingCategory ? 'Adding…' : '+ Add'}

+ + + + + + +
+
+ Merge strictness + +
+

How similar articles must be before they're combined into one story.

+
+ Loose + + Strict + {local.mergeStrictness} +
+
+ +
+ Hold before publish +

Wait window to gather more sources before finalizing a story.

+ +
+ +
+ Follow-up articles +

+ Instead of editing a published article, a distinct follow-up is created once enough new + corroborating sources arrive after enough time has passed. +

+
+
+ + +
+
+ + +
+
diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index 5e8c709..db3fad8 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -196,7 +196,6 @@ - {#if form.type === 'youtube'} @@ -396,7 +395,7 @@ } .row { display: grid; - grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 120px; + grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 170px; gap: 10px; padding: 10px; align-items: center; diff --git a/frontend/src/lib/components/admin/WidgetSection.svelte b/frontend/src/lib/components/admin/WidgetSection.svelte new file mode 100644 index 0000000..78d5ca6 --- /dev/null +++ b/frontend/src/lib/components/admin/WidgetSection.svelte @@ -0,0 +1,116 @@ + + +
+
+ + + + + {enabled ? 'Active' : 'Disabled'} + +
+ {#if expanded} +
+ {@render children()} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/admin/WidgetsTab.svelte b/frontend/src/lib/components/admin/WidgetsTab.svelte new file mode 100644 index 0000000..8918255 --- /dev/null +++ b/frontend/src/lib/components/admin/WidgetsTab.svelte @@ -0,0 +1,69 @@ + + +{#each widgetOrder as key, i (key)} + toggle(key)} + canMoveUp={i > 0} + canMoveDown={i < widgetOrder.length - 1} + onMoveUp={() => move(i, -1)} + onMoveDown={() => move(i, 1)} + > + {#if key === 'weather'} + + {:else if key === 'stocks'} + + {:else if key === 'bookmarks'} + + {:else if key === 'poe2'} + + {/if} + +{/each} diff --git a/frontend/src/lib/components/sidebar/Sidebar.svelte b/frontend/src/lib/components/sidebar/Sidebar.svelte index 250d620..8dcda03 100644 --- a/frontend/src/lib/components/sidebar/Sidebar.svelte +++ b/frontend/src/lib/components/sidebar/Sidebar.svelte @@ -1,12 +1,24 @@