From bf0cb1107009c029c73054b4120c40ea2556a5bf Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 00:27:27 +0000 Subject: [PATCH 01/24] Add pluggable widget system; migrate PoE2 as the pilot implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widgets are no longer hand-wired per-feature across scheduler.ts, settings.ts, and the API routes. A new WidgetPlugin interface (backend/src/widgets/types.ts) lets a widget declare its own schema migration, poll interval, routes, and uninstall hook; an installed_widgets registry table replaces the closed widgets/widgetOrder unions on global_settings, and the scheduler/route registration now iterate loaded widgets generically instead of one hardcoded block per widget. PoE2 moves into backend/src/widgets/poe2/ as the first real plugin (its tables renamed to the widget_poe2_ convention, league cache moved into a new generic widget_kv store). Weather/Stocks/Bookmarks get thin wrapper plugins so they share the same dispatch loop without migrating their schema. New admin routes let a widget be uploaded live (POST /api/admin/widgets, JSON body with inline file contents — no archive dependency needed) and removed with full data pruning (DELETE /api/admin/widgets/:id): a host-side safety-net sweep drops any widget__* table and widget_kv rows regardless of whether the widget's own uninstall() hook runs, so deleted widgets don't leave dead data behind. Built-in widgets can't be deleted through this route. Verified against a copy of the real dev DB: old poe2_watchlist data survives the rename, the migration is idempotent on a second boot, and a live-uploaded test widget was polled, queried, and fully deleted (table/kv/on-disk directory all gone) without a restart. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/api/admin.ts | 155 +++++------------- backend/src/api/public.ts | 21 +-- backend/src/bookmarks/plugin.ts | 47 ++++++ backend/src/index.ts | 11 ++ backend/src/poe2/poller.ts | 63 ------- backend/src/queue/scheduler.ts | 80 ++++----- backend/src/stocks/plugin.ts | 54 ++++++ backend/src/storage/db/index.ts | 115 ++++++++++--- backend/src/storage/db/installedWidgets.ts | 70 ++++++++ backend/src/storage/db/settings.ts | 75 ++++++--- backend/src/storage/db/types.ts | 15 ++ backend/src/storage/db/widgetKv.ts | 24 +++ backend/src/weather/plugin.ts | 36 ++++ backend/src/widgets/install.ts | 59 +++++++ backend/src/widgets/manifest.ts | 39 +++++ backend/src/{ => widgets}/poe2/client.ts | 9 +- .../poe2Watchlist.ts => widgets/poe2/db.ts} | 26 +-- backend/src/widgets/poe2/plugin.ts | 107 ++++++++++++ backend/src/widgets/poe2/poll.ts | 74 +++++++++ backend/src/widgets/registry.ts | 69 ++++++++ backend/src/widgets/sweep.ts | 48 ++++++ backend/src/widgets/types.ts | 31 ++++ backend/src/widgets/uninstall.ts | 38 +++++ 23 files changed, 962 insertions(+), 304 deletions(-) create mode 100644 backend/src/bookmarks/plugin.ts delete mode 100644 backend/src/poe2/poller.ts create mode 100644 backend/src/stocks/plugin.ts create mode 100644 backend/src/storage/db/installedWidgets.ts create mode 100644 backend/src/storage/db/widgetKv.ts create mode 100644 backend/src/weather/plugin.ts create mode 100644 backend/src/widgets/install.ts create mode 100644 backend/src/widgets/manifest.ts rename backend/src/{ => widgets}/poe2/client.ts (90%) rename backend/src/{storage/db/poe2Watchlist.ts => widgets/poe2/db.ts} (75%) create mode 100644 backend/src/widgets/poe2/plugin.ts create mode 100644 backend/src/widgets/poe2/poll.ts create mode 100644 backend/src/widgets/registry.ts create mode 100644 backend/src/widgets/sweep.ts create mode 100644 backend/src/widgets/types.ts create mode 100644 backend/src/widgets/uninstall.ts diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index cf2b1f3..868a00d 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -3,20 +3,17 @@ import * as settingsDb from '../storage/db/settings.js'; import * as sourcesDb from '../storage/db/sources.js'; import * as eventsDb from '../storage/db/events.js'; import * as categoriesDb from '../storage/db/categories.js'; -import * as stocksDb from '../storage/db/stocks.js'; -import * as bookmarksDb from '../storage/db/bookmarks.js'; -import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; import * as telegramClient from '../telegram/client.js'; -import { geocodeLocation } from '../weather/client.js'; -import { pollWeatherNow } from '../weather/poller.js'; -import { pollStocksNow } from '../stocks/poller.js'; -import { browseCurrencies, fetchCurrentLeague } from '../poe2/client.js'; -import { pollPoe2Now } from '../poe2/poller.js'; +import { loadedWidgets } from '../widgets/registry.js'; +import { installUploadedWidget } from '../widgets/install.js'; +import { uninstallWidget } from '../widgets/uninstall.js'; +import type { GlobalSettings } from '../storage/db/types.js'; // Not part of GlobalSettings itself (nothing to persist) — computed fresh on every // settings read/write so the Retention tab's "currently using" line and usage bar @@ -43,21 +40,24 @@ export async function registerAdminRoutes(app: FastifyInstance) { if (body.weather) { // Poll immediately rather than waiting for the next scheduler tick (up to 45 // minutes) — the admin just changed the location/unit and expects to see it reflected. - pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); + loadedWidgets + .get('weather') + ?.poll?.run() + .catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); } if (body.widgets) { // Re-enabling a widget (see the Widgets tab) should show fresh data right away - // instead of waiting out its normal cadence (up to 45m/15m/1h) — scheduler.ts - // skips polling entirely while a widget is disabled, so there's nothing recent - // to fall back on otherwise. - if (body.widgets.weather && !before.widgets.weather) { - pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); - } - if (body.widgets.stocks && !before.widgets.stocks) { - pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); - } - if (body.widgets.poe2 && !before.widgets.poe2) { - pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`)); + // instead of waiting out its normal cadence — scheduler.ts skips polling + // entirely while a widget is disabled, so there's nothing recent to fall back + // on otherwise. Generic over every loaded widget with a poll hook, rather than + // one hardcoded branch per widget. + for (const id of Object.keys(body.widgets) as (keyof GlobalSettings['widgets'])[]) { + if (body.widgets[id] && !before.widgets[id]) { + loadedWidgets + .get(id) + ?.poll?.run() + .catch((err) => logger.error(id, `Immediate poll failed: ${err.message}`)); + } } } return { ...settings, categoryPriority: categoriesDb.listCategories() }; @@ -231,106 +231,27 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reply.code(200).send(telegramClient.getStatus()); }); - // --- Weather (config lives in global_settings — see PATCH /api/admin/settings above) --- - app.get('/api/admin/weather/geocode', async (req, reply) => { - const { query } = req.query as { query?: string }; - if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' }); - try { - return await geocodeLocation(query.trim()); - } catch (err) { - return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` }); - } + // Per-widget routes (weather geocode, stocks CRUD, bookmarks CRUD, poe2 browse/ + // watchlist) are registered by each widget's own plugin — see widgets/registry.ts's + // generic registerAdminRoutes loop in index.ts. What's left here is the + // upload/delete lifecycle for pluggable widgets themselves. + + // --- Pluggable widgets (upload/list/delete — see widgets/install.ts, uninstall.ts) --- + app.get('/api/admin/widgets', async () => installedWidgetsDb.listInstalled()); + + app.post('/api/admin/widgets', async (req, reply) => { + const { manifest, files } = req.body as { manifest?: unknown; files?: unknown }; + const result = await installUploadedWidget(manifest, files); + if (!result.ok) return reply.code(400).send({ error: result.error }); + return reply.code(201).send({ id: result.id }); }); - // --- Stocks --- - app.get('/api/admin/stocks', async () => stocksDb.listStockTickers()); - - app.post('/api/admin/stocks', async (req, reply) => { - const { label, symbol } = req.body as { label?: string; symbol?: string }; - if (!label || !label.trim() || !symbol || !symbol.trim()) { - return reply.code(400).send({ error: 'label and symbol are required' }); - } - const created = stocksDb.createStockTicker(label.trim(), symbol.trim()); - // Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap, - // and refreshes every existing ticker's price too. - pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); - return reply.code(201).send(created); - }); - - app.patch('/api/admin/stocks/:id', async (req, reply) => { + app.delete('/api/admin/widgets/:id', async (req, reply) => { const { id } = req.params as { id: string }; - const updated = stocksDb.updateStockTicker(id, req.body as any); - if (!updated) return reply.code(404).send({ error: 'not found' }); - return updated; - }); - - app.delete('/api/admin/stocks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - stocksDb.deleteStockTicker(id); - return reply.code(204).send(); - }); - - // --- Bookmarks --- - app.get('/api/admin/bookmarks', async () => bookmarksDb.listBookmarks()); - - app.post('/api/admin/bookmarks', async (req, reply) => { - const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean }; - if (!name || !name.trim() || !url || !url.trim()) { - return reply.code(400).send({ error: 'name and url are required' }); - } - const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate); - return reply.code(201).send(created); - }); - - app.patch('/api/admin/bookmarks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - const updated = bookmarksDb.updateBookmark(id, req.body as any); - if (!updated) return reply.code(404).send({ error: 'not found' }); - return updated; - }); - - app.delete('/api/admin/bookmarks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - bookmarksDb.deleteBookmark(id); - return reply.code(204).send(); - }); - - // --- PoE2 (league is always auto-detected, never admin-set — see poe2/poller.ts) --- - app.get('/api/admin/poe2/browse', async (_req, reply) => { - try { - const league = await fetchCurrentLeague(); - return await browseCurrencies(league.id); - } catch (err) { - return reply.code(502).send({ error: `poe.ninja unreachable: ${(err as Error).message}` }); - } - }); - - app.get('/api/admin/poe2/watchlist', async () => poe2WatchlistDb.listWatchlist()); - - app.post('/api/admin/poe2/watchlist', async (req, reply) => { - const { base, quote } = req.body as { - base?: { currencyId?: string; name?: string }; - quote?: { currencyId?: string; name?: string }; - }; - if (!base?.currencyId || !base?.name || !quote?.currencyId || !quote?.name) { - return reply.code(400).send({ error: 'base and quote currencies (currencyId, name) are required' }); - } - if (base.currencyId === quote.currencyId) { - return reply.code(400).send({ error: 'Base and quote currencies must be different' }); - } - const created = poe2WatchlistDb.addWatchlistEntry( - { currencyId: base.currencyId, name: base.name }, - { currencyId: quote.currencyId, name: quote.name } - ); - // Poll immediately rather than waiting for the next tick (up to 1 hour) — cheap, - // and refreshes every existing entry's rate too. - pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`)); - return reply.code(201).send(created); - }); - - app.delete('/api/admin/poe2/watchlist/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - poe2WatchlistDb.removeWatchlistEntry(id); + const widget = installedWidgetsDb.getInstalled(id); + if (!widget) return reply.code(404).send({ error: 'not found' }); + if (widget.source === 'builtin') return reply.code(400).send({ error: 'built-in widgets cannot be deleted' }); + await uninstallWidget(id); return reply.code(204).send(); }); diff --git a/backend/src/api/public.ts b/backend/src/api/public.ts index e2a0898..c31c6d5 100644 --- a/backend/src/api/public.ts +++ b/backend/src/api/public.ts @@ -4,9 +4,6 @@ import * as tagsDb from '../storage/db/tags.js'; import * as eventsDb from '../storage/db/events.js'; import * as categoriesDb from '../storage/db/categories.js'; import * as settingsDb from '../storage/db/settings.js'; -import * as stocksDb from '../storage/db/stocks.js'; -import * as bookmarksDb from '../storage/db/bookmarks.js'; -import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js'; import { hasPrivateAccess } from './privateAccess.js'; export async function registerPublicRoutes(app: FastifyInstance) { @@ -70,19 +67,7 @@ export async function registerPublicRoutes(app: FastifyInstance) { return { ...widgets, order: widgetOrder }; }); - // Sidebar widgets — see WeatherTab/StocksTab/BookmarksTab in the admin panel. - app.get('/api/weather', async () => settingsDb.getSettings().weather); - - app.get('/api/stocks', async () => stocksDb.listStockTickers()); - - app.get('/api/bookmarks', async (req) => { - const bookmarks = bookmarksDb.listBookmarks(); - if (hasPrivateAccess(req)) return bookmarks; - return bookmarks.filter((b) => !b.isPrivate); - }); - - app.get('/api/poe2', async () => { - const { leagueName, updatedAt } = settingsDb.getSettings().poe2; - return { leagueName, updatedAt, entries: poe2WatchlistDb.listWatchlist() }; - }); + // Per-widget public routes (GET /api/weather, /api/stocks, /api/bookmarks, /api/poe2) + // are registered by each widget's own plugin — see widgets/registry.ts's generic + // registerPublicRoutes loop in index.ts. } diff --git a/backend/src/bookmarks/plugin.ts b/backend/src/bookmarks/plugin.ts new file mode 100644 index 0000000..bcd546b --- /dev/null +++ b/backend/src/bookmarks/plugin.ts @@ -0,0 +1,47 @@ +import type { WidgetPlugin } from '../widgets/types.js'; +import * as bookmarksDb from '../storage/db/bookmarks.js'; +import { hasPrivateAccess } from '../api/privateAccess.js'; + +// Built-in sidebar "Bookmarks" widget — thin wrapper so it funnels into the same +// route-registration loop as pluggable widgets (see widgets/registry.ts); no poll (purely +// admin-curated links) and no migrate()/uninstall() since its schema isn't moving. Not +// going through the upload/delete lifecycle — see widgets/types.ts. +export const bookmarksPlugin: WidgetPlugin = { + id: 'bookmarks', + displayName: 'Bookmarks', + version: '1.0.0', + + registerPublicRoutes(app) { + app.get('/api/bookmarks', async (req) => { + const bookmarks = bookmarksDb.listBookmarks(); + if (hasPrivateAccess(req)) return bookmarks; + return bookmarks.filter((b) => !b.isPrivate); + }); + }, + + registerAdminRoutes(app) { + app.get('/api/admin/bookmarks', async () => bookmarksDb.listBookmarks()); + + app.post('/api/admin/bookmarks', async (req, reply) => { + const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean }; + if (!name || !name.trim() || !url || !url.trim()) { + return reply.code(400).send({ error: 'name and url are required' }); + } + const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate); + return reply.code(201).send(created); + }); + + app.patch('/api/admin/bookmarks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + const updated = bookmarksDb.updateBookmark(id, req.body as any); + if (!updated) return reply.code(404).send({ error: 'not found' }); + return updated; + }); + + app.delete('/api/admin/bookmarks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + bookmarksDb.deleteBookmark(id); + return reply.code(204).send(); + }); + } +}; diff --git a/backend/src/index.ts b/backend/src/index.ts index 6fbe6d2..e3bfb93 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -14,6 +14,7 @@ import { registerPrivateAccess, privateAccessConfigured } from './api/privateAcc import { startScheduler } from './queue/scheduler.js'; import { initFromSavedSession } from './telegram/client.js'; import { logger } from './storage/db/logs.js'; +import { loadAllWidgets, loadedWidgets } from './widgets/registry.js'; const PORT = Number(process.env.PORT) || 4000; const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173'; @@ -35,6 +36,7 @@ async function main() { migrate(); printApiKeyBanner(); await initFromSavedSession(); + await loadAllWidgets(); const app = Fastify({ logger: false }); @@ -74,6 +76,15 @@ async function main() { await registerAdminRoutes(app); await registerPrivateAccess(app); + // Each loaded widget (built-in or uploaded — see widgets/registry.ts) registers its + // own routes here rather than being hardcoded into public.ts/admin.ts. Runs after + // registerAuth so any /api/admin/* route a widget registers is gated by the same + // X-Api-Key preHandler automatically. + for (const plugin of loadedWidgets.values()) { + plugin.registerPublicRoutes?.(app); + plugin.registerAdminRoutes?.(app); + } + // Fastify's own logger is off (see below) — without this, an unhandled exception // in any route handler produces a bare 500 with zero trace anywhere, including the // admin panel's own Logs tab. This is what "Save failed" with no log entry was. diff --git a/backend/src/poe2/poller.ts b/backend/src/poe2/poller.ts deleted file mode 100644 index e6cd503..0000000 --- a/backend/src/poe2/poller.ts +++ /dev/null @@ -1,63 +0,0 @@ -import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js'; -import * as settingsDb from '../storage/db/settings.js'; -import { logger } from '../storage/db/logs.js'; -import { fetchCurrentLeague, fetchCurrencyValues } from './client.js'; - -const DAY_MS = 24 * 60 * 60_000; - -function pctChange(current: number, past: number | null): number | null { - if (past === null || past === 0) return null; - return ((current - past) / past) * 100; -} - -// Called on a schedule (see queue/scheduler.ts) and immediately after the admin adds a pair -// (see api/admin.ts) — always re-detects the current challenge league fresh (cheap, guarantees -// correctness across league rotations with no separate staleness logic), then one overview -// request covers the whole watchlist. A pair whose base or quote currency is no longer traded -// this league gets its own lastError, it never aborts the rest of the batch. -export async function pollPoe2Now(): Promise { - let league; - try { - league = await fetchCurrentLeague(); - } catch (err) { - logger.error('poe2', `League lookup failed: ${(err as Error).message}`); - return; - } - - const entries = poe2WatchlistDb.listWatchlist(); - if (entries.length === 0) { - const { poe2 } = settingsDb.getSettings(); - settingsDb.updateSettings({ - poe2: { ...poe2, leagueId: league.id, leagueName: league.name, updatedAt: new Date().toISOString() } - }); - return; - } - - try { - const valuesById = await fetchCurrencyValues(league.id); - const now = new Date(); - const nowIso = now.toISOString(); - const cutoff24h = new Date(now.getTime() - DAY_MS).toISOString(); - - for (const entry of entries) { - const baseValue = valuesById.get(entry.baseCurrencyId); - const quoteValue = valuesById.get(entry.quoteCurrencyId); - if (baseValue === undefined || quoteValue === undefined) { - poe2WatchlistDb.markPolled(entry.id, null, null, 'One or both currencies no longer traded in this league'); - continue; - } - - const rate = baseValue / quoteValue; - const change24h = pctChange(rate, poe2WatchlistDb.rateAtOrBefore(entry.id, cutoff24h)); - poe2WatchlistDb.recordRate(entry.id, rate, nowIso); - poe2WatchlistDb.markPolled(entry.id, rate, change24h, null); - } - - poe2WatchlistDb.pruneOldHistory(); - settingsDb.updateSettings({ - poe2: { leagueId: league.id, leagueName: league.name, updatedAt: nowIso } - }); - } catch (err) { - logger.error('poe2', `Watchlist poll failed: ${(err as Error).message}`); - } -} diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index 76dd754..c20a905 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -4,17 +4,48 @@ import { runEventRecaps } from './eventsRecap.js'; import { runRetentionSweep } from './retention.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import * as settingsDb from '../storage/db/settings.js'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; import { logger } from '../storage/db/logs.js'; -import { pollWeatherNow } from '../weather/poller.js'; -import { pollStocksNow } from '../stocks/poller.js'; -import { pollPoe2Now } from '../poe2/poller.js'; +import { loadedWidgets } from '../widgets/registry.js'; +import type { WidgetPlugin } from '../widgets/types.js'; const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency const SYNTHESIS_TICK_MS = 60_000; const RETENTION_TICK_MS = 60 * 60_000; // hourly -const WEATHER_TICK_MS = 45 * 60_000; -const STOCKS_TICK_MS = 15 * 60_000; // per admin spec — stock prices move faster than weather -const POE2_TICK_MS = 60 * 60_000; // poe.ninja's own overview data doesn't refresh faster than hourly, so polling more often than this just re-fetches the same numbers + +// Per-widget setInterval handles, keyed by widget id — lets a single widget's polling be +// started/stopped independently (on live upload/delete, or an enable toggle) without +// touching any other widget's interval. Exported so widgets/install.ts and +// widgets/uninstall.ts can drive it directly. +export const widgetIntervals = new Map(); + +// Starts (or re-starts) polling for one widget — an immediate poll if it's currently +// enabled (unlike RSS sources, whose "due" check makes a brand-new source eligible on the +// very next 1-minute tick, a widget has no such shortcut; without this the sidebar would +// sit empty for up to a full poll interval after every restart or fresh install), then a +// recurring interval that re-checks the enabled flag on every tick — so disabling a widget +// stops the actual external polling, not just hides it in the sidebar. +export function startWidgetPolling(plugin: WidgetPlugin) { + if (!plugin.poll) return; + stopWidgetPolling(plugin.id); + + if (installedWidgetsDb.getInstalled(plugin.id)?.enabled) { + plugin.poll.run().catch((err) => logger.error(plugin.id, `Initial poll failed: ${(err as Error).message}`)); + } + const handle = setInterval(() => { + if (!installedWidgetsDb.getInstalled(plugin.id)?.enabled) return; + plugin.poll!.run().catch((err) => logger.error(plugin.id, `Poll tick failed: ${(err as Error).message}`)); + }, plugin.poll.intervalMs); + widgetIntervals.set(plugin.id, handle); +} + +export function stopWidgetPolling(id: string) { + const handle = widgetIntervals.get(id); + if (handle) { + clearInterval(handle); + widgetIntervals.delete(id); + } +} export function startScheduler() { const provider = () => { @@ -66,37 +97,12 @@ export function startScheduler() { } }, RETENTION_TICK_MS); - // Immediate first call for all three — unlike RSS sources (whose "due" check makes a - // brand-new source eligible on the very next 1-minute tick), weather/stocks/poe2 have - // no such shortcut; without this the sidebar is empty for up to 45/15/60 minutes after - // every restart. Each is also gated on its Widgets-tab enabled flag (see - // admin/settings' consolidated Widgets tab) — disabling a widget stops these external - // calls entirely rather than just hiding the sidebar box, so there's no pointless - // polling for something nobody's looking at. Re-enabling it triggers an immediate - // poll instead (see admin.ts's PATCH /api/admin/settings), same as this initial call. - if (settingsDb.getSettings().widgets.weather) { - pollWeatherNow().catch((err) => logger.error('weather', `Initial poll failed: ${err.message}`)); + for (const plugin of loadedWidgets.values()) { + startWidgetPolling(plugin); } - setInterval(() => { - if (!settingsDb.getSettings().widgets.weather) return; - pollWeatherNow().catch((err) => logger.error('weather', `Poll tick failed: ${err.message}`)); - }, WEATHER_TICK_MS); - if (settingsDb.getSettings().widgets.stocks) { - pollStocksNow().catch((err) => logger.error('stocks', `Initial poll failed: ${err.message}`)); - } - setInterval(() => { - if (!settingsDb.getSettings().widgets.stocks) return; - pollStocksNow().catch((err) => logger.error('stocks', `Poll tick failed: ${err.message}`)); - }, STOCKS_TICK_MS); - - if (settingsDb.getSettings().widgets.poe2) { - pollPoe2Now().catch((err) => logger.error('poe2', `Initial poll failed: ${err.message}`)); - } - setInterval(() => { - if (!settingsDb.getSettings().widgets.poe2) return; - pollPoe2Now().catch((err) => logger.error('poe2', `Poll tick failed: ${err.message}`)); - }, POE2_TICK_MS); - - logger.info('scheduler', 'Started: poll every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h'); + logger.info( + 'scheduler', + `Started: poll every 1m, synthesis every 1m, retention every 1h, ${loadedWidgets.size} widget(s) polling on their own intervals` + ); } diff --git a/backend/src/stocks/plugin.ts b/backend/src/stocks/plugin.ts new file mode 100644 index 0000000..bfab96b --- /dev/null +++ b/backend/src/stocks/plugin.ts @@ -0,0 +1,54 @@ +import type { WidgetPlugin } from '../widgets/types.js'; +import * as stocksDb from '../storage/db/stocks.js'; +import { logger } from '../storage/db/logs.js'; +import { pollStocksNow } from './poller.js'; + +// Built-in sidebar "Stocks" widget — thin wrapper so it funnels into the same +// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts); its +// tickers stay in the dedicated stock_tickers table exactly as before. Not going through +// the upload/delete lifecycle — see widgets/types.ts; no migrate()/uninstall() since its +// schema isn't moving. +export const stocksPlugin: WidgetPlugin = { + id: 'stocks', + displayName: 'Stocks', + version: '1.0.0', + + poll: { + // Per admin spec — stock prices move faster than weather. + intervalMs: 15 * 60_000, + run: pollStocksNow + }, + + registerPublicRoutes(app) { + app.get('/api/stocks', async () => stocksDb.listStockTickers()); + }, + + registerAdminRoutes(app) { + app.get('/api/admin/stocks', async () => stocksDb.listStockTickers()); + + app.post('/api/admin/stocks', async (req, reply) => { + const { label, symbol } = req.body as { label?: string; symbol?: string }; + if (!label || !label.trim() || !symbol || !symbol.trim()) { + return reply.code(400).send({ error: 'label and symbol are required' }); + } + const created = stocksDb.createStockTicker(label.trim(), symbol.trim()); + // Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap, + // and refreshes every existing ticker's price too. + pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); + return reply.code(201).send(created); + }); + + app.patch('/api/admin/stocks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + const updated = stocksDb.updateStockTicker(id, req.body as any); + if (!updated) return reply.code(404).send({ error: 'not found' }); + return updated; + }); + + app.delete('/api/admin/stocks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + stocksDb.deleteStockTicker(id); + return reply.code(204).send(); + }); + } +}; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 2de9caa..692ba01 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -33,6 +33,18 @@ export function migrate() { db.exec('DROP TABLE IF EXISTS poe2_watchlist;'); } + // PoE2 moved into the pluggable-widget system (backend/src/widgets/poe2/) and its + // schema now follows the widget__ naming convention its plugin.migrate() owns — + // a plain rename (metadata-only in SQLite, no row copy) rather than grandfathering + // the old names in via an exceptions list, since it's cheap and PoE2 is meant to be + // the reference implementation of the convention it establishes. + const hasOldPoe2Table = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='poe2_watchlist'`).get(); + if (hasOldPoe2Table) { + db.exec('ALTER TABLE poe2_watchlist RENAME TO widget_poe2_watchlist;'); + db.exec('ALTER TABLE poe2_rate_history RENAME TO widget_poe2_rate_history;'); + db.exec('DROP INDEX IF EXISTS idx_poe2_rate_history_watchlist;'); + } + db.exec(` CREATE TABLE IF NOT EXISTS sources ( id TEXT PRIMARY KEY, @@ -223,35 +235,32 @@ export function migrate() { created_at TEXT NOT NULL ); - -- Sidebar "PoE2" widget — tracks exchange rates between arbitrary currency pairs - -- (see poe2/poller.ts), always against the current challenge league (auto-detected, - -- no admin config). Rate is "1 base = last_rate quote"; both currencies' names are - -- captured at add-time from the browse picker, not re-resolved. No icon columns — - -- the UI doesn't display them. - CREATE TABLE IF NOT EXISTS poe2_watchlist ( - id TEXT PRIMARY KEY, - base_currency_id TEXT NOT NULL, -- opaque id from the exchange overview's lines[].id, e.g. "exalted" - base_name TEXT NOT NULL, - quote_currency_id TEXT NOT NULL, - quote_name TEXT NOT NULL, - priority_rank INTEGER NOT NULL, - last_rate REAL, - last_change_24h REAL, - last_polled_at TEXT, - last_error TEXT, - created_at TEXT NOT NULL + -- Generic config/cache store for pluggable widgets (see widgets/types.ts, + -- storage/db/widgetKv.ts) — lets a widget stay fully prunable by widget_id alone on + -- uninstall without a bespoke table for simple key/value state. + CREATE TABLE IF NOT EXISTS widget_kv ( + widget_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, -- JSON + updated_at TEXT NOT NULL, + PRIMARY KEY (widget_id, key) ); - -- Per-poll rate snapshots for the pairs above — poe.ninja doesn't expose a matching - -- 24h change window, so it's computed ourselves from this history (see - -- poe2/poller.ts). Pruned to the last 2 days on every poll. - CREATE TABLE IF NOT EXISTS poe2_rate_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - watchlist_id TEXT NOT NULL, - rate REAL NOT NULL, - recorded_at TEXT NOT NULL + -- Registry of installed sidebar widgets, both built-in (weather/stocks/bookmarks, + -- and poe2 as the pluggable reference implementation) and uploaded (see + -- widgets/registry.ts, widgets/install.ts). Replaces the old closed + -- widgets/widgetOrder unions on global_settings — see storage/db/settings.ts. + CREATE TABLE IF NOT EXISTS installed_widgets ( + id TEXT PRIMARY KEY, + display_name TEXT NOT NULL, + source TEXT NOT NULL, -- 'builtin' | 'uploaded' + code_path TEXT NOT NULL, -- builtin: identifying module path segment; uploaded: on-disk install dir under ./data/ + enabled INTEGER NOT NULL DEFAULT 1, + priority_rank INTEGER NOT NULL, + version TEXT NOT NULL DEFAULT '1.0.0', + owned_tables TEXT NOT NULL DEFAULT '[]', -- JSON array, self-reported by the plugin — used by the uninstall safety-net sweep + installed_at TEXT NOT NULL ); - CREATE INDEX IF NOT EXISTS idx_poe2_rate_history_watchlist ON poe2_rate_history(watchlist_id, recorded_at); -- Sidebar "Bookmarks" widget — admin-curated links, each independently hidden/public -- via is_private (same private-access lock feature as categories.is_private). @@ -417,4 +426,58 @@ export function migrate() { ); } } + + // One-time seed of the installed_widgets registry from whatever the pre-registry + // install had (upgrade path) or sensible defaults (fresh install) — see + // widgets/registry.ts, storage/db/installedWidgets.ts. Raw SQL rather than importing + // installedWidgets.ts here to avoid a circular import (it imports `db` from this + // file). + const widgetCount = db.prepare('SELECT COUNT(*) as c FROM installed_widgets').get() as { c: number }; + if (widgetCount.c === 0) { + const settingsRow = db.prepare('SELECT * FROM global_settings WHERE id = 1').get() as any; + const order: string[] = settingsRow + ? JSON.parse(settingsRow.widget_order ?? '["weather","stocks","poe2","bookmarks"]') + : ['weather', 'stocks', 'poe2', 'bookmarks']; + const displayNames: Record = { weather: 'Weather', stocks: 'Stocks', bookmarks: 'Bookmarks', poe2: 'PoE2' }; + const codePaths: Record = { weather: 'weather', stocks: 'stocks', bookmarks: 'bookmarks', poe2: 'widgets/poe2' }; + const ownedTables: Record = { poe2: ['widget_poe2_watchlist', 'widget_poe2_rate_history'] }; + const enabledOf = (id: string) => (settingsRow ? !!settingsRow[`widget_${id}_enabled`] : true); + const insertWidget = db.prepare( + `INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, owned_tables, installed_at) + VALUES (?, ?, 'builtin', ?, ?, ?, ?, ?)` + ); + const installedAt = new Date().toISOString(); + order.forEach((id, i) => { + insertWidget.run( + id, + displayNames[id] ?? id, + codePaths[id] ?? id, + enabledOf(id) ? 1 : 0, + i + 1, + JSON.stringify(ownedTables[id] ?? []), + installedAt + ); + }); + } + + // PoE2's league cache (previously bare global_settings columns) moves into the + // generic widget_kv store so it's prunable via the same deleteAllKv('poe2') path as + // everything else the widget owns, rather than needing bespoke column-nulling logic + // on uninstall. One-time copy, guarded on the widget_kv row not already existing. + const poe2CacheRow = db.prepare('SELECT poe2_league_id, poe2_league_name, poe2_updated_at FROM global_settings WHERE id = 1').get() as + | { poe2_league_id: string | null; poe2_league_name: string | null; poe2_updated_at: string | null } + | undefined; + const hasPoe2Kv = db.prepare("SELECT 1 FROM widget_kv WHERE widget_id = 'poe2' AND key = 'leagueCache'").get(); + if (poe2CacheRow?.poe2_league_id && !hasPoe2Kv) { + db.prepare( + `INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES ('poe2', 'leagueCache', ?, ?)` + ).run( + JSON.stringify({ + leagueId: poe2CacheRow.poe2_league_id, + leagueName: poe2CacheRow.poe2_league_name, + updatedAt: poe2CacheRow.poe2_updated_at + }), + new Date().toISOString() + ); + } } diff --git a/backend/src/storage/db/installedWidgets.ts b/backend/src/storage/db/installedWidgets.ts new file mode 100644 index 0000000..64a49c2 --- /dev/null +++ b/backend/src/storage/db/installedWidgets.ts @@ -0,0 +1,70 @@ +import { db } from './index.js'; +import type { InstalledWidget } from './types.js'; + +function rowToWidget(row: any): InstalledWidget { + return { + id: row.id, + displayName: row.display_name, + source: row.source, + codePath: row.code_path, + enabled: !!row.enabled, + priorityRank: row.priority_rank, + version: row.version, + ownedTables: JSON.parse(row.owned_tables), + installedAt: row.installed_at + }; +} + +export function listInstalled(): InstalledWidget[] { + return (db.prepare('SELECT * FROM installed_widgets ORDER BY priority_rank').all() as any[]).map(rowToWidget); +} + +export function getInstalled(id: string): InstalledWidget | null { + const row = db.prepare('SELECT * FROM installed_widgets WHERE id = ?').get(id); + return row ? rowToWidget(row) : null; +} + +export function insertWidget(widget: { + id: string; + displayName: string; + source: 'builtin' | 'uploaded'; + codePath: string; + enabled?: boolean; + priorityRank?: number; + version?: string; + ownedTables?: string[]; +}): InstalledWidget { + const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM installed_widgets').get() as { m: number }; + const priorityRank = widget.priorityRank ?? maxRank.m + 1; + const installedAt = new Date().toISOString(); + db.prepare( + `INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, version, owned_tables, installed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + widget.id, + widget.displayName, + widget.source, + widget.codePath, + widget.enabled === false ? 0 : 1, + priorityRank, + widget.version ?? '1.0.0', + JSON.stringify(widget.ownedTables ?? []), + installedAt + ); + return getInstalled(widget.id)!; +} + +export function setEnabled(id: string, enabled: boolean) { + db.prepare('UPDATE installed_widgets SET enabled = ? WHERE id = ?').run(enabled ? 1 : 0, id); +} + +// Ids in display order — every id must already exist as a row; unlisted ids keep their +// current rank (mirrors the old widget_order JSON array's "admin-sortable" behavior). +export function reorder(ids: string[]) { + const stmt = db.prepare('UPDATE installed_widgets SET priority_rank = ? WHERE id = ?'); + ids.forEach((id, i) => stmt.run(i + 1, id)); +} + +export function deleteInstalled(id: string) { + db.prepare('DELETE FROM installed_widgets WHERE id = ?').run(id); +} diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 604422e..07d2cbd 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -1,7 +1,39 @@ import { db } from './index.js'; import type { GlobalSettings } from './types.js'; +import * as installedWidgetsDb from './installedWidgets.js'; +import { getKv, setKv } from './widgetKv.js'; + +const BUILTIN_WIDGET_IDS = ['weather', 'stocks', 'bookmarks', 'poe2'] as const; + +interface Poe2LeagueCache { + leagueId: string | null; + leagueName: string | null; + updatedAt: string | null; +} + +// widgets/widgetOrder are computed from the installed_widgets registry (see +// storage/db/installedWidgets.ts, widgets/registry.ts) rather than stored as their own +// global_settings columns — the registry is the single source of truth for enable state +// and ordering for every widget, built-in or uploaded. Only the four built-in ids are +// reflected here since GlobalSettings.widgets/widgetOrder are closed unions the frontend +// depends on (uploaded widgets have no frontend representation yet). +function widgetsAndOrder(): Pick { + const installed = installedWidgetsDb.listInstalled(); + const byId = new Map(installed.map((w) => [w.id, w])); + const widgets = { + weather: !!byId.get('weather')?.enabled, + stocks: !!byId.get('stocks')?.enabled, + bookmarks: !!byId.get('bookmarks')?.enabled, + poe2: !!byId.get('poe2')?.enabled + }; + const widgetOrder = installed.map((w) => w.id).filter((id): id is (typeof BUILTIN_WIDGET_IDS)[number] => + (BUILTIN_WIDGET_IDS as readonly string[]).includes(id) + ); + return { widgets, widgetOrder }; +} function rowToSettings(row: any): GlobalSettings { + const poe2Cache = getKv('poe2', 'leagueCache') ?? { leagueId: null, leagueName: null, updatedAt: null }; return { mergeStrictness: row.merge_strictness, holdBeforePublishMinutes: row.hold_before_publish_minutes, @@ -16,13 +48,7 @@ function rowToSettings(row: any): GlobalSettings { fxtwitterBaseUrl: row.fxtwitter_base_url, nitterInstanceUrl: row.nitter_instance_url, telegramMediaMode: row.telegram_media_mode, - widgets: { - weather: !!row.widget_weather_enabled, - stocks: !!row.widget_stocks_enabled, - bookmarks: !!row.widget_bookmarks_enabled, - poe2: !!row.widget_poe2_enabled - }, - widgetOrder: JSON.parse(row.widget_order), + ...widgetsAndOrder(), retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days, @@ -44,11 +70,7 @@ function rowToSettings(row: any): GlobalSettings { alerts: JSON.parse(row.weather_alerts), updatedAt: row.weather_updated_at }, - poe2: { - leagueId: row.poe2_league_id, - leagueName: row.poe2_league_name, - updatedAt: row.poe2_updated_at - } + poe2: poe2Cache }; } @@ -68,6 +90,19 @@ export function updateSettings(patch: Partial): GlobalSettings { poe2: { ...current.poe2, ...(patch.poe2 ?? {}) }, widgets: { ...current.widgets, ...(patch.widgets ?? {}) } }; + + if (patch.widgets) { + for (const id of BUILTIN_WIDGET_IDS) { + if (patch.widgets[id] !== undefined) installedWidgetsDb.setEnabled(id, patch.widgets[id]); + } + } + if (patch.widgetOrder) { + installedWidgetsDb.reorder(patch.widgetOrder); + } + if (patch.poe2) { + setKv('poe2', 'leagueCache', merged.poe2); + } + // Named params (rather than positional `?`) so this list can be reordered or // extended without the column list and the bound-values list silently drifting // out of sync — node:sqlite binds each by its `$name` key, not position. @@ -80,16 +115,12 @@ export function updateSettings(patch: Partial): GlobalSettings { ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models, nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url, telegram_media_mode=$telegram_media_mode, - widget_weather_enabled=$widget_weather_enabled, widget_stocks_enabled=$widget_stocks_enabled, - widget_bookmarks_enabled=$widget_bookmarks_enabled, widget_poe2_enabled=$widget_poe2_enabled, - widget_order=$widget_order, published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days, storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit, weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude, weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit, weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily, - weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at, - poe2_league_id=$poe2_league_id, poe2_league_name=$poe2_league_name, poe2_updated_at=$poe2_updated_at + weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at WHERE id = 1` ).run({ $merge_strictness: merged.mergeStrictness, @@ -105,11 +136,6 @@ export function updateSettings(patch: Partial): GlobalSettings { $fxtwitter_base_url: merged.fxtwitterBaseUrl, $nitter_instance_url: merged.nitterInstanceUrl, $telegram_media_mode: merged.telegramMediaMode, - $widget_weather_enabled: merged.widgets.weather ? 1 : 0, - $widget_stocks_enabled: merged.widgets.stocks ? 1 : 0, - $widget_bookmarks_enabled: merged.widgets.bookmarks ? 1 : 0, - $widget_poe2_enabled: merged.widgets.poe2 ? 1 : 0, - $widget_order: JSON.stringify(merged.widgetOrder), $published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays, $raw_item_max_age_days: merged.retention.rawItemMaxAgeDays, $storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0, @@ -125,10 +151,7 @@ export function updateSettings(patch: Partial): GlobalSettings { $weather_hourly: JSON.stringify(merged.weather.hourly), $weather_daily: JSON.stringify(merged.weather.daily), $weather_alerts: JSON.stringify(merged.weather.alerts), - $weather_updated_at: merged.weather.updatedAt, - $poe2_league_id: merged.poe2.leagueId, - $poe2_league_name: merged.poe2.leagueName, - $poe2_updated_at: merged.poe2.updatedAt + $weather_updated_at: merged.weather.updatedAt }); return getSettings(); } diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 82836f4..b2b8751 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -256,6 +256,21 @@ export interface Poe2WatchlistEntry { createdAt: string; } +/** A row in `installed_widgets` — the registry of both built-in and uploaded sidebar widgets (see widgets/registry.ts). */ +export interface InstalledWidget { + id: string; + displayName: string; + source: 'builtin' | 'uploaded'; + /** Builtin: the module's identifying path segment (e.g. 'weather', 'widgets/poe2'). Uploaded: the on-disk install directory, e.g. './data/widgets-installed/'. */ + codePath: string; + enabled: boolean; + priorityRank: number; + version: string; + /** Self-reported by the plugin (WidgetPlugin.ownedTables) at install/load time — used by the uninstall safety-net sweep. */ + ownedTables: string[]; + installedAt: string; +} + export interface Bookmark { id: string; name: string; diff --git a/backend/src/storage/db/widgetKv.ts b/backend/src/storage/db/widgetKv.ts new file mode 100644 index 0000000..b2a3bf2 --- /dev/null +++ b/backend/src/storage/db/widgetKv.ts @@ -0,0 +1,24 @@ +import { db } from './index.js'; + +// Generic config/cache store for pluggable widgets (see widgets/types.ts) — a widget with +// simple needs (weather-shaped: "last fetched value + timestamp") uses this instead of +// getting bolted-on global_settings columns, so it stays fully prunable by widget_id alone +// on uninstall (see widgets/uninstall.ts) without any host-side schema change. + +export function getKv(widgetId: string, key: string): T | null { + const row = db.prepare('SELECT value FROM widget_kv WHERE widget_id = ? AND key = ?').get(widgetId, key) as + | { value: string } + | undefined; + return row ? (JSON.parse(row.value) as T) : null; +} + +export function setKv(widgetId: string, key: string, value: unknown) { + db.prepare( + `INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(widget_id, key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at` + ).run(widgetId, key, JSON.stringify(value), new Date().toISOString()); +} + +export function deleteAllKv(widgetId: string) { + db.prepare('DELETE FROM widget_kv WHERE widget_id = ?').run(widgetId); +} diff --git a/backend/src/weather/plugin.ts b/backend/src/weather/plugin.ts new file mode 100644 index 0000000..ee8ed77 --- /dev/null +++ b/backend/src/weather/plugin.ts @@ -0,0 +1,36 @@ +import type { WidgetPlugin } from '../widgets/types.js'; +import * as settingsDb from '../storage/db/settings.js'; +import { geocodeLocation } from './client.js'; +import { pollWeatherNow } from './poller.js'; + +// Built-in sidebar "Weather" widget — thin wrapper so it funnels into the same +// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts), while +// its config/cache stay on global_settings' weather_* columns exactly as before. Not going +// through the upload/delete lifecycle — see widgets/types.ts and the plan's "built-in vs +// pluggable" split; no migrate()/uninstall() since its schema isn't moving. +export const weatherPlugin: WidgetPlugin = { + id: 'weather', + displayName: 'Weather', + version: '1.0.0', + + poll: { + intervalMs: 45 * 60_000, + run: pollWeatherNow + }, + + registerPublicRoutes(app) { + app.get('/api/weather', async () => settingsDb.getSettings().weather); + }, + + registerAdminRoutes(app) { + app.get('/api/admin/weather/geocode', async (req, reply) => { + const { query } = req.query as { query?: string }; + if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' }); + try { + return await geocodeLocation(query.trim()); + } catch (err) { + return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` }); + } + }); + } +}; diff --git a/backend/src/widgets/install.ts b/backend/src/widgets/install.ts new file mode 100644 index 0000000..b8a34be --- /dev/null +++ b/backend/src/widgets/install.ts @@ -0,0 +1,59 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; +import { loadUploadedWidget } from './registry.js'; +import { startWidgetPolling } from '../queue/scheduler.js'; +import { validateManifest, type WidgetManifest } from './manifest.js'; + +const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; + +export type InstallResult = { ok: true; id: string } | { ok: false; error: string }; + +// Installs and hot-loads a widget uploaded live to the running backend (see +// api/admin.ts's POST /api/admin/widgets) — writes its files under ./data/, never +// dist/ or src/, so it survives a rebuild/redeploy of the core app. Its migrate() +// runs and its poll interval (if declared) starts immediately, with no restart +// required. NOTE: its registerPublicRoutes/registerAdminRoutes, if declared, do NOT +// take effect until the next restart — Fastify throws "instance is already +// listening" if you try to add a route after app.listen() has resolved, and there's +// no supported way around that short of a much larger request-dispatch redesign. A +// live-installed widget's data/poll side works immediately; its custom HTTP routes +// don't until the process restarts (see widgets/registry.ts's startup discovery, +// which re-registers everything, routes included, on every boot). +export async function installUploadedWidget(manifest: unknown, files: unknown): Promise { + const validationError = validateManifest(manifest, files); + if (validationError) return { ok: false, error: validationError }; + const m = manifest as WidgetManifest; + const f = files as Record; + + if (installedWidgetsDb.getInstalled(m.id)) { + return { ok: false, error: `widget "${m.id}" is already installed` }; + } + + const dir = path.join(WIDGETS_INSTALLED_DIR, m.id); + fs.mkdirSync(dir, { recursive: true }); + for (const [relPath, content] of Object.entries(f)) { + const filePath = path.join(dir, relPath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, content, 'utf8'); + } + fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(m, null, 2), 'utf8'); + + const plugin = await loadUploadedWidget(m.id, dir); + if (!plugin) { + fs.rmSync(dir, { recursive: true, force: true }); + return { ok: false, error: "widget failed to load — check its entry file's default export satisfies WidgetPlugin" }; + } + + installedWidgetsDb.insertWidget({ + id: m.id, + displayName: m.displayName, + source: 'uploaded', + codePath: dir, + version: m.version, + ownedTables: plugin.ownedTables ?? [] + }); + + startWidgetPolling(plugin); + return { ok: true, id: m.id }; +} diff --git a/backend/src/widgets/manifest.ts b/backend/src/widgets/manifest.ts new file mode 100644 index 0000000..540e49a --- /dev/null +++ b/backend/src/widgets/manifest.ts @@ -0,0 +1,39 @@ +const ID_PATTERN = /^[a-z0-9-]{1,40}$/; +const RESERVED_IDS = new Set(['weather', 'stocks', 'bookmarks', 'poe2']); +const MAX_TOTAL_BYTES = 5 * 1024 * 1024; + +export interface WidgetManifest { + id: string; + displayName: string; + version: string; + /** Relative path within `files` to the ESM entry point, e.g. "index.mjs" — a default export satisfying WidgetPlugin. */ + entry: string; +} + +// Upload body shape: { manifest, files: { "index.mjs": "", ... } } — plain +// JSON rather than a literal zip, so no archive/multipart dependency is needed (see +// widgets/install.ts). Returns a human-readable error string, or null if valid. +export function validateManifest(manifest: unknown, files: unknown): string | null { + if (!manifest || typeof manifest !== 'object') return 'manifest is required'; + const m = manifest as Record; + if (typeof m.id !== 'string' || !ID_PATTERN.test(m.id)) return 'manifest.id must match /^[a-z0-9-]{1,40}$/'; + if (RESERVED_IDS.has(m.id)) return `id "${m.id}" is reserved for a built-in widget`; + if (typeof m.displayName !== 'string' || !m.displayName.trim()) return 'manifest.displayName is required'; + if (typeof m.version !== 'string' || !m.version.trim()) return 'manifest.version is required'; + if (typeof m.entry !== 'string' || !m.entry.trim()) return 'manifest.entry is required'; + + if (!files || typeof files !== 'object' || Array.isArray(files)) return 'files must be a non-empty object'; + const entries = Object.entries(files as Record); + if (entries.length === 0) return 'files must be a non-empty object'; + if (!(m.entry in (files as Record))) return `entry "${m.entry as string}" not found in files`; + + let totalBytes = 0; + for (const [relPath, content] of entries) { + if (typeof content !== 'string') return `files["${relPath}"] must be a string`; + if (relPath.startsWith('/') || relPath.split('/').some((seg) => seg === '..')) return `invalid file path "${relPath}"`; + totalBytes += Buffer.byteLength(content, 'utf8'); + } + if (totalBytes > MAX_TOTAL_BYTES) return `bundle exceeds ${MAX_TOTAL_BYTES}-byte limit`; + + return null; +} diff --git a/backend/src/poe2/client.ts b/backend/src/widgets/poe2/client.ts similarity index 90% rename from backend/src/poe2/client.ts rename to backend/src/widgets/poe2/client.ts index d3755db..74ef227 100644 --- a/backend/src/poe2/client.ts +++ b/backend/src/widgets/poe2/client.ts @@ -1,5 +1,5 @@ // poe.ninja's public PoE2 economy API — free, no account or API key required. This is the -// only file that talks to it; poller.ts orchestrates when/how results get saved, same +// only file that talks to it; poll.ts orchestrates when/how results get saved, same // separation as backend/src/telegram/ and backend/src/weather/ keep between the raw client // and their callers. // @@ -13,7 +13,7 @@ // change, which poe.ninja's overview doesn't expose directly. Every line's `primaryValue` is // expressed in the same (unspecified, and irrelevant) reference currency, so any pair's rate // is just baseValue / quoteValue with the reference cancelling out — see fetchCurrencyValues -// below and poe2/poller.ts, which self-computes change from its own polling history instead. +// below and poll.ts, which self-computes change from its own polling history instead. const BASE_URL = 'https://poe.ninja'; export interface LeagueInfo { @@ -57,8 +57,9 @@ async function fetchCurrencyOverview(leagueId: string): Promise { const { lines, items } = await fetchCurrencyOverview(leagueId); const nameById = new Map(items.map((item) => [item.id, item.name])); diff --git a/backend/src/storage/db/poe2Watchlist.ts b/backend/src/widgets/poe2/db.ts similarity index 75% rename from backend/src/storage/db/poe2Watchlist.ts rename to backend/src/widgets/poe2/db.ts index beba793..8cd6500 100644 --- a/backend/src/storage/db/poe2Watchlist.ts +++ b/backend/src/widgets/poe2/db.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { db } from './index.js'; -import type { Poe2WatchlistEntry } from './types.js'; +import { db } from '../../storage/db/index.js'; +import type { Poe2WatchlistEntry } from '../../storage/db/types.js'; interface CurrencyRef { currencyId: string; @@ -24,21 +24,21 @@ function rowToEntry(row: any): Poe2WatchlistEntry { } export function listWatchlist(): Poe2WatchlistEntry[] { - const rows = db.prepare('SELECT * FROM poe2_watchlist ORDER BY priority_rank').all(); + const rows = db.prepare('SELECT * FROM widget_poe2_watchlist ORDER BY priority_rank').all(); return rows.map(rowToEntry); } -// No update() — currencies are picked from a live browse list (see poe2/client.ts's +// No update() — currencies are picked from a live browse list (see widgets/poe2/client.ts's // browseCurrencies), not typed, so there's nothing to edit; remove and re-add covers the // rare "picked the wrong one" case. export function addWatchlistEntry(base: CurrencyRef, quote: CurrencyRef): Poe2WatchlistEntry { const id = `poe2-${base.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${quote.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${randomUUID().slice(0, 6)}` .replace(/-+/g, '-') .replace(/(^-|-$)/g, ''); - const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM poe2_watchlist').get() as { m: number }; + const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM widget_poe2_watchlist').get() as { m: number }; const createdAt = new Date().toISOString(); db.prepare( - `INSERT INTO poe2_watchlist + `INSERT INTO widget_poe2_watchlist (id, base_currency_id, base_name, quote_currency_id, quote_name, priority_rank, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)` ).run(id, base.currencyId, base.name, quote.currencyId, quote.name, maxRank.m + 1, createdAt); @@ -58,14 +58,14 @@ export function addWatchlistEntry(base: CurrencyRef, quote: CurrencyRef): Poe2Wa } export function removeWatchlistEntry(id: string) { - db.prepare('DELETE FROM poe2_rate_history WHERE watchlist_id = ?').run(id); - db.prepare('DELETE FROM poe2_watchlist WHERE id = ?').run(id); + db.prepare('DELETE FROM widget_poe2_rate_history WHERE watchlist_id = ?').run(id); + db.prepare('DELETE FROM widget_poe2_watchlist WHERE id = ?').run(id); } -// One snapshot per poll (see poe2/poller.ts) — the raw material 24h change is computed +// One snapshot per poll (see widgets/poe2/poll.ts) — the raw material 24h change is computed // from, since poe.ninja itself doesn't expose per-pair rates or a matching change window. export function recordRate(watchlistId: string, rate: number, recordedAt: string) { - db.prepare('INSERT INTO poe2_rate_history (watchlist_id, rate, recorded_at) VALUES (?, ?, ?)').run( + db.prepare('INSERT INTO widget_poe2_rate_history (watchlist_id, rate, recorded_at) VALUES (?, ?, ?)').run( watchlistId, rate, recordedAt @@ -77,14 +77,14 @@ export function recordRate(watchlistId: string, rate: number, recordedAt: string // than fabricating a 0% figure. export function rateAtOrBefore(watchlistId: string, cutoffIso: string): number | null { const row = db - .prepare('SELECT rate FROM poe2_rate_history WHERE watchlist_id = ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1') + .prepare('SELECT rate FROM widget_poe2_rate_history WHERE watchlist_id = ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1') .get(watchlistId, cutoffIso) as { rate: number } | undefined; return row ? row.rate : null; } export function markPolled(id: string, rate: number | null, change24h: number | null, error: string | null) { db.prepare( - `UPDATE poe2_watchlist + `UPDATE widget_poe2_watchlist SET last_rate = ?, last_change_24h = ?, last_polled_at = ?, last_error = ? WHERE id = ?` ).run(rate, change24h, new Date().toISOString(), error, id); @@ -94,5 +94,5 @@ export function markPolled(id: string, rate: number | null, change24h: number | // poll to be briefly late without losing the data point it needs. export function pruneOldHistory() { const cutoff = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(); - db.prepare('DELETE FROM poe2_rate_history WHERE recorded_at < ?').run(cutoff); + db.prepare('DELETE FROM widget_poe2_rate_history WHERE recorded_at < ?').run(cutoff); } diff --git a/backend/src/widgets/poe2/plugin.ts b/backend/src/widgets/poe2/plugin.ts new file mode 100644 index 0000000..92623df --- /dev/null +++ b/backend/src/widgets/poe2/plugin.ts @@ -0,0 +1,107 @@ +import type { DatabaseSync } from 'node:sqlite'; +import type { WidgetPlugin } from '../types.js'; +import { deleteAllKv } from '../../storage/db/widgetKv.js'; +import { logger } from '../../storage/db/logs.js'; +import * as poe2Db from './db.js'; +import { browseCurrencies, fetchCurrentLeague } from './client.js'; +import { pollPoe2Now, getLeagueCache } from './poll.js'; + +const OWNED_TABLES = ['widget_poe2_watchlist', 'widget_poe2_rate_history']; + +// Sidebar "PoE2" widget — tracks exchange rates between arbitrary currency pairs, always +// against the current challenge league (auto-detected, no admin config). The pilot +// implementation of the pluggable-widget system (see widgets/types.ts) — every other +// widget still ships in-process, but this one exercises the full migrate/poll/routes/ +// uninstall lifecycle a live-uploaded widget would also go through. +export const poe2Plugin: WidgetPlugin = { + id: 'poe2', + displayName: 'PoE2', + version: '1.0.0', + ownedTables: OWNED_TABLES, + + migrate(db: DatabaseSync) { + db.exec(` + CREATE TABLE IF NOT EXISTS widget_poe2_watchlist ( + id TEXT PRIMARY KEY, + base_currency_id TEXT NOT NULL, + base_name TEXT NOT NULL, + quote_currency_id TEXT NOT NULL, + quote_name TEXT NOT NULL, + priority_rank INTEGER NOT NULL, + last_rate REAL, + last_change_24h REAL, + last_polled_at TEXT, + last_error TEXT, + created_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS widget_poe2_rate_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + watchlist_id TEXT NOT NULL, + rate REAL NOT NULL, + recorded_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_widget_poe2_rate_history_watchlist ON widget_poe2_rate_history(watchlist_id, recorded_at); + `); + }, + + poll: { + // poe.ninja's own overview data doesn't refresh faster than hourly, so polling + // more often than this just re-fetches the same numbers. + intervalMs: 60 * 60_000, + run: pollPoe2Now + }, + + registerPublicRoutes(app) { + app.get('/api/poe2', async () => { + const { leagueName, updatedAt } = getLeagueCache(); + return { leagueName, updatedAt, entries: poe2Db.listWatchlist() }; + }); + }, + + registerAdminRoutes(app) { + // League is always auto-detected, never admin-set. + app.get('/api/admin/poe2/browse', async (_req, reply) => { + try { + const league = await fetchCurrentLeague(); + return await browseCurrencies(league.id); + } catch (err) { + return reply.code(502).send({ error: `poe.ninja unreachable: ${(err as Error).message}` }); + } + }); + + app.get('/api/admin/poe2/watchlist', async () => poe2Db.listWatchlist()); + + app.post('/api/admin/poe2/watchlist', async (req, reply) => { + const { base, quote } = req.body as { + base?: { currencyId?: string; name?: string }; + quote?: { currencyId?: string; name?: string }; + }; + if (!base?.currencyId || !base?.name || !quote?.currencyId || !quote?.name) { + return reply.code(400).send({ error: 'base and quote currencies (currencyId, name) are required' }); + } + if (base.currencyId === quote.currencyId) { + return reply.code(400).send({ error: 'Base and quote currencies must be different' }); + } + const created = poe2Db.addWatchlistEntry( + { currencyId: base.currencyId, name: base.name }, + { currencyId: quote.currencyId, name: quote.name } + ); + // Poll immediately rather than waiting for the next tick (up to 1 hour) — cheap, + // and refreshes every existing entry's rate too. + pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`)); + return reply.code(201).send(created); + }); + + app.delete('/api/admin/poe2/watchlist/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + poe2Db.removeWatchlistEntry(id); + return reply.code(204).send(); + }); + }, + + uninstall(db: DatabaseSync) { + db.exec('DROP TABLE IF EXISTS widget_poe2_rate_history;'); + db.exec('DROP TABLE IF EXISTS widget_poe2_watchlist;'); + deleteAllKv('poe2'); + } +}; diff --git a/backend/src/widgets/poe2/poll.ts b/backend/src/widgets/poe2/poll.ts new file mode 100644 index 0000000..86804a9 --- /dev/null +++ b/backend/src/widgets/poe2/poll.ts @@ -0,0 +1,74 @@ +import * as poe2Db from './db.js'; +import { getKv, setKv } from '../../storage/db/widgetKv.js'; +import { logger } from '../../storage/db/logs.js'; +import { fetchCurrentLeague, fetchCurrencyValues } from './client.js'; + +const DAY_MS = 24 * 60 * 60_000; +const WIDGET_ID = 'poe2'; + +interface LeagueCache { + leagueId: string | null; + leagueName: string | null; + updatedAt: string | null; +} + +// Cache of what the last poll learned about the current league — no admin-set config +// (unlike weather): the league is always auto-detected, so this is purely a cache. Lives +// in the generic widget_kv store (see storage/db/widgetKv.ts) so it's prunable via the +// same deleteAllKv('poe2') path as everything else this widget owns. +export function getLeagueCache(): LeagueCache { + return getKv(WIDGET_ID, 'leagueCache') ?? { leagueId: null, leagueName: null, updatedAt: null }; +} + +function pctChange(current: number, past: number | null): number | null { + if (past === null || past === 0) return null; + return ((current - past) / past) * 100; +} + +// Called on a schedule (see queue/scheduler.ts, via plugin.poll) and immediately after the +// admin adds a pair (see plugin.ts's admin routes) — always re-detects the current +// challenge league fresh (cheap, guarantees correctness across league rotations with no +// separate staleness logic), then one overview request covers the whole watchlist. A pair +// whose base or quote currency is no longer traded this league gets its own lastError, it +// never aborts the rest of the batch. +export async function pollPoe2Now(): Promise { + let league; + try { + league = await fetchCurrentLeague(); + } catch (err) { + logger.error('poe2', `League lookup failed: ${(err as Error).message}`); + return; + } + + const entries = poe2Db.listWatchlist(); + if (entries.length === 0) { + setKv(WIDGET_ID, 'leagueCache', { leagueId: league.id, leagueName: league.name, updatedAt: new Date().toISOString() }); + return; + } + + try { + const valuesById = await fetchCurrencyValues(league.id); + const now = new Date(); + const nowIso = now.toISOString(); + const cutoff24h = new Date(now.getTime() - DAY_MS).toISOString(); + + for (const entry of entries) { + const baseValue = valuesById.get(entry.baseCurrencyId); + const quoteValue = valuesById.get(entry.quoteCurrencyId); + if (baseValue === undefined || quoteValue === undefined) { + poe2Db.markPolled(entry.id, null, null, 'One or both currencies no longer traded in this league'); + continue; + } + + const rate = baseValue / quoteValue; + const change24h = pctChange(rate, poe2Db.rateAtOrBefore(entry.id, cutoff24h)); + poe2Db.recordRate(entry.id, rate, nowIso); + poe2Db.markPolled(entry.id, rate, change24h, null); + } + + poe2Db.pruneOldHistory(); + setKv(WIDGET_ID, 'leagueCache', { leagueId: league.id, leagueName: league.name, updatedAt: nowIso }); + } catch (err) { + logger.error('poe2', `Watchlist poll failed: ${(err as Error).message}`); + } +} diff --git a/backend/src/widgets/registry.ts b/backend/src/widgets/registry.ts new file mode 100644 index 0000000..a45931d --- /dev/null +++ b/backend/src/widgets/registry.ts @@ -0,0 +1,69 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { db } from '../storage/db/index.js'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; +import { logger } from '../storage/db/logs.js'; +import type { WidgetPlugin } from './types.js'; +import { sweepOrphanedWidgetData } from './sweep.js'; +import { weatherPlugin } from '../weather/plugin.js'; +import { stocksPlugin } from '../stocks/plugin.js'; +import { bookmarksPlugin } from '../bookmarks/plugin.js'; +import { poe2Plugin } from './poe2/plugin.js'; + +// Both built-in and uploaded widgets funnel into this one map — scheduler dispatch and +// route registration (see index.ts, queue/scheduler.ts) iterate it generically instead of +// hardcoding a block per widget, mirroring ingestion/poller.ts's +// Record dispatch, just keyed by a dynamic string id. +export const loadedWidgets = new Map(); + +function registerBuiltinWidget(plugin: WidgetPlugin) { + plugin.migrate?.(db); + loadedWidgets.set(plugin.id, plugin); +} + +// Called once at startup (see index.ts, after migrate() and before route registration / +// startScheduler()) and again after a live upload (see widgets/install.ts) to pick up just +// the newly-installed one without reloading everything else. +export async function loadAllWidgets(): Promise { + loadedWidgets.clear(); + + registerBuiltinWidget(weatherPlugin); + registerBuiltinWidget(stocksPlugin); + registerBuiltinWidget(bookmarksPlugin); + registerBuiltinWidget(poe2Plugin); + + for (const row of installedWidgetsDb.listInstalled().filter((w) => w.source === 'uploaded')) { + await loadUploadedWidget(row.id, row.codePath); + } + + sweepOrphanedWidgetData( + db, + installedWidgetsDb.listInstalled().map((w) => w.id) + ); +} + +// A widget whose code fails to load logs an error and is simply absent from +// loadedWidgets — scheduler/route registration skip an id with no entry. Its +// installed_widgets row stays, so it's still visible and deletable from the admin side +// (see widgets/uninstall.ts, which doesn't require the widget's own code to be loadable). +export async function loadUploadedWidget(id: string, codePath: string): Promise { + try { + const manifest = JSON.parse(fs.readFileSync(path.join(codePath, 'manifest.json'), 'utf8')) as { entry: string }; + const entryUrl = pathToFileURL(path.join(codePath, manifest.entry)).href; + const mod = await import(entryUrl); + const plugin: WidgetPlugin = mod.default; + if (!plugin || typeof plugin.id !== 'string') { + throw new Error('module has no default-exported WidgetPlugin'); + } + if (plugin.id !== id) { + throw new Error(`plugin id "${plugin.id}" does not match installed id "${id}"`); + } + plugin.migrate?.(db); + loadedWidgets.set(id, plugin); + return plugin; + } catch (err) { + logger.error('widgets', `Failed to load uploaded widget "${id}": ${(err as Error).message}`); + return null; + } +} diff --git a/backend/src/widgets/sweep.ts b/backend/src/widgets/sweep.ts new file mode 100644 index 0000000..0184ad1 --- /dev/null +++ b/backend/src/widgets/sweep.ts @@ -0,0 +1,48 @@ +import type { DatabaseSync } from 'node:sqlite'; + +// Host-side safety net for widget data pruning (see widgets/types.ts's uninstall contract +// and the plan's "data isolation strategy"). This alone is sufficient to fully clean a +// widget's data even with zero cooperation from its own code — a widget's own uninstall() +// hook (if present) is an optimization/extension point, not a requirement for correctness. + +function ownedTablesForId(db: DatabaseSync, widgetId: string, extraOwnedTables: string[] = []): string[] { + const prefix = `widget_${widgetId}_`; + const rows = db + .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? ESCAPE '\\'`) + .all(`${prefix.replace(/_/g, '\\_')}%`) as { name: string }[]; + return Array.from(new Set([...rows.map((r) => r.name), ...extraOwnedTables])); +} + +// Drops every table owned by a single widget (by the widget__ naming convention, plus +// any self-reported extraOwnedTables for a grandfathered name), and clears its widget_kv +// rows. Safe to call even if the widget's own code is missing/broken/never uninstall()ed. +export function sweepWidgetData(db: DatabaseSync, widgetId: string, extraOwnedTables: string[] = []) { + for (const table of ownedTablesForId(db, widgetId, extraOwnedTables)) { + db.exec(`DROP TABLE IF EXISTS "${table}";`); + } + db.prepare('DELETE FROM widget_kv WHERE widget_id = ?').run(widgetId); +} + +// Full-registry sweep, run once at every startup after the registry loads (see +// widgets/registry.ts) — catches anything left behind by a process crash mid-uninstall, or +// a manually-edited installed_widgets table, that the per-widget sweep above never got a +// chance to run for. +export function sweepOrphanedWidgetData(db: DatabaseSync, knownWidgetIds: string[]) { + const known = new Set(knownWidgetIds); + const tables = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'widget\\_%' ESCAPE '\\'`).all() as { + name: string; + }[]; + for (const { name } of tables) { + if (name === 'widget_kv') continue; // the shared kv table itself, not a per-widget owned table + const id = name.slice('widget_'.length).split('_')[0]; + if (!known.has(id)) { + db.exec(`DROP TABLE IF EXISTS "${name}";`); + } + } + const kvWidgetIds = db.prepare('SELECT DISTINCT widget_id FROM widget_kv').all() as { widget_id: string }[]; + const orphanedKvIds = kvWidgetIds.map((r) => r.widget_id).filter((id) => !known.has(id)); + if (orphanedKvIds.length > 0) { + const placeholders = orphanedKvIds.map(() => '?').join(','); + db.prepare(`DELETE FROM widget_kv WHERE widget_id IN (${placeholders})`).run(...orphanedKvIds); + } +} diff --git a/backend/src/widgets/types.ts b/backend/src/widgets/types.ts new file mode 100644 index 0000000..b445d2b --- /dev/null +++ b/backend/src/widgets/types.ts @@ -0,0 +1,31 @@ +import type { DatabaseSync } from 'node:sqlite'; +import type { FastifyInstance } from 'fastify'; + +/** + * A widget's own lifecycle hooks — deliberately all-optional (unlike + * ingestion/adapters/base.ts's SourceAdapter, which has one mandatory fetch()) because + * unlike source adapters, widgets don't share a common downstream table or data shape + * to normalize into; each is a self-contained module that opts into only what it needs. + */ +export interface WidgetPlugin { + /** Stable id — also the table-name (`widget__*`) and directory-name namespace. Validated on install: /^[a-z0-9-]{1,40}$/ */ + id: string; + displayName: string; + version: string; + + /** Widget-owned schema. Must be idempotent (CREATE TABLE IF NOT EXISTS style), same contract as the host's own migrate(). */ + migrate?(db: DatabaseSync): void; + + /** Self-reported table names, for the uninstall safety-net sweep — belt-and-suspenders alongside the widget__ naming convention (see widgets/sweep.ts). */ + ownedTables?: string[]; + + /** Host owns the setInterval/enable-gating (see queue/scheduler.ts); the plugin just does the fetch-and-persist work. */ + poll?: { intervalMs: number; run(): Promise }; + + /** Called once at load time (startup, or immediately after a live upload). Admin routes registered here are auto-gated by the existing X-Api-Key preHandler (see api/auth.ts), since it matches on any /api/admin/* path. */ + registerPublicRoutes?(app: FastifyInstance): void; + registerAdminRoutes?(app: FastifyInstance): void; + + /** Best-effort cleanup, called before the host's safety-net sweep on delete. Should not assume its own migrate() succeeded or that external APIs are reachable — the sweep is the real guarantee, this is just an extension point. */ + uninstall?(db: DatabaseSync): void; +} diff --git a/backend/src/widgets/uninstall.ts b/backend/src/widgets/uninstall.ts new file mode 100644 index 0000000..46f5b1a --- /dev/null +++ b/backend/src/widgets/uninstall.ts @@ -0,0 +1,38 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { db } from '../storage/db/index.js'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; +import { logger } from '../storage/db/logs.js'; +import { loadedWidgets } from './registry.js'; +import { sweepWidgetData } from './sweep.js'; +import { stopWidgetPolling } from '../queue/scheduler.js'; + +const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; +const WIDGETS_DATA_DIR = process.env.WIDGETS_DATA_DIR || './data/widgets-data'; + +// Fully removes an uploaded widget — safe to call even if its code is already +// broken/missing (the plugin.uninstall() step is best-effort; sweepWidgetData is the real +// guarantee, matching every table/kv row/on-disk file regardless of the widget's own +// cooperation). Callers (see api/admin.ts's DELETE /api/admin/widgets/:id) are +// responsible for rejecting built-in widgets before calling this. +export async function uninstallWidget(id: string): Promise { + stopWidgetPolling(id); + + const plugin = loadedWidgets.get(id); + if (plugin?.uninstall) { + try { + plugin.uninstall(db); + } catch (err) { + logger.error('widgets', `uninstall() hook failed for "${id}": ${(err as Error).message}`); + } + } + + const row = installedWidgetsDb.getInstalled(id); + sweepWidgetData(db, id, row?.ownedTables ?? []); + + fs.rmSync(path.join(WIDGETS_INSTALLED_DIR, id), { recursive: true, force: true }); + fs.rmSync(path.join(WIDGETS_DATA_DIR, id), { recursive: true, force: true }); + + loadedWidgets.delete(id); + installedWidgetsDb.deleteInstalled(id); +} From 837aa77bfca20b135f5866589449360efb42160b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 01:16:47 +0000 Subject: [PATCH 02/24] Full widget parity, /api/widget/ route namespace, live sidebar rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Weather, Stocks, and Bookmarks move into backend/src/widgets// alongside PoE2, each a full WidgetPlugin rather than a thin wrapper: their tables are renamed to the widget__ convention (stock_tickers, bookmarks) or moved off global_settings entirely into the generic widget_kv store (weather's location/unit config and forecast cache). Every widget's routes move under a consistent /api/widget/ (public) and /api/admin/widget/ (admin) namespace, replacing the previous flat /api/weather, /api/admin/poe2/browse, etc. The registry-management routes (list/upload/enable/delete any widget) stay at /api/admin/widgets since they address the collection, not one widget's own data. This also closes the gap where an uploaded widget had backend data plumbing but no visible presence anywhere: a widget's poll() can now publish to a generic GET /api/widget/:id/report feed (registered once, works for any id with zero per-widget route registration, so it's live immediately after upload with no restart); the sidebar renders it via a new GenericWidgetCard, or via a new DynamicWidgetSlot that dynamic-imports an optional pre-built vanilla-JS frontend bundle the widget can ship (served from a new /widget-assets/:id/* route) — plain browser import(), not blocked by SvelteKit's ahead-of-time Svelte compilation the way raw .svelte source would be. A widget's own custom API routes still need a restart to register (a hard Fastify limitation), but its data/poll/report and any custom frontend UI now work fully live. The admin Widgets tab gained an upload form and a list of installed pluggable widgets with enable/delete. Verified against a copy of the real dev DB (all three renames + the weather config/cache migration fire once and are idempotent on a second boot) and through a real browser: uploaded a widget with both a report-driven poll and a custom frontend bundle, confirmed it renders live in the sidebar with no backend restart, then deleted it and confirmed it disappears along with all of its data (table/kv rows/on-disk files). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/api/admin.ts | 28 ++- backend/src/api/public.ts | 34 ++- backend/src/bookmarks/plugin.ts | 47 ---- backend/src/index.ts | 21 ++ backend/src/stocks/plugin.ts | 54 ----- backend/src/storage/db/index.ts | 97 ++++---- backend/src/storage/db/installedWidgets.ts | 7 +- backend/src/storage/db/settings.ts | 52 +---- backend/src/storage/db/types.ts | 69 +----- backend/src/weather/plugin.ts | 36 --- .../bookmarks.ts => widgets/bookmarks/db.ts} | 18 +- backend/src/widgets/bookmarks/plugin.ts | 68 ++++++ backend/src/widgets/install.ts | 3 +- backend/src/widgets/manifest.ts | 8 + backend/src/widgets/poe2/plugin.ts | 10 +- backend/src/widgets/registry.ts | 6 +- backend/src/widgets/report.ts | 13 ++ backend/src/{ => widgets}/stocks/client.ts | 0 backend/src/widgets/stocks/db.ts | 54 +++++ backend/src/widgets/stocks/plugin.ts | 94 ++++++++ .../poller.ts => widgets/stocks/poll.ts} | 12 +- backend/src/{ => widgets}/weather/client.ts | 0 backend/src/widgets/weather/db.ts | 103 +++++++++ backend/src/widgets/weather/plugin.ts | 47 ++++ .../poller.ts => widgets/weather/poll.ts} | 44 ++-- frontend/src/lib/adminApi.ts | 57 +++-- frontend/src/lib/adminTypes.ts | 29 ++- frontend/src/lib/api.ts | 8 +- .../src/lib/components/admin/Poe2Tab.svelte | 9 +- .../lib/components/admin/WeatherTab.svelte | 17 +- .../lib/components/admin/WidgetsTab.svelte | 213 +++++++++++++++++- .../sidebar/DynamicWidgetSlot.svelte | 67 ++++++ .../sidebar/GenericWidgetCard.svelte | 124 ++++++++++ .../src/lib/components/sidebar/Sidebar.svelte | 9 + frontend/src/lib/types.ts | 18 ++ .../src/routes/admin/settings/+page.svelte | 10 +- frontend/src/routes/admin/settings/+page.ts | 43 +++- 37 files changed, 1119 insertions(+), 410 deletions(-) delete mode 100644 backend/src/bookmarks/plugin.ts delete mode 100644 backend/src/stocks/plugin.ts delete mode 100644 backend/src/weather/plugin.ts rename backend/src/{storage/db/bookmarks.ts => widgets/bookmarks/db.ts} (62%) create mode 100644 backend/src/widgets/bookmarks/plugin.ts create mode 100644 backend/src/widgets/report.ts rename backend/src/{ => widgets}/stocks/client.ts (100%) create mode 100644 backend/src/widgets/stocks/db.ts create mode 100644 backend/src/widgets/stocks/plugin.ts rename backend/src/{stocks/poller.ts => widgets/stocks/poll.ts} (65%) rename backend/src/{ => widgets}/weather/client.ts (100%) create mode 100644 backend/src/widgets/weather/db.ts create mode 100644 backend/src/widgets/weather/plugin.ts rename backend/src/{weather/poller.ts => widgets/weather/poll.ts} (52%) create mode 100644 frontend/src/lib/components/sidebar/DynamicWidgetSlot.svelte create mode 100644 frontend/src/lib/components/sidebar/GenericWidgetCard.svelte diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 868a00d..0912941 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -37,14 +37,6 @@ export async function registerAdminRoutes(app: FastifyInstance) { } const before = settingsDb.getSettings(); const settings = withStorageUsed(settingsDb.updateSettings(body)); - if (body.weather) { - // Poll immediately rather than waiting for the next scheduler tick (up to 45 - // minutes) — the admin just changed the location/unit and expects to see it reflected. - loadedWidgets - .get('weather') - ?.poll?.run() - .catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); - } if (body.widgets) { // Re-enabling a widget (see the Widgets tab) should show fresh data right away // instead of waiting out its normal cadence — scheduler.ts skips polling @@ -246,6 +238,26 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reply.code(201).send({ id: result.id }); }); + app.patch('/api/admin/widgets/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + const { enabled } = req.body as { enabled?: boolean }; + const widget = installedWidgetsDb.getInstalled(id); + if (!widget) return reply.code(404).send({ error: 'not found' }); + if (typeof enabled === 'boolean') { + installedWidgetsDb.setEnabled(id, enabled); + // Re-enabling should show fresh data right away rather than waiting out the + // widget's own poll interval — same "immediate poll on enable" behavior the + // 4 built-ins get via PATCH /api/admin/settings above. + if (enabled && !widget.enabled) { + loadedWidgets + .get(id) + ?.poll?.run() + .catch((err) => logger.error(id, `Immediate poll failed: ${err.message}`)); + } + } + return installedWidgetsDb.getInstalled(id); + }); + app.delete('/api/admin/widgets/:id', async (req, reply) => { const { id } = req.params as { id: string }; const widget = installedWidgetsDb.getInstalled(id); diff --git a/backend/src/api/public.ts b/backend/src/api/public.ts index c31c6d5..97122dd 100644 --- a/backend/src/api/public.ts +++ b/backend/src/api/public.ts @@ -4,6 +4,9 @@ import * as tagsDb from '../storage/db/tags.js'; import * as eventsDb from '../storage/db/events.js'; import * as categoriesDb from '../storage/db/categories.js'; import * as settingsDb from '../storage/db/settings.js'; +import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; +import { getKv } from '../storage/db/widgetKv.js'; +import type { WidgetReport } from '../widgets/report.js'; import { hasPrivateAccess } from './privateAccess.js'; export async function registerPublicRoutes(app: FastifyInstance) { @@ -59,15 +62,34 @@ export async function registerPublicRoutes(app: FastifyInstance) { return categories.filter((c) => !c.isPrivate); }); - // Per-widget enable flags + display order — see the admin panel's consolidated - // "Widgets" tab. Weather/Stocks/PoE2's backend pollers are also gated on these - // flags (see scheduler.ts); Sidebar.svelte renders in exactly this order. + // Per-widget enable flags + display order for the 4 built-ins — see the admin panel's + // consolidated "Widgets" tab. Weather/Stocks/PoE2's backend pollers are also gated on + // these flags (see scheduler.ts); Sidebar.svelte renders in exactly this order. The + // `pluggable` array lists enabled *uploaded* widgets separately (their ids aren't part + // of the closed weather|stocks|bookmarks|poe2 union `order` uses) — see + // widgets/registry.ts, Sidebar.svelte's GenericWidgetCard/DynamicWidgetSlot. app.get('/api/widgets', async () => { const { widgets, widgetOrder } = settingsDb.getSettings(); - return { ...widgets, order: widgetOrder }; + const pluggable = installedWidgetsDb + .listInstalled() + .filter((w) => w.source === 'uploaded' && w.enabled) + .map((w) => ({ id: w.id, displayName: w.displayName, frontendEntry: w.frontendEntry })); + return { ...widgets, order: widgetOrder, pluggable }; }); - // Per-widget public routes (GET /api/weather, /api/stocks, /api/bookmarks, /api/poe2) - // are registered by each widget's own plugin — see widgets/registry.ts's generic + // Generic live-data feed any widget (built-in or uploaded) can publish to via + // setKv(id, 'report', ...) in its own poll.run() — see widgets/report.ts. Registered + // once here rather than per-widget, so it works for a widget uploaded after this + // process started, with zero new route registration (Fastify refuses routes added + // after app.listen(), so this is what makes "poll live -> sidebar shows it live" work + // for an uploaded widget without a restart). + app.get('/api/widget/:id/report', async (req, reply) => { + const { id } = req.params as { id: string }; + if (!installedWidgetsDb.getInstalled(id)?.enabled) return reply.code(404).send(); + return { data: getKv(id, 'report') }; + }); + + // Per-widget public routes (GET /api/widget/weather, /stocks, /bookmarks, /poe2) are + // registered by each widget's own plugin — see widgets/registry.ts's generic // registerPublicRoutes loop in index.ts. } diff --git a/backend/src/bookmarks/plugin.ts b/backend/src/bookmarks/plugin.ts deleted file mode 100644 index bcd546b..0000000 --- a/backend/src/bookmarks/plugin.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { WidgetPlugin } from '../widgets/types.js'; -import * as bookmarksDb from '../storage/db/bookmarks.js'; -import { hasPrivateAccess } from '../api/privateAccess.js'; - -// Built-in sidebar "Bookmarks" widget — thin wrapper so it funnels into the same -// route-registration loop as pluggable widgets (see widgets/registry.ts); no poll (purely -// admin-curated links) and no migrate()/uninstall() since its schema isn't moving. Not -// going through the upload/delete lifecycle — see widgets/types.ts. -export const bookmarksPlugin: WidgetPlugin = { - id: 'bookmarks', - displayName: 'Bookmarks', - version: '1.0.0', - - registerPublicRoutes(app) { - app.get('/api/bookmarks', async (req) => { - const bookmarks = bookmarksDb.listBookmarks(); - if (hasPrivateAccess(req)) return bookmarks; - return bookmarks.filter((b) => !b.isPrivate); - }); - }, - - registerAdminRoutes(app) { - app.get('/api/admin/bookmarks', async () => bookmarksDb.listBookmarks()); - - app.post('/api/admin/bookmarks', async (req, reply) => { - const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean }; - if (!name || !name.trim() || !url || !url.trim()) { - return reply.code(400).send({ error: 'name and url are required' }); - } - const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate); - return reply.code(201).send(created); - }); - - app.patch('/api/admin/bookmarks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - const updated = bookmarksDb.updateBookmark(id, req.body as any); - if (!updated) return reply.code(404).send({ error: 'not found' }); - return updated; - }); - - app.delete('/api/admin/bookmarks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - bookmarksDb.deleteBookmark(id); - return reply.code(204).send(); - }); - } -}; diff --git a/backend/src/index.ts b/backend/src/index.ts index e3bfb93..5563f60 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -19,6 +19,7 @@ import { loadAllWidgets, loadedWidgets } from './widgets/registry.js'; const PORT = Number(process.env.PORT) || 4000; const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173'; const MEDIA_DIR = process.env.MEDIA_DIR || './data/media'; +const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; function printApiKeyBanner() { const line = '='.repeat(64); @@ -103,6 +104,26 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); + // A widget's optional pre-built frontend bundle (see widgets/manifest.ts's + // frontendEntry) — one generic wildcard route rather than one per widget, so it works + // for a widget uploaded after this process started, with no restart (unlike a + // widget's own custom API routes, which do need one — see widgets/install.ts). + // Explicit Content-Type is required here (unlike /media/:filename above) — browsers + // reject a dynamically-imported module whose response isn't served as a JS MIME + // type. The wildcard also lets a bundle's own relative imports (e.g. `import + // './helper.mjs'`) resolve automatically, since the browser requests those against + // this same route. + app.get('/widget-assets/:id/*', async (req, reply) => { + const { id } = req.params as { id: string }; + const rel = (req.params as { '*': string })['*']; + if (rel.includes('..')) return reply.code(400).send(); + const filePath = path.join(WIDGETS_INSTALLED_DIR, id, rel); + if (!fs.existsSync(filePath)) return reply.code(404).send(); + if (rel.endsWith('.mjs') || rel.endsWith('.js')) reply.type('text/javascript'); + else if (rel.endsWith('.css')) reply.type('text/css'); + return reply.send(fs.createReadStream(filePath)); + }); + // Static "/media/proxy" and "/media/telegram-proxy" take priority over the // "/media/:filename" param route above regardless of registration order // (find-my-way, Fastify's router, always prefers a static segment over a parametric diff --git a/backend/src/stocks/plugin.ts b/backend/src/stocks/plugin.ts deleted file mode 100644 index bfab96b..0000000 --- a/backend/src/stocks/plugin.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { WidgetPlugin } from '../widgets/types.js'; -import * as stocksDb from '../storage/db/stocks.js'; -import { logger } from '../storage/db/logs.js'; -import { pollStocksNow } from './poller.js'; - -// Built-in sidebar "Stocks" widget — thin wrapper so it funnels into the same -// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts); its -// tickers stay in the dedicated stock_tickers table exactly as before. Not going through -// the upload/delete lifecycle — see widgets/types.ts; no migrate()/uninstall() since its -// schema isn't moving. -export const stocksPlugin: WidgetPlugin = { - id: 'stocks', - displayName: 'Stocks', - version: '1.0.0', - - poll: { - // Per admin spec — stock prices move faster than weather. - intervalMs: 15 * 60_000, - run: pollStocksNow - }, - - registerPublicRoutes(app) { - app.get('/api/stocks', async () => stocksDb.listStockTickers()); - }, - - registerAdminRoutes(app) { - app.get('/api/admin/stocks', async () => stocksDb.listStockTickers()); - - app.post('/api/admin/stocks', async (req, reply) => { - const { label, symbol } = req.body as { label?: string; symbol?: string }; - if (!label || !label.trim() || !symbol || !symbol.trim()) { - return reply.code(400).send({ error: 'label and symbol are required' }); - } - const created = stocksDb.createStockTicker(label.trim(), symbol.trim()); - // Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap, - // and refreshes every existing ticker's price too. - pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); - return reply.code(201).send(created); - }); - - app.patch('/api/admin/stocks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - const updated = stocksDb.updateStockTicker(id, req.body as any); - if (!updated) return reply.code(404).send({ error: 'not found' }); - return updated; - }); - - app.delete('/api/admin/stocks/:id', async (req, reply) => { - const { id } = req.params as { id: string }; - stocksDb.deleteStockTicker(id); - return reply.code(204).send(); - }); - } -}; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 692ba01..b113c97 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -45,6 +45,20 @@ export function migrate() { db.exec('DROP INDEX IF EXISTS idx_poe2_rate_history_watchlist;'); } + // Stocks and Bookmarks moved into the pluggable-widget system too (backend/src/widgets/ + // stocks/, widgets/bookmarks/) — same plain-rename treatment as PoE2 above, now owned by + // each widget's own plugin.migrate() instead of this file's CREATE TABLE block. + const hasOldStockTickersTable = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='stock_tickers'`).get(); + if (hasOldStockTickersTable) { + db.exec('ALTER TABLE stock_tickers RENAME TO widget_stocks_tickers;'); + } + // "bookmarks" is generic enough to collide with something else entirely — check for the + // old bookmarks shape specifically (its distinctive is_private column) before renaming. + const oldBookmarksCols = db.prepare(`PRAGMA table_info(bookmarks)`).all() as { name: string }[]; + if (oldBookmarksCols.some((c) => c.name === 'is_private')) { + db.exec('ALTER TABLE bookmarks RENAME TO widget_bookmarks_items;'); + } + db.exec(` CREATE TABLE IF NOT EXISTS sources ( id TEXT PRIMARY KEY, @@ -220,21 +234,6 @@ export function migrate() { poe2_updated_at TEXT ); - -- Sidebar "Stocks" widget — polled every 15 minutes from Yahoo Finance (see - -- stocks/poller.ts). Price/change/poll-state live directly on the row, same as - -- sources.last_polled_at, rather than a separate quote-cache table. - CREATE TABLE IF NOT EXISTS stock_tickers ( - id TEXT PRIMARY KEY, - label TEXT NOT NULL, - symbol TEXT NOT NULL, -- Yahoo symbol syntax, e.g. "^DJI", "AAPL", "BTC-USD" - priority_rank INTEGER NOT NULL, - last_price REAL, - last_change_percent REAL, - last_polled_at TEXT, - last_error TEXT, - created_at TEXT NOT NULL - ); - -- Generic config/cache store for pluggable widgets (see widgets/types.ts, -- storage/db/widgetKv.ts) — lets a widget stay fully prunable by widget_id alone on -- uninstall without a bespoke table for simple key/value state. @@ -259,20 +258,10 @@ export function migrate() { priority_rank INTEGER NOT NULL, version TEXT NOT NULL DEFAULT '1.0.0', owned_tables TEXT NOT NULL DEFAULT '[]', -- JSON array, self-reported by the plugin — used by the uninstall safety-net sweep + frontend_entry TEXT, -- relative path to an optional pre-built browser JS bundle, served from /widget-assets//* installed_at TEXT NOT NULL ); - -- Sidebar "Bookmarks" widget — admin-curated links, each independently hidden/public - -- via is_private (same private-access lock feature as categories.is_private). - CREATE TABLE IF NOT EXISTS bookmarks ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - url TEXT NOT NULL, - priority_rank INTEGER NOT NULL, - is_private INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL - ); - -- Singleton row (see storage/crypto.ts) — encrypted Telegram API credentials and -- the resulting login session. Deliberately its own table, not part of -- global_settings, so these encrypted blobs never ride along in the generic @@ -382,22 +371,8 @@ export function migrate() { `ALTER TABLE global_settings ADD COLUMN widget_order TEXT NOT NULL DEFAULT '["weather","stocks","poe2","bookmarks"]'` ); } - - // Seed a handful of sensible default tickers so the Stocks widget isn't empty on a - // fresh install — the admin can remove/replace any of them via the Stocks tab. - const tickerCount = db.prepare('SELECT COUNT(*) as c FROM stock_tickers').get() as { c: number }; - if (tickerCount.c === 0) { - const defaults: [string, string][] = [ - ['Dow Jones', '^DJI'], - ['S&P 500', '^GSPC'], - ['Bitcoin', 'BTC-USD'] - ]; - const stmt = db.prepare( - 'INSERT INTO stock_tickers (id, label, symbol, priority_rank, created_at) VALUES (?, ?, ?, ?, ?)' - ); - defaults.forEach(([label, symbol], i) => { - stmt.run(`stk-${symbol.replace(/[^a-z0-9]+/gi, '-')}`, label, symbol, i + 1, new Date().toISOString()); - }); + if (!hasColumn('installed_widgets', 'frontend_entry')) { + db.exec('ALTER TABLE installed_widgets ADD COLUMN frontend_entry TEXT'); } // Seed default categories if none exist yet. "News" sits right under "Top stories" — @@ -439,7 +414,12 @@ export function migrate() { ? JSON.parse(settingsRow.widget_order ?? '["weather","stocks","poe2","bookmarks"]') : ['weather', 'stocks', 'poe2', 'bookmarks']; const displayNames: Record = { weather: 'Weather', stocks: 'Stocks', bookmarks: 'Bookmarks', poe2: 'PoE2' }; - const codePaths: Record = { weather: 'weather', stocks: 'stocks', bookmarks: 'bookmarks', poe2: 'widgets/poe2' }; + const codePaths: Record = { + weather: 'widgets/weather', + stocks: 'widgets/stocks', + bookmarks: 'widgets/bookmarks', + poe2: 'widgets/poe2' + }; const ownedTables: Record = { poe2: ['widget_poe2_watchlist', 'widget_poe2_rate_history'] }; const enabledOf = (id: string) => (settingsRow ? !!settingsRow[`widget_${id}_enabled`] : true); const insertWidget = db.prepare( @@ -480,4 +460,35 @@ export function migrate() { new Date().toISOString() ); } + + // Weather's config (admin-set location/units) and cache (last-polled forecast) move off + // their bare global_settings columns into two widget_kv entries — same "every widget + // owns its own data" consistency as PoE2's league cache above. Old columns left in + // place, unread. + const weatherRow = db.prepare('SELECT * FROM global_settings WHERE id = 1').get() as any; + const hasWeatherConfigKv = db.prepare("SELECT 1 FROM widget_kv WHERE widget_id = 'weather' AND key = 'config'").get(); + if (weatherRow?.weather_location_name && !hasWeatherConfigKv) { + const nowIso = new Date().toISOString(); + db.prepare(`INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES ('weather', 'config', ?, ?)`).run( + JSON.stringify({ + locationName: weatherRow.weather_location_name, + latitude: weatherRow.weather_latitude, + longitude: weatherRow.weather_longitude, + unit: weatherRow.weather_unit, + windUnit: weatherRow.weather_wind_unit, + pressureUnit: weatherRow.weather_pressure_unit + }), + nowIso + ); + db.prepare(`INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES ('weather', 'cache', ?, ?)`).run( + JSON.stringify({ + current: weatherRow.weather_current ? JSON.parse(weatherRow.weather_current) : null, + hourly: JSON.parse(weatherRow.weather_hourly ?? '[]'), + daily: JSON.parse(weatherRow.weather_daily ?? '[]'), + alerts: JSON.parse(weatherRow.weather_alerts ?? '[]'), + updatedAt: weatherRow.weather_updated_at + }), + nowIso + ); + } } diff --git a/backend/src/storage/db/installedWidgets.ts b/backend/src/storage/db/installedWidgets.ts index 64a49c2..ee1850c 100644 --- a/backend/src/storage/db/installedWidgets.ts +++ b/backend/src/storage/db/installedWidgets.ts @@ -11,6 +11,7 @@ function rowToWidget(row: any): InstalledWidget { priorityRank: row.priority_rank, version: row.version, ownedTables: JSON.parse(row.owned_tables), + frontendEntry: row.frontend_entry ?? null, installedAt: row.installed_at }; } @@ -33,13 +34,14 @@ export function insertWidget(widget: { priorityRank?: number; version?: string; ownedTables?: string[]; + frontendEntry?: string | null; }): InstalledWidget { const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM installed_widgets').get() as { m: number }; const priorityRank = widget.priorityRank ?? maxRank.m + 1; const installedAt = new Date().toISOString(); db.prepare( - `INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, version, owned_tables, installed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + `INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, version, owned_tables, frontend_entry, installed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( widget.id, widget.displayName, @@ -49,6 +51,7 @@ export function insertWidget(widget: { priorityRank, widget.version ?? '1.0.0', JSON.stringify(widget.ownedTables ?? []), + widget.frontendEntry ?? null, installedAt ); return getInstalled(widget.id)!; diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 07d2cbd..05e7c17 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -1,16 +1,9 @@ import { db } from './index.js'; import type { GlobalSettings } from './types.js'; import * as installedWidgetsDb from './installedWidgets.js'; -import { getKv, setKv } from './widgetKv.js'; const BUILTIN_WIDGET_IDS = ['weather', 'stocks', 'bookmarks', 'poe2'] as const; -interface Poe2LeagueCache { - leagueId: string | null; - leagueName: string | null; - updatedAt: string | null; -} - // widgets/widgetOrder are computed from the installed_widgets registry (see // storage/db/installedWidgets.ts, widgets/registry.ts) rather than stored as their own // global_settings columns — the registry is the single source of truth for enable state @@ -32,8 +25,10 @@ function widgetsAndOrder(): Pick { return { widgets, widgetOrder }; } +// Every widget's own config/data now lives behind its own /api/widget/ routes (see +// widgets/weather/db.ts, widgets/poe2/poll.ts's league cache, etc.) — this settings blob +// is just the scalar pipeline knobs plus the 4 builtins' enable/order flags. function rowToSettings(row: any): GlobalSettings { - const poe2Cache = getKv('poe2', 'leagueCache') ?? { leagueId: null, leagueName: null, updatedAt: null }; return { mergeStrictness: row.merge_strictness, holdBeforePublishMinutes: row.hold_before_publish_minutes, @@ -55,22 +50,7 @@ function rowToSettings(row: any): GlobalSettings { storageCapEnabled: !!row.storage_cap_enabled, storageCapValue: row.storage_cap_value, storageCapUnit: row.storage_cap_unit - }, - weather: { - locationName: row.weather_location_name, - latitude: row.weather_latitude, - longitude: row.weather_longitude, - unit: row.weather_unit, - windUnit: row.weather_wind_unit, - pressureUnit: row.weather_pressure_unit, - // Unlike retention, this is genuinely absent pre-first-poll (and pre-location-config) — null-safe parse. - current: row.weather_current ? JSON.parse(row.weather_current) : null, - hourly: JSON.parse(row.weather_hourly), - daily: JSON.parse(row.weather_daily), - alerts: JSON.parse(row.weather_alerts), - updatedAt: row.weather_updated_at - }, - poe2: poe2Cache + } }; } @@ -86,8 +66,6 @@ export function updateSettings(patch: Partial): GlobalSettings { ...patch, retention: { ...current.retention, ...(patch.retention ?? {}) }, selectedModels: { ...current.selectedModels, ...(patch.selectedModels ?? {}) }, - weather: { ...current.weather, ...(patch.weather ?? {}) }, - poe2: { ...current.poe2, ...(patch.poe2 ?? {}) }, widgets: { ...current.widgets, ...(patch.widgets ?? {}) } }; @@ -99,9 +77,6 @@ export function updateSettings(patch: Partial): GlobalSettings { if (patch.widgetOrder) { installedWidgetsDb.reorder(patch.widgetOrder); } - if (patch.poe2) { - setKv('poe2', 'leagueCache', merged.poe2); - } // Named params (rather than positional `?`) so this list can be reordered or // extended without the column list and the bound-values list silently drifting @@ -116,11 +91,7 @@ export function updateSettings(patch: Partial): GlobalSettings { nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url, telegram_media_mode=$telegram_media_mode, published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days, - storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit, - weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude, - weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit, - weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily, - weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at + storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit WHERE id = 1` ).run({ $merge_strictness: merged.mergeStrictness, @@ -140,18 +111,7 @@ export function updateSettings(patch: Partial): GlobalSettings { $raw_item_max_age_days: merged.retention.rawItemMaxAgeDays, $storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0, $storage_cap_value: merged.retention.storageCapValue, - $storage_cap_unit: merged.retention.storageCapUnit, - $weather_location_name: merged.weather.locationName, - $weather_latitude: merged.weather.latitude, - $weather_longitude: merged.weather.longitude, - $weather_unit: merged.weather.unit, - $weather_wind_unit: merged.weather.windUnit, - $weather_pressure_unit: merged.weather.pressureUnit, - $weather_current: merged.weather.current ? JSON.stringify(merged.weather.current) : null, - $weather_hourly: JSON.stringify(merged.weather.hourly), - $weather_daily: JSON.stringify(merged.weather.daily), - $weather_alerts: JSON.stringify(merged.weather.alerts), - $weather_updated_at: merged.weather.updatedAt + $storage_cap_unit: merged.retention.storageCapUnit }); return getSettings(); } diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index b2b8751..53fd98d 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -208,21 +208,6 @@ export interface Category { isSpillover: boolean; } -export interface WeatherHourEntry { - time: string; - temp: number; - conditionText: string; - icon: string; -} - -export interface WeatherDayEntry { - date: string; - tempMax: number; - tempMin: number; - conditionText: string; - icon: string; -} - export interface StockTicker { id: string; label: string; @@ -268,6 +253,8 @@ export interface InstalledWidget { version: string; /** Self-reported by the plugin (WidgetPlugin.ownedTables) at install/load time — used by the uninstall safety-net sweep. */ ownedTables: string[]; + /** Relative path (within the widget's install dir) to an optional pre-built browser JS bundle, served from /widget-assets//* and dynamic-import()ed by the sidebar's DynamicWidgetSlot. Null for a widget with no custom frontend (falls back to the generic report card). */ + frontendEntry: string | null; installedAt: string; } @@ -314,56 +301,4 @@ export interface GlobalSettings { storageCapValue: number; storageCapUnit: 'MB' | 'GB'; }; - /** Sidebar "Weather" widget config + cache — see weather/poller.ts. Singleton, since there's only ever one configured location. */ - weather: { - locationName: string | null; - latitude: number | null; - longitude: number | null; - unit: 'celsius' | 'fahrenheit'; - windUnit: 'mph' | 'kph'; - pressureUnit: 'inHg' | 'hPa'; - current: { - temp: number; - /** Apparent temperature (Open-Meteo's own heat-index/wind-chill blend) — "Feels like". */ - feelsLike: number; - conditionText: string; - icon: string; - /** Percent, 0-100. */ - humidity: number; - /** Percent, 0-100 — the current hour's forecast precipitation probability (there's no true instantaneous "chance of rain" measurement). */ - precipitationChance: number; - /** Already in the admin's configured windUnit. */ - windSpeed: number; - /** 8-point compass abbreviation, e.g. "NW". */ - windDirection: string; - /** Already in the admin's configured pressureUnit. */ - pressure: number; - sunrise: string; - sunset: string; - } | null; - hourly: WeatherHourEntry[]; - daily: WeatherDayEntry[]; - /** Active NWS alerts (flash flood, hurricane, blizzard, etc.) for the configured location — US-only, empty elsewhere. See weather/client.ts's fetchActiveAlerts. */ - alerts: WeatherAlert[]; - updatedAt: string | null; - }; - /** - * Sidebar "PoE2" widget cache — see poe2/poller.ts. No admin-set config (unlike weather): - * the league is always auto-detected as the current challenge league, so this is purely - * a cache of what the last poll learned. Watchlist entries themselves live in the - * poe2_watchlist table, not here — same split as stock_tickers vs. this settings row. - */ - poe2: { - leagueId: string | null; - leagueName: string | null; - updatedAt: string | null; - }; -} - -export interface WeatherAlert { - id: string; - event: string; - headline: string; - severity: string; - expires: string; } diff --git a/backend/src/weather/plugin.ts b/backend/src/weather/plugin.ts deleted file mode 100644 index ee8ed77..0000000 --- a/backend/src/weather/plugin.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { WidgetPlugin } from '../widgets/types.js'; -import * as settingsDb from '../storage/db/settings.js'; -import { geocodeLocation } from './client.js'; -import { pollWeatherNow } from './poller.js'; - -// Built-in sidebar "Weather" widget — thin wrapper so it funnels into the same -// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts), while -// its config/cache stay on global_settings' weather_* columns exactly as before. Not going -// through the upload/delete lifecycle — see widgets/types.ts and the plan's "built-in vs -// pluggable" split; no migrate()/uninstall() since its schema isn't moving. -export const weatherPlugin: WidgetPlugin = { - id: 'weather', - displayName: 'Weather', - version: '1.0.0', - - poll: { - intervalMs: 45 * 60_000, - run: pollWeatherNow - }, - - registerPublicRoutes(app) { - app.get('/api/weather', async () => settingsDb.getSettings().weather); - }, - - registerAdminRoutes(app) { - app.get('/api/admin/weather/geocode', async (req, reply) => { - const { query } = req.query as { query?: string }; - if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' }); - try { - return await geocodeLocation(query.trim()); - } catch (err) { - return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` }); - } - }); - } -}; diff --git a/backend/src/storage/db/bookmarks.ts b/backend/src/widgets/bookmarks/db.ts similarity index 62% rename from backend/src/storage/db/bookmarks.ts rename to backend/src/widgets/bookmarks/db.ts index 0882ca9..1fcedeb 100644 --- a/backend/src/storage/db/bookmarks.ts +++ b/backend/src/widgets/bookmarks/db.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { db } from './index.js'; -import type { Bookmark } from './types.js'; +import { db } from '../../storage/db/index.js'; +import type { Bookmark } from '../../storage/db/types.js'; function rowToBookmark(row: any): Bookmark { return { @@ -14,33 +14,33 @@ function rowToBookmark(row: any): Bookmark { } // Always returns every bookmark, private or not — filtering for unauthenticated visitors -// happens at the route layer (GET /api/bookmarks), same as categoriesDb.listCategories(). +// happens at the route layer (GET /api/widget/bookmarks), same as categoriesDb.listCategories(). export function listBookmarks(): Bookmark[] { - const rows = db.prepare('SELECT * FROM bookmarks ORDER BY priority_rank').all(); + const rows = db.prepare('SELECT * FROM widget_bookmarks_items ORDER BY priority_rank').all(); return rows.map(rowToBookmark); } export function createBookmark(name: string, url: string, isPrivate = false): Bookmark { const id = `bm-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`; - const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM bookmarks').get() as { m: number }; + const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM widget_bookmarks_items').get() as { m: number }; const createdAt = new Date().toISOString(); db.prepare( - 'INSERT INTO bookmarks (id, name, url, priority_rank, is_private, created_at) VALUES (?, ?, ?, ?, ?, ?)' + 'INSERT INTO widget_bookmarks_items (id, name, url, priority_rank, is_private, created_at) VALUES (?, ?, ?, ?, ?, ?)' ).run(id, name, url, maxRank.m + 1, isPrivate ? 1 : 0, createdAt); return { id, name, url, priorityRank: maxRank.m + 1, isPrivate, createdAt }; } export function updateBookmark(id: string, patch: { name?: string; url?: string; isPrivate?: boolean }): Bookmark | null { - const existing = db.prepare('SELECT * FROM bookmarks WHERE id = ?').get(id); + const existing = db.prepare('SELECT * FROM widget_bookmarks_items WHERE id = ?').get(id); if (!existing) return null; const current = rowToBookmark(existing); const merged = { ...current, ...patch }; - db.prepare('UPDATE bookmarks SET name = ?, url = ?, is_private = ? WHERE id = ?').run( + db.prepare('UPDATE widget_bookmarks_items SET name = ?, url = ?, is_private = ? WHERE id = ?').run( merged.name, merged.url, merged.isPrivate ? 1 : 0, id ); return merged; } export function deleteBookmark(id: string) { - db.prepare('DELETE FROM bookmarks WHERE id = ?').run(id); + db.prepare('DELETE FROM widget_bookmarks_items WHERE id = ?').run(id); } diff --git a/backend/src/widgets/bookmarks/plugin.ts b/backend/src/widgets/bookmarks/plugin.ts new file mode 100644 index 0000000..cff7137 --- /dev/null +++ b/backend/src/widgets/bookmarks/plugin.ts @@ -0,0 +1,68 @@ +import type { DatabaseSync } from 'node:sqlite'; +import type { WidgetPlugin } from '../types.js'; +import * as bookmarksDb from './db.js'; +import { hasPrivateAccess } from '../../api/privateAccess.js'; + +const OWNED_TABLES = ['widget_bookmarks_items']; + +// Sidebar "Bookmarks" widget — admin-curated links, each independently hidden/public via +// is_private (same private-access lock feature as categories.is_private). No poll (purely +// admin-curated, no external fetch). Built-in and non-deletable, but otherwise a full +// WidgetPlugin like an uploaded one. +export const bookmarksPlugin: WidgetPlugin = { + id: 'bookmarks', + displayName: 'Bookmarks', + version: '1.0.0', + ownedTables: OWNED_TABLES, + + migrate(db: DatabaseSync) { + db.exec(` + CREATE TABLE IF NOT EXISTS widget_bookmarks_items ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + url TEXT NOT NULL, + priority_rank INTEGER NOT NULL, + is_private INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + ); + `); + }, + + registerPublicRoutes(app) { + app.get('/api/widget/bookmarks', async (req) => { + const bookmarks = bookmarksDb.listBookmarks(); + if (hasPrivateAccess(req)) return bookmarks; + return bookmarks.filter((b) => !b.isPrivate); + }); + }, + + registerAdminRoutes(app) { + app.get('/api/admin/widget/bookmarks', async () => bookmarksDb.listBookmarks()); + + app.post('/api/admin/widget/bookmarks', async (req, reply) => { + const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean }; + if (!name || !name.trim() || !url || !url.trim()) { + return reply.code(400).send({ error: 'name and url are required' }); + } + const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate); + return reply.code(201).send(created); + }); + + app.patch('/api/admin/widget/bookmarks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + const updated = bookmarksDb.updateBookmark(id, req.body as any); + if (!updated) return reply.code(404).send({ error: 'not found' }); + return updated; + }); + + app.delete('/api/admin/widget/bookmarks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + bookmarksDb.deleteBookmark(id); + return reply.code(204).send(); + }); + }, + + uninstall(db: DatabaseSync) { + db.exec('DROP TABLE IF EXISTS widget_bookmarks_items;'); + } +}; diff --git a/backend/src/widgets/install.ts b/backend/src/widgets/install.ts index b8a34be..789e4f5 100644 --- a/backend/src/widgets/install.ts +++ b/backend/src/widgets/install.ts @@ -51,7 +51,8 @@ export async function installUploadedWidget(manifest: unknown, files: unknown): source: 'uploaded', codePath: dir, version: m.version, - ownedTables: plugin.ownedTables ?? [] + ownedTables: plugin.ownedTables ?? [], + frontendEntry: m.frontendEntry ?? null }); startWidgetPolling(plugin); diff --git a/backend/src/widgets/manifest.ts b/backend/src/widgets/manifest.ts index 540e49a..c7fc125 100644 --- a/backend/src/widgets/manifest.ts +++ b/backend/src/widgets/manifest.ts @@ -8,6 +8,8 @@ export interface WidgetManifest { version: string; /** Relative path within `files` to the ESM entry point, e.g. "index.mjs" — a default export satisfying WidgetPlugin. */ entry: string; + /** Optional relative path within `files` to a pre-built browser JS bundle (plain vanilla JS, not raw Svelte source — see widgets/registry.ts's frontend-loading notes) exporting a default `{ mount(container, ctx) }`. Served from /widget-assets//* and dynamic-import()ed by the sidebar's DynamicWidgetSlot. Omit for a widget with no custom UI — it falls back to the generic report card. */ + frontendEntry?: string; } // Upload body shape: { manifest, files: { "index.mjs": "", ... } } — plain @@ -21,11 +23,17 @@ export function validateManifest(manifest: unknown, files: unknown): string | nu if (typeof m.displayName !== 'string' || !m.displayName.trim()) return 'manifest.displayName is required'; if (typeof m.version !== 'string' || !m.version.trim()) return 'manifest.version is required'; if (typeof m.entry !== 'string' || !m.entry.trim()) return 'manifest.entry is required'; + if (m.frontendEntry !== undefined && (typeof m.frontendEntry !== 'string' || !m.frontendEntry.trim())) { + return 'manifest.frontendEntry must be a non-empty string when present'; + } if (!files || typeof files !== 'object' || Array.isArray(files)) return 'files must be a non-empty object'; const entries = Object.entries(files as Record); if (entries.length === 0) return 'files must be a non-empty object'; if (!(m.entry in (files as Record))) return `entry "${m.entry as string}" not found in files`; + if (m.frontendEntry !== undefined && !(m.frontendEntry in (files as Record))) { + return `frontendEntry "${m.frontendEntry as string}" not found in files`; + } let totalBytes = 0; for (const [relPath, content] of entries) { diff --git a/backend/src/widgets/poe2/plugin.ts b/backend/src/widgets/poe2/plugin.ts index 92623df..730bdbd 100644 --- a/backend/src/widgets/poe2/plugin.ts +++ b/backend/src/widgets/poe2/plugin.ts @@ -52,7 +52,7 @@ export const poe2Plugin: WidgetPlugin = { }, registerPublicRoutes(app) { - app.get('/api/poe2', async () => { + app.get('/api/widget/poe2', async () => { const { leagueName, updatedAt } = getLeagueCache(); return { leagueName, updatedAt, entries: poe2Db.listWatchlist() }; }); @@ -60,7 +60,7 @@ export const poe2Plugin: WidgetPlugin = { registerAdminRoutes(app) { // League is always auto-detected, never admin-set. - app.get('/api/admin/poe2/browse', async (_req, reply) => { + app.get('/api/admin/widget/poe2/browse', async (_req, reply) => { try { const league = await fetchCurrentLeague(); return await browseCurrencies(league.id); @@ -69,9 +69,9 @@ export const poe2Plugin: WidgetPlugin = { } }); - app.get('/api/admin/poe2/watchlist', async () => poe2Db.listWatchlist()); + app.get('/api/admin/widget/poe2/watchlist', async () => poe2Db.listWatchlist()); - app.post('/api/admin/poe2/watchlist', async (req, reply) => { + app.post('/api/admin/widget/poe2/watchlist', async (req, reply) => { const { base, quote } = req.body as { base?: { currencyId?: string; name?: string }; quote?: { currencyId?: string; name?: string }; @@ -92,7 +92,7 @@ export const poe2Plugin: WidgetPlugin = { return reply.code(201).send(created); }); - app.delete('/api/admin/poe2/watchlist/:id', async (req, reply) => { + app.delete('/api/admin/widget/poe2/watchlist/:id', async (req, reply) => { const { id } = req.params as { id: string }; poe2Db.removeWatchlistEntry(id); return reply.code(204).send(); diff --git a/backend/src/widgets/registry.ts b/backend/src/widgets/registry.ts index a45931d..de9b4e9 100644 --- a/backend/src/widgets/registry.ts +++ b/backend/src/widgets/registry.ts @@ -6,9 +6,9 @@ import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; import { logger } from '../storage/db/logs.js'; import type { WidgetPlugin } from './types.js'; import { sweepOrphanedWidgetData } from './sweep.js'; -import { weatherPlugin } from '../weather/plugin.js'; -import { stocksPlugin } from '../stocks/plugin.js'; -import { bookmarksPlugin } from '../bookmarks/plugin.js'; +import { weatherPlugin } from './weather/plugin.js'; +import { stocksPlugin } from './stocks/plugin.js'; +import { bookmarksPlugin } from './bookmarks/plugin.js'; import { poe2Plugin } from './poe2/plugin.js'; // Both built-in and uploaded widgets funnel into this one map — scheduler dispatch and diff --git a/backend/src/widgets/report.ts b/backend/src/widgets/report.ts new file mode 100644 index 0000000..2a1e837 --- /dev/null +++ b/backend/src/widgets/report.ts @@ -0,0 +1,13 @@ +/** + * Generic live-data shape a widget's poll.run() can publish via + * setKv(id, 'report', report) (see storage/db/widgetKv.ts) for the sidebar's generic + * report card to render — see api/public.ts's GET /api/widget/:id/report, registered once + * at startup so it works for any widget id including ones uploaded after boot, with zero + * per-widget route registration (sidesteps Fastify's "no routes after listen()" limit). + */ +export interface WidgetReport { + title: string; + headline?: { value: string; delta?: string } | null; + rows?: { label: string; value: string }[]; + updatedAt: string | null; +} diff --git a/backend/src/stocks/client.ts b/backend/src/widgets/stocks/client.ts similarity index 100% rename from backend/src/stocks/client.ts rename to backend/src/widgets/stocks/client.ts diff --git a/backend/src/widgets/stocks/db.ts b/backend/src/widgets/stocks/db.ts new file mode 100644 index 0000000..0b78608 --- /dev/null +++ b/backend/src/widgets/stocks/db.ts @@ -0,0 +1,54 @@ +import { randomUUID } from 'node:crypto'; +import { db } from '../../storage/db/index.js'; +import type { StockTicker } from '../../storage/db/types.js'; + +function rowToTicker(row: any): StockTicker { + return { + id: row.id, + label: row.label, + symbol: row.symbol, + priorityRank: row.priority_rank, + lastPrice: row.last_price, + lastChangePercent: row.last_change_percent, + lastPolledAt: row.last_polled_at, + lastError: row.last_error, + createdAt: row.created_at + }; +} + +export function listStockTickers(): StockTicker[] { + const rows = db.prepare('SELECT * FROM widget_stocks_tickers ORDER BY priority_rank').all(); + return rows.map(rowToTicker); +} + +export function createStockTicker(label: string, symbol: string): StockTicker { + const id = `stk-${symbol.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`; + const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM widget_stocks_tickers').get() as { m: number }; + const createdAt = new Date().toISOString(); + db.prepare( + 'INSERT INTO widget_stocks_tickers (id, label, symbol, priority_rank, created_at) VALUES (?, ?, ?, ?, ?)' + ).run(id, label, symbol, maxRank.m + 1, createdAt); + return { + id, label, symbol, priorityRank: maxRank.m + 1, + lastPrice: null, lastChangePercent: null, lastPolledAt: null, lastError: null, createdAt + }; +} + +export function updateStockTicker(id: string, patch: { label?: string; symbol?: string }): StockTicker | null { + const existing = db.prepare('SELECT * FROM widget_stocks_tickers WHERE id = ?').get(id); + if (!existing) return null; + const current = rowToTicker(existing); + const merged = { ...current, ...patch }; + db.prepare('UPDATE widget_stocks_tickers SET label = ?, symbol = ? WHERE id = ?').run(merged.label, merged.symbol, id); + return { ...merged }; +} + +export function deleteStockTicker(id: string) { + db.prepare('DELETE FROM widget_stocks_tickers WHERE id = ?').run(id); +} + +export function markStockPolled(id: string, price: number | null, changePercent: number | null, error: string | null) { + db.prepare( + 'UPDATE widget_stocks_tickers SET last_price = ?, last_change_percent = ?, last_polled_at = ?, last_error = ? WHERE id = ?' + ).run(price, changePercent, new Date().toISOString(), error, id); +} diff --git a/backend/src/widgets/stocks/plugin.ts b/backend/src/widgets/stocks/plugin.ts new file mode 100644 index 0000000..519f75c --- /dev/null +++ b/backend/src/widgets/stocks/plugin.ts @@ -0,0 +1,94 @@ +import type { DatabaseSync } from 'node:sqlite'; +import type { WidgetPlugin } from '../types.js'; +import { logger } from '../../storage/db/logs.js'; +import * as stocksDb from './db.js'; +import { pollStocksNow } from './poll.js'; + +const OWNED_TABLES = ['widget_stocks_tickers']; + +// Sidebar "Stocks" widget — polled every 15 minutes from Yahoo Finance (see poll.ts). +// Price/change/poll-state live directly on its own table, same as sources.last_polled_at, +// rather than a separate quote-cache table. Built-in and non-deletable, but otherwise a +// full WidgetPlugin like an uploaded one. +export const stocksPlugin: WidgetPlugin = { + id: 'stocks', + displayName: 'Stocks', + version: '1.0.0', + ownedTables: OWNED_TABLES, + + migrate(db: DatabaseSync) { + db.exec(` + CREATE TABLE IF NOT EXISTS widget_stocks_tickers ( + id TEXT PRIMARY KEY, + label TEXT NOT NULL, + symbol TEXT NOT NULL, + priority_rank INTEGER NOT NULL, + last_price REAL, + last_change_percent REAL, + last_polled_at TEXT, + last_error TEXT, + created_at TEXT NOT NULL + ); + `); + + // Seed a handful of sensible default tickers so the widget isn't empty on a fresh + // install — the admin can remove/replace any of them. + const tickerCount = db.prepare('SELECT COUNT(*) as c FROM widget_stocks_tickers').get() as { c: number }; + if (tickerCount.c === 0) { + const defaults: [string, string][] = [ + ['Dow Jones', '^DJI'], + ['S&P 500', '^GSPC'], + ['Bitcoin', 'BTC-USD'] + ]; + const stmt = db.prepare( + 'INSERT INTO widget_stocks_tickers (id, label, symbol, priority_rank, created_at) VALUES (?, ?, ?, ?, ?)' + ); + defaults.forEach(([label, symbol], i) => { + stmt.run(`stk-${symbol.replace(/[^a-z0-9]+/gi, '-')}`, label, symbol, i + 1, new Date().toISOString()); + }); + } + }, + + poll: { + // Per admin spec — stock prices move faster than weather. + intervalMs: 15 * 60_000, + run: pollStocksNow + }, + + registerPublicRoutes(app) { + app.get('/api/widget/stocks', async () => stocksDb.listStockTickers()); + }, + + registerAdminRoutes(app) { + app.get('/api/admin/widget/stocks', async () => stocksDb.listStockTickers()); + + app.post('/api/admin/widget/stocks', async (req, reply) => { + const { label, symbol } = req.body as { label?: string; symbol?: string }; + if (!label || !label.trim() || !symbol || !symbol.trim()) { + return reply.code(400).send({ error: 'label and symbol are required' }); + } + const created = stocksDb.createStockTicker(label.trim(), symbol.trim()); + // Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap, + // and refreshes every existing ticker's price too. + pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`)); + return reply.code(201).send(created); + }); + + app.patch('/api/admin/widget/stocks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + const updated = stocksDb.updateStockTicker(id, req.body as any); + if (!updated) return reply.code(404).send({ error: 'not found' }); + return updated; + }); + + app.delete('/api/admin/widget/stocks/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + stocksDb.deleteStockTicker(id); + return reply.code(204).send(); + }); + }, + + uninstall(db: DatabaseSync) { + db.exec('DROP TABLE IF EXISTS widget_stocks_tickers;'); + } +}; diff --git a/backend/src/stocks/poller.ts b/backend/src/widgets/stocks/poll.ts similarity index 65% rename from backend/src/stocks/poller.ts rename to backend/src/widgets/stocks/poll.ts index 84ccb26..58fdd75 100644 --- a/backend/src/stocks/poller.ts +++ b/backend/src/widgets/stocks/poll.ts @@ -1,11 +1,11 @@ -import * as stocksDb from '../storage/db/stocks.js'; -import { logger } from '../storage/db/logs.js'; +import * as stocksDb from './db.js'; +import { logger } from '../../storage/db/logs.js'; import { fetchQuotes } from './client.js'; -// Called on a schedule (see queue/scheduler.ts) and immediately after the admin adds a -// ticker (see api/admin.ts) — one request per configured ticker (see client.ts for why -// there's no batch endpoint here). A symbol Yahoo can't resolve gets its own lastError, -// it never aborts the rest of the batch. +// Called on a schedule (see queue/scheduler.ts, via plugin.poll) and immediately after the +// admin adds a ticker (see plugin.ts's admin routes) — one request per configured ticker +// (see client.ts for why there's no batch endpoint here). A symbol Yahoo can't resolve +// gets its own lastError, it never aborts the rest of the batch. export async function pollStocksNow(): Promise { const tickers = stocksDb.listStockTickers(); if (tickers.length === 0) return; diff --git a/backend/src/weather/client.ts b/backend/src/widgets/weather/client.ts similarity index 100% rename from backend/src/weather/client.ts rename to backend/src/widgets/weather/client.ts diff --git a/backend/src/widgets/weather/db.ts b/backend/src/widgets/weather/db.ts new file mode 100644 index 0000000..52f4189 --- /dev/null +++ b/backend/src/widgets/weather/db.ts @@ -0,0 +1,103 @@ +import { getKv, setKv } from '../../storage/db/widgetKv.js'; + +export interface WeatherHourEntry { + time: string; + temp: number; + conditionText: string; + icon: string; +} + +export interface WeatherDayEntry { + date: string; + tempMax: number; + tempMin: number; + conditionText: string; + icon: string; +} + +export interface WeatherAlert { + id: string; + event: string; + headline: string; + severity: string; + expires: string; +} + +export interface WeatherConfig { + locationName: string | null; + latitude: number | null; + longitude: number | null; + unit: 'celsius' | 'fahrenheit'; + windUnit: 'mph' | 'kph'; + pressureUnit: 'inHg' | 'hPa'; +} + +export interface WeatherCache { + current: { + temp: number; + /** Apparent temperature (Open-Meteo's own heat-index/wind-chill blend) — "Feels like". */ + feelsLike: number; + conditionText: string; + icon: string; + /** Percent, 0-100. */ + humidity: number; + /** Percent, 0-100 — the current hour's forecast precipitation probability (there's no true instantaneous "chance of rain" measurement). */ + precipitationChance: number; + /** Already in the admin's configured windUnit. */ + windSpeed: number; + /** 8-point compass abbreviation, e.g. "NW". */ + windDirection: string; + /** Already in the admin's configured pressureUnit. */ + pressure: number; + sunrise: string; + sunset: string; + } | null; + hourly: WeatherHourEntry[]; + daily: WeatherDayEntry[]; + /** Active NWS alerts (flash flood, hurricane, blizzard, etc.) for the configured location — US-only, empty elsewhere. See client.ts's fetchActiveAlerts. */ + alerts: WeatherAlert[]; + updatedAt: string | null; +} + +const WIDGET_ID = 'weather'; + +const DEFAULT_CONFIG: WeatherConfig = { + locationName: null, + latitude: null, + longitude: null, + unit: 'fahrenheit', + windUnit: 'mph', + pressureUnit: 'inHg' +}; + +const DEFAULT_CACHE: WeatherCache = { + current: null, + hourly: [], + daily: [], + alerts: [], + updatedAt: null +}; + +// Config (admin-settable: location/units) and cache (poll-computed forecast) are stored as +// two separate widget_kv keys — mirrors the singleton-row split stock_tickers/bookmarks +// tables already had from global_settings, just via the generic kv store instead of a +// bespoke table (this widget has no data shaped like rows, so no dedicated table is needed). +export function getConfig(): WeatherConfig { + return getKv(WIDGET_ID, 'config') ?? DEFAULT_CONFIG; +} + +export function setConfig(patch: Partial): WeatherConfig { + const merged = { ...getConfig(), ...patch }; + setKv(WIDGET_ID, 'config', merged); + return merged; +} + +export function getCache(): WeatherCache { + return getKv(WIDGET_ID, 'cache') ?? DEFAULT_CACHE; +} + +export function setCache(patch: Partial): WeatherCache { + const merged = { ...getCache(), ...patch }; + setKv(WIDGET_ID, 'cache', merged); + return merged; +} diff --git a/backend/src/widgets/weather/plugin.ts b/backend/src/widgets/weather/plugin.ts new file mode 100644 index 0000000..3a786b3 --- /dev/null +++ b/backend/src/widgets/weather/plugin.ts @@ -0,0 +1,47 @@ +import type { WidgetPlugin } from '../types.js'; +import { logger } from '../../storage/db/logs.js'; +import { geocodeLocation } from './client.js'; +import * as weatherDb from './db.js'; +import { pollWeatherNow } from './poll.js'; + +// Sidebar "Weather" widget — config (location/units) and cache (forecast) live entirely in +// widget_kv (see db.ts); no bespoke table needed. Built-in and non-deletable (source: +// 'builtin' in installed_widgets), but otherwise a full WidgetPlugin like an uploaded one. +export const weatherPlugin: WidgetPlugin = { + id: 'weather', + displayName: 'Weather', + version: '1.0.0', + + poll: { + intervalMs: 45 * 60_000, + run: pollWeatherNow + }, + + registerPublicRoutes(app) { + app.get('/api/widget/weather', async () => ({ ...weatherDb.getConfig(), ...weatherDb.getCache() })); + }, + + registerAdminRoutes(app) { + // Returns config + cache together (same shape as the public route) so the admin + // tab can show "currently showing X" status alongside the location/unit form. + app.get('/api/admin/widget/weather', async () => ({ ...weatherDb.getConfig(), ...weatherDb.getCache() })); + + app.patch('/api/admin/widget/weather', async (req) => { + weatherDb.setConfig(req.body as any); + // Poll immediately rather than waiting for the next scheduler tick (up to 45 + // minutes) — the admin just changed the location/unit and expects to see it reflected. + pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); + return { ...weatherDb.getConfig(), ...weatherDb.getCache() }; + }); + + app.get('/api/admin/widget/weather/geocode', async (req, reply) => { + const { query } = req.query as { query?: string }; + if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' }); + try { + return await geocodeLocation(query.trim()); + } catch (err) { + return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` }); + } + }); + } +}; diff --git a/backend/src/weather/poller.ts b/backend/src/widgets/weather/poll.ts similarity index 52% rename from backend/src/weather/poller.ts rename to backend/src/widgets/weather/poll.ts index 18ac676..e1ffbe0 100644 --- a/backend/src/weather/poller.ts +++ b/backend/src/widgets/weather/poll.ts @@ -1,26 +1,29 @@ -import * as settingsDb from '../storage/db/settings.js'; -import { logger } from '../storage/db/logs.js'; +import { logger } from '../../storage/db/logs.js'; import { fetchForecast, fetchActiveAlerts } from './client.js'; +import * as weatherDb from './db.js'; +import type { WeatherCache } from './db.js'; -// Called on a schedule (see queue/scheduler.ts) and immediately after the admin changes -// the weather location/units (see api/admin.ts) — writes straight into global_settings' -// weather_* columns via settingsDb, same singleton-row approach as retention. +// Called on a schedule (see queue/scheduler.ts, via plugin.poll) and immediately after the +// admin changes the weather location/units (see plugin.ts's admin routes) — writes into +// the widget's own widget_kv cache entry via weatherDb, same singleton-cache approach as +// before, just no longer riding on global_settings. export async function pollWeatherNow(): Promise { - const { weather } = settingsDb.getSettings(); - if (weather.latitude === null || weather.longitude === null) { + const config = weatherDb.getConfig(); + if (config.latitude === null || config.longitude === null) { // No location configured yet — not an error, just nothing to do. return; } - let forecastUpdate: Partial = {}; + const cache = weatherDb.getCache(); + let forecastUpdate: Partial = {}; let forecastSucceeded = false; try { const { current, hourly, daily } = await fetchForecast( - weather.latitude, - weather.longitude, - weather.unit, - weather.windUnit, - weather.pressureUnit + config.latitude, + config.longitude, + config.unit, + config.windUnit, + config.pressureUnit ); forecastUpdate = { current, hourly, daily }; forecastSucceeded = true; @@ -33,20 +36,17 @@ export async function pollWeatherNow(): Promise { // reliably (and expectedly) for every non-US location. A failure here shouldn't // touch the forecast update above, and unlike a stale forecast, a stale alert that's // since expired is worse to keep showing than none at all — clear to empty on failure. - let alerts = weather.alerts; + let alerts = cache.alerts; try { - alerts = await fetchActiveAlerts(weather.latitude, weather.longitude); + alerts = await fetchActiveAlerts(config.latitude, config.longitude); } catch (err) { alerts = []; logger.warn('weather', `Alerts poll failed (expected outside the US): ${(err as Error).message}`); } - settingsDb.updateSettings({ - weather: { - ...weather, - ...forecastUpdate, - alerts, - updatedAt: forecastSucceeded ? new Date().toISOString() : weather.updatedAt - } + weatherDb.setCache({ + ...forecastUpdate, + alerts, + updatedAt: forecastSucceeded ? new Date().toISOString() : cache.updatedAt }); } diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 304d197..0cbc7a1 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -13,7 +13,10 @@ import type { AdminStockTicker, AdminBookmark, Poe2BrowseEntry, - AdminPoe2Entry + AdminPoe2Entry, + AdminWeatherSettings, + InstalledWidget, + WidgetUploadManifest } from './adminTypes'; async function request(path: string, options: RequestInit = {}, fetchFn: typeof fetch = fetch): Promise { @@ -170,47 +173,53 @@ export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: nu return request(`/api/admin/logs${qs ? `?${qs}` : ''}`, {}, fetchFn); }; -// Weather — config/cache lives on AdminSettings.weather (see updateSettings above); this -// is just the geocoding lookup used to resolve a typed city name to lat/lon. +// Weather — config/cache now live behind the widget's own dedicated admin route (see +// backend/src/widgets/weather/plugin.ts) rather than riding along on AdminSettings. +export const getWeatherConfig = (fetchFn?: typeof fetch) => + request('/api/admin/widget/weather', {}, fetchFn); + +export const updateWeatherConfig = (patch: Partial, fetchFn?: typeof fetch) => + request('/api/admin/widget/weather', { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); + export const geocodeLocation = (query: string, fetchFn?: typeof fetch) => - request(`/api/admin/weather/geocode?query=${encodeURIComponent(query)}`, {}, fetchFn); + request(`/api/admin/widget/weather/geocode?query=${encodeURIComponent(query)}`, {}, fetchFn); // Stocks export const getStockTickers = (fetchFn?: typeof fetch) => - request('/api/admin/stocks', {}, fetchFn); + request('/api/admin/widget/stocks', {}, fetchFn); export const addStockTicker = (label: string, symbol: string, fetchFn?: typeof fetch) => - request('/api/admin/stocks', { method: 'POST', body: JSON.stringify({ label, symbol }) }, fetchFn); + request('/api/admin/widget/stocks', { method: 'POST', body: JSON.stringify({ label, symbol }) }, fetchFn); export const updateStockTicker = (id: string, patch: { label?: string; symbol?: string }, fetchFn?: typeof fetch) => - request(`/api/admin/stocks/${id}`, { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); + request(`/api/admin/widget/stocks/${id}`, { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); export const deleteStockTicker = (id: string, fetchFn?: typeof fetch) => - request(`/api/admin/stocks/${id}`, { method: 'DELETE' }, fetchFn); + request(`/api/admin/widget/stocks/${id}`, { method: 'DELETE' }, fetchFn); // Bookmarks export const getAdminBookmarks = (fetchFn?: typeof fetch) => - request('/api/admin/bookmarks', {}, fetchFn); + request('/api/admin/widget/bookmarks', {}, fetchFn); export const addBookmark = (name: string, url: string, isPrivate = false, fetchFn?: typeof fetch) => request( - '/api/admin/bookmarks', + '/api/admin/widget/bookmarks', { method: 'POST', body: JSON.stringify({ name, url, isPrivate }) }, fetchFn ); export const updateBookmark = (id: string, patch: { name?: string; url?: string; isPrivate?: boolean }, fetchFn?: typeof fetch) => - request(`/api/admin/bookmarks/${id}`, { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); + request(`/api/admin/widget/bookmarks/${id}`, { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); export const deleteBookmark = (id: string, fetchFn?: typeof fetch) => - request(`/api/admin/bookmarks/${id}`, { method: 'DELETE' }, fetchFn); + request(`/api/admin/widget/bookmarks/${id}`, { method: 'DELETE' }, fetchFn); -// PoE2 — league is always auto-detected, never admin-set (see poe2/poller.ts). +// PoE2 — league is always auto-detected, never admin-set (see widgets/poe2/poll.ts). export const browsePoe2Currencies = (fetchFn?: typeof fetch) => - request('/api/admin/poe2/browse', {}, fetchFn); + request('/api/admin/widget/poe2/browse', {}, fetchFn); export const getPoe2Watchlist = (fetchFn?: typeof fetch) => - request('/api/admin/poe2/watchlist', {}, fetchFn); + request('/api/admin/widget/poe2/watchlist', {}, fetchFn); export const addPoe2WatchlistEntry = ( base: { currencyId: string; name: string }, @@ -218,10 +227,24 @@ export const addPoe2WatchlistEntry = ( fetchFn?: typeof fetch ) => request( - '/api/admin/poe2/watchlist', + '/api/admin/widget/poe2/watchlist', { method: 'POST', body: JSON.stringify({ base, quote }) }, fetchFn ); export const removePoe2WatchlistEntry = (id: string, fetchFn?: typeof fetch) => - request(`/api/admin/poe2/watchlist/${id}`, { method: 'DELETE' }, fetchFn); + request(`/api/admin/widget/poe2/watchlist/${id}`, { method: 'DELETE' }, fetchFn); + +// Pluggable widgets (upload/list/enable/delete) — see backend/src/widgets/install.ts, +// uninstall.ts. Built-in widgets (source: 'builtin') 400 on deleteWidget. +export const listWidgets = (fetchFn?: typeof fetch) => + request('/api/admin/widgets', {}, fetchFn); + +export const installWidget = (manifest: WidgetUploadManifest, files: Record, fetchFn?: typeof fetch) => + request<{ id: string }>('/api/admin/widgets', { method: 'POST', body: JSON.stringify({ manifest, files }) }, fetchFn); + +export const setWidgetEnabled = (id: string, enabled: boolean, fetchFn?: typeof fetch) => + request(`/api/admin/widgets/${id}`, { method: 'PATCH', body: JSON.stringify({ enabled }) }, fetchFn); + +export const deleteWidget = (id: string, fetchFn?: typeof fetch) => + request(`/api/admin/widgets/${id}`, { method: 'DELETE' }, fetchFn); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index a703596..d5d6b22 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -112,12 +112,6 @@ export interface AdminPoe2Entry { lastError: string | null; } -export interface AdminPoe2Settings { - leagueId: string | null; - leagueName: string | null; - updatedAt: string | null; -} - export interface AdminWidgetsEnabled { weather: boolean; stocks: boolean; @@ -125,6 +119,27 @@ export interface AdminWidgetsEnabled { poe2: boolean; } +/** A row from the installed_widgets registry — see backend/src/storage/db/installedWidgets.ts. */ +export interface InstalledWidget { + id: string; + displayName: string; + source: 'builtin' | 'uploaded'; + enabled: boolean; + priorityRank: number; + version: string; + frontendEntry: string | null; + installedAt: string; +} + +/** Body for POST /api/admin/widgets — see backend/src/widgets/manifest.ts. */ +export interface WidgetUploadManifest { + id: string; + displayName: string; + version: string; + entry: string; + frontendEntry?: string; +} + export interface AdminSettings { mergeStrictness: 1 | 2 | 3 | 4 | 5; holdBeforePublishMinutes: number; @@ -143,8 +158,6 @@ export interface AdminSettings { widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; retention: RetentionSettings; categoryPriority: CategoryPriority[]; - weather: AdminWeatherSettings; - poe2: AdminPoe2Settings; } export interface AdminSource { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index dba92e5..46d67e3 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -43,19 +43,19 @@ export function getCategories(fetchFn?: typeof fetch): Promise { } export function getWeather(fetchFn?: typeof fetch): Promise { - return get('/api/weather', fetchFn); + return get('/api/widget/weather', fetchFn); } export function getStocks(fetchFn?: typeof fetch): Promise { - return get('/api/stocks', fetchFn); + return get('/api/widget/stocks', fetchFn); } export function getBookmarks(fetchFn?: typeof fetch): Promise { - return get('/api/bookmarks', fetchFn); + return get('/api/widget/bookmarks', fetchFn); } export function getPoe2(fetchFn?: typeof fetch): Promise { - return get('/api/poe2', fetchFn); + return get('/api/widget/poe2', fetchFn); } export function getWidgetsEnabled(fetchFn?: typeof fetch): Promise { diff --git a/frontend/src/lib/components/admin/Poe2Tab.svelte b/frontend/src/lib/components/admin/Poe2Tab.svelte index 7cfd71d..b6b45a3 100644 --- a/frontend/src/lib/components/admin/Poe2Tab.svelte +++ b/frontend/src/lib/components/admin/Poe2Tab.svelte @@ -1,9 +1,10 @@ {#each widgetOrder as key, i (key)} @@ -57,13 +134,139 @@ onMoveDown={() => move(i, 1)} > {#if key === 'weather'} - + {:else if key === 'stocks'} {:else if key === 'bookmarks'} {:else if key === 'poe2'} - + {/if} {/each} + +
+
+ Pluggable widgets + +
+ + {#if showUpload} +
+ + + + + + {#if uploadError}

{uploadError}

{/if} + +
+ {/if} + + {#if pluggable.length === 0} +

No uploaded widgets installed.

+ {:else} +
+ {#each pluggable as w (w.id)} +
+ {w.displayName} + togglePluggable(w)} role="button" tabindex="0"> + {w.enabled ? 'Active' : 'Disabled'} + + +
+ {/each} +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/sidebar/DynamicWidgetSlot.svelte b/frontend/src/lib/components/sidebar/DynamicWidgetSlot.svelte new file mode 100644 index 0000000..4bbb7bb --- /dev/null +++ b/frontend/src/lib/components/sidebar/DynamicWidgetSlot.svelte @@ -0,0 +1,67 @@ + + +
+ {#if error} +
{displayName}
+

Failed to load: {error}

+ {/if} +
+
+ + diff --git a/frontend/src/lib/components/sidebar/GenericWidgetCard.svelte b/frontend/src/lib/components/sidebar/GenericWidgetCard.svelte new file mode 100644 index 0000000..528be1f --- /dev/null +++ b/frontend/src/lib/components/sidebar/GenericWidgetCard.svelte @@ -0,0 +1,124 @@ + + +
+
+ {report?.title ?? displayName} +
+ {#if report?.headline} +
+ {report.headline.value} + {#if report.headline.delta}{report.headline.delta}{/if} +
+ {/if} + {#if report?.rows && report.rows.length > 0} +
+ {#each report.rows as row, i (i)} +
+ {row.label} + {row.value} +
+ {/each} +
+ {:else if !report?.headline} +

{loaded ? 'No data yet' : 'Loading…'}

+ {/if} +
+ + diff --git a/frontend/src/lib/components/sidebar/Sidebar.svelte b/frontend/src/lib/components/sidebar/Sidebar.svelte index 8dcda03..4f50397 100644 --- a/frontend/src/lib/components/sidebar/Sidebar.svelte +++ b/frontend/src/lib/components/sidebar/Sidebar.svelte @@ -5,6 +5,8 @@ import StocksWidget from './StocksWidget.svelte'; import BookmarksWidget from './BookmarksWidget.svelte'; import Poe2Widget from './Poe2Widget.svelte'; + import GenericWidgetCard from './GenericWidgetCard.svelte'; + import DynamicWidgetSlot from './DynamicWidgetSlot.svelte'; let { weather, @@ -101,6 +103,13 @@ {/if} {/each} + {#each widgetsEnabled.pluggable as w (w.id)} + {#if w.frontendEntry} + + {:else} + + {/if} + {/each} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 7a9d9b1..5f82215 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -173,6 +173,22 @@ export interface Poe2Data { entries: Poe2WatchlistEntry[]; } +/** An uploaded (non-core) widget the sidebar renders generically — see GenericWidgetCard.svelte / DynamicWidgetSlot.svelte. */ +export interface PluggableWidgetSummary { + id: string; + displayName: string; + /** Relative path under /widget-assets// to a custom mount() bundle — null means render the generic report card instead. */ + frontendEntry: string | null; +} + +/** Generic live-data shape a widget publishes via its own poll — see GET /api/widget/:id/report. */ +export interface WidgetReport { + title: string; + headline?: { value: string; delta?: string } | null; + rows?: { label: string; value: string }[]; + updatedAt: string | null; +} + /** Per-widget sidebar visibility + display order, admin-set from the consolidated "Widgets" tab. */ export interface WidgetsEnabled { weather: boolean; @@ -180,4 +196,6 @@ export interface WidgetsEnabled { bookmarks: boolean; poe2: boolean; order: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; + /** Enabled uploaded widgets, in their own priority order — rendered after the 4 built-ins (see Sidebar.svelte). */ + pluggable: PluggableWidgetSummary[]; } diff --git a/frontend/src/routes/admin/settings/+page.svelte b/frontend/src/routes/admin/settings/+page.svelte index 6d695ef..0a7382f 100644 --- a/frontend/src/routes/admin/settings/+page.svelte +++ b/frontend/src/routes/admin/settings/+page.svelte @@ -44,7 +44,15 @@ {:else if active === 'events'} {:else if active === 'widgets'} - + {:else if active === 'connections'} {:else if active === 'logs'} diff --git a/frontend/src/routes/admin/settings/+page.ts b/frontend/src/routes/admin/settings/+page.ts index eb4c82d..e3a00dd 100644 --- a/frontend/src/routes/admin/settings/+page.ts +++ b/frontend/src/routes/admin/settings/+page.ts @@ -10,23 +10,30 @@ import { getLogs, getStockTickers, getAdminBookmarks, - getPoe2Watchlist + getPoe2Watchlist, + getWeatherConfig, + listWidgets } from '$lib/adminApi'; +import { getPoe2 } from '$lib/api'; import type { ModelCatalog, AiStatus, TelegramStatus } from '$lib/adminTypes'; const EMPTY_MODELS: ModelCatalog = { embedding: [], image: [], synthesis: [] }; export const load: PageLoad = async ({ fetch }) => { try { - const [settings, sources, events, logs, stockTickers, bookmarks, poe2Watchlist] = await Promise.all([ - getSettings(fetch), - getSources(fetch), - getEvents(fetch), - getLogs({}, fetch), - getStockTickers(fetch), - getAdminBookmarks(fetch), - getPoe2Watchlist(fetch) - ]); + const [settings, sources, events, logs, stockTickers, bookmarks, poe2Watchlist, weatherConfig, poe2, installedWidgets] = + await Promise.all([ + getSettings(fetch), + getSources(fetch), + getEvents(fetch), + getLogs({}, fetch), + getStockTickers(fetch), + getAdminBookmarks(fetch), + getPoe2Watchlist(fetch), + getWeatherConfig(fetch), + getPoe2(fetch), + listWidgets(fetch) + ]); // The AI service (Ollama) may not be running yet — that shouldn't take down the // whole settings page, just leave the Models/Connections tabs showing "unreachable". @@ -42,7 +49,21 @@ export const load: PageLoad = async ({ fetch }) => { () => ({ credentialsConfigured: false, connected: false, phone: null }) ); - return { settings, sources, events, models, aiStatus, telegramStatus, logs, stockTickers, bookmarks, poe2Watchlist }; + return { + settings, + sources, + events, + models, + aiStatus, + telegramStatus, + logs, + stockTickers, + bookmarks, + poe2Watchlist, + weatherConfig, + poe2, + installedWidgets + }; } catch (err) { if ((err as { status?: number }).status === 401) { throw redirect(302, '/admin/login?redirectTo=/admin/settings'); From b1557d2368c04ca81535f25cd6ac9398e71e412d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 01:42:55 +0000 Subject: [PATCH 03/24] Hot-swap Fastify routes on widget install/delete instead of restarting the process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A widget's own custom routes previously required a full process restart to register — impractical in practice, since the admin API key regenerates on every restart and would log the admin out of the panel they were just using to install the widget. New backend/src/server.ts owns building and swapping the Fastify instance, split into two steps: validateRoutesBuildable() builds a candidate app and listens on a throwaway ephemeral port to catch a broken widget's route registration (e.g. a path collision) before anything live is touched, and swapLiveServer() does the real close-old/build-new/listen-new cycle on the actual port. Only the HTTP server and its router are rebuilt — the DB connection, in-memory widget registry, scheduler intervals, Telegram session, and admin API key all stay untouched in the same running process. The split exists because of a real bug hit in testing: the admin routes that trigger install/delete are themselves served by the live Fastify instance, so awaiting the full swap inline closed the connection before the response could be sent (a DELETE that should have returned 204 came back as a bare connection reset). Now install/uninstall only awaiit the safe ephemeral-port validation inline (letting a broken widget be rejected and rolled back within its own request), and the admin routes schedule the actual swap via setImmediate after their response is already on the wire. Verified live: uploaded a widget with a custom route, confirmed a clean 201 and the route working moments later with no restart (same PID, same admin API key); deleted it and confirmed a clean 204, the route gone, and core widget routes unaffected; and uploaded a deliberately broken widget whose route collided with /health, confirming it was rejected with a 400, fully rolled back, and /health kept responding normally throughout. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/api/admin.ts | 20 +++- backend/src/index.ts | 122 ++------------------- backend/src/server.ts | 178 +++++++++++++++++++++++++++++++ backend/src/widgets/install.ts | 50 +++++++-- backend/src/widgets/uninstall.ts | 12 ++- 5 files changed, 252 insertions(+), 130 deletions(-) create mode 100644 backend/src/server.ts diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 0912941..24820ca 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -13,8 +13,20 @@ import * as telegramClient from '../telegram/client.js'; import { loadedWidgets } from '../widgets/registry.js'; import { installUploadedWidget } from '../widgets/install.js'; import { uninstallWidget } from '../widgets/uninstall.js'; +import { swapLiveServer } from '../server.js'; import type { GlobalSettings } from '../storage/db/types.js'; +// Rebuilds and swaps in the live Fastify instance to pick up a widget's newly +// (de)registered routes — MUST run after the triggering request has already sent its +// response, never awaited inline in that handler, since the swap closes the very +// instance serving it (see widgets/install.ts's and uninstall.ts's doc comments for +// why — this dropped the response entirely when tried inline during testing). +function scheduleServerSwap(context: string) { + setImmediate(() => { + swapLiveServer().catch((err) => logger.error('server', `Route swap after ${context} failed: ${(err as Error).message}`)); + }); +} + // Not part of GlobalSettings itself (nothing to persist) — computed fresh on every // settings read/write so the Retention tab's "currently using" line and usage bar // always reflect the real total, not whatever was true when the row was last saved. @@ -235,7 +247,8 @@ export async function registerAdminRoutes(app: FastifyInstance) { const { manifest, files } = req.body as { manifest?: unknown; files?: unknown }; const result = await installUploadedWidget(manifest, files); if (!result.ok) return reply.code(400).send({ error: result.error }); - return reply.code(201).send({ id: result.id }); + reply.code(201).send({ id: result.id }); + if (result.needsServerSwap) scheduleServerSwap(`installing "${result.id}"`); }); app.patch('/api/admin/widgets/:id', async (req, reply) => { @@ -263,8 +276,9 @@ export async function registerAdminRoutes(app: FastifyInstance) { const widget = installedWidgetsDb.getInstalled(id); if (!widget) return reply.code(404).send({ error: 'not found' }); if (widget.source === 'builtin') return reply.code(400).send({ error: 'built-in widgets cannot be deleted' }); - await uninstallWidget(id); - return reply.code(204).send(); + const hadRoutes = await uninstallWidget(id); + reply.code(204).send(); + if (hadRoutes) scheduleServerSwap(`deleting "${id}"`); }); // --- Logs --- diff --git a/backend/src/index.ts b/backend/src/index.ts index 5563f60..e51250c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,25 +1,11 @@ -import Fastify from 'fastify'; -import cors from '@fastify/cors'; -import cookie from '@fastify/cookie'; -import fs from 'node:fs'; -import path from 'node:path'; import { migrate } from './storage/db/index.js'; import { ADMIN_API_KEY } from './api/apiKey.js'; -import { registerAuth } from './api/auth.js'; -import { registerPublicRoutes } from './api/public.js'; -import { registerAdminRoutes } from './api/admin.js'; -import { registerMediaProxy } from './api/mediaProxy.js'; -import { registerTelegramMediaProxy } from './api/telegramMediaProxy.js'; -import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js'; +import { privateAccessConfigured } from './api/privateAccess.js'; import { startScheduler } from './queue/scheduler.js'; import { initFromSavedSession } from './telegram/client.js'; import { logger } from './storage/db/logs.js'; -import { loadAllWidgets, loadedWidgets } from './widgets/registry.js'; - -const PORT = Number(process.env.PORT) || 4000; -const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173'; -const MEDIA_DIR = process.env.MEDIA_DIR || './data/media'; -const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; +import { loadAllWidgets } from './widgets/registry.js'; +import { reloadServerRoutes } from './server.js'; function printApiKeyBanner() { const line = '='.repeat(64); @@ -29,7 +15,9 @@ function printApiKeyBanner() { console.log(`\n${line}`); console.log(' Homefeed admin API key (required for every /api/admin/* request)'); console.log(` ${ADMIN_API_KEY}`); - console.log(' This key is generated fresh on every restart — it will not be the same next time.'); + console.log(' This key is generated fresh on every process restart — it will not be'); + console.log(' the same next time. Installing/deleting a widget does NOT restart the'); + console.log(' process (see server.ts) and does not change this key.'); console.log(`${line}\n`); } @@ -38,103 +26,7 @@ async function main() { printApiKeyBanner(); await initFromSavedSession(); await loadAllWidgets(); - - const app = Fastify({ logger: false }); - - // Cross-origin is expected — see project-structure.md "Cross-origin and security - // implications". Not a wildcard: only the configured frontend origin is allowed. - // @fastify/cors defaults to GET/HEAD/POST only — without an explicit methods list, - // every PATCH (settings saves) and DELETE (removing sources/events) gets silently - // blocked by the browser at the CORS preflight stage, before the request ever - // reaches a route handler. - // credentials: true is required for the browser to send/accept the private-category - // login cookie cross-origin — safe only because origin is a specific value above, - // never a wildcard (the two are mutually exclusive per the CORS spec anyway). - await app.register(cors, { - origin: FRONTEND_ORIGIN, - credentials: true, - methods: ['GET', 'POST', 'PATCH', 'DELETE', 'PUT', 'OPTIONS'] - }); - - await app.register(cookie); - - // Overrides Fastify's default JSON body parser, which throws "Body cannot be empty - // when content-type is set to 'application/json'" for any bodyless request (DELETE, - // or POST with no payload) that still carries a Content-Type header — exactly what - // browsers' fetch() does when a client sets that header unconditionally. An empty - // body is just as valid as `{}` for routes that don't read req.body at all. - app.addContentTypeParser('application/json', { parseAs: 'string' }, (_req, body, done) => { - if (typeof body !== 'string' || body.trim() === '') return done(null, {}); - try { - done(null, JSON.parse(body)); - } catch (err) { - done(err as Error, undefined); - } - }); - - await registerAuth(app); - await registerPublicRoutes(app); - await registerAdminRoutes(app); - await registerPrivateAccess(app); - - // Each loaded widget (built-in or uploaded — see widgets/registry.ts) registers its - // own routes here rather than being hardcoded into public.ts/admin.ts. Runs after - // registerAuth so any /api/admin/* route a widget registers is gated by the same - // X-Api-Key preHandler automatically. - for (const plugin of loadedWidgets.values()) { - plugin.registerPublicRoutes?.(app); - plugin.registerAdminRoutes?.(app); - } - - // Fastify's own logger is off (see below) — without this, an unhandled exception - // in any route handler produces a bare 500 with zero trace anywhere, including the - // admin panel's own Logs tab. This is what "Save failed" with no log entry was. - app.setErrorHandler((err: Error & { statusCode?: number }, req, reply) => { - logger.error('server', `${req.method} ${req.url} failed: ${err.message}`); - reply.code(err.statusCode ?? 500).send({ error: err.message }); - }); - - // Locally hosted media (see storage/media) — served directly rather than via a - // heavier static-file plugin, since this is a small, flat directory. - app.get('/media/:filename', async (req, reply) => { - const { filename } = req.params as { filename: string }; - if (filename.includes('..') || filename.includes('/')) return reply.code(400).send(); - const filePath = path.join(MEDIA_DIR, filename); - if (!fs.existsSync(filePath)) return reply.code(404).send(); - return reply.send(fs.createReadStream(filePath)); - }); - - // A widget's optional pre-built frontend bundle (see widgets/manifest.ts's - // frontendEntry) — one generic wildcard route rather than one per widget, so it works - // for a widget uploaded after this process started, with no restart (unlike a - // widget's own custom API routes, which do need one — see widgets/install.ts). - // Explicit Content-Type is required here (unlike /media/:filename above) — browsers - // reject a dynamically-imported module whose response isn't served as a JS MIME - // type. The wildcard also lets a bundle's own relative imports (e.g. `import - // './helper.mjs'`) resolve automatically, since the browser requests those against - // this same route. - app.get('/widget-assets/:id/*', async (req, reply) => { - const { id } = req.params as { id: string }; - const rel = (req.params as { '*': string })['*']; - if (rel.includes('..')) return reply.code(400).send(); - const filePath = path.join(WIDGETS_INSTALLED_DIR, id, rel); - if (!fs.existsSync(filePath)) return reply.code(404).send(); - if (rel.endsWith('.mjs') || rel.endsWith('.js')) reply.type('text/javascript'); - else if (rel.endsWith('.css')) reply.type('text/css'); - return reply.send(fs.createReadStream(filePath)); - }); - - // Static "/media/proxy" and "/media/telegram-proxy" take priority over the - // "/media/:filename" param route above regardless of registration order - // (find-my-way, Fastify's router, always prefers a static segment over a parametric - // one at the same depth). - await registerMediaProxy(app); - await registerTelegramMediaProxy(app); - - app.get('/health', async () => ({ ok: true })); - - await app.listen({ port: PORT, host: '0.0.0.0' }); - logger.info('server', `Listening on :${PORT} (frontend origin: ${FRONTEND_ORIGIN})`); + await reloadServerRoutes(); if (!privateAccessConfigured()) { logger.info('server', 'Private categories disabled — set PRIVATE_ACCESS_PASSWORD to enable'); } diff --git a/backend/src/server.ts b/backend/src/server.ts new file mode 100644 index 0000000..3b01a44 --- /dev/null +++ b/backend/src/server.ts @@ -0,0 +1,178 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import cors from '@fastify/cors'; +import cookie from '@fastify/cookie'; +import fs from 'node:fs'; +import path from 'node:path'; +import { registerAuth } from './api/auth.js'; +import { registerPublicRoutes } from './api/public.js'; +import { registerAdminRoutes } from './api/admin.js'; +import { registerMediaProxy } from './api/mediaProxy.js'; +import { registerTelegramMediaProxy } from './api/telegramMediaProxy.js'; +import { registerPrivateAccess } from './api/privateAccess.js'; +import { loadedWidgets } from './widgets/registry.js'; +import { logger } from './storage/db/logs.js'; + +const PORT = Number(process.env.PORT) || 4000; +const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173'; +const MEDIA_DIR = process.env.MEDIA_DIR || './data/media'; +const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; + +let currentApp: FastifyInstance | null = null; + +async function buildApp(): Promise { + const app = Fastify({ logger: false }); + + // Cross-origin is expected — see project-structure.md "Cross-origin and security + // implications". Not a wildcard: only the configured frontend origin is allowed. + // @fastify/cors defaults to GET/HEAD/POST only — without an explicit methods list, + // every PATCH (settings saves) and DELETE (removing sources/events) gets silently + // blocked by the browser at the CORS preflight stage, before the request ever + // reaches a route handler. + // credentials: true is required for the browser to send/accept the private-category + // login cookie cross-origin — safe only because origin is a specific value above, + // never a wildcard (the two are mutually exclusive per the CORS spec anyway). + await app.register(cors, { + origin: FRONTEND_ORIGIN, + credentials: true, + methods: ['GET', 'POST', 'PATCH', 'DELETE', 'PUT', 'OPTIONS'] + }); + + await app.register(cookie); + + // Overrides Fastify's default JSON body parser, which throws "Body cannot be empty + // when content-type is set to 'application/json'" for any bodyless request (DELETE, + // or POST with no payload) that still carries a Content-Type header — exactly what + // browsers' fetch() does when a client sets that header unconditionally. An empty + // body is just as valid as `{}` for routes that don't read req.body at all. + app.addContentTypeParser('application/json', { parseAs: 'string' }, (_req, body, done) => { + if (typeof body !== 'string' || body.trim() === '') return done(null, {}); + try { + done(null, JSON.parse(body)); + } catch (err) { + done(err as Error, undefined); + } + }); + + await registerAuth(app); + await registerPublicRoutes(app); + await registerAdminRoutes(app); + await registerPrivateAccess(app); + + // Each loaded widget (built-in or uploaded — see widgets/registry.ts) registers its + // own routes here rather than being hardcoded into public.ts/admin.ts. Runs after + // registerAuth so any /api/admin/* route a widget registers is gated by the same + // X-Api-Key preHandler automatically. + for (const plugin of loadedWidgets.values()) { + plugin.registerPublicRoutes?.(app); + plugin.registerAdminRoutes?.(app); + } + + // Fastify's own logger is off (see below) — without this, an unhandled exception + // in any route handler produces a bare 500 with zero trace anywhere, including the + // admin panel's own Logs tab. This is what "Save failed" with no log entry was. + app.setErrorHandler((err: Error & { statusCode?: number }, req, reply) => { + logger.error('server', `${req.method} ${req.url} failed: ${err.message}`); + reply.code(err.statusCode ?? 500).send({ error: err.message }); + }); + + // Locally hosted media (see storage/media) — served directly rather than via a + // heavier static-file plugin, since this is a small, flat directory. + app.get('/media/:filename', async (req, reply) => { + const { filename } = req.params as { filename: string }; + if (filename.includes('..') || filename.includes('/')) return reply.code(400).send(); + const filePath = path.join(MEDIA_DIR, filename); + if (!fs.existsSync(filePath)) return reply.code(404).send(); + return reply.send(fs.createReadStream(filePath)); + }); + + // A widget's optional pre-built frontend bundle (see widgets/manifest.ts's + // frontendEntry) — one generic wildcard route rather than one per widget, so it works + // for a widget uploaded after this process started, with no restart (unlike a + // widget's own custom API routes, which need reloadServerRoutes() below to activate). + // Explicit Content-Type is required here (unlike /media/:filename above) — browsers + // reject a dynamically-imported module whose response isn't served as a JS MIME + // type. The wildcard also lets a bundle's own relative imports (e.g. `import + // './helper.mjs'`) resolve automatically, since the browser requests those against + // this same route. + app.get('/widget-assets/:id/*', async (req, reply) => { + const { id } = req.params as { id: string }; + const rel = (req.params as { '*': string })['*']; + if (rel.includes('..')) return reply.code(400).send(); + const filePath = path.join(WIDGETS_INSTALLED_DIR, id, rel); + if (!fs.existsSync(filePath)) return reply.code(404).send(); + if (rel.endsWith('.mjs') || rel.endsWith('.js')) reply.type('text/javascript'); + else if (rel.endsWith('.css')) reply.type('text/css'); + return reply.send(fs.createReadStream(filePath)); + }); + + // Static "/media/proxy" and "/media/telegram-proxy" take priority over the + // "/media/:filename" param route above regardless of registration order + // (find-my-way, Fastify's router, always prefers a static segment over a parametric + // one at the same depth). + await registerMediaProxy(app); + await registerTelegramMediaProxy(app); + + app.get('/health', async () => ({ ok: true })); + + return app; +} + +/** + * Builds a fresh Fastify instance with every currently loaded widget's routes and swaps + * it in for the running one — the only way to pick up a route a live-uploaded widget + * declares, since Fastify refuses to add routes to an already-listening instance (throws + * "instance is already listening" synchronously). Called once at process startup (with no + * previous instance to close) and again after any widget install/uninstall that changes + * the route set (see widgets/install.ts, uninstall.ts). + * + * Deliberately reuses everything else already live in this process — the DB connection, + * the in-memory widget registry, the scheduler's setInterval loops, the Telegram client's + * session, and the admin API key all stay untouched. Only the HTTP server + its router are + * rebuilt, which is what makes this meaningfully better than a full process restart: none + * of that state is lost, and in particular the admin API key (regenerated only on true + * process start) stays valid, so installing a widget never logs the admin out. + * + * Split into two steps rather than one, because of a real deadlock/dropped-response bug + * hit in testing: the install/delete admin routes that trigger a reload are themselves + * served BY the live app instance. Awaiting the full swap (which closes that very instance) + * from inside its own still-executing request handler closed the connection before the + * response could be flushed — the client saw a bare connection reset, not a 204/201. + * + * validateRoutesBuildable() never touches the live server at all (builds on an OS-assigned + * ephemeral port and closes it again), so it's safe to await synchronously inside a + * request handler — that's what lets a broken widget's install be rejected/rolled back in + * the same response. swapLiveServer() is the part that actually closes the current + * instance; callers that are themselves inside a request handler for the live instance + * MUST defer this past sending their response (e.g. via setImmediate — see + * api/admin.ts's widget install/delete routes). Boot-time startup (see index.ts) has no + * in-flight request to worry about, so it just awaits both in sequence via + * reloadServerRoutes() below. + */ +export async function validateRoutesBuildable(): Promise { + const candidate = await buildApp(); + await candidate.listen({ port: 0, host: '127.0.0.1' }); + await candidate.close(); +} + +export async function swapLiveServer(): Promise { + const oldApp = currentApp; + // The new instance binds the same fixed PORT the old one holds, so the old one has to + // let go of it first — there's a brief window (typically well under a second) where + // nothing is listening on PORT. Acceptable for a self-hosted single-admin tool where + // this only fires right after an admin's own widget install/delete action. + if (oldApp) await oldApp.close(); + const newApp = await buildApp(); + await newApp.listen({ port: PORT, host: '0.0.0.0' }); + currentApp = newApp; + + logger.info( + 'server', + `Listening on :${PORT} (frontend origin: ${FRONTEND_ORIGIN}) — ${loadedWidgets.size} widget(s) registered` + ); +} + +/** Boot-time convenience — validate then swap in one call. Only safe when there's no in-flight request being served by the instance being replaced (i.e. process startup). */ +export async function reloadServerRoutes(): Promise { + await validateRoutesBuildable(); + await swapLiveServer(); +} diff --git a/backend/src/widgets/install.ts b/backend/src/widgets/install.ts index 789e4f5..11b2a1a 100644 --- a/backend/src/widgets/install.ts +++ b/backend/src/widgets/install.ts @@ -1,25 +1,36 @@ import fs from 'node:fs'; import path from 'node:path'; import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; -import { loadUploadedWidget } from './registry.js'; +import { loadUploadedWidget, loadedWidgets } from './registry.js'; import { startWidgetPolling } from '../queue/scheduler.js'; import { validateManifest, type WidgetManifest } from './manifest.js'; +import { validateRoutesBuildable } from '../server.js'; +import { logger } from '../storage/db/logs.js'; const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed'; -export type InstallResult = { ok: true; id: string } | { ok: false; error: string }; +export type InstallResult = + | { ok: true; id: string; needsServerSwap: boolean } + | { ok: false; error: string }; // Installs and hot-loads a widget uploaded live to the running backend (see // api/admin.ts's POST /api/admin/widgets) — writes its files under ./data/, never // dist/ or src/, so it survives a rebuild/redeploy of the core app. Its migrate() -// runs and its poll interval (if declared) starts immediately, with no restart -// required. NOTE: its registerPublicRoutes/registerAdminRoutes, if declared, do NOT -// take effect until the next restart — Fastify throws "instance is already -// listening" if you try to add a route after app.listen() has resolved, and there's -// no supported way around that short of a much larger request-dispatch redesign. A -// live-installed widget's data/poll side works immediately; its custom HTTP routes -// don't until the process restarts (see widgets/registry.ts's startup discovery, -// which re-registers everything, routes included, on every boot). +// runs and its poll interval (if declared) starts immediately. If it declares +// registerPublicRoutes/registerAdminRoutes, `needsServerSwap` comes back true — the +// caller (api/admin.ts's POST route) must call server.ts's swapLiveServer() itself, +// AFTER sending its own response, never inline here: this function runs inside the +// very request handler whose underlying Fastify instance a swap would close, so +// awaiting the swap here would drop the response before the client ever sees it (hit +// this for real in testing). validateRoutesBuildable() is safe to await here — it +// never touches the live server, only a throwaway instance on an ephemeral port — so +// a widget whose routes are actually broken (e.g. a path collision) is still caught +// and rolled back within this same call, before anything user-visible commits. +// +// Either way this is a full HTTP-server rebuild within the running process, not a +// process restart — the DB connection, scheduler intervals, Telegram session, and +// (critically) the admin API key all survive; a process restart would regenerate the +// key and log the admin out. export async function installUploadedWidget(manifest: unknown, files: unknown): Promise { const validationError = validateManifest(manifest, files); if (validationError) return { ok: false, error: validationError }; @@ -55,6 +66,23 @@ export async function installUploadedWidget(manifest: unknown, files: unknown): frontendEntry: m.frontendEntry ?? null }); + const needsServerSwap = !!(plugin.registerPublicRoutes || plugin.registerAdminRoutes); + if (needsServerSwap) { + try { + await validateRoutesBuildable(); + } catch (err) { + // A widget whose routes break Fastify's registration (e.g. a path collision) + // isn't a successful install — the site itself was never at risk since this + // only ever touched a throwaway ephemeral-port instance, but this widget still + // needs to be fully rolled back rather than left half-installed. + logger.error('widgets', `Install of "${m.id}" rolled back — its routes failed to register: ${(err as Error).message}`); + loadedWidgets.delete(m.id); + installedWidgetsDb.deleteInstalled(m.id); + fs.rmSync(dir, { recursive: true, force: true }); + return { ok: false, error: `widget's routes failed to register: ${(err as Error).message}` }; + } + } + startWidgetPolling(plugin); - return { ok: true, id: m.id }; + return { ok: true, id: m.id, needsServerSwap }; } diff --git a/backend/src/widgets/uninstall.ts b/backend/src/widgets/uninstall.ts index 46f5b1a..6d6426e 100644 --- a/backend/src/widgets/uninstall.ts +++ b/backend/src/widgets/uninstall.ts @@ -15,10 +15,18 @@ const WIDGETS_DATA_DIR = process.env.WIDGETS_DATA_DIR || './data/widgets-data'; // guarantee, matching every table/kv row/on-disk file regardless of the widget's own // cooperation). Callers (see api/admin.ts's DELETE /api/admin/widgets/:id) are // responsible for rejecting built-in widgets before calling this. -export async function uninstallWidget(id: string): Promise { +// +// Returns whether the deleted widget had declared routes, i.e. whether the caller needs +// to swap the live server afterward (see server.ts's swapLiveServer()) — deliberately NOT +// done inline here: this runs inside the DELETE route's own request handler, and that +// handler is served by the very Fastify instance a swap would close, which drops the +// response before the client ever sees it (hit this for real in testing). The caller must +// send its response first, then swap — see api/admin.ts. +export async function uninstallWidget(id: string): Promise { stopWidgetPolling(id); const plugin = loadedWidgets.get(id); + const hadRoutes = !!(plugin?.registerPublicRoutes || plugin?.registerAdminRoutes); if (plugin?.uninstall) { try { plugin.uninstall(db); @@ -35,4 +43,6 @@ export async function uninstallWidget(id: string): Promise { loadedWidgets.delete(id); installedWidgetsDb.deleteInstalled(id); + + return hadRoutes; } From ee585ea65cc0652b85aeeb7449e87eb413d78e62 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 03:31:31 +0000 Subject: [PATCH 04/24] Fix silent Ollama prompt truncation in synthesis pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ollama was defaulting to a 4096-token context (vs. the model's 32768 training context) and silently truncating any oversized prompt by dropping content from the middle, with no error surfaced anywhere — observed losing ~53% of a merge-cluster prompt in production. Two prompt builders (buildPrompt/buildRecapPrompt) concatenated all source summaries/article bodies with no size cap, so a cluster with enough sources (or a recap spanning enough articles) could easily exceed the window. Fix: OllamaProvider.generate() now always sends explicit num_ctx/ num_predict options (sized for CPU-only inference — i5-6600K, no GPU, ~17 tok/s prompt processing) instead of leaving Ollama to pick a default. synthesis.ts now caps prompt size itself before it ever reaches Ollama, giving each source/article an equal character budget and trimming individual entries rather than dropping whole ones off the end — every source stays at least partially represented and attributable. Trims are logged via the existing admin log stream instead of failing silently. --- backend/src/inference/ollama-provider.ts | 31 ++++++++++++- backend/src/inference/provider.ts | 2 +- backend/src/pipeline/synthesis.ts | 58 ++++++++++++++++++++---- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index b735c5f..6817232 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,25 @@ import type { InferenceProvider } from './provider.js'; +/** + * Default context window / max-generation length requested from Ollama when a caller + * doesn't specify its own. Ollama otherwise falls back to whatever the model's + * Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct + * here, well under that model's 32768-token training context) and SILENTLY truncates + * any prompt that doesn't fit — dropping the middle of the prompt with no error + * surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the + * limit deliberate and stable instead of whatever Ollama happens to pick. + * + * 8192 is sized for CPU-only inference (the reference box is an i5-6600K running + * Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the + * constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but + * prompt-processing time scales with context, so this trades headroom against + * per-request latency rather than maxing out the model's full 32768-token capacity. + * Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit + * within this budget up front, rather than relying on Ollama to truncate for them. + */ +export const DEFAULT_NUM_CTX = 8192; +export const DEFAULT_NUM_PREDICT = 700; + /** * Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend * setting (GlobalSettings.aiServiceHost/Port), editable via the admin panel — @@ -15,7 +35,10 @@ export class OllamaProvider implements InferenceProvider { return `${this.host}:${this.port}`; } - async generate(prompt: string, opts: { model?: string; system?: string } = {}): Promise { + async generate( + prompt: string, + opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {} + ): Promise { const res = await fetch(`${this.base()}/api/generate`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -23,7 +46,11 @@ export class OllamaProvider implements InferenceProvider { model: opts.model, prompt, system: opts.system, - stream: false + stream: false, + options: { + num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, + num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT + } }) }); if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); diff --git a/backend/src/inference/provider.ts b/backend/src/inference/provider.ts index d994b1d..aaa61f8 100644 --- a/backend/src/inference/provider.ts +++ b/backend/src/inference/provider.ts @@ -1,5 +1,5 @@ export interface InferenceProvider { - generate(prompt: string, opts?: { model?: string; system?: string }): Promise; + generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise; embed(text: string, opts?: { model?: string }): Promise; listModels(): Promise; isReachable(): Promise; diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index d52a539..2cbac86 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -1,8 +1,28 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { ContentItem, MergedArticle } from '../storage/db/types.js'; +import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; +import { logger } from '../storage/db/logs.js'; const TAG_DELIMITER = '---TAGS---'; +// Ollama truncates prompts that don't fit its context window by keeping a small prefix +// and dropping everything else in the middle — silently, with no error, and with no +// regard for which sources end up cut (see ollama-provider.ts for the incident that +// prompted this). Rather than relying on that, prompts here are sized to fit +// DEFAULT_NUM_CTX up front: each source/article gets an equal character budget, cut only +// when the whole prompt would otherwise overflow, so every source stays at least +// partially represented (and attributable) instead of some being dropped outright. +// ~4 chars/token is a rough heuristic (no tokenizer available here) — good enough for a +// safety margin, not meant to be exact. +const CHARS_PER_TOKEN = 4; +const RESERVED_OVERHEAD_TOKENS = 300; // system prompt + per-entry headers/formatting +const MAX_INPUT_CHARS = (DEFAULT_NUM_CTX - DEFAULT_NUM_PREDICT - RESERVED_OVERHEAD_TOKENS) * CHARS_PER_TOKEN; +const MIN_ENTRY_CHARS = 300; // floor so a huge cluster/recap doesn't shrink every entry to nothing + +function capEntryText(text: string, budgetChars: number): string { + return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; +} + const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: - Summarizes what has happened across the period covered, in chronological order - Highlights the most significant developments rather than restating every article @@ -27,9 +47,17 @@ export interface SynthesisResult { } function buildPrompt(items: ContentItem[]): string { - return items - .map((item, i) => `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${item.summary}`) - .join('\n\n'); + const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); + let truncated = 0; + const entries = items.map((item, i) => { + const summary = capEntryText(item.summary, budgetPerItem); + if (summary !== item.summary) truncated++; + return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`; + }); + if (truncated > 0) { + logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return entries.join('\n\n'); } function parseResult(raw: string): SynthesisResult { @@ -48,15 +76,22 @@ export async function synthesizeArticle( items: ContentItem[] ): Promise { const prompt = buildPrompt(items); - const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string { - const entries = articles - .map((article, i) => `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${article.body}`) - .join('\n\n'); - return `Tracked event: ${eventName}\n\n${entries}`; + const budgetPerArticle = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / articles.length)); + let truncated = 0; + const entries = articles.map((article, i) => { + const body = capEntryText(article.body, budgetPerArticle); + if (body !== article.body) truncated++; + return `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${body}`; + }); + if (truncated > 0) { + logger.warn('events', `Trimmed ${truncated}/${articles.length} recap article bod${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return `Tracked event: ${eventName}\n\n${entries.join('\n\n')}`; } /** @@ -74,6 +109,11 @@ export async function synthesizeRecap( articles: MergedArticle[] ): Promise { const prompt = buildRecapPrompt(eventName, articles); - const raw = await provider.generate(prompt, { model, system: RECAP_SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { + model, + system: RECAP_SYSTEM_PROMPT, + numCtx: DEFAULT_NUM_CTX, + numPredict: DEFAULT_NUM_PREDICT + }); return parseResult(raw); } From 53ebb683397bd1b56d7843dd50335f42934a0609 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 03:52:42 +0000 Subject: [PATCH 05/24] Use full article body, not just the RSS blurb, in synthesis prompts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildPrompt() only ever sent ContentItem.summary (a ~500-char RSS description) to the model, never .body (the full article text when the feed provides ) — even though publishDirect already preferred body over summary for the no-AI-merge path. A single-source cluster was effectively asking the model to "lightly rewrite" a one-paragraph blurb, which it did almost verbatim, producing a short repeated synopsis instead of an actual article. Now mirrors publishDirect's item.body || item.summary fallback. Body is already HTML-stripped at ingestion (ingestion/adapters/base.ts), so no new sanitization needed. The per-entry character budget added in the previous truncation fix now does real work here, since full bodies can be much longer than summaries. --- backend/src/pipeline/synthesis.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 2cbac86..1405563 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -50,12 +50,17 @@ function buildPrompt(items: ContentItem[]): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; const entries = items.map((item, i) => { - const summary = capEntryText(item.summary, budgetPerItem); - if (summary !== item.summary) truncated++; - return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`; + // Same fallback publishDirect uses (publish.ts) — body is the full article text + // when the feed supplies it (e.g. RSS ), summary is a ~500-char + // blurb. Using summary alone starved the model of real content to synthesize + // from, so a single-source cluster just echoed the blurb back nearly verbatim. + const full = item.body || item.summary; + const text = capEntryText(full, budgetPerItem); + if (text !== full) truncated++; + return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`; }); if (truncated > 0) { - logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`); } return entries.join('\n\n'); } From e063d90c9784df3e83a1219fbd7d803f15f6aa60 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 04:10:01 +0000 Subject: [PATCH 06/24] Add per-category "No AI" toggle to skip clustering/synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category priority admin pane gains a "No AI" checkbox alongside Private/More. When set, items whose source falls under that category skip embedding, clustering, and LLM synthesis entirely — each publishes on its own, verbatim from its source (title + body/summary), the same direct-publish path YouTube/Nitter/Telegram items always use. Backend: new categories.disable_ai column (default off, migrated in for existing installs), threaded through categories.ts CRUD and the POST /api/admin/categories + PATCH /api/admin/settings routes. priorityQueue.ts's runSynthesisCycle now partitions items three ways before clustering: source-type direct (youtube/nitter/telegram), category-disabled direct (new), then whatever's left goes through the normal embed/cluster/synthesize pipeline. Tracked-event recaps are a separate, already-existing per-event toggle (TrackedEvent.recapIntervalHours) since events aren't tied to a single category — unaffected by this change. --- backend/src/api/admin.ts | 9 ++++- backend/src/queue/priorityQueue.ts | 37 ++++++++++++++++--- backend/src/storage/db/categories.ts | 19 ++++++---- backend/src/storage/db/index.ts | 6 ++- backend/src/storage/db/types.ts | 2 + frontend/src/lib/adminApi.ts | 10 ++++- frontend/src/lib/adminTypes.ts | 1 + .../src/lib/components/admin/MergeTab.svelte | 21 ++++++++++- 8 files changed, 84 insertions(+), 21 deletions(-) diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 24820ca..0d2b353 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -69,9 +69,14 @@ export async function registerAdminRoutes(app: FastifyInstance) { // --- Categories (add/remove — reordering/privacy is via PATCH /settings above) --- app.post('/api/admin/categories', async (req, reply) => { - const { name, isPrivate, isSpillover } = req.body as { name?: string; isPrivate?: boolean; isSpillover?: boolean }; + const { name, isPrivate, isSpillover, disableAi } = req.body as { + name?: string; + isPrivate?: boolean; + isSpillover?: boolean; + disableAi?: boolean; + }; if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' }); - const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover); + const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover, !!disableAi); return reply.code(201).send(created); }); diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 1eded36..3894bc8 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -45,6 +45,16 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map, return best; } +/** True if any of the item's source's categories (same leading-segment match as primaryCategoryRank) has AI disabled. */ +function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sourcesById: Map): boolean { + const source = sourcesById.get(item.sourceId); + for (const cat of source?.category ?? []) { + const leading = cat.split(':')[0].trim().toLowerCase(); + if (disabledNames.has(leading)) return true; + } + return false; +} + /** * Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same * publish-then-tag-then-log/error shape, differing only in how the success/failure @@ -114,6 +124,8 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // direct-publish partition and each item's category/type lookups — avoids a // separate sourcesDb.getSource() round-trip per item. const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with // anything else — each is always its own article, same shape whether the AI service @@ -121,18 +133,31 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const directPublishSourceIds = new Set( [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); - const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); + const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - const publishedDirect = await publishItemsDirect( - directItems, + // A category with disableAi set (see the Category priority admin pane) opts its + // items out of clustering/synthesis entirely — each publishes on its own, using its + // own source's text, same as the source-type-driven direct items above. + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + const [categoryDirectItems, mergeableItems] = partition(remaining, (item) => + inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) + ); + + const publishedTypeDirect = await publishItemsDirect( + typeDirectItems, settings, activeEvents, (item) => sourcesById.get(item.sourceId)?.type ?? 'unknown', 'Direct publish failed' ); - const categories = categoriesDb.listCategories(); - const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); + const publishedCategoryDirect = await publishItemsDirect( + categoryDirectItems, + settings, + activeEvents, + () => 'AI disabled for category', + 'Direct publish failed' + ); const ranked = mergeableItems .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) @@ -184,5 +209,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } - return published + publishedDirect; + return published + publishedTypeDirect + publishedCategoryDirect; } diff --git a/backend/src/storage/db/categories.ts b/backend/src/storage/db/categories.ts index 2397610..42fb7d3 100644 --- a/backend/src/storage/db/categories.ts +++ b/backend/src/storage/db/categories.ts @@ -9,7 +9,8 @@ function rowToCategory(row: any): Category { priorityRank: row.priority_rank, isDefault: !!row.is_default, isPrivate: !!row.is_private, - isSpillover: !!row.is_spillover + isSpillover: !!row.is_spillover, + disableAi: !!row.disable_ai }; } @@ -24,18 +25,20 @@ export function listPrivateCategoryNames(): string[] { return rows.map((r) => r.name); } -export function setCategoryOrder(order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean }[]) { - const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ? WHERE id = ?'); - for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.id); +export function setCategoryOrder( + order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean; disableAi: boolean }[] +) { + const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ?, disable_ai = ? WHERE id = ?'); + for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.disableAi ? 1 : 0, c.id); } -export function createCategory(name: string, isPrivate = false, isSpillover = false): Category { +export function createCategory(name: string, isPrivate = false, isSpillover = false, disableAi = false): Category { const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`; const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number }; db.prepare( - 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover) VALUES (?, ?, ?, 0, ?, ?)' - ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0); - return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover }; + 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover, disable_ai) VALUES (?, ?, ?, 0, ?, ?, ?)' + ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0, disableAi ? 1 : 0); + return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover, disableAi }; } export function deleteCategory(id: string) { diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index b113c97..e4f0c3f 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -179,7 +179,8 @@ export function migrate() { priority_rank INTEGER NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, is_private INTEGER NOT NULL DEFAULT 0, - is_spillover INTEGER NOT NULL DEFAULT 0 -- collapsed into the nav's "More »" overflow page instead of its own tab + is_spillover INTEGER NOT NULL DEFAULT 0, -- collapsed into the nav's "More »" overflow page instead of its own tab + disable_ai INTEGER NOT NULL DEFAULT 0 -- skip clustering/synthesis for this category's items; publish each one directly ); CREATE TABLE IF NOT EXISTS logs ( @@ -319,6 +320,9 @@ export function migrate() { if (!hasColumn('categories', 'is_spillover')) { db.exec('ALTER TABLE categories ADD COLUMN is_spillover INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('categories', 'disable_ai')) { + db.exec('ALTER TABLE categories ADD COLUMN disable_ai INTEGER NOT NULL DEFAULT 0'); + } if (!hasColumn('content_items', 'telegram_message')) { db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT'); } diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 53fd98d..a6a31eb 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -206,6 +206,8 @@ export interface Category { isPrivate: boolean; /** Grouped into the nav's "More »" overflow page instead of getting its own top-level tab — see +layout.svelte and /more. */ isSpillover: boolean; + /** Skips clustering/AI synthesis for this category's items — each one publishes directly (own article, own source's text), same as YouTube/Nitter/Telegram items always do. See priorityQueue.ts's runSynthesisCycle. */ + disableAi: boolean; } export interface StockTicker { diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 0cbc7a1..b52060f 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -69,10 +69,16 @@ export const updateSettings = (patch: Partial, fetchFn?: typeof f request('/api/admin/settings', { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); // Categories -export const createCategory = (name: string, isPrivate = false, isSpillover = false, fetchFn?: typeof fetch) => +export const createCategory = ( + name: string, + isPrivate = false, + isSpillover = false, + disableAi = false, + fetchFn?: typeof fetch +) => request( '/api/admin/categories', - { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover }) }, + { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover, disableAi }) }, fetchFn ); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index d5d6b22..3ed0dc7 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -14,6 +14,7 @@ export interface CategoryPriority { isDefault: boolean; isPrivate: boolean; isSpillover: boolean; + disableAi: boolean; } export interface WeatherHourEntry { diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index ab4d0fa..731f143 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -13,6 +13,7 @@ let newCategoryName = $state(''); let newCategoryPrivate = $state(false); let newCategorySpillover = $state(false); + let newCategoryDisableAi = $state(false); let addingCategory = $state(false); // Advisory only — the nav starts getting too wide / wrapping past ~10 tabs, so this @@ -48,11 +49,12 @@ if (!name) return; addingCategory = true; try { - const created = await createCategory(name, newCategoryPrivate, newCategorySpillover); + const created = await createCategory(name, newCategoryPrivate, newCategorySpillover, newCategoryDisableAi); local.categoryPriority = [...local.categoryPriority, created]; newCategoryName = ''; newCategoryPrivate = false; newCategorySpillover = false; + newCategoryDisableAi = false; } finally { addingCategory = false; } @@ -68,6 +70,11 @@ scheduleSave(); } + function toggleDisableAi(id: string) { + local.categoryPriority = local.categoryPriority.map((c) => (c.id === id ? { ...c, disableAi: !c.disableAi } : c)); + scheduleSave(); + } + async function removeCategory(id: string, isDefault: boolean, name: string) { if (isDefault) { // Sensible-default categories can still be removed — e.g. a fresh install's @@ -98,7 +105,9 @@ private category (and everything in it) is hidden from the public site until a visitor logs in with the lock icon in the masthead. A "More" category is collapsed into a single "More »" nav tab instead of getting its own, and shows up on that overflow page with its - latest few articles. + latest few articles. "No AI" skips clustering and synthesis for that category — each item + publishes on its own, using its own source's text, instead of being merged/rewritten by the + model.

{#if primaryCategoryCount > 10}

@@ -120,6 +129,10 @@ toggleSpillover(cat.id)} /> More + {/if} From adb2783f1b950de819abe13c62de6cc86bb1483d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:10:50 +0000 Subject: [PATCH 07/24] Fix synthesis fetch failures from Node's default 5-minute HTTP timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing published for hours, every cluster failing with "fetch failed". Ollama's own log showed the real story: requests being cancelled at exactly 5m0s with a 500, not a model or server error. Node's global fetch (undici) defaults to a 5-minute headers/body timeout, and CPU-only prompt processing on the reference hardware (i5-6600K, no GPU, ~17 tok/s) legitimately takes longer than that once prompts carry full article bodies instead of short blurbs (the previous fix in this same line of work) — every generate() call past a few thousand tokens got killed client-side before Ollama could finish. OllamaProvider.generate() now passes a dedicated undici Agent with headersTimeout/bodyTimeout disabled as the fetch dispatcher, so the request runs as long as it actually needs to. Verified the failure mode and the fix directly: a short-timeout dispatcher against a deliberately slow server reproduces the exact same "fetch failed" / UND_ERR_HEADERS_TIMEOUT error seen in production, and a zero-timeout dispatcher completes the same slow request without issue. undici was already a transitive dependency (via jsdom); added directly since ollama-provider.ts now imports from it. --- backend/package-lock.json | 3 ++- backend/package.json | 3 ++- backend/src/inference/ollama-provider.ts | 21 +++++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 3bff8b6..120652a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,7 +15,8 @@ "fastify": "^5.10.0", "jsdom": "^29.1.1", "rss-parser": "^3.13.0", - "telegram": "^2.26.22" + "telegram": "^2.26.22", + "undici": "^7.28.0" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/package.json b/backend/package.json index 9eca3ab..fb0285e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -18,7 +18,8 @@ "fastify": "^5.10.0", "jsdom": "^29.1.1", "rss-parser": "^3.13.0", - "telegram": "^2.26.22" + "telegram": "^2.26.22", + "undici": "^7.28.0" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 6817232..7173922 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,19 @@ +import { Agent } from 'undici'; import type { InferenceProvider } from './provider.js'; +/** + * Node's global fetch (undici) defaults to a 5-minute headers/body timeout — fine for + * ordinary HTTP calls, but a real problem for /api/generate on CPU-only inference: a + * near-full context window can legitimately take longer than that just for prompt + * processing on the reference hardware (i5-6600K, no GPU, ~17 tokens/sec). Once + * synthesis prompts started carrying full article bodies instead of short blurbs, every + * generate() call past a few thousand tokens got killed at exactly 5m0s — visible in + * Ollama's own log as the request being cancelled, not a genuine model/server error — + * so no cluster could ever finish synthesizing. No timeout at all here; Ollama's own + * process is the natural backstop, not a clock tuned for hardware this doesn't run on. + */ +const noTimeoutDispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); + /** * Default context window / max-generation length requested from Ollama when a caller * doesn't specify its own. Ollama otherwise falls back to whatever the model's @@ -51,8 +65,11 @@ export class OllamaProvider implements InferenceProvider { num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT } - }) - }); + }), + // Not in the ambient RequestInit type this project resolves to, but Node's global + // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. + dispatcher: noTimeoutDispatcher + } as RequestInit); if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); const data = (await res.json()) as { response: string }; return data.response; From 3eeee956cb0a89534d32a3a7c9706af3004b567f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:22:23 +0000 Subject: [PATCH 08/24] Fix scheduler racing itself into publishing duplicate articles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthesis tick fires every 60 seconds via setInterval with no reentrancy guard. An item only gets marked "clustered" after its article finishes synthesizing and publishing — so once generate() calls started legitimately taking longer than 60 seconds (bigger prompts + no client timeout, both from earlier fixes in this line of work), the next tick would fire mid-generation, see the same item still "unclustered", and synthesize + publish it again as a fresh, differently-worded article. Repeated overlaps produced a run of near-identical articles from the same single source item, seconds apart. everyTickSkippingOverlap() now guards all three scheduler intervals (poll, synthesis, retention): a tick is skipped outright if the previous invocation hasn't finished, rather than overlapping it. Verified in isolation — a task slower than its own tick interval never overlaps itself (measured max concurrency of 1). --- backend/src/queue/scheduler.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index c20a905..5b69c8e 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -13,6 +13,28 @@ const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each const SYNTHESIS_TICK_MS = 60_000; const RETENTION_TICK_MS = 60 * 60_000; // hourly +/** + * Runs fn on every tick, but skips a tick outright if the previous one is still in + * flight instead of overlapping it. Matters most for the synthesis tick: an item stays + * "unclustered" (cluster_id IS NULL — see contentItems.unclusteredItemsExcludingSources) + * until AFTER its cluster finishes synthesizing and publishing, so a generate() call + * that runs past the next tick (easily minutes, on CPU-only inference — see + * ollama-provider.ts) used to let the same item get picked up and republished as a + * fresh, differently-worded article by an overlapping cycle, repeatedly, until the + * first cycle's assignCluster() finally landed. Node is single-threaded, so the only + * source of "concurrent" runs here is exactly this interval overlap. + */ +function everyTickSkippingOverlap(ms: number, fn: () => Promise) { + let running = false; + setInterval(() => { + if (running) return; + running = true; + fn().finally(() => { + running = false; + }); + }, ms); +} + // Per-widget setInterval handles, keyed by widget id — lets a single widget's polling be // started/stopped independently (on live upload/delete, or an enable toggle) without // touching any other widget's interval. Exported so widgets/install.ts and @@ -53,16 +75,16 @@ export function startScheduler() { return new OllamaProvider(s.aiServiceHost, s.aiServicePort); }; - setInterval(async () => { + everyTickSkippingOverlap(POLL_TICK_MS, async () => { try { const ingested = await pollDueSources(); if (ingested > 0) logger.info('scheduler', `Poll tick: ingested ${ingested} new item(s)`); } catch (err) { logger.error('scheduler', `Poll tick failed: ${(err as Error).message}`); } - }, POLL_TICK_MS); + }); - setInterval(async () => { + everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); const p = provider(); @@ -86,16 +108,16 @@ export function startScheduler() { } catch (err) { logger.error('scheduler', `Synthesis tick failed: ${(err as Error).message}`); } - }, SYNTHESIS_TICK_MS); + }); - setInterval(() => { + everyTickSkippingOverlap(RETENTION_TICK_MS, async () => { try { runRetentionSweep(settingsDb.getSettings()); logger.info('retention', 'Retention sweep completed'); } catch (err) { logger.error('retention', `Retention tick failed: ${(err as Error).message}`); } - }, RETENTION_TICK_MS); + }); for (const plugin of loadedWidgets.values()) { startWidgetPolling(plugin); From 3d47aec353ac1977136a90d3f68cafb83d9ce6f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:26:43 +0000 Subject: [PATCH 09/24] Add 15-minute and 1-hour options to Hold before publish setting --- frontend/src/lib/components/admin/MergeTab.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index 731f143..7439a51 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -204,7 +204,9 @@

Wait window to gather more sources before finalizing a story.

From 4b2def21511829efcbc0df4dc5019aa3156d038f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:37:37 +0000 Subject: [PATCH 10/24] Fix synthesis prompt labeling sources by opaque ID, causing hallucinated attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildPrompt() labeled each source with item.sourceId — an internal DB foreign key like "src-e8dbf745-..." — never the outlet's actual name. The model had no real outlet to attribute to, so on a single-source item it fell back to copying the illustrative example names straight out of its own system prompt ("Reuters reported...", "AP notes...") and fabricated a two-outlet merge out of one real 6abc article. The article's sources metadata (built separately from real DB records) was correct the whole time; only the AI-written body text invented sources that were never in the input. synthesizeArticle now takes a sourceId->name map (built in publish.ts via the same sources.getSource() lookup already used for the sources metadata) and buildPrompt labels each entry with the real name. SYSTEM_PROMPT no longer gives concrete example outlet names to copy — it references "each source's exact name as given below" and explicitly forbids attributing to any outlet not actually provided. Verified directly: captured the exact prompt text sent to a mock provider and confirmed it now contains the real source name and never the raw internal id. --- backend/src/pipeline/publish.ts | 3 ++- backend/src/pipeline/synthesis.ts | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index ab19350..d09a7b4 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -360,7 +360,8 @@ export async function publishCluster( ): Promise { const items = cluster.items; - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items); + const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); + const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames); const resolvedTags = []; for (const label of tagLabels) { diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 1405563..fd1898f 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -32,12 +32,12 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: -- Attributes specific claims to the outlet that reported them (e.g. "Reuters reported...", "AP notes...") +- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. - Does not copy phrasing verbatim from any source - Stays neutral and factual, without editorializing - Is 2-4 short paragraphs -If only one source is provided, lightly rewrite it in your own words rather than merging. +If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source. After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; @@ -46,7 +46,7 @@ export interface SynthesisResult { tagLabels: string[]; } -function buildPrompt(items: ContentItem[]): string { +function buildPrompt(items: ContentItem[], sourceNames: Map): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; const entries = items.map((item, i) => { @@ -57,7 +57,13 @@ function buildPrompt(items: ContentItem[]): string { const full = item.body || item.summary; const text = capEntryText(full, budgetPerItem); if (text !== full) truncated++; - return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`; + // The label here (not item.sourceId, an opaque internal id the model can't use) + // is the only real outlet name the model ever sees — without it, a small model + // has nothing to attribute to and falls back to copying the illustrative outlet + // names out of its own system prompt instructions instead (seen in production: + // a single-source item fabricating "Reuters reported..."/"AP notes..." wholesale). + const name = sourceNames.get(item.sourceId) ?? 'Unknown source'; + return `Source ${i + 1} (${name}):\nTitle: ${item.title}\nSummary: ${text}`; }); if (truncated > 0) { logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`); @@ -78,9 +84,10 @@ function parseResult(raw: string): SynthesisResult { export async function synthesizeArticle( provider: InferenceProvider, model: string, - items: ContentItem[] + items: ContentItem[], + sourceNames: Map ): Promise { - const prompt = buildPrompt(items); + const prompt = buildPrompt(items, sourceNames); const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } From 964762d1b087dc36960550f194c40cf251d229bc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:41:11 +0000 Subject: [PATCH 11/24] Skip the AI rewrite entirely for single-source clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cluster of one item still went through synthesizeArticle to be "lightly rewritten" — the only recent real-world example fabricated a fake two-outlet merge out of one genuine article (see the opaque-sourceId attribution fix). There's no actual synthesis to do with one source, so the rewrite step only added risk (hallucinated attribution, subtly altered facts) for no benefit. priorityQueue.ts's runSynthesisCycle now routes a 1-item cluster to publishDirect instead of publishCluster — same verbatim-text path already used for youtube/nitter/telegram items and AI-disabled categories. publishCluster is now only ever called with 2+ items, so its doc comment and synthesis.ts's system prompt no longer reference the single-source case. Verified directly: a 1-item cluster now publishes with the original body untouched and zero calls to the model, while a 2-item cluster still goes through the AI merge path unchanged. --- backend/src/pipeline/publish.ts | 5 +++-- backend/src/pipeline/synthesis.ts | 2 -- backend/src/queue/priorityQueue.ts | 9 ++++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index d09a7b4..27ac515 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -349,8 +349,9 @@ export async function publishDirect( /** * Publishing is always automatic — there's no draft/review state (see schema doc). - * A cluster of size 1 publishes as-is via the same path; synthesizeArticle lightly - * rewrites rather than merges when there's only one source. + * Callers should route a size-1 cluster to publishDirect instead — there's nothing to + * merge, so an LLM rewrite would only add risk (hallucinated attribution, altered + * facts) for no synthesis benefit. See priorityQueue.ts's runSynthesisCycle. */ export async function publishCluster( provider: InferenceProvider, diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index fd1898f..21bacbf 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -37,8 +37,6 @@ const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summari - Stays neutral and factual, without editorializing - Is 2-4 short paragraphs -If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source. - After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; export interface SynthesisResult { diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 3894bc8..15323e3 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -186,7 +186,14 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // in practice a cluster's items are all near-duplicate coverage of the same // story, so they'd all match the same event's filter anyway when they match at all. const eventId = cluster.items.map((i) => claimedEventId(i, activeEvents)).find((id) => id !== null) ?? undefined; - const article = await publishCluster(provider, settings, cluster, { eventId }); + // A single-item cluster has nothing to merge — publish the source's own text + // verbatim instead of asking the LLM to "lightly rewrite" it, which only risked + // introducing errors (or fabricated attribution — see synthesis.ts) with no + // actual synthesis to justify the risk. + const article = + cluster.items.length === 1 + ? await publishDirect(cluster.items[0], settings, { eventId }) + : await publishCluster(provider, settings, cluster, { eventId }); contentItemsDb.assignCluster( cluster.items.map((i) => i.id), cluster.id From 8170c00bf0a4d41eacd56872fc87b4e2f19e6091 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:55:26 +0000 Subject: [PATCH 12/24] Add admin-configurable writing style for AI synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthesis system prompts were previously the only "instructions" the AI ever got, hardcoded and invisible from the admin panel — no way to control tone, and no way to know what was actually being sent without reading the source. Adds a "Writing style" panel to the Merge tab: a preset dropdown (Default/Casual/Formal) plus a free-text field for arbitrary additional instructions (e.g. "keep paragraphs under 3 sentences"). Both are appended as an addendum to the existing base system prompts in synthesis.ts — the structural rules (attribution, paragraph count, tag format) are never overridden, only style on top of them. Applies to AI-merged articles and event recaps; single-source items still publish verbatim with no AI involved either way. Backend: new global_settings.synthesis_style_preset (default) and .synthesis_custom_instructions ('') columns, migrated in for existing installs, threaded through settings.ts and into synthesizeArticle/ synthesizeRecap's system prompt construction. Verified: settings round-trip through GET/PATCH /api/admin/settings with correct defaults; a captured prompt confirms 'default' with no custom text produces the exact original prompt unchanged, while 'casual' + custom text appends both correctly; migration against an old-schema global_settings table adds both columns with correct defaults. --- backend/src/pipeline/publish.ts | 4 +-- backend/src/pipeline/synthesis.ts | 35 +++++++++++++++---- backend/src/storage/db/index.ts | 8 +++++ backend/src/storage/db/settings.ts | 5 +++ backend/src/storage/db/types.ts | 4 +++ frontend/src/lib/adminTypes.ts | 2 ++ .../src/lib/components/admin/MergeTab.svelte | 33 ++++++++++++++++- 7 files changed, 81 insertions(+), 10 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 27ac515..b5348ef 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -362,7 +362,7 @@ export async function publishCluster( const items = cluster.items; const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames); + const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -461,7 +461,7 @@ export async function publishEventRecap( event: TrackedEvent, constituents: MergedArticle[] ): Promise { - const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents); + const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); const resolvedTags = []; for (const label of tagLabels) { diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 21bacbf..a3be66c 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -1,5 +1,5 @@ import type { InferenceProvider } from '../inference/provider.js'; -import type { ContentItem, MergedArticle } from '../storage/db/types.js'; +import type { ContentItem, GlobalSettings, MergedArticle } from '../storage/db/types.js'; import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; import { logger } from '../storage/db/logs.js'; @@ -23,7 +23,7 @@ function capEntryText(text: string, budgetChars: number): string { return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; } -const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: +const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: - Summarizes what has happened across the period covered, in chronological order - Highlights the most significant developments rather than restating every article - Stays neutral and factual, without editorializing @@ -31,7 +31,7 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; -const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: +const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: - Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. - Does not copy phrasing verbatim from any source - Stays neutral and factual, without editorializing @@ -39,6 +39,24 @@ const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summari After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; +// Admin-selectable presets (Merge tab, "Writing style") — appended to whichever base +// prompt applies. 'default' adds nothing: the base prompts above already describe the +// original neutral wire-service tone this pipeline shipped with. +const STYLE_PRESETS: Record = { + default: '', + casual: 'Write in a casual, conversational tone, like a knowledgeable friend catching you up on what happened — contractions and plain language are fine. Still stay factual and keep outlet attribution accurate.', + formal: 'Write in a formal, measured register — precise language, no contractions, no colloquialisms.' +}; + +/** Admin-configurable tone: a preset plus optional free-text instructions, both from GlobalSettings — the only two knobs that affect HOW the model writes, as opposed to WHAT gets clustered/published. Appended to the base prompt, never replacing its structural rules (attribution, paragraph count, tag format). */ +function styleAddendum(settings: GlobalSettings): string { + const preset = STYLE_PRESETS[settings.synthesisStylePreset] ?? ''; + const custom = settings.synthesisCustomInstructions.trim(); + const lines = [preset, custom].filter(Boolean); + if (lines.length === 0) return ''; + return `\n\nAdditional style instructions from the site admin (follow these without breaking the rules above):\n${lines.join('\n')}`; +} + export interface SynthesisResult { body: string; tagLabels: string[]; @@ -83,10 +101,12 @@ export async function synthesizeArticle( provider: InferenceProvider, model: string, items: ContentItem[], - sourceNames: Map + sourceNames: Map, + settings: GlobalSettings ): Promise { const prompt = buildPrompt(items, sourceNames); - const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); + const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); + const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } @@ -116,12 +136,13 @@ export async function synthesizeRecap( provider: InferenceProvider, model: string, eventName: string, - articles: MergedArticle[] + articles: MergedArticle[], + settings: GlobalSettings ): Promise { const prompt = buildRecapPrompt(eventName, articles); const raw = await provider.generate(prompt, { model, - system: RECAP_SYSTEM_PROMPT, + system: RECAP_SYSTEM_PROMPT_BASE + styleAddendum(settings), numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index e4f0c3f..0af104f 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -212,6 +212,8 @@ export function migrate() { fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com', nitter_instance_url TEXT NOT NULL DEFAULT 'https://nitter.net', -- admin's preferred instance, prefills new Nitter sources (Connections tab) telegram_media_mode TEXT NOT NULL DEFAULT 'self-host', -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL) + synthesis_style_preset TEXT NOT NULL DEFAULT 'default', -- default | casual | formal — see pipeline/synthesis.ts's STYLE_PRESETS + synthesis_custom_instructions TEXT NOT NULL DEFAULT '', -- free-text addendum appended to the synthesis system prompt, on top of the preset widget_weather_enabled INTEGER NOT NULL DEFAULT 1, widget_stocks_enabled INTEGER NOT NULL DEFAULT 1, widget_bookmarks_enabled INTEGER NOT NULL DEFAULT 1, @@ -378,6 +380,12 @@ export function migrate() { if (!hasColumn('installed_widgets', 'frontend_entry')) { db.exec('ALTER TABLE installed_widgets ADD COLUMN frontend_entry TEXT'); } + if (!hasColumn('global_settings', 'synthesis_style_preset')) { + db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_style_preset TEXT NOT NULL DEFAULT 'default'"); + } + if (!hasColumn('global_settings', 'synthesis_custom_instructions')) { + db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_custom_instructions TEXT NOT NULL DEFAULT ''"); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 05e7c17..02a8035 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -43,6 +43,8 @@ function rowToSettings(row: any): GlobalSettings { fxtwitterBaseUrl: row.fxtwitter_base_url, nitterInstanceUrl: row.nitter_instance_url, telegramMediaMode: row.telegram_media_mode, + synthesisStylePreset: row.synthesis_style_preset, + synthesisCustomInstructions: row.synthesis_custom_instructions, ...widgetsAndOrder(), retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, @@ -90,6 +92,7 @@ export function updateSettings(patch: Partial): GlobalSettings { ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models, nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url, telegram_media_mode=$telegram_media_mode, + synthesis_style_preset=$synthesis_style_preset, synthesis_custom_instructions=$synthesis_custom_instructions, published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days, storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit WHERE id = 1` @@ -107,6 +110,8 @@ export function updateSettings(patch: Partial): GlobalSettings { $fxtwitter_base_url: merged.fxtwitterBaseUrl, $nitter_instance_url: merged.nitterInstanceUrl, $telegram_media_mode: merged.telegramMediaMode, + $synthesis_style_preset: merged.synthesisStylePreset, + $synthesis_custom_instructions: merged.synthesisCustomInstructions, $published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays, $raw_item_max_age_days: merged.retention.rawItemMaxAgeDays, $storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index a6a31eb..de5c96f 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -287,6 +287,10 @@ export interface GlobalSettings { nitterInstanceUrl: string; /** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */ telegramMediaMode: 'self-host' | 'proxy'; + /** Tone preset applied to every AI-synthesized article/recap (see pipeline/synthesis.ts's STYLE_PRESETS) — 'default' is the original neutral wire-service tone with no addendum. Never applies to single-source items, which always publish verbatim without going through the AI at all. */ + synthesisStylePreset: 'default' | 'casual' | 'formal'; + /** Free-text instructions appended to the synthesis system prompt alongside the style preset — e.g. "keep it under 3 sentences per paragraph". Empty string means no addendum. */ + synthesisCustomInstructions: string; /** Per-widget enable flags — see admin/settings' consolidated "Widgets" tab. Weather/Stocks/PoE2's backend pollers (scheduler.ts) are gated on these too, not just sidebar visibility; Bookmarks has no poller so its flag only affects the sidebar. */ widgets: { weather: boolean; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 3ed0dc7..0a91abe 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -155,6 +155,8 @@ export interface AdminSettings { fxtwitterBaseUrl: string; nitterInstanceUrl: string; telegramMediaMode: 'self-host' | 'proxy'; + synthesisStylePreset: 'default' | 'casual' | 'formal'; + synthesisCustomInstructions: string; widgets: AdminWidgetsEnabled; widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; retention: RetentionSettings; diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index 7439a51..137e32f 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -34,7 +34,9 @@ followUpMinNewSources: local.followUpMinNewSources, tagDedupThreshold: local.tagDedupThreshold, tagExpiryDays: local.tagExpiryDays, - categoryPriority: local.categoryPriority + categoryPriority: local.categoryPriority, + synthesisStylePreset: local.synthesisStylePreset, + synthesisCustomInstructions: local.synthesisCustomInstructions }); status = 'saved'; setTimeout(() => (status = 'idle'), 1500); @@ -199,6 +201,29 @@ +
+ Writing style +

+ Applies to AI-merged articles and event recaps only — a story with just one source + publishes with its original text untouched, no AI involved. +

+ + + +
+
Hold before publish

Wait window to gather more sources before finalizing a story.

@@ -328,6 +353,12 @@ select { width: 100%; } + textarea { + width: 100%; + margin-top: 6px; + font: inherit; + resize: vertical; + } .priority-list { display: flex; flex-direction: column; From eeb8abd7d794cfd9a8119b4c1c125b0a74240f04 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:29:45 +0000 Subject: [PATCH 13/24] Have the model synthesize a real title instead of truncating the body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every article title ended in "…" because there was never an actual title — deriveTitle() just took the body's first paragraph and cut it at 97 characters. The AI was never asked for a headline at all. Both system prompts now ask for a response in three parts (headline, then the article/recap, then tags), each separated by a delimiter. parseResult() extracts all three; if the model doesn't follow the format at all, it falls back to the old truncated-first-line heuristic rather than breaking. Delimiter matching is now a loose regex instead of an exact string — production had already shown a small model reproducing "---TAGS---" inexactly (e.g. "---\n\nTAGS---"), which the old exact-string split missed entirely and leaked into the published body. Same tolerance now applies to the new title delimiter. publishCluster uses the synthesized title directly; publishEventRecap uses it too, falling back to the previous ": recap" format only if the model returns an empty title. Verified: exact-format output, sloppy-delimiter output, and no-delimiter-at-all output all parse into sensible {title, body, tags}; a full runSynthesisCycle pass against a mock provider publishes an article with the real synthesized headline as its title. --- backend/src/pipeline/publish.ts | 14 +++----- backend/src/pipeline/synthesis.ts | 59 +++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index b5348ef..0c18fcf 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -36,12 +36,6 @@ function anyPushesToTopStories(items: ContentItem[]): boolean { return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false); } -/** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */ -function deriveTitle(body: string): string { - const firstLine = body.split('\n')[0]; - return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine; -} - /** * Resolves the hero image for a regular (non-tweet) article: try the best candidate * from the source items, download and locally host it; if there isn't one, fall back @@ -362,7 +356,7 @@ export async function publishCluster( const items = cluster.items; const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); + const { title, body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -418,7 +412,7 @@ export async function publishCluster( const now = new Date().toISOString(); const article = articles.insertArticle({ - title: deriveTitle(body), + title, body, heroImage, video, @@ -461,7 +455,7 @@ export async function publishEventRecap( event: TrackedEvent, constituents: MergedArticle[] ): Promise { - const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); + const { title, body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -478,7 +472,7 @@ export async function publishEventRecap( const now = new Date().toISOString(); return articles.insertArticle({ - title: `${event.name}: recap`, + title: title || `${event.name}: recap`, body, heroImage, video: null, diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index a3be66c..3c04413 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -3,8 +3,17 @@ import type { ContentItem, GlobalSettings, MergedArticle } from '../storage/db/t import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; import { logger } from '../storage/db/logs.js'; +const TITLE_DELIMITER = '---TITLE---'; const TAG_DELIMITER = '---TAGS---'; +// Small/quantized models don't always reproduce a literal delimiter exactly — extra +// dashes, an inserted blank line, different case (seen in production with the tag +// delimiter: "---\n\nTAGS---" instead of "---TAGS---", which an exact-string split +// missed entirely, leaking the raw delimiter text into the published body). Splitting +// on a loose regex instead tolerates that variance. +const TITLE_DELIMITER_RE = /-{2,}\s*TITLE\s*-{2,}/i; +const TAG_DELIMITER_RE = /-{2,}\s*TAGS\s*-{2,}/i; + // Ollama truncates prompts that don't fit its context window by keeping a small prefix // and dropping everything else in the middle — silently, with no error, and with no // regard for which sources end up cut (see ollama-provider.ts for the incident that @@ -23,21 +32,25 @@ function capEntryText(text: string, budgetChars: number): string { return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; } -const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: -- Summarizes what has happened across the period covered, in chronological order -- Highlights the most significant developments rather than restating every article -- Stays neutral and factual, without editorializing -- Is 3-5 short paragraphs +const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write your response in exactly three parts, in this order: -After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; +1. A short, specific headline for this recap (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period). +2. On a new line, write exactly "${TITLE_DELIMITER}", then the recap article: + - Summarizes what has happened across the period covered, in chronological order + - Highlights the most significant developments rather than restating every article + - Stays neutral and factual, without editorializing + - Is 3-5 short paragraphs +3. On a new line after the recap, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; -const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: -- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. -- Does not copy phrasing verbatim from any source -- Stays neutral and factual, without editorializing -- Is 2-4 short paragraphs +const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write your response in exactly three parts, in this order: -After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; +1. A short, specific headline for this story (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period, no site/outlet name). +2. On a new line, write exactly "${TITLE_DELIMITER}", then the article: + - Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. + - Does not copy phrasing verbatim from any source + - Stays neutral and factual, without editorializing + - Is 2-4 short paragraphs +3. On a new line after the article, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; // Admin-selectable presets (Merge tab, "Writing style") — appended to whichever base // prompt applies. 'default' adds nothing: the base prompts above already describe the @@ -58,10 +71,17 @@ function styleAddendum(settings: GlobalSettings): string { } export interface SynthesisResult { + title: string; body: string; tagLabels: string[]; } +/** Only used when the model doesn't follow the requested title/delimiter format at all — a real headline beats a truncated sentence fragment, but publishing with no title at all is worse than either. */ +function fallbackTitle(body: string): string { + const firstLine = body.split('\n')[0]; + return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine; +} + function buildPrompt(items: ContentItem[], sourceNames: Map): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; @@ -88,13 +108,24 @@ function buildPrompt(items: ContentItem[], sourceNames: Map): st } function parseResult(raw: string): SynthesisResult { - const [body, tagSection] = raw.split(TAG_DELIMITER); + const [beforeTags, tagSection] = raw.split(TAG_DELIMITER_RE); const tagLabels = (tagSection ?? '') .split(',') .map((t) => t.trim()) .filter((t) => t.length > 0 && t.length < 60); - return { body: body.trim(), tagLabels }; + const titleSplit = (beforeTags ?? raw).split(TITLE_DELIMITER_RE); + const titlePart = titleSplit[0]; + // join() rather than titleSplit[1] in case the delimiter text somehow appears again + // inside the body itself — keeps that content rather than silently dropping it. + const bodyPart = titleSplit.length > 1 ? titleSplit.slice(1).join('') : undefined; + // If the title delimiter never showed up, the model didn't follow the requested + // format — treat the whole thing as body rather than mistaking the article itself + // for a "title", and fall back to the old truncated-first-line heuristic. + const body = (bodyPart ?? titlePart).trim(); + const title = bodyPart !== undefined ? titlePart.trim() : fallbackTitle(body); + + return { title, body, tagLabels }; } export async function synthesizeArticle( From 6ecd247ce797b5209aa06a312a143374e67e83e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:36:27 +0000 Subject: [PATCH 14/24] Fix embed() calls silently timing out, dropping single-source items forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported symptom: articles that never got AI-merged (single source, nothing else to combine with) simply never published at all. Root cause: the same default-5-minute-fetch-timeout bug fixed for generate() earlier was never applied to embed(). Ollama serves one inference request at a time (n_slots = 1) — an embed() call issued while a slow generate() call is in flight has to wait in queue for that same slot, and on this CPU-only hardware a generate() call can easily run past 5 minutes. That wait alone was enough to trip Node's default fetch timeout on the embed request. embedPendingItems() catches that failure and just drops the item from its result (logged, not thrown) — clusterItems() only ever sees items that already have an embedding, so a dropped item never joins a cluster, never gets assignCluster() called, and stays "unclustered" forever, retried every cycle with the same failure for as long as Ollama stays busy. An item that happened to embed during an idle window still merges or publishes fine — which is exactly the split reported: synthesized articles show up, standalone ones don't. Fix: embed() now uses the same noTimeoutDispatcher already wired into generate(). Verified the request completes correctly end-to-end against a real HTTP server that delays its response. --- backend/src/inference/ollama-provider.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 7173922..002f68b 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -79,8 +79,17 @@ export class OllamaProvider implements InferenceProvider { const res = await fetch(`${this.base()}/api/embeddings`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: opts.model, prompt: text }) - }); + body: JSON.stringify({ model: opts.model, prompt: text }), + // Ollama serves one inference request at a time (n_slots = 1) — an embed call + // queued behind a slow generate() call waits for that same slot, and on this + // CPU-only hardware a generate() call can easily run past 5 minutes. Without + // this, that wait alone was enough to trip the same default fetch timeout + // generate() had (see noTimeoutDispatcher above), silently dropping the item + // from embedPendingItems — it never got clustered, so a single-source item + // unlucky enough to be embedded while Ollama was busy never published at all, + // retried every cycle with the same result for as long as Ollama stayed busy. + dispatcher: noTimeoutDispatcher + } as RequestInit); if (!res.ok) throw new Error(`Ollama embed failed: ${res.status} ${await res.text()}`); const data = (await res.json()) as { embedding: number[] }; return data.embedding; From 5be5f0ce940582764755b8d8642a95503646d8e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:48:38 +0000 Subject: [PATCH 15/24] Give direct-publish items their own tick so a slow AI backlog can't block them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported: after clearing all articles/media and rescanning every source, items in "No AI" categories weren't publishing instantly like they should. Root cause: runSynthesisCycle bundled three unrelated jobs into one function, all guarded by a single reentrancy lock (added earlier this session to stop the AI-merge path from racing itself into duplicate articles): (1) YouTube/Nitter/Telegram direct-publish, (2) "No AI" category direct-publish, (3) embed/cluster/AI-merge. A mass rescan produces a big backlog of slow generate() calls for (3) — each one can run minutes on this CPU-only hardware — and since the whole function shared one guard, a newly-ingested "No AI" item had to wait for that entire backlog to drain before its own (fast, no-AI-needed) publish step even got a turn. Split into two independently-scheduled, independently-guarded ticks: runDirectPublishCycle (source-type-driven + "No AI"-category items, regardless of Ollama's reachability) and runSynthesisCycle (now only the embed/cluster/merge path). They operate on disjoint item sets, so running them "concurrently" is safe — no risk of the duplicate-publish race the shared guard was originally added to prevent. Verified directly: with a mock provider whose generate() call takes 3 seconds (standing in for a multi-minute real one), a "No AI" category item published in 68ms — before the slow merge was even close to finishing — while the merge itself still completed correctly on its own schedule. --- backend/src/queue/priorityQueue.ts | 75 +++++++++++++++++++++--------- backend/src/queue/scheduler.ts | 24 +++++++++- 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 15323e3..679f594 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -88,7 +88,10 @@ async function publishItemsDirect( * pipeline, this doesn't wait out the hold-before-publish window: that window exists to * give corroborating sources time to arrive before an AI merge locks in, which doesn't * apply here since there's no merging happening at all — each item is just itself. - * Still respects category priority. + * Still respects category priority. Harmless overlap with runDirectPublishCycle (which + * runs regardless of reachability) — an item already published by one is simply gone + * from the other's next "unclustered" query, since assignCluster lands before either + * moves on to its next item. */ export async function runPassthroughCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); @@ -107,41 +110,31 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { +export async function runDirectPublishCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); if (items.length === 0) return 0; - // One fetch of the full source list per cycle, reused below for both the - // direct-publish partition and each item's category/type lookups — avoids a - // separate sourcesDb.getSource() round-trip per item. const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); const categories = categoriesDb.listCategories(); - const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); - // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with - // anything else — each is always its own article, same shape whether the AI service - // is up or not. Route them straight to publishDirect, same as the no-AI passthrough path. const directPublishSourceIds = new Set( [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - // A category with disableAi set (see the Category priority admin pane) opts its - // items out of clustering/synthesis entirely — each publishes on its own, using its - // own source's text, same as the source-type-driven direct items above. const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); - const [categoryDirectItems, mergeableItems] = partition(remaining, (item) => - inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) - ); + const [categoryDirectItems] = partition(remaining, (item) => inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)); const publishedTypeDirect = await publishItemsDirect( typeDirectItems, @@ -159,6 +152,42 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G 'Direct publish failed' ); + return publishedTypeDirect + publishedCategoryDirect; +} + +/** + * One pass of the synthesis queue: cluster whatever's unclustered (excluding items + * runDirectPublishCycle already owns — see there), ordered by admin-defined category + * priority, and publish clusters that have cleared the hold-before-publish window. + * Items claimed by an active tracked event (belonging to one of its sources and + * matching its keyword filter, if any) publish exactly like everything else — + * individually or merged with same-story coverage — just tagged with the event's id so + * they're browsable under it and eligible for eventsRecap.ts's periodic AI wrap-up. + */ +export async function runSynthesisCycle(provider: InferenceProvider, settings: GlobalSettings): Promise { + const activeEvents = eventsDb.listActiveEvents(); + const items = contentItemsDb.unclusteredItemsExcludingSources([]); + if (items.length === 0) return 0; + + // One fetch of the full source list per cycle, reused below for both the + // direct-publish exclusion and each item's category/rank lookups — avoids a + // separate sourcesDb.getSource() round-trip per item. + const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); + + // YouTube/Nitter/Telegram items and AI-disabled-category items are runDirectPublishCycle's + // job (its own guarded tick, so a slow merge backlog here never blocks them) — excluded + // here too since a batch just ingested this instant may still be unclustered when this + // runs before that cycle's own pass gets to it. + const directPublishSourceIds = new Set( + [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) + ); + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + const mergeableItems = items.filter( + (item) => !directPublishSourceIds.has(item.sourceId) && !inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) + ); + const ranked = mergeableItems .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) .sort((a, b) => a.rank - b.rank) @@ -216,5 +245,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } - return published + publishedTypeDirect + publishedCategoryDirect; + return published; } diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index 5b69c8e..577dd5f 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -1,5 +1,5 @@ import { pollDueSources } from '../ingestion/poller.js'; -import { runSynthesisCycle, runPassthroughCycle } from './priorityQueue.js'; +import { runSynthesisCycle, runPassthroughCycle, runDirectPublishCycle } from './priorityQueue.js'; import { runEventRecaps } from './eventsRecap.js'; import { runRetentionSweep } from './retention.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; @@ -10,6 +10,7 @@ import { loadedWidgets } from '../widgets/registry.js'; import type { WidgetPlugin } from '../widgets/types.js'; const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency +const DIRECT_PUBLISH_TICK_MS = 60_000; const SYNTHESIS_TICK_MS = 60_000; const RETENTION_TICK_MS = 60 * 60_000; // hourly @@ -23,6 +24,13 @@ const RETENTION_TICK_MS = 60 * 60_000; // hourly * fresh, differently-worded article by an overlapping cycle, repeatedly, until the * first cycle's assignCluster() finally landed. Node is single-threaded, so the only * source of "concurrent" runs here is exactly this interval overlap. + * + * Each call gets its own independent `running` flag/timer — the direct-publish and + * synthesis ticks are deliberately two separate calls to this (not one shared guard) + * precisely so a slow AI-merge backlog on one never blocks the other's fast, + * no-AI-needed items from publishing on schedule. They operate on disjoint item sets + * (see priorityQueue.ts), so there's no risk of the two racing each other into a + * duplicate publish the way an overlapping call to the *same* fn would. */ function everyTickSkippingOverlap(ms: number, fn: () => Promise) { let running = false; @@ -84,6 +92,18 @@ export function startScheduler() { } }); + everyTickSkippingOverlap(DIRECT_PUBLISH_TICK_MS, async () => { + try { + const settings = settingsDb.getSettings(); + const published = await runDirectPublishCycle(settings); + if (published > 0) { + logger.info('scheduler', `Direct-publish tick: published ${published} article(s)`); + } + } catch (err) { + logger.error('scheduler', `Direct-publish tick failed: ${(err as Error).message}`); + } + }); + everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); @@ -125,6 +145,6 @@ export function startScheduler() { logger.info( 'scheduler', - `Started: poll every 1m, synthesis every 1m, retention every 1h, ${loadedWidgets.size} widget(s) polling on their own intervals` + `Started: poll every 1m, direct-publish every 1m, synthesis every 1m, retention every 1h, ${loadedWidgets.size} widget(s) polling on their own intervals` ); } From b88a39e09b1c06d821ea02bcd090db3d4882b294 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:27:01 +0000 Subject: [PATCH 16/24] Add pipeline backlog/throughput dashboard to admin Logs tab Admins had no visibility into how many articles were queued for AI synthesis or waiting out the hold-before-publish window, nor how fast Ollama could clear that backlog. GET /api/admin/pipeline-stats reports a live backlog snapshot (items awaiting embedding, clusters on hold vs. ready, items still held) computed straight from the DB with no AI calls, plus real Ollama generate() throughput (tokens/sec, in-flight call) tracked from actual requests, and estimates minutes-to-clear from recent generate() call durations. Surfaced as a stat-tile dashboard atop the Logs tab. --- backend/src/api/admin.ts | 35 +++++ backend/src/inference/ollama-provider.ts | 67 ++++++--- backend/src/inference/provider.ts | 5 +- backend/src/inference/stats.ts | 57 ++++++++ backend/src/pipeline/synthesis.ts | 6 +- backend/src/queue/backlogStats.ts | 114 +++++++++++++++ backend/src/queue/priorityQueue.ts | 17 ++- frontend/src/lib/adminApi.ts | 5 +- frontend/src/lib/adminTypes.ts | 27 ++++ .../src/lib/components/admin/LogsTab.svelte | 138 +++++++++++++++++- 10 files changed, 440 insertions(+), 31 deletions(-) create mode 100644 backend/src/inference/stats.ts create mode 100644 backend/src/queue/backlogStats.ts diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 0d2b353..289e8ae 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -9,6 +9,8 @@ import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; +import * as backlogStats from '../queue/backlogStats.js'; +import * as ollamaStats from '../inference/stats.js'; import * as telegramClient from '../telegram/client.js'; import { loadedWidgets } from '../widgets/registry.js'; import { installUploadedWidget } from '../widgets/install.js'; @@ -294,4 +296,37 @@ export async function registerAdminRoutes(app: FastifyInstance) { limit: limit ? Number(limit) : undefined }); }); + + // Backlog/throughput dashboard for the Logs tab — backlog counts are recomputed live + // from the DB on every request (cheap: no AI calls, see backlogStats.ts), while Ollama + // throughput/in-flight status comes from a rolling in-memory sample of recent + // generate() calls (see inference/stats.ts) since that can only be observed as calls + // actually happen, not recomputed on demand. + app.get('/api/admin/pipeline-stats', async () => { + const settings = settingsDb.getSettings(); + const backlog = backlogStats.getBacklogSnapshot(settings); + const throughput = ollamaStats.getThroughput(); + const inFlight = ollamaStats.getInFlight(); + const { lastDirectCycle, lastSynthesisCycle } = backlogStats.getLastCycles(); + + // Estimate is deliberately conservative: only clusters that actually need an LLM + // call (2+ items — see backlogStats.ts) count toward it, and it's null (rather than + // a misleading guess) until at least one real generate() call has completed, since + // there's no token-speed data to estimate from yet. + const estimatedMinutesToClear = + backlog.clusters.readyNowNeedingSynthesis === 0 + ? 0 + : throughput.avgGenerateDurationMs !== null + ? Math.ceil((backlog.clusters.readyNowNeedingSynthesis * throughput.avgGenerateDurationMs) / 60_000) + : null; + + return { + timestamp: new Date().toISOString(), + ollama: { inFlight, ...throughput }, + backlog, + estimatedMinutesToClear, + lastDirectCycle, + lastSynthesisCycle + }; + }); } diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 002f68b..df32f58 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,6 @@ import { Agent } from 'undici'; import type { InferenceProvider } from './provider.js'; +import * as stats from './stats.js'; /** * Node's global fetch (undici) defaults to a 5-minute headers/body timeout — fine for @@ -51,28 +52,52 @@ export class OllamaProvider implements InferenceProvider { async generate( prompt: string, - opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {} + opts: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string } = {} ): Promise { - const res = await fetch(`${this.base()}/api/generate`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: opts.model, - prompt, - system: opts.system, - stream: false, - options: { - num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, - num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT - } - }), - // Not in the ambient RequestInit type this project resolves to, but Node's global - // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. - dispatcher: noTimeoutDispatcher - } as RequestInit); - if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); - const data = (await res.json()) as { response: string }; - return data.response; + const startedAt = Date.now(); + stats.recordGenerateStart(opts.label ?? 'synthesis'); + try { + const res = await fetch(`${this.base()}/api/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: opts.model, + prompt, + system: opts.system, + stream: false, + options: { + num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, + num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT + } + }), + // Not in the ambient RequestInit type this project resolves to, but Node's global + // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. + dispatcher: noTimeoutDispatcher + } as RequestInit); + if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); + const data = (await res.json()) as { + response: string; + eval_count?: number; + eval_duration?: number; + prompt_eval_count?: number; + prompt_eval_duration?: number; + total_duration?: number; + }; + // Ollama reports these *_duration fields in nanoseconds — dividing eval_count by + // (eval_duration/1e9) gives generation tokens/sec, and total_duration/1e6 gives + // wall-clock milliseconds (falling back to a local measurement if a given Ollama + // version's response ever omits it). + stats.recordGenerateEnd({ + genTokensPerSec: data.eval_count && data.eval_duration ? data.eval_count / (data.eval_duration / 1e9) : null, + promptTokensPerSec: + data.prompt_eval_count && data.prompt_eval_duration ? data.prompt_eval_count / (data.prompt_eval_duration / 1e9) : null, + totalDurationMs: data.total_duration ? data.total_duration / 1e6 : Date.now() - startedAt + }); + return data.response; + } catch (err) { + stats.recordGenerateEnd(null); + throw err; + } } async embed(text: string, opts: { model?: string } = {}): Promise { diff --git a/backend/src/inference/provider.ts b/backend/src/inference/provider.ts index aaa61f8..a542722 100644 --- a/backend/src/inference/provider.ts +++ b/backend/src/inference/provider.ts @@ -1,5 +1,8 @@ export interface InferenceProvider { - generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise; + generate( + prompt: string, + opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string } + ): Promise; embed(text: string, opts?: { model?: string }): Promise; listModels(): Promise; isReachable(): Promise; diff --git a/backend/src/inference/stats.ts b/backend/src/inference/stats.ts new file mode 100644 index 0000000..4f1d68a --- /dev/null +++ b/backend/src/inference/stats.ts @@ -0,0 +1,57 @@ +/** + * In-memory-only tracking of Ollama generate() throughput and in-flight status, for the + * admin "Logs" dashboard (see queue/backlogStats.ts, api/admin.ts's GET + * /api/admin/pipeline-stats). Deliberately not persisted to disk — a restart losing a + * few minutes of rolling samples is fine, since the next few generate() calls rebuild it. + */ + +const MAX_SAMPLES = 20; + +export interface GenerateSample { + /** Generation speed (tokens/sec) from Ollama's eval_count/eval_duration — null if the response omitted them. */ + genTokensPerSec: number | null; + /** Prompt-processing speed (tokens/sec) from prompt_eval_count/prompt_eval_duration — usually the dominant cost on CPU-only inference. */ + promptTokensPerSec: number | null; + totalDurationMs: number; +} + +const samples: GenerateSample[] = []; +let inFlight: { label: string; startedAt: number } | null = null; + +/** Call immediately before issuing a generate() request. */ +export function recordGenerateStart(label: string): void { + inFlight = { label, startedAt: Date.now() }; +} + +/** Call in a finally block after the request settles — pass null on failure/abort. */ +export function recordGenerateEnd(sample: GenerateSample | null): void { + inFlight = null; + if (!sample) return; + samples.push(sample); + if (samples.length > MAX_SAMPLES) samples.shift(); +} + +export function getInFlight(): { label: string; elapsedMs: number } | null { + return inFlight ? { label: inFlight.label, elapsedMs: Date.now() - inFlight.startedAt } : null; +} + +function average(nums: number[]): number | null { + if (nums.length === 0) return null; + return nums.reduce((a, b) => a + b, 0) / nums.length; +} + +export interface ThroughputStats { + sampleCount: number; + avgGenTokensPerSec: number | null; + avgPromptTokensPerSec: number | null; + avgGenerateDurationMs: number | null; +} + +export function getThroughput(): ThroughputStats { + return { + sampleCount: samples.length, + avgGenTokensPerSec: average(samples.map((s) => s.genTokensPerSec).filter((n): n is number => n !== null)), + avgPromptTokensPerSec: average(samples.map((s) => s.promptTokensPerSec).filter((n): n is number => n !== null)), + avgGenerateDurationMs: average(samples.map((s) => s.totalDurationMs)) + }; +} diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 3c04413..6006132 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -137,7 +137,8 @@ export async function synthesizeArticle( ): Promise { const prompt = buildPrompt(items, sourceNames); const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); - const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); + const label = `Merging ${items.length} source${items.length === 1 ? '' : 's'}: "${items[0]?.title.slice(0, 60) ?? ''}"`; + const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT, label }); return parseResult(raw); } @@ -175,7 +176,8 @@ export async function synthesizeRecap( model, system: RECAP_SYSTEM_PROMPT_BASE + styleAddendum(settings), numCtx: DEFAULT_NUM_CTX, - numPredict: DEFAULT_NUM_PREDICT + numPredict: DEFAULT_NUM_PREDICT, + label: `Recapping event: "${eventName.slice(0, 60)}"` }); return parseResult(raw); } diff --git a/backend/src/queue/backlogStats.ts b/backend/src/queue/backlogStats.ts new file mode 100644 index 0000000..ff69d4f --- /dev/null +++ b/backend/src/queue/backlogStats.ts @@ -0,0 +1,114 @@ +import * as contentItemsDb from '../storage/db/contentItems.js'; +import * as sourcesDb from '../storage/db/sources.js'; +import * as categoriesDb from '../storage/db/categories.js'; +import { clusterItems } from '../pipeline/clustering.js'; +import type { ContentItem, GlobalSettings, Source } from '../storage/db/types.js'; + +interface CycleRecord { + at: string; + published: number; +} + +let lastDirectCycle: CycleRecord | null = null; +let lastSynthesisCycle: CycleRecord | null = null; + +/** Called by priorityQueue.ts at the end of runDirectPublishCycle. */ +export function recordDirectPublishCycle(published: number): void { + lastDirectCycle = { at: new Date().toISOString(), published }; +} + +/** Called by priorityQueue.ts at the end of runSynthesisCycle. */ +export function recordSynthesisCycle(published: number): void { + lastSynthesisCycle = { at: new Date().toISOString(), published }; +} + +export function getLastCycles(): { lastDirectCycle: CycleRecord | null; lastSynthesisCycle: CycleRecord | null } { + return { lastDirectCycle, lastSynthesisCycle }; +} + +function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sourcesById: Map): boolean { + const source = sourcesById.get(item.sourceId); + for (const cat of source?.category ?? []) { + if (disabledNames.has(cat.split(':')[0].trim().toLowerCase())) return true; + } + return false; +} + +export interface BacklogSnapshot { + totalUnclusteredItems: number; + /** Items that need no AI at all (YouTube/Nitter/Telegram sources, or "No AI" categories) — publish on the next direct-publish tick. */ + directEligibleItems: number; + /** Mergeable items that haven't been embedded yet (embed() failed/pending, or just ingested since the last synthesis tick). */ + awaitingEmbeddingItems: number; + clusters: { + total: number; + /** Cleared the hold-before-publish window — will publish on the next synthesis tick. */ + readyNow: number; + /** Of readyNow, clusters with 2+ items — these are the ones that actually need an LLM generate() call (single-item clusters publish verbatim, no AI). */ + readyNowNeedingSynthesis: number; + /** Still waiting out the hold-before-publish window. */ + onHold: number; + itemsOnHold: number; + earliestHoldRemainingMs: number | null; + }; +} + +/** + * Read-only snapshot of the current backlog for the admin dashboard — mirrors the same + * categorization runDirectPublishCycle/runSynthesisCycle use (priorityQueue.ts), but never + * calls the AI itself: items with no embedding yet are just counted, not embedded, and + * clustering only runs over items that already have one (cosine similarity over stored + * vectors — no network call). Cheap enough to call on every dashboard refresh. + */ +export function getBacklogSnapshot(settings: GlobalSettings): BacklogSnapshot { + const items = contentItemsDb.unclusteredItemsExcludingSources([]); + const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + + const directPublishSourceIds = new Set( + [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) + ); + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + + const directEligible: ContentItem[] = []; + const mergeable: ContentItem[] = []; + for (const item of items) { + if (directPublishSourceIds.has(item.sourceId) || inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)) { + directEligible.push(item); + } else { + mergeable.push(item); + } + } + + const awaitingEmbedding = mergeable.filter((item) => !item.embedding); + const embedded = mergeable.filter((item) => item.embedding); + + const clusters = clusterItems(embedded, settings.mergeStrictness); + const holdMs = settings.holdBeforePublishMinutes * 60_000; + + let readyNow = 0; + let readyNowNeedingSynthesis = 0; + let onHold = 0; + let itemsOnHold = 0; + let earliestHoldRemainingMs: number | null = null; + + for (const cluster of clusters) { + const earliestFetch = Math.min(...cluster.items.map((i) => new Date(i.fetchedAt).getTime())); + const remaining = holdMs - (Date.now() - earliestFetch); + if (remaining > 0) { + onHold++; + itemsOnHold += cluster.items.length; + earliestHoldRemainingMs = earliestHoldRemainingMs === null ? remaining : Math.min(earliestHoldRemainingMs, remaining); + } else { + readyNow++; + if (cluster.items.length > 1) readyNowNeedingSynthesis++; + } + } + + return { + totalUnclusteredItems: items.length, + directEligibleItems: directEligible.length, + awaitingEmbeddingItems: awaitingEmbedding.length, + clusters: { total: clusters.length, readyNow, readyNowNeedingSynthesis, onHold, itemsOnHold, earliestHoldRemainingMs } + }; +} diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 679f594..e832241 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -7,6 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js'; import { clusterItems } from '../pipeline/clustering.js'; import { publishCluster, publishDirect } from '../pipeline/publish.js'; import { logger } from '../storage/db/logs.js'; +import * as backlogStats from './backlogStats.js'; import type { GlobalSettings, ContentItem, TrackedEvent, Source } from '../storage/db/types.js'; function partition(items: T[], predicate: (item: T) => boolean): [T[], T[]] { @@ -123,7 +124,10 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); - if (items.length === 0) return 0; + if (items.length === 0) { + backlogStats.recordDirectPublishCycle(0); + return 0; + } const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); const categories = categoriesDb.listCategories(); @@ -152,7 +156,9 @@ export async function runDirectPublishCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); - if (items.length === 0) return 0; + if (items.length === 0) { + backlogStats.recordSynthesisCycle(0); + return 0; + } // One fetch of the full source list per cycle, reused below for both the // direct-publish exclusion and each item's category/rank lookups — avoids a @@ -245,5 +254,7 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } + backlogStats.recordSynthesisCycle(published); + return published; } diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index b52060f..bc963f9 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -16,7 +16,8 @@ import type { AdminPoe2Entry, AdminWeatherSettings, InstalledWidget, - WidgetUploadManifest + WidgetUploadManifest, + PipelineStats } from './adminTypes'; async function request(path: string, options: RequestInit = {}, fetchFn: typeof fetch = fetch): Promise { @@ -179,6 +180,8 @@ export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: nu return request(`/api/admin/logs${qs ? `?${qs}` : ''}`, {}, fetchFn); }; +export const getPipelineStats = (fetchFn?: typeof fetch) => request('/api/admin/pipeline-stats', {}, fetchFn); + // Weather — config/cache now live behind the widget's own dedicated admin route (see // backend/src/widgets/weather/plugin.ts) rather than riding along on AdminSettings. export const getWeatherConfig = (fetchFn?: typeof fetch) => diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 0a91abe..b15e373 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -211,6 +211,33 @@ export interface TelegramStatus { phone: string | null; } +export interface PipelineStats { + timestamp: string; + ollama: { + inFlight: { label: string; elapsedMs: number } | null; + sampleCount: number; + avgGenTokensPerSec: number | null; + avgPromptTokensPerSec: number | null; + avgGenerateDurationMs: number | null; + }; + backlog: { + totalUnclusteredItems: number; + directEligibleItems: number; + awaitingEmbeddingItems: number; + clusters: { + total: number; + readyNow: number; + readyNowNeedingSynthesis: number; + onHold: number; + itemsOnHold: number; + earliestHoldRemainingMs: number | null; + }; + }; + estimatedMinutesToClear: number | null; + lastDirectCycle: { at: string; published: number } | null; + lastSynthesisCycle: { at: string; published: number } | null; +} + export interface LogEntry { id: number; timestamp: string; diff --git a/frontend/src/lib/components/admin/LogsTab.svelte b/frontend/src/lib/components/admin/LogsTab.svelte index 8406d9e..30dc117 100644 --- a/frontend/src/lib/components/admin/LogsTab.svelte +++ b/frontend/src/lib/components/admin/LogsTab.svelte @@ -1,10 +1,11 @@ +{#if stats} +
+
+ Backlog + {stats.backlog.totalUnclusteredItems} + item{stats.backlog.totalUnclusteredItems === 1 ? '' : 's'} not yet published +
+
+ Awaiting embedding + {stats.backlog.awaitingEmbeddingItems} + need an embed() call before they can cluster +
+
+ Held for publishing + {stats.backlog.clusters.itemsOnHold} + + {stats.backlog.clusters.onHold} cluster{stats.backlog.clusters.onHold === 1 ? '' : 's'} on hold-before-publish + {#if stats.backlog.clusters.earliestHoldRemainingMs !== null} + · earliest clears in {formatDuration(stats.backlog.clusters.earliestHoldRemainingMs)} + {/if} + +
+
+ Awaiting synthesis + {stats.backlog.clusters.readyNowNeedingSynthesis} + multi-source cluster{stats.backlog.clusters.readyNowNeedingSynthesis === 1 ? '' : 's'} ready, needs an AI merge +
+
+ Estimated to clear + {formatEta(stats.estimatedMinutesToClear)} + + {#if stats.ollama.avgGenerateDurationMs !== null} + based on {stats.ollama.sampleCount} recent generate call{stats.ollama.sampleCount === 1 ? '' : 's'}, avg {formatDuration(stats.ollama.avgGenerateDurationMs)} each + {:else} + no completed generate calls yet + {/if} + +
+
+ Ollama right now + {#if stats.ollama.inFlight} + Synthesizing + {stats.ollama.inFlight.label} · {formatDuration(stats.ollama.inFlight.elapsedMs)} elapsed + {:else} + Idle + + {#if stats.ollama.avgGenTokensPerSec !== null} + ~{stats.ollama.avgGenTokensPerSec.toFixed(1)} gen tok/s, ~{stats.ollama.avgPromptTokensPerSec?.toFixed(1) ?? '?'} prompt tok/s + {:else} + no throughput data yet + {/if} + + {/if} +
+
+ Last direct-publish tick + {stats.lastDirectCycle ? stats.lastDirectCycle.published : '—'} + {formatAgo(stats.lastDirectCycle?.at ?? null)} +
+
+ Last synthesis tick + {stats.lastSynthesisCycle ? stats.lastSynthesisCycle.published : '—'} + {formatAgo(stats.lastSynthesisCycle?.at ?? null)} +
+
+{/if} +
@@ -70,6 +167,41 @@
diff --git a/frontend/src/routes/tag/[slug]/+page.ts b/frontend/src/routes/tag/[slug]/+page.ts new file mode 100644 index 0000000..04f9471 --- /dev/null +++ b/frontend/src/routes/tag/[slug]/+page.ts @@ -0,0 +1,22 @@ +import { error } from '@sveltejs/kit'; +import type { PageLoad } from './$types'; +import { getFeed, getTagBySlug } from '$lib/api'; + +const PAGE_SIZE = 15; + +// Mirrors /category/[name] and /event/[id] — a tag chip links by slug, so the slug is +// resolved to the real tag (id + label) via a dedicated backend lookup (GET +// /api/tag/:slug) rather than a preloaded list, since tags aren't loaded by the root +// layout the way categories/events are. +export const load: PageLoad = async ({ params, fetch }) => { + let tag; + try { + tag = await getTagBySlug(params.slug, fetch); + } catch { + throw error(404, 'Tag not found'); + } + + const filters = { tag: tag.id }; + const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); + return { initial, filters, tag, pageSize: PAGE_SIZE }; +}; From 15716cff5e45e0163c5f68aa2760384f46cbe3ee Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 19:14:50 +0000 Subject: [PATCH 18/24] Fix blank-article publishing bug + add per-article reissue tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A quantized model can occasionally return just the delimiter scaffold ("---TITLE---" / "---TAGS---") with no real headline or article text in between — parseResult treated that as a structurally valid response and published a blank article with empty title/body but real sources and a hero image attached. synthesizeArticle/synthesizeRecap now throw on an empty parsed body instead, so the existing catch-and-retry logic in runSynthesisCycle leaves the cluster unclustered for the next tick rather than ever inserting one of these. Also adds POST /api/admin/articles/:id/reissue to fix articles already published this way: the existing per-source reissue tool explicitly refuses to touch a multi-source article, which this failure mode always produces (an empty synthesis only happens on an actual multi-item merge — a single-item cluster publishes verbatim with no AI call at all), so there was no way to recover one without this. --- backend/src/api/admin.ts | 13 ++++++++++++- backend/src/pipeline/synthesis.ts | 21 +++++++++++++++++++-- backend/src/storage/contentCascade.ts | 23 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 289e8ae..0b66960 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -4,7 +4,7 @@ import * as sourcesDb from '../storage/db/sources.js'; import * as eventsDb from '../storage/db/events.js'; import * as categoriesDb from '../storage/db/categories.js'; import * as installedWidgetsDb from '../storage/db/installedWidgets.js'; -import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; +import { clearSourceContent, reissueSourceContent, reissueArticle, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; @@ -155,6 +155,17 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reissueSourceContent(id); }); + // Fixes one specific bad article (e.g. a degenerate/empty AI synthesis — see + // synthesis.ts's assertNonEmpty) by deleting it and requeuing every item it merged, + // regardless of how many different sources contributed — reissueSourceContent above + // deliberately won't touch a multi-source article at all. + app.post('/api/admin/articles/:id/reissue', async (req, reply) => { + const { id } = req.params as { id: string }; + const result = reissueArticle(id); + if (!result) return reply.code(404).send({ error: 'not found' }); + return result; + }); + // --- Tracked events --- app.get('/api/admin/events', async () => eventsDb.listEvents()); diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index dfdd5be..d9bcf83 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -163,6 +163,23 @@ function parseResult(raw: string): SynthesisResult { return { title, body, tagLabels }; } +/** + * A quantized/small model occasionally reproduces just the requested delimiter + * scaffold ("---TITLE---\n\n---TAGS---") with no real headline or article text in + * between — a structurally "valid" response by parseResult's own logic (delimiters + * found, nothing crashed) but empty in substance. Left unchecked this published a + * blank article (empty title/body, still with real sources/hero image attached) once + * in production. Treating an empty body as a hard failure lets the caller's existing + * catch-and-retry logic (see priorityQueue.ts's runSynthesisCycle) leave the cluster + * unclustered for the next cycle instead of ever inserting one of these. + */ +function assertNonEmpty(result: SynthesisResult, context: string): SynthesisResult { + if (!result.body.trim()) { + throw new Error(`Model returned an empty article body for ${context}`); + } + return result; +} + export async function synthesizeArticle( provider: InferenceProvider, model: string, @@ -174,7 +191,7 @@ export async function synthesizeArticle( const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); const label = `Merging ${items.length} source${items.length === 1 ? '' : 's'}: "${items[0]?.title.slice(0, 60) ?? ''}"`; const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT, label }); - return parseResult(raw); + return assertNonEmpty(parseResult(raw), `"${items[0]?.title.slice(0, 60) ?? ''}"`); } function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string { @@ -214,5 +231,5 @@ export async function synthesizeRecap( numPredict: DEFAULT_NUM_PREDICT, label: `Recapping event: "${eventName.slice(0, 60)}"` }); - return parseResult(raw); + return assertNonEmpty(parseResult(raw), `event recap "${eventName.slice(0, 60)}"`); } diff --git a/backend/src/storage/contentCascade.ts b/backend/src/storage/contentCascade.ts index 3757245..15ef491 100644 --- a/backend/src/storage/contentCascade.ts +++ b/backend/src/storage/contentCascade.ts @@ -86,6 +86,29 @@ export function reissueSourceContent(sourceId: string): ReissueResult { return { articlesDeleted, itemsRequeued: requeueIds.size }; } +/** + * Deletes one specific article (and its media) and requeues every content item that + * contributed to it — unlike reissueSourceContent, this works regardless of how many + * different sources the article merged together, since it's scoped to the article + * itself rather than "everything from source X". Exists for exactly the failure mode + * synthesis.ts's assertNonEmpty guards against going forward: a bad synthesis call + * that already made it into a published (garbage) article before that guard existed, + * where the source-scoped reissue tools can't help because the article spans sources. + * Returns null if the article doesn't exist. + */ +export function reissueArticle(articleId: string): ReissueResult | null { + const article = articlesDb.getArticle(articleId); + if (!article) return null; + + const itemIds = article.sources.map((s) => s.itemId); + deleteMediaByArticleId(article.id); + articlesDb.deleteArticle(article.id); + contentItemsDb.resetClusterForItems(itemIds); + + logger.info('admin', `Reissuing article ${articleId}: deleted, ${itemIds.length} item(s) requeued`); + return { articlesDeleted: 1, itemsRequeued: itemIds.length }; +} + /** Wipes every published article and its media, keeping raw ingested items intact so they can be re-synthesized fresh. */ export function clearAllArticles(): number { const articles = articlesDb.allArticlesNewestFirst(); From 7518e6f81e4d620e63bfc65b405ea55f5b015901 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 19:21:28 +0000 Subject: [PATCH 19/24] Add "Reissue an article" panel to the Retention admin tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires up POST /api/admin/articles/:id/reissue (added alongside the blank-article fix) as a UI panel instead of requiring curl: paste an article ID, it deletes the article and requeues its source items for re-publish. Verified live in a browser against a real backend/DB — both the success path and the "no article with that ID" 404 case. --- frontend/src/lib/adminApi.ts | 5 ++ .../lib/components/admin/RetentionTab.svelte | 61 ++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index bc963f9..a95bdb6 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -107,6 +107,11 @@ export const pollSourceNow = (id: string, fetchFn?: typeof fetch) => export const reissueSourceContent = (id: string, fetchFn?: typeof fetch) => request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/sources/${id}/reissue`, { method: 'POST' }, fetchFn); +// Fixes one specific bad article regardless of how many sources it merged — unlike +// reissueSourceContent above, which deliberately won't touch a multi-source article. +export const reissueArticle = (id: string, fetchFn?: typeof fetch) => + request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/articles/${id}/reissue`, { method: 'POST' }, fetchFn); + // Content clearing — wipe articles/media/a source's raw items so they can be repopulated fresh. export const clearSourceContent = (id: string, fetchFn?: typeof fetch) => request<{ itemsDeleted: number; articlesDeleted: number }>(`/api/admin/content/sources/${id}`, { method: 'DELETE' }, fetchFn); diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index f2a1058..b583620 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -1,6 +1,6 @@
@@ -84,6 +141,57 @@
+
+
+ Context window + +
+

+ How much text the synthesis model can take in (context window) and how long its response + can be (max response length). Too low a response limit is why an article or event recap + sometimes cuts off mid-sentence instead of finishing. + {#if detecting} + Detecting {selected.synthesis}'s limit… + {:else if detectedMax} + Detected max for {selected.synthesis}: {detectedMax.toLocaleString()} tokens. + {:else} + Couldn't detect a limit for {selected.synthesis} — defaulting the slider's ceiling to + {FALLBACK_MAX_CTX.toLocaleString()}. Setting num_ctx above what the model actually + supports will make Ollama reject or silently degrade requests. + {/if} +

+ + +
+ +
+ + +
+ +
+
+