Salastil/homefeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add pluggable widget system; migrate PoE2 as the pilot implementation

Browse Source 
Widgets are no longer hand-wired per-feature across scheduler.ts, settings.ts,
and the API routes. A new WidgetPlugin interface (backend/src/widgets/types.ts)
lets a widget declare its own schema migration, poll interval, routes, and
uninstall hook; an installed_widgets registry table replaces the closed
widgets/widgetOrder unions on global_settings, and the scheduler/route
registration now iterate loaded widgets generically instead of one hardcoded
block per widget.

PoE2 moves into backend/src/widgets/poe2/ as the first real plugin (its tables
renamed to the widget_poe2_ convention, league cache moved into a new generic
widget_kv store). Weather/Stocks/Bookmarks get thin wrapper plugins so they
share the same dispatch loop without migrating their schema.

New admin routes let a widget be uploaded live (POST /api/admin/widgets, JSON
body with inline file contents — no archive dependency needed) and removed
with full data pruning (DELETE /api/admin/widgets/:id): a host-side safety-net
sweep drops any widget_<id>_* table and widget_kv rows regardless of whether
the widget's own uninstall() hook runs, so deleted widgets don't leave dead
data behind. Built-in widgets can't be deleted through this route.

Verified against a copy of the real dev DB: old poe2_watchlist data survives
the rename, the migration is idempotent on a second boot, and a live-uploaded
test widget was polled, queried, and fully deleted (table/kv/on-disk directory
all gone) without a restart.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8
This commit is contained in:
Claude
2026-07-27 00:27:27 +00:00
parent b30657a161
commit bf0cb11070
23 changed files with 962 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/api/admin.ts
+37 -116
View File
@@ -3,20 +3,17 @@ import * as settingsDb from '../storage/db/settings.js';
import * as sourcesDb from '../storage/db/sources.js'; import * as sourcesDb from '../storage/db/sources.js';
import * as eventsDb from '../storage/db/events.js'; import * as eventsDb from '../storage/db/events.js';
import * as categoriesDb from '../storage/db/categories.js'; import * as categoriesDb from '../storage/db/categories.js';
import * as stocksDb from '../storage/db/stocks.js'; import * as installedWidgetsDb from '../storage/db/installedWidgets.js';
import * as bookmarksDb from '../storage/db/bookmarks.js';
import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js';
import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
import { totalStorageBytes } from '../storage/media/index.js'; import { totalStorageBytes } from '../storage/media/index.js';
import { OllamaProvider } from '../inference/ollama-provider.js'; import { OllamaProvider } from '../inference/ollama-provider.js';
import { pollSourceNow } from '../ingestion/poller.js'; import { pollSourceNow } from '../ingestion/poller.js';
import { logger, listLogs } from '../storage/db/logs.js'; import { logger, listLogs } from '../storage/db/logs.js';
import * as telegramClient from '../telegram/client.js'; import * as telegramClient from '../telegram/client.js';
import { geocodeLocation } from '../weather/client.js'; import { loadedWidgets } from '../widgets/registry.js';
import { pollWeatherNow } from '../weather/poller.js'; import { installUploadedWidget } from '../widgets/install.js';
import { pollStocksNow } from '../stocks/poller.js'; import { uninstallWidget } from '../widgets/uninstall.js';
import { browseCurrencies, fetchCurrentLeague } from '../poe2/client.js'; import type { GlobalSettings } from '../storage/db/types.js';
import { pollPoe2Now } from '../poe2/poller.js';
// Not part of GlobalSettings itself (nothing to persist) — computed fresh on every // Not part of GlobalSettings itself (nothing to persist) — computed fresh on every
// settings read/write so the Retention tab's "currently using" line and usage bar // settings read/write so the Retention tab's "currently using" line and usage bar
@@ -43,21 +40,24 @@ export async function registerAdminRoutes(app: FastifyInstance) {
if (body.weather) { if (body.weather) {
// Poll immediately rather than waiting for the next scheduler tick (up to 45 // Poll immediately rather than waiting for the next scheduler tick (up to 45
// minutes) — the admin just changed the location/unit and expects to see it reflected. // minutes) — the admin just changed the location/unit and expects to see it reflected.
pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); loadedWidgets
.get('weather')
?.poll?.run()
.catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`));
} }
if (body.widgets) { if (body.widgets) {
// Re-enabling a widget (see the Widgets tab) should show fresh data right away // Re-enabling a widget (see the Widgets tab) should show fresh data right away
// instead of waiting out its normal cadence (up to 45m/15m/1h) — scheduler.ts // instead of waiting out its normal cadence — scheduler.ts skips polling
// skips polling entirely while a widget is disabled, so there's nothing recent // entirely while a widget is disabled, so there's nothing recent to fall back
// to fall back on otherwise. // on otherwise. Generic over every loaded widget with a poll hook, rather than
if (body.widgets.weather && !before.widgets.weather) { // one hardcoded branch per widget.
pollWeatherNow().catch((err) => logger.error('weather', `Immediate poll failed: ${err.message}`)); for (const id of Object.keys(body.widgets) as (keyof GlobalSettings['widgets'])[]) {
if (body.widgets[id] && !before.widgets[id]) {
loadedWidgets
.get(id)
?.poll?.run()
.catch((err) => logger.error(id, `Immediate poll failed: ${err.message}`));
} }
if (body.widgets.stocks && !before.widgets.stocks) {
pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`));
}
if (body.widgets.poe2 && !before.widgets.poe2) {
pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`));
} }
} }
return { ...settings, categoryPriority: categoriesDb.listCategories() }; return { ...settings, categoryPriority: categoriesDb.listCategories() };
@@ -231,106 +231,27 @@ export async function registerAdminRoutes(app: FastifyInstance) {
return reply.code(200).send(telegramClient.getStatus()); return reply.code(200).send(telegramClient.getStatus());
}); });
// --- Weather (config lives in global_settings — see PATCH /api/admin/settings above) --- // Per-widget routes (weather geocode, stocks CRUD, bookmarks CRUD, poe2 browse/
app.get('/api/admin/weather/geocode', async (req, reply) => { // watchlist) are registered by each widget's own plugin — see widgets/registry.ts's
const { query } = req.query as { query?: string }; // generic registerAdminRoutes loop in index.ts. What's left here is the
if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' }); // upload/delete lifecycle for pluggable widgets themselves.
try {
return await geocodeLocation(query.trim()); // --- Pluggable widgets (upload/list/delete — see widgets/install.ts, uninstall.ts) ---
} catch (err) { app.get('/api/admin/widgets', async () => installedWidgetsDb.listInstalled());
return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` });
} app.post('/api/admin/widgets', async (req, reply) => {
const { manifest, files } = req.body as { manifest?: unknown; files?: unknown };
const result = await installUploadedWidget(manifest, files);
if (!result.ok) return reply.code(400).send({ error: result.error });
return reply.code(201).send({ id: result.id });
}); });
// --- Stocks --- app.delete('/api/admin/widgets/:id', async (req, reply) => {
app.get('/api/admin/stocks', async () => stocksDb.listStockTickers());
app.post('/api/admin/stocks', async (req, reply) => {
const { label, symbol } = req.body as { label?: string; symbol?: string };
if (!label || !label.trim() || !symbol || !symbol.trim()) {
return reply.code(400).send({ error: 'label and symbol are required' });
}
const created = stocksDb.createStockTicker(label.trim(), symbol.trim());
// Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap,
// and refreshes every existing ticker's price too.
pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`));
return reply.code(201).send(created);
});
app.patch('/api/admin/stocks/:id', async (req, reply) => {
const { id } = req.params as { id: string }; const { id } = req.params as { id: string };
const updated = stocksDb.updateStockTicker(id, req.body as any); const widget = installedWidgetsDb.getInstalled(id);
if (!updated) return reply.code(404).send({ error: 'not found' }); if (!widget) return reply.code(404).send({ error: 'not found' });
return updated; if (widget.source === 'builtin') return reply.code(400).send({ error: 'built-in widgets cannot be deleted' });
}); await uninstallWidget(id);
app.delete('/api/admin/stocks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
stocksDb.deleteStockTicker(id);
return reply.code(204).send();
});
// --- Bookmarks ---
app.get('/api/admin/bookmarks', async () => bookmarksDb.listBookmarks());
app.post('/api/admin/bookmarks', async (req, reply) => {
const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean };
if (!name || !name.trim() || !url || !url.trim()) {
return reply.code(400).send({ error: 'name and url are required' });
}
const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate);
return reply.code(201).send(created);
});
app.patch('/api/admin/bookmarks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
const updated = bookmarksDb.updateBookmark(id, req.body as any);
if (!updated) return reply.code(404).send({ error: 'not found' });
return updated;
});
app.delete('/api/admin/bookmarks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
bookmarksDb.deleteBookmark(id);
return reply.code(204).send();
});
// --- PoE2 (league is always auto-detected, never admin-set — see poe2/poller.ts) ---
app.get('/api/admin/poe2/browse', async (_req, reply) => {
try {
const league = await fetchCurrentLeague();
return await browseCurrencies(league.id);
} catch (err) {
return reply.code(502).send({ error: `poe.ninja unreachable: ${(err as Error).message}` });
}
});
app.get('/api/admin/poe2/watchlist', async () => poe2WatchlistDb.listWatchlist());
app.post('/api/admin/poe2/watchlist', async (req, reply) => {
const { base, quote } = req.body as {
base?: { currencyId?: string; name?: string };
quote?: { currencyId?: string; name?: string };
};
if (!base?.currencyId || !base?.name || !quote?.currencyId || !quote?.name) {
return reply.code(400).send({ error: 'base and quote currencies (currencyId, name) are required' });
}
if (base.currencyId === quote.currencyId) {
return reply.code(400).send({ error: 'Base and quote currencies must be different' });
}
const created = poe2WatchlistDb.addWatchlistEntry(
{ currencyId: base.currencyId, name: base.name },
{ currencyId: quote.currencyId, name: quote.name }
);
// Poll immediately rather than waiting for the next tick (up to 1 hour) — cheap,
// and refreshes every existing entry's rate too.
pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`));
return reply.code(201).send(created);
});
app.delete('/api/admin/poe2/watchlist/:id', async (req, reply) => {
const { id } = req.params as { id: string };
poe2WatchlistDb.removeWatchlistEntry(id);
return reply.code(204).send(); return reply.code(204).send();
}); });
backend/src/api/public.ts
+3 -18
View File
@@ -4,9 +4,6 @@ import * as tagsDb from '../storage/db/tags.js';
import * as eventsDb from '../storage/db/events.js'; import * as eventsDb from '../storage/db/events.js';
import * as categoriesDb from '../storage/db/categories.js'; import * as categoriesDb from '../storage/db/categories.js';
import * as settingsDb from '../storage/db/settings.js'; import * as settingsDb from '../storage/db/settings.js';
import * as stocksDb from '../storage/db/stocks.js';
import * as bookmarksDb from '../storage/db/bookmarks.js';
import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js';
import { hasPrivateAccess } from './privateAccess.js'; import { hasPrivateAccess } from './privateAccess.js';
export async function registerPublicRoutes(app: FastifyInstance) { export async function registerPublicRoutes(app: FastifyInstance) {
@@ -70,19 +67,7 @@ export async function registerPublicRoutes(app: FastifyInstance) {
return { ...widgets, order: widgetOrder }; return { ...widgets, order: widgetOrder };
}); });
// Sidebar widgets — see WeatherTab/StocksTab/BookmarksTab in the admin panel. // Per-widget public routes (GET /api/weather, /api/stocks, /api/bookmarks, /api/poe2)
app.get('/api/weather', async () => settingsDb.getSettings().weather); // are registered by each widget's own plugin — see widgets/registry.ts's generic
// registerPublicRoutes loop in index.ts.
app.get('/api/stocks', async () => stocksDb.listStockTickers());
app.get('/api/bookmarks', async (req) => {
const bookmarks = bookmarksDb.listBookmarks();
if (hasPrivateAccess(req)) return bookmarks;
return bookmarks.filter((b) => !b.isPrivate);
});
app.get('/api/poe2', async () => {
const { leagueName, updatedAt } = settingsDb.getSettings().poe2;
return { leagueName, updatedAt, entries: poe2WatchlistDb.listWatchlist() };
});
} }
backend/src/bookmarks/plugin.ts
+47
View File
@@ -0,0 +1,47 @@
import type { WidgetPlugin } from '../widgets/types.js';
import * as bookmarksDb from '../storage/db/bookmarks.js';
import { hasPrivateAccess } from '../api/privateAccess.js';
// Built-in sidebar "Bookmarks" widget — thin wrapper so it funnels into the same
// route-registration loop as pluggable widgets (see widgets/registry.ts); no poll (purely
// admin-curated links) and no migrate()/uninstall() since its schema isn't moving. Not
// going through the upload/delete lifecycle — see widgets/types.ts.
export const bookmarksPlugin: WidgetPlugin = {
id: 'bookmarks',
displayName: 'Bookmarks',
version: '1.0.0',
registerPublicRoutes(app) {
app.get('/api/bookmarks', async (req) => {
const bookmarks = bookmarksDb.listBookmarks();
if (hasPrivateAccess(req)) return bookmarks;
return bookmarks.filter((b) => !b.isPrivate);
});
},
registerAdminRoutes(app) {
app.get('/api/admin/bookmarks', async () => bookmarksDb.listBookmarks());
app.post('/api/admin/bookmarks', async (req, reply) => {
const { name, url, isPrivate } = req.body as { name?: string; url?: string; isPrivate?: boolean };
if (!name || !name.trim() || !url || !url.trim()) {
return reply.code(400).send({ error: 'name and url are required' });
}
const created = bookmarksDb.createBookmark(name.trim(), url.trim(), !!isPrivate);
return reply.code(201).send(created);
});
app.patch('/api/admin/bookmarks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
const updated = bookmarksDb.updateBookmark(id, req.body as any);
if (!updated) return reply.code(404).send({ error: 'not found' });
return updated;
});
app.delete('/api/admin/bookmarks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
bookmarksDb.deleteBookmark(id);
return reply.code(204).send();
});
}
};
backend/src/index.ts
+11
View File
@@ -14,6 +14,7 @@ import { registerPrivateAccess, privateAccessConfigured } from './api/privateAcc
import { startScheduler } from './queue/scheduler.js'; import { startScheduler } from './queue/scheduler.js';
import { initFromSavedSession } from './telegram/client.js'; import { initFromSavedSession } from './telegram/client.js';
import { logger } from './storage/db/logs.js'; import { logger } from './storage/db/logs.js';
import { loadAllWidgets, loadedWidgets } from './widgets/registry.js';
const PORT = Number(process.env.PORT) || 4000; const PORT = Number(process.env.PORT) || 4000;
const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173'; const FRONTEND_ORIGIN = process.env.FRONTEND_ORIGIN || 'http://localhost:5173';
@@ -35,6 +36,7 @@ async function main() {
migrate(); migrate();
printApiKeyBanner(); printApiKeyBanner();
await initFromSavedSession(); await initFromSavedSession();
await loadAllWidgets();
const app = Fastify({ logger: false }); const app = Fastify({ logger: false });
@@ -74,6 +76,15 @@ async function main() {
await registerAdminRoutes(app); await registerAdminRoutes(app);
await registerPrivateAccess(app); await registerPrivateAccess(app);
// Each loaded widget (built-in or uploaded — see widgets/registry.ts) registers its
// own routes here rather than being hardcoded into public.ts/admin.ts. Runs after
// registerAuth so any /api/admin/* route a widget registers is gated by the same
// X-Api-Key preHandler automatically.
for (const plugin of loadedWidgets.values()) {
plugin.registerPublicRoutes?.(app);
plugin.registerAdminRoutes?.(app);
}
// Fastify's own logger is off (see below) — without this, an unhandled exception // Fastify's own logger is off (see below) — without this, an unhandled exception
// in any route handler produces a bare 500 with zero trace anywhere, including the // in any route handler produces a bare 500 with zero trace anywhere, including the
// admin panel's own Logs tab. This is what "Save failed" with no log entry was. // admin panel's own Logs tab. This is what "Save failed" with no log entry was.
backend/src/poe2/poller.ts
-63
View File
@@ -1,63 +0,0 @@
import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js';
import * as settingsDb from '../storage/db/settings.js';
import { logger } from '../storage/db/logs.js';
import { fetchCurrentLeague, fetchCurrencyValues } from './client.js';
const DAY_MS = 24 * 60 * 60_000;
function pctChange(current: number, past: number | null): number | null {
if (past === null || past === 0) return null;
return ((current - past) / past) * 100;
}
// Called on a schedule (see queue/scheduler.ts) and immediately after the admin adds a pair
// (see api/admin.ts) — always re-detects the current challenge league fresh (cheap, guarantees
// correctness across league rotations with no separate staleness logic), then one overview
// request covers the whole watchlist. A pair whose base or quote currency is no longer traded
// this league gets its own lastError, it never aborts the rest of the batch.
export async function pollPoe2Now(): Promise<void> {
let league;
try {
league = await fetchCurrentLeague();
} catch (err) {
logger.error('poe2', `League lookup failed: ${(err as Error).message}`);
return;
}
const entries = poe2WatchlistDb.listWatchlist();
if (entries.length === 0) {
const { poe2 } = settingsDb.getSettings();
settingsDb.updateSettings({
poe2: { ...poe2, leagueId: league.id, leagueName: league.name, updatedAt: new Date().toISOString() }
});
return;
}
try {
const valuesById = await fetchCurrencyValues(league.id);
const now = new Date();
const nowIso = now.toISOString();
const cutoff24h = new Date(now.getTime() - DAY_MS).toISOString();
for (const entry of entries) {
const baseValue = valuesById.get(entry.baseCurrencyId);
const quoteValue = valuesById.get(entry.quoteCurrencyId);
if (baseValue === undefined || quoteValue === undefined) {
poe2WatchlistDb.markPolled(entry.id, null, null, 'One or both currencies no longer traded in this league');
continue;
}
const rate = baseValue / quoteValue;
const change24h = pctChange(rate, poe2WatchlistDb.rateAtOrBefore(entry.id, cutoff24h));
poe2WatchlistDb.recordRate(entry.id, rate, nowIso);
poe2WatchlistDb.markPolled(entry.id, rate, change24h, null);
}
poe2WatchlistDb.pruneOldHistory();
settingsDb.updateSettings({
poe2: { leagueId: league.id, leagueName: league.name, updatedAt: nowIso }
});
} catch (err) {
logger.error('poe2', `Watchlist poll failed: ${(err as Error).message}`);
}
}
backend/src/queue/scheduler.ts
+43 -37
View File
@@ -4,17 +4,48 @@ import { runEventRecaps } from './eventsRecap.js';
import { runRetentionSweep } from './retention.js'; import { runRetentionSweep } from './retention.js';
import { OllamaProvider } from '../inference/ollama-provider.js'; import { OllamaProvider } from '../inference/ollama-provider.js';
import * as settingsDb from '../storage/db/settings.js'; import * as settingsDb from '../storage/db/settings.js';
import * as installedWidgetsDb from '../storage/db/installedWidgets.js';
import { logger } from '../storage/db/logs.js'; import { logger } from '../storage/db/logs.js';
import { pollWeatherNow } from '../weather/poller.js'; import { loadedWidgets } from '../widgets/registry.js';
import { pollStocksNow } from '../stocks/poller.js'; import type { WidgetPlugin } from '../widgets/types.js';
import { pollPoe2Now } from '../poe2/poller.js';
const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency
const SYNTHESIS_TICK_MS = 60_000; const SYNTHESIS_TICK_MS = 60_000;
const RETENTION_TICK_MS = 60 * 60_000; // hourly const RETENTION_TICK_MS = 60 * 60_000; // hourly
const WEATHER_TICK_MS = 45 * 60_000;
const STOCKS_TICK_MS = 15 * 60_000; // per admin spec — stock prices move faster than weather // Per-widget setInterval handles, keyed by widget id — lets a single widget's polling be
const POE2_TICK_MS = 60 * 60_000; // poe.ninja's own overview data doesn't refresh faster than hourly, so polling more often than this just re-fetches the same numbers // started/stopped independently (on live upload/delete, or an enable toggle) without
// touching any other widget's interval. Exported so widgets/install.ts and
// widgets/uninstall.ts can drive it directly.
export const widgetIntervals = new Map<string, NodeJS.Timeout>();
// Starts (or re-starts) polling for one widget — an immediate poll if it's currently
// enabled (unlike RSS sources, whose "due" check makes a brand-new source eligible on the
// very next 1-minute tick, a widget has no such shortcut; without this the sidebar would
// sit empty for up to a full poll interval after every restart or fresh install), then a
// recurring interval that re-checks the enabled flag on every tick — so disabling a widget
// stops the actual external polling, not just hides it in the sidebar.
export function startWidgetPolling(plugin: WidgetPlugin) {
if (!plugin.poll) return;
stopWidgetPolling(plugin.id);
if (installedWidgetsDb.getInstalled(plugin.id)?.enabled) {
plugin.poll.run().catch((err) => logger.error(plugin.id, `Initial poll failed: ${(err as Error).message}`));
}
const handle = setInterval(() => {
if (!installedWidgetsDb.getInstalled(plugin.id)?.enabled) return;
plugin.poll!.run().catch((err) => logger.error(plugin.id, `Poll tick failed: ${(err as Error).message}`));
}, plugin.poll.intervalMs);
widgetIntervals.set(plugin.id, handle);
}
export function stopWidgetPolling(id: string) {
const handle = widgetIntervals.get(id);
if (handle) {
clearInterval(handle);
widgetIntervals.delete(id);
}
}
export function startScheduler() { export function startScheduler() {
const provider = () => { const provider = () => {
@@ -66,37 +97,12 @@ export function startScheduler() {
} }
}, RETENTION_TICK_MS); }, RETENTION_TICK_MS);
// Immediate first call for all three — unlike RSS sources (whose "due" check makes a for (const plugin of loadedWidgets.values()) {
// brand-new source eligible on the very next 1-minute tick), weather/stocks/poe2 have startWidgetPolling(plugin);
// no such shortcut; without this the sidebar is empty for up to 45/15/60 minutes after
// every restart. Each is also gated on its Widgets-tab enabled flag (see
// admin/settings' consolidated Widgets tab) — disabling a widget stops these external
// calls entirely rather than just hiding the sidebar box, so there's no pointless
// polling for something nobody's looking at. Re-enabling it triggers an immediate
// poll instead (see admin.ts's PATCH /api/admin/settings), same as this initial call.
if (settingsDb.getSettings().widgets.weather) {
pollWeatherNow().catch((err) => logger.error('weather', `Initial poll failed: ${err.message}`));
} }
setInterval(() => {
if (!settingsDb.getSettings().widgets.weather) return;
pollWeatherNow().catch((err) => logger.error('weather', `Poll tick failed: ${err.message}`));
}, WEATHER_TICK_MS);
if (settingsDb.getSettings().widgets.stocks) { logger.info(
pollStocksNow().catch((err) => logger.error('stocks', `Initial poll failed: ${err.message}`)); 'scheduler',
} `Started: poll every 1m, synthesis every 1m, retention every 1h, ${loadedWidgets.size} widget(s) polling on their own intervals`
setInterval(() => { );
if (!settingsDb.getSettings().widgets.stocks) return;
pollStocksNow().catch((err) => logger.error('stocks', `Poll tick failed: ${err.message}`));
}, STOCKS_TICK_MS);
if (settingsDb.getSettings().widgets.poe2) {
pollPoe2Now().catch((err) => logger.error('poe2', `Initial poll failed: ${err.message}`));
}
setInterval(() => {
if (!settingsDb.getSettings().widgets.poe2) return;
pollPoe2Now().catch((err) => logger.error('poe2', `Poll tick failed: ${err.message}`));
}, POE2_TICK_MS);
logger.info('scheduler', 'Started: poll every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h');
} }
backend/src/stocks/plugin.ts
+54
View File
@@ -0,0 +1,54 @@
import type { WidgetPlugin } from '../widgets/types.js';
import * as stocksDb from '../storage/db/stocks.js';
import { logger } from '../storage/db/logs.js';
import { pollStocksNow } from './poller.js';
// Built-in sidebar "Stocks" widget — thin wrapper so it funnels into the same
// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts); its
// tickers stay in the dedicated stock_tickers table exactly as before. Not going through
// the upload/delete lifecycle — see widgets/types.ts; no migrate()/uninstall() since its
// schema isn't moving.
export const stocksPlugin: WidgetPlugin = {
id: 'stocks',
displayName: 'Stocks',
version: '1.0.0',
poll: {
// Per admin spec — stock prices move faster than weather.
intervalMs: 15 * 60_000,
run: pollStocksNow
},
registerPublicRoutes(app) {
app.get('/api/stocks', async () => stocksDb.listStockTickers());
},
registerAdminRoutes(app) {
app.get('/api/admin/stocks', async () => stocksDb.listStockTickers());
app.post('/api/admin/stocks', async (req, reply) => {
const { label, symbol } = req.body as { label?: string; symbol?: string };
if (!label || !label.trim() || !symbol || !symbol.trim()) {
return reply.code(400).send({ error: 'label and symbol are required' });
}
const created = stocksDb.createStockTicker(label.trim(), symbol.trim());
// Poll immediately rather than waiting for the next tick (up to 15 minutes) — cheap,
// and refreshes every existing ticker's price too.
pollStocksNow().catch((err) => logger.error('stocks', `Immediate poll failed: ${err.message}`));
return reply.code(201).send(created);
});
app.patch('/api/admin/stocks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
const updated = stocksDb.updateStockTicker(id, req.body as any);
if (!updated) return reply.code(404).send({ error: 'not found' });
return updated;
});
app.delete('/api/admin/stocks/:id', async (req, reply) => {
const { id } = req.params as { id: string };
stocksDb.deleteStockTicker(id);
return reply.code(204).send();
});
}
};
backend/src/storage/db/index.ts
+89 -26
View File
@@ -33,6 +33,18 @@ export function migrate() {
db.exec('DROP TABLE IF EXISTS poe2_watchlist;'); db.exec('DROP TABLE IF EXISTS poe2_watchlist;');
} }
// PoE2 moved into the pluggable-widget system (backend/src/widgets/poe2/) and its
// schema now follows the widget_<id>_ naming convention its plugin.migrate() owns —
// a plain rename (metadata-only in SQLite, no row copy) rather than grandfathering
// the old names in via an exceptions list, since it's cheap and PoE2 is meant to be
// the reference implementation of the convention it establishes.
const hasOldPoe2Table = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='poe2_watchlist'`).get();
if (hasOldPoe2Table) {
db.exec('ALTER TABLE poe2_watchlist RENAME TO widget_poe2_watchlist;');
db.exec('ALTER TABLE poe2_rate_history RENAME TO widget_poe2_rate_history;');
db.exec('DROP INDEX IF EXISTS idx_poe2_rate_history_watchlist;');
}
db.exec(` db.exec(`
CREATE TABLE IF NOT EXISTS sources ( CREATE TABLE IF NOT EXISTS sources (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
@@ -223,35 +235,32 @@ export function migrate() {
created_at TEXT NOT NULL created_at TEXT NOT NULL
); );
-- Sidebar "PoE2" widget — tracks exchange rates between arbitrary currency pairs -- Generic config/cache store for pluggable widgets (see widgets/types.ts,
-- (see poe2/poller.ts), always against the current challenge league (auto-detected, -- storage/db/widgetKv.ts) — lets a widget stay fully prunable by widget_id alone on
-- no admin config). Rate is "1 base = last_rate quote"; both currencies' names are -- uninstall without a bespoke table for simple key/value state.
-- captured at add-time from the browse picker, not re-resolved. No icon columns — CREATE TABLE IF NOT EXISTS widget_kv (
-- the UI doesn't display them. widget_id TEXT NOT NULL,
CREATE TABLE IF NOT EXISTS poe2_watchlist ( key TEXT NOT NULL,
id TEXT PRIMARY KEY, value TEXT NOT NULL, -- JSON
base_currency_id TEXT NOT NULL, -- opaque id from the exchange overview's lines[].id, e.g. "exalted" updated_at TEXT NOT NULL,
base_name TEXT NOT NULL, PRIMARY KEY (widget_id, key)
quote_currency_id TEXT NOT NULL,
quote_name TEXT NOT NULL,
priority_rank INTEGER NOT NULL,
last_rate REAL,
last_change_24h REAL,
last_polled_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL
); );
-- Per-poll rate snapshots for the pairs above — poe.ninja doesn't expose a matching -- Registry of installed sidebar widgets, both built-in (weather/stocks/bookmarks,
-- 24h change window, so it's computed ourselves from this history (see -- and poe2 as the pluggable reference implementation) and uploaded (see
-- poe2/poller.ts). Pruned to the last 2 days on every poll. -- widgets/registry.ts, widgets/install.ts). Replaces the old closed
CREATE TABLE IF NOT EXISTS poe2_rate_history ( -- widgets/widgetOrder unions on global_settings — see storage/db/settings.ts.
id INTEGER PRIMARY KEY AUTOINCREMENT, CREATE TABLE IF NOT EXISTS installed_widgets (
watchlist_id TEXT NOT NULL, id TEXT PRIMARY KEY,
rate REAL NOT NULL, display_name TEXT NOT NULL,
recorded_at TEXT NOT NULL source TEXT NOT NULL, -- 'builtin' | 'uploaded'
code_path TEXT NOT NULL, -- builtin: identifying module path segment; uploaded: on-disk install dir under ./data/
enabled INTEGER NOT NULL DEFAULT 1,
priority_rank INTEGER NOT NULL,
version TEXT NOT NULL DEFAULT '1.0.0',
owned_tables TEXT NOT NULL DEFAULT '[]', -- JSON array, self-reported by the plugin — used by the uninstall safety-net sweep
installed_at TEXT NOT NULL
); );
CREATE INDEX IF NOT EXISTS idx_poe2_rate_history_watchlist ON poe2_rate_history(watchlist_id, recorded_at);
-- Sidebar "Bookmarks" widget — admin-curated links, each independently hidden/public -- Sidebar "Bookmarks" widget — admin-curated links, each independently hidden/public
-- via is_private (same private-access lock feature as categories.is_private). -- via is_private (same private-access lock feature as categories.is_private).
@@ -417,4 +426,58 @@ export function migrate() {
); );
} }
} }
// One-time seed of the installed_widgets registry from whatever the pre-registry
// install had (upgrade path) or sensible defaults (fresh install) — see
// widgets/registry.ts, storage/db/installedWidgets.ts. Raw SQL rather than importing
// installedWidgets.ts here to avoid a circular import (it imports `db` from this
// file).
const widgetCount = db.prepare('SELECT COUNT(*) as c FROM installed_widgets').get() as { c: number };
if (widgetCount.c === 0) {
const settingsRow = db.prepare('SELECT * FROM global_settings WHERE id = 1').get() as any;
const order: string[] = settingsRow
? JSON.parse(settingsRow.widget_order ?? '["weather","stocks","poe2","bookmarks"]')
: ['weather', 'stocks', 'poe2', 'bookmarks'];
const displayNames: Record<string, string> = { weather: 'Weather', stocks: 'Stocks', bookmarks: 'Bookmarks', poe2: 'PoE2' };
const codePaths: Record<string, string> = { weather: 'weather', stocks: 'stocks', bookmarks: 'bookmarks', poe2: 'widgets/poe2' };
const ownedTables: Record<string, string[]> = { poe2: ['widget_poe2_watchlist', 'widget_poe2_rate_history'] };
const enabledOf = (id: string) => (settingsRow ? !!settingsRow[`widget_${id}_enabled`] : true);
const insertWidget = db.prepare(
`INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, owned_tables, installed_at)
VALUES (?, ?, 'builtin', ?, ?, ?, ?, ?)`
);
const installedAt = new Date().toISOString();
order.forEach((id, i) => {
insertWidget.run(
id,
displayNames[id] ?? id,
codePaths[id] ?? id,
enabledOf(id) ? 1 : 0,
i + 1,
JSON.stringify(ownedTables[id] ?? []),
installedAt
);
});
}
// PoE2's league cache (previously bare global_settings columns) moves into the
// generic widget_kv store so it's prunable via the same deleteAllKv('poe2') path as
// everything else the widget owns, rather than needing bespoke column-nulling logic
// on uninstall. One-time copy, guarded on the widget_kv row not already existing.
const poe2CacheRow = db.prepare('SELECT poe2_league_id, poe2_league_name, poe2_updated_at FROM global_settings WHERE id = 1').get() as
| { poe2_league_id: string | null; poe2_league_name: string | null; poe2_updated_at: string | null }
| undefined;
const hasPoe2Kv = db.prepare("SELECT 1 FROM widget_kv WHERE widget_id = 'poe2' AND key = 'leagueCache'").get();
if (poe2CacheRow?.poe2_league_id && !hasPoe2Kv) {
db.prepare(
`INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES ('poe2', 'leagueCache', ?, ?)`
).run(
JSON.stringify({
leagueId: poe2CacheRow.poe2_league_id,
leagueName: poe2CacheRow.poe2_league_name,
updatedAt: poe2CacheRow.poe2_updated_at
}),
new Date().toISOString()
);
}
} }
backend/src/storage/db/installedWidgets.ts
+70
View File
@@ -0,0 +1,70 @@
import { db } from './index.js';
import type { InstalledWidget } from './types.js';
function rowToWidget(row: any): InstalledWidget {
return {
id: row.id,
displayName: row.display_name,
source: row.source,
codePath: row.code_path,
enabled: !!row.enabled,
priorityRank: row.priority_rank,
version: row.version,
ownedTables: JSON.parse(row.owned_tables),
installedAt: row.installed_at
};
}
export function listInstalled(): InstalledWidget[] {
return (db.prepare('SELECT * FROM installed_widgets ORDER BY priority_rank').all() as any[]).map(rowToWidget);
}
export function getInstalled(id: string): InstalledWidget | null {
const row = db.prepare('SELECT * FROM installed_widgets WHERE id = ?').get(id);
return row ? rowToWidget(row) : null;
}
export function insertWidget(widget: {
id: string;
displayName: string;
source: 'builtin' | 'uploaded';
codePath: string;
enabled?: boolean;
priorityRank?: number;
version?: string;
ownedTables?: string[];
}): InstalledWidget {
const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM installed_widgets').get() as { m: number };
const priorityRank = widget.priorityRank ?? maxRank.m + 1;
const installedAt = new Date().toISOString();
db.prepare(
`INSERT INTO installed_widgets (id, display_name, source, code_path, enabled, priority_rank, version, owned_tables, installed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
widget.id,
widget.displayName,
widget.source,
widget.codePath,
widget.enabled === false ? 0 : 1,
priorityRank,
widget.version ?? '1.0.0',
JSON.stringify(widget.ownedTables ?? []),
installedAt
);
return getInstalled(widget.id)!;
}
export function setEnabled(id: string, enabled: boolean) {
db.prepare('UPDATE installed_widgets SET enabled = ? WHERE id = ?').run(enabled ? 1 : 0, id);
}
// Ids in display order — every id must already exist as a row; unlisted ids keep their
// current rank (mirrors the old widget_order JSON array's "admin-sortable" behavior).
export function reorder(ids: string[]) {
const stmt = db.prepare('UPDATE installed_widgets SET priority_rank = ? WHERE id = ?');
ids.forEach((id, i) => stmt.run(i + 1, id));
}
export function deleteInstalled(id: string) {
db.prepare('DELETE FROM installed_widgets WHERE id = ?').run(id);
}
backend/src/storage/db/settings.ts
+49 -26
View File
@@ -1,7 +1,39 @@
import { db } from './index.js'; import { db } from './index.js';
import type { GlobalSettings } from './types.js'; import type { GlobalSettings } from './types.js';
import * as installedWidgetsDb from './installedWidgets.js';
import { getKv, setKv } from './widgetKv.js';
const BUILTIN_WIDGET_IDS = ['weather', 'stocks', 'bookmarks', 'poe2'] as const;
interface Poe2LeagueCache {
leagueId: string | null;
leagueName: string | null;
updatedAt: string | null;
}
// widgets/widgetOrder are computed from the installed_widgets registry (see
// storage/db/installedWidgets.ts, widgets/registry.ts) rather than stored as their own
// global_settings columns — the registry is the single source of truth for enable state
// and ordering for every widget, built-in or uploaded. Only the four built-in ids are
// reflected here since GlobalSettings.widgets/widgetOrder are closed unions the frontend
// depends on (uploaded widgets have no frontend representation yet).
function widgetsAndOrder(): Pick<GlobalSettings, 'widgets' | 'widgetOrder'> {
const installed = installedWidgetsDb.listInstalled();
const byId = new Map(installed.map((w) => [w.id, w]));
const widgets = {
weather: !!byId.get('weather')?.enabled,
stocks: !!byId.get('stocks')?.enabled,
bookmarks: !!byId.get('bookmarks')?.enabled,
poe2: !!byId.get('poe2')?.enabled
};
const widgetOrder = installed.map((w) => w.id).filter((id): id is (typeof BUILTIN_WIDGET_IDS)[number] =>
(BUILTIN_WIDGET_IDS as readonly string[]).includes(id)
);
return { widgets, widgetOrder };
}
function rowToSettings(row: any): GlobalSettings { function rowToSettings(row: any): GlobalSettings {
const poe2Cache = getKv<Poe2LeagueCache>('poe2', 'leagueCache') ?? { leagueId: null, leagueName: null, updatedAt: null };
return { return {
mergeStrictness: row.merge_strictness, mergeStrictness: row.merge_strictness,
holdBeforePublishMinutes: row.hold_before_publish_minutes, holdBeforePublishMinutes: row.hold_before_publish_minutes,
@@ -16,13 +48,7 @@ function rowToSettings(row: any): GlobalSettings {
fxtwitterBaseUrl: row.fxtwitter_base_url, fxtwitterBaseUrl: row.fxtwitter_base_url,
nitterInstanceUrl: row.nitter_instance_url, nitterInstanceUrl: row.nitter_instance_url,
telegramMediaMode: row.telegram_media_mode, telegramMediaMode: row.telegram_media_mode,
widgets: { ...widgetsAndOrder(),
weather: !!row.widget_weather_enabled,
stocks: !!row.widget_stocks_enabled,
bookmarks: !!row.widget_bookmarks_enabled,
poe2: !!row.widget_poe2_enabled
},
widgetOrder: JSON.parse(row.widget_order),
retention: { retention: {
publishedArticleMaxAgeDays: row.published_article_max_age_days, publishedArticleMaxAgeDays: row.published_article_max_age_days,
rawItemMaxAgeDays: row.raw_item_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days,
@@ -44,11 +70,7 @@ function rowToSettings(row: any): GlobalSettings {
alerts: JSON.parse(row.weather_alerts), alerts: JSON.parse(row.weather_alerts),
updatedAt: row.weather_updated_at updatedAt: row.weather_updated_at
}, },
poe2: { poe2: poe2Cache
leagueId: row.poe2_league_id,
leagueName: row.poe2_league_name,
updatedAt: row.poe2_updated_at
}
}; };
} }
@@ -68,6 +90,19 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
poe2: { ...current.poe2, ...(patch.poe2 ?? {}) }, poe2: { ...current.poe2, ...(patch.poe2 ?? {}) },
widgets: { ...current.widgets, ...(patch.widgets ?? {}) } widgets: { ...current.widgets, ...(patch.widgets ?? {}) }
}; };
if (patch.widgets) {
for (const id of BUILTIN_WIDGET_IDS) {
if (patch.widgets[id] !== undefined) installedWidgetsDb.setEnabled(id, patch.widgets[id]);
}
}
if (patch.widgetOrder) {
installedWidgetsDb.reorder(patch.widgetOrder);
}
if (patch.poe2) {
setKv('poe2', 'leagueCache', merged.poe2);
}
// Named params (rather than positional `?`) so this list can be reordered or // Named params (rather than positional `?`) so this list can be reordered or
// extended without the column list and the bound-values list silently drifting // extended without the column list and the bound-values list silently drifting
// out of sync — node:sqlite binds each by its `$name` key, not position. // out of sync — node:sqlite binds each by its `$name` key, not position.
@@ -80,16 +115,12 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models, ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models,
nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url, nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url,
telegram_media_mode=$telegram_media_mode, telegram_media_mode=$telegram_media_mode,
widget_weather_enabled=$widget_weather_enabled, widget_stocks_enabled=$widget_stocks_enabled,
widget_bookmarks_enabled=$widget_bookmarks_enabled, widget_poe2_enabled=$widget_poe2_enabled,
widget_order=$widget_order,
published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days, published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days,
storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit, storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit,
weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude, weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude,
weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit, weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit,
weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily, weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily,
weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at, weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at
poe2_league_id=$poe2_league_id, poe2_league_name=$poe2_league_name, poe2_updated_at=$poe2_updated_at
WHERE id = 1` WHERE id = 1`
).run({ ).run({
$merge_strictness: merged.mergeStrictness, $merge_strictness: merged.mergeStrictness,
@@ -105,11 +136,6 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
$fxtwitter_base_url: merged.fxtwitterBaseUrl, $fxtwitter_base_url: merged.fxtwitterBaseUrl,
$nitter_instance_url: merged.nitterInstanceUrl, $nitter_instance_url: merged.nitterInstanceUrl,
$telegram_media_mode: merged.telegramMediaMode, $telegram_media_mode: merged.telegramMediaMode,
$widget_weather_enabled: merged.widgets.weather ? 1 : 0,
$widget_stocks_enabled: merged.widgets.stocks ? 1 : 0,
$widget_bookmarks_enabled: merged.widgets.bookmarks ? 1 : 0,
$widget_poe2_enabled: merged.widgets.poe2 ? 1 : 0,
$widget_order: JSON.stringify(merged.widgetOrder),
$published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays, $published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays,
$raw_item_max_age_days: merged.retention.rawItemMaxAgeDays, $raw_item_max_age_days: merged.retention.rawItemMaxAgeDays,
$storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0, $storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0,
@@ -125,10 +151,7 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
$weather_hourly: JSON.stringify(merged.weather.hourly), $weather_hourly: JSON.stringify(merged.weather.hourly),
$weather_daily: JSON.stringify(merged.weather.daily), $weather_daily: JSON.stringify(merged.weather.daily),
$weather_alerts: JSON.stringify(merged.weather.alerts), $weather_alerts: JSON.stringify(merged.weather.alerts),
$weather_updated_at: merged.weather.updatedAt, $weather_updated_at: merged.weather.updatedAt
$poe2_league_id: merged.poe2.leagueId,
$poe2_league_name: merged.poe2.leagueName,
$poe2_updated_at: merged.poe2.updatedAt
}); });
return getSettings(); return getSettings();
} }
backend/src/storage/db/types.ts
+15
View File
@@ -256,6 +256,21 @@ export interface Poe2WatchlistEntry {
createdAt: string; createdAt: string;
} }
/** A row in `installed_widgets` — the registry of both built-in and uploaded sidebar widgets (see widgets/registry.ts). */
export interface InstalledWidget {
id: string;
displayName: string;
source: 'builtin' | 'uploaded';
/** Builtin: the module's identifying path segment (e.g. 'weather', 'widgets/poe2'). Uploaded: the on-disk install directory, e.g. './data/widgets-installed/<id>'. */
codePath: string;
enabled: boolean;
priorityRank: number;
version: string;
/** Self-reported by the plugin (WidgetPlugin.ownedTables) at install/load time — used by the uninstall safety-net sweep. */
ownedTables: string[];
installedAt: string;
}
export interface Bookmark { export interface Bookmark {
id: string; id: string;
name: string; name: string;
backend/src/storage/db/widgetKv.ts
+24
View File
@@ -0,0 +1,24 @@
import { db } from './index.js';
// Generic config/cache store for pluggable widgets (see widgets/types.ts) — a widget with
// simple needs (weather-shaped: "last fetched value + timestamp") uses this instead of
// getting bolted-on global_settings columns, so it stays fully prunable by widget_id alone
// on uninstall (see widgets/uninstall.ts) without any host-side schema change.
export function getKv<T>(widgetId: string, key: string): T | null {
const row = db.prepare('SELECT value FROM widget_kv WHERE widget_id = ? AND key = ?').get(widgetId, key) as
| { value: string }
| undefined;
return row ? (JSON.parse(row.value) as T) : null;
}
export function setKv(widgetId: string, key: string, value: unknown) {
db.prepare(
`INSERT INTO widget_kv (widget_id, key, value, updated_at) VALUES (?, ?, ?, ?)
ON CONFLICT(widget_id, key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`
).run(widgetId, key, JSON.stringify(value), new Date().toISOString());
}
export function deleteAllKv(widgetId: string) {
db.prepare('DELETE FROM widget_kv WHERE widget_id = ?').run(widgetId);
}
backend/src/weather/plugin.ts
+36
View File
@@ -0,0 +1,36 @@
import type { WidgetPlugin } from '../widgets/types.js';
import * as settingsDb from '../storage/db/settings.js';
import { geocodeLocation } from './client.js';
import { pollWeatherNow } from './poller.js';
// Built-in sidebar "Weather" widget — thin wrapper so it funnels into the same
// scheduler/route-registration loop as pluggable widgets (see widgets/registry.ts), while
// its config/cache stay on global_settings' weather_* columns exactly as before. Not going
// through the upload/delete lifecycle — see widgets/types.ts and the plan's "built-in vs
// pluggable" split; no migrate()/uninstall() since its schema isn't moving.
export const weatherPlugin: WidgetPlugin = {
id: 'weather',
displayName: 'Weather',
version: '1.0.0',
poll: {
intervalMs: 45 * 60_000,
run: pollWeatherNow
},
registerPublicRoutes(app) {
app.get('/api/weather', async () => settingsDb.getSettings().weather);
},
registerAdminRoutes(app) {
app.get('/api/admin/weather/geocode', async (req, reply) => {
const { query } = req.query as { query?: string };
if (!query || !query.trim()) return reply.code(400).send({ error: 'query required' });
try {
return await geocodeLocation(query.trim());
} catch (err) {
return reply.code(502).send({ error: `Geocoding service unreachable: ${(err as Error).message}` });
}
});
}
};
backend/src/widgets/install.ts
+59
View File
@@ -0,0 +1,59 @@
import fs from 'node:fs';
import path from 'node:path';
import * as installedWidgetsDb from '../storage/db/installedWidgets.js';
import { loadUploadedWidget } from './registry.js';
import { startWidgetPolling } from '../queue/scheduler.js';
import { validateManifest, type WidgetManifest } from './manifest.js';
const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed';
export type InstallResult = { ok: true; id: string } | { ok: false; error: string };
// Installs and hot-loads a widget uploaded live to the running backend (see
// api/admin.ts's POST /api/admin/widgets) — writes its files under ./data/, never
// dist/ or src/, so it survives a rebuild/redeploy of the core app. Its migrate()
// runs and its poll interval (if declared) starts immediately, with no restart
// required. NOTE: its registerPublicRoutes/registerAdminRoutes, if declared, do NOT
// take effect until the next restart — Fastify throws "instance is already
// listening" if you try to add a route after app.listen() has resolved, and there's
// no supported way around that short of a much larger request-dispatch redesign. A
// live-installed widget's data/poll side works immediately; its custom HTTP routes
// don't until the process restarts (see widgets/registry.ts's startup discovery,
// which re-registers everything, routes included, on every boot).
export async function installUploadedWidget(manifest: unknown, files: unknown): Promise<InstallResult> {
const validationError = validateManifest(manifest, files);
if (validationError) return { ok: false, error: validationError };
const m = manifest as WidgetManifest;
const f = files as Record<string, string>;
if (installedWidgetsDb.getInstalled(m.id)) {
return { ok: false, error: `widget "${m.id}" is already installed` };
}
const dir = path.join(WIDGETS_INSTALLED_DIR, m.id);
fs.mkdirSync(dir, { recursive: true });
for (const [relPath, content] of Object.entries(f)) {
const filePath = path.join(dir, relPath);
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, content, 'utf8');
}
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(m, null, 2), 'utf8');
const plugin = await loadUploadedWidget(m.id, dir);
if (!plugin) {
fs.rmSync(dir, { recursive: true, force: true });
return { ok: false, error: "widget failed to load — check its entry file's default export satisfies WidgetPlugin" };
}
installedWidgetsDb.insertWidget({
id: m.id,
displayName: m.displayName,
source: 'uploaded',
codePath: dir,
version: m.version,
ownedTables: plugin.ownedTables ?? []
});
startWidgetPolling(plugin);
return { ok: true, id: m.id };
}
backend/src/widgets/manifest.ts
+39
View File
@@ -0,0 +1,39 @@
const ID_PATTERN = /^[a-z0-9-]{1,40}$/;
const RESERVED_IDS = new Set(['weather', 'stocks', 'bookmarks', 'poe2']);
const MAX_TOTAL_BYTES = 5 * 1024 * 1024;
export interface WidgetManifest {
id: string;
displayName: string;
version: string;
/** Relative path within `files` to the ESM entry point, e.g. "index.mjs" — a default export satisfying WidgetPlugin. */
entry: string;
}
// Upload body shape: { manifest, files: { "index.mjs": "<source text>", ... } } — plain
// JSON rather than a literal zip, so no archive/multipart dependency is needed (see
// widgets/install.ts). Returns a human-readable error string, or null if valid.
export function validateManifest(manifest: unknown, files: unknown): string | null {
if (!manifest || typeof manifest !== 'object') return 'manifest is required';
const m = manifest as Record<string, unknown>;
if (typeof m.id !== 'string' || !ID_PATTERN.test(m.id)) return 'manifest.id must match /^[a-z0-9-]{1,40}$/';
if (RESERVED_IDS.has(m.id)) return `id "${m.id}" is reserved for a built-in widget`;
if (typeof m.displayName !== 'string' || !m.displayName.trim()) return 'manifest.displayName is required';
if (typeof m.version !== 'string' || !m.version.trim()) return 'manifest.version is required';
if (typeof m.entry !== 'string' || !m.entry.trim()) return 'manifest.entry is required';
if (!files || typeof files !== 'object' || Array.isArray(files)) return 'files must be a non-empty object';
const entries = Object.entries(files as Record<string, unknown>);
if (entries.length === 0) return 'files must be a non-empty object';
if (!(m.entry in (files as Record<string, unknown>))) return `entry "${m.entry as string}" not found in files`;
let totalBytes = 0;
for (const [relPath, content] of entries) {
if (typeof content !== 'string') return `files["${relPath}"] must be a string`;
if (relPath.startsWith('/') || relPath.split('/').some((seg) => seg === '..')) return `invalid file path "${relPath}"`;
totalBytes += Buffer.byteLength(content, 'utf8');
}
if (totalBytes > MAX_TOTAL_BYTES) return `bundle exceeds ${MAX_TOTAL_BYTES}-byte limit`;
return null;
}
backend/src/poe2/client.ts → backend/src/widgets/poe2/client.ts
+5 -4
View File
@@ -1,5 +1,5 @@
// poe.ninja's public PoE2 economy API — free, no account or API key required. This is the // poe.ninja's public PoE2 economy API — free, no account or API key required. This is the
// only file that talks to it; poller.ts orchestrates when/how results get saved, same // only file that talks to it; poll.ts orchestrates when/how results get saved, same
// separation as backend/src/telegram/ and backend/src/weather/ keep between the raw client // separation as backend/src/telegram/ and backend/src/weather/ keep between the raw client
// and their callers. // and their callers.
// //
@@ -13,7 +13,7 @@
// change, which poe.ninja's overview doesn't expose directly. Every line's `primaryValue` is // change, which poe.ninja's overview doesn't expose directly. Every line's `primaryValue` is
// expressed in the same (unspecified, and irrelevant) reference currency, so any pair's rate // expressed in the same (unspecified, and irrelevant) reference currency, so any pair's rate
// is just baseValue / quoteValue with the reference cancelling out — see fetchCurrencyValues // is just baseValue / quoteValue with the reference cancelling out — see fetchCurrencyValues
// below and poe2/poller.ts, which self-computes change from its own polling history instead. // below and poll.ts, which self-computes change from its own polling history instead.
const BASE_URL = 'https://poe.ninja'; const BASE_URL = 'https://poe.ninja';
export interface LeagueInfo { export interface LeagueInfo {
@@ -57,8 +57,9 @@ async function fetchCurrencyOverview(leagueId: string): Promise<RawCurrencyOverv
} }
// Fetches the whole traded-currency list for the admin's search-and-pick UI (see // Fetches the whole traded-currency list for the admin's search-and-pick UI (see
// api/admin.ts's GET /api/admin/poe2/browse) — only currencies that actually have a `lines` // widgets/poe2/plugin.ts's GET /api/admin/poe2/browse) — only currencies that actually have
// entry (i.e. are currently traded), not every currency poe.ninja has ever known about. // a `lines` entry (i.e. are currently traded), not every currency poe.ninja has ever known
// about.
export async function browseCurrencies(leagueId: string): Promise<CurrencyBrowseEntry[]> { export async function browseCurrencies(leagueId: string): Promise<CurrencyBrowseEntry[]> {
const { lines, items } = await fetchCurrencyOverview(leagueId); const { lines, items } = await fetchCurrencyOverview(leagueId);
const nameById = new Map(items.map((item) => [item.id, item.name])); const nameById = new Map(items.map((item) => [item.id, item.name]));
backend/src/storage/db/poe2Watchlist.ts → backend/src/widgets/poe2/db.ts
+13 -13
View File
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { db } from './index.js'; import { db } from '../../storage/db/index.js';
import type { Poe2WatchlistEntry } from './types.js'; import type { Poe2WatchlistEntry } from '../../storage/db/types.js';
interface CurrencyRef { interface CurrencyRef {
currencyId: string; currencyId: string;
@@ -24,21 +24,21 @@ function rowToEntry(row: any): Poe2WatchlistEntry {
} }
export function listWatchlist(): Poe2WatchlistEntry[] { export function listWatchlist(): Poe2WatchlistEntry[] {
const rows = db.prepare('SELECT * FROM poe2_watchlist ORDER BY priority_rank').all(); const rows = db.prepare('SELECT * FROM widget_poe2_watchlist ORDER BY priority_rank').all();
return rows.map(rowToEntry); return rows.map(rowToEntry);
} }
// No update() — currencies are picked from a live browse list (see poe2/client.ts's // No update() — currencies are picked from a live browse list (see widgets/poe2/client.ts's
// browseCurrencies), not typed, so there's nothing to edit; remove and re-add covers the // browseCurrencies), not typed, so there's nothing to edit; remove and re-add covers the
// rare "picked the wrong one" case. // rare "picked the wrong one" case.
export function addWatchlistEntry(base: CurrencyRef, quote: CurrencyRef): Poe2WatchlistEntry { export function addWatchlistEntry(base: CurrencyRef, quote: CurrencyRef): Poe2WatchlistEntry {
const id = `poe2-${base.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${quote.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${randomUUID().slice(0, 6)}` const id = `poe2-${base.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${quote.currencyId.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-${randomUUID().slice(0, 6)}`
.replace(/-+/g, '-') .replace(/-+/g, '-')
.replace(/(^-|-$)/g, ''); .replace(/(^-|-$)/g, '');
const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM poe2_watchlist').get() as { m: number }; const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM widget_poe2_watchlist').get() as { m: number };
const createdAt = new Date().toISOString(); const createdAt = new Date().toISOString();
db.prepare( db.prepare(
`INSERT INTO poe2_watchlist `INSERT INTO widget_poe2_watchlist
(id, base_currency_id, base_name, quote_currency_id, quote_name, priority_rank, created_at) (id, base_currency_id, base_name, quote_currency_id, quote_name, priority_rank, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)` VALUES (?, ?, ?, ?, ?, ?, ?)`
).run(id, base.currencyId, base.name, quote.currencyId, quote.name, maxRank.m + 1, createdAt); ).run(id, base.currencyId, base.name, quote.currencyId, quote.name, maxRank.m + 1, createdAt);
@@ -58,14 +58,14 @@ export function addWatchlistEntry(base: CurrencyRef, quote: CurrencyRef): Poe2Wa
} }
export function removeWatchlistEntry(id: string) { export function removeWatchlistEntry(id: string) {
db.prepare('DELETE FROM poe2_rate_history WHERE watchlist_id = ?').run(id); db.prepare('DELETE FROM widget_poe2_rate_history WHERE watchlist_id = ?').run(id);
db.prepare('DELETE FROM poe2_watchlist WHERE id = ?').run(id); db.prepare('DELETE FROM widget_poe2_watchlist WHERE id = ?').run(id);
} }
// One snapshot per poll (see poe2/poller.ts) — the raw material 24h change is computed // One snapshot per poll (see widgets/poe2/poll.ts) — the raw material 24h change is computed
// from, since poe.ninja itself doesn't expose per-pair rates or a matching change window. // from, since poe.ninja itself doesn't expose per-pair rates or a matching change window.
export function recordRate(watchlistId: string, rate: number, recordedAt: string) { export function recordRate(watchlistId: string, rate: number, recordedAt: string) {
db.prepare('INSERT INTO poe2_rate_history (watchlist_id, rate, recorded_at) VALUES (?, ?, ?)').run( db.prepare('INSERT INTO widget_poe2_rate_history (watchlist_id, rate, recorded_at) VALUES (?, ?, ?)').run(
watchlistId, watchlistId,
rate, rate,
recordedAt recordedAt
@@ -77,14 +77,14 @@ export function recordRate(watchlistId: string, rate: number, recordedAt: string
// than fabricating a 0% figure. // than fabricating a 0% figure.
export function rateAtOrBefore(watchlistId: string, cutoffIso: string): number | null { export function rateAtOrBefore(watchlistId: string, cutoffIso: string): number | null {
const row = db const row = db
.prepare('SELECT rate FROM poe2_rate_history WHERE watchlist_id = ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1') .prepare('SELECT rate FROM widget_poe2_rate_history WHERE watchlist_id = ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1')
.get(watchlistId, cutoffIso) as { rate: number } | undefined; .get(watchlistId, cutoffIso) as { rate: number } | undefined;
return row ? row.rate : null; return row ? row.rate : null;
} }
export function markPolled(id: string, rate: number | null, change24h: number | null, error: string | null) { export function markPolled(id: string, rate: number | null, change24h: number | null, error: string | null) {
db.prepare( db.prepare(
`UPDATE poe2_watchlist `UPDATE widget_poe2_watchlist
SET last_rate = ?, last_change_24h = ?, last_polled_at = ?, last_error = ? SET last_rate = ?, last_change_24h = ?, last_polled_at = ?, last_error = ?
WHERE id = ?` WHERE id = ?`
).run(rate, change24h, new Date().toISOString(), error, id); ).run(rate, change24h, new Date().toISOString(), error, id);
@@ -94,5 +94,5 @@ export function markPolled(id: string, rate: number | null, change24h: number |
// poll to be briefly late without losing the data point it needs. // poll to be briefly late without losing the data point it needs.
export function pruneOldHistory() { export function pruneOldHistory() {
const cutoff = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(); const cutoff = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString();
db.prepare('DELETE FROM poe2_rate_history WHERE recorded_at < ?').run(cutoff); db.prepare('DELETE FROM widget_poe2_rate_history WHERE recorded_at < ?').run(cutoff);
} }
backend/src/widgets/poe2/plugin.ts
+107
View File
@@ -0,0 +1,107 @@
import type { DatabaseSync } from 'node:sqlite';
import type { WidgetPlugin } from '../types.js';
import { deleteAllKv } from '../../storage/db/widgetKv.js';
import { logger } from '../../storage/db/logs.js';
import * as poe2Db from './db.js';
import { browseCurrencies, fetchCurrentLeague } from './client.js';
import { pollPoe2Now, getLeagueCache } from './poll.js';
const OWNED_TABLES = ['widget_poe2_watchlist', 'widget_poe2_rate_history'];
// Sidebar "PoE2" widget — tracks exchange rates between arbitrary currency pairs, always
// against the current challenge league (auto-detected, no admin config). The pilot
// implementation of the pluggable-widget system (see widgets/types.ts) — every other
// widget still ships in-process, but this one exercises the full migrate/poll/routes/
// uninstall lifecycle a live-uploaded widget would also go through.
export const poe2Plugin: WidgetPlugin = {
id: 'poe2',
displayName: 'PoE2',
version: '1.0.0',
ownedTables: OWNED_TABLES,
migrate(db: DatabaseSync) {
db.exec(`
CREATE TABLE IF NOT EXISTS widget_poe2_watchlist (
id TEXT PRIMARY KEY,
base_currency_id TEXT NOT NULL,
base_name TEXT NOT NULL,
quote_currency_id TEXT NOT NULL,
quote_name TEXT NOT NULL,
priority_rank INTEGER NOT NULL,
last_rate REAL,
last_change_24h REAL,
last_polled_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS widget_poe2_rate_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
watchlist_id TEXT NOT NULL,
rate REAL NOT NULL,
recorded_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_widget_poe2_rate_history_watchlist ON widget_poe2_rate_history(watchlist_id, recorded_at);
`);
},
poll: {
// poe.ninja's own overview data doesn't refresh faster than hourly, so polling
// more often than this just re-fetches the same numbers.
intervalMs: 60 * 60_000,
run: pollPoe2Now
},
registerPublicRoutes(app) {
app.get('/api/poe2', async () => {
const { leagueName, updatedAt } = getLeagueCache();
return { leagueName, updatedAt, entries: poe2Db.listWatchlist() };
});
},
registerAdminRoutes(app) {
// League is always auto-detected, never admin-set.
app.get('/api/admin/poe2/browse', async (_req, reply) => {
try {
const league = await fetchCurrentLeague();
return await browseCurrencies(league.id);
} catch (err) {
return reply.code(502).send({ error: `poe.ninja unreachable: ${(err as Error).message}` });
}
});
app.get('/api/admin/poe2/watchlist', async () => poe2Db.listWatchlist());
app.post('/api/admin/poe2/watchlist', async (req, reply) => {
const { base, quote } = req.body as {
base?: { currencyId?: string; name?: string };
quote?: { currencyId?: string; name?: string };
};
if (!base?.currencyId || !base?.name || !quote?.currencyId || !quote?.name) {
return reply.code(400).send({ error: 'base and quote currencies (currencyId, name) are required' });
}
if (base.currencyId === quote.currencyId) {
return reply.code(400).send({ error: 'Base and quote currencies must be different' });
}
const created = poe2Db.addWatchlistEntry(
{ currencyId: base.currencyId, name: base.name },
{ currencyId: quote.currencyId, name: quote.name }
);
// Poll immediately rather than waiting for the next tick (up to 1 hour) — cheap,
// and refreshes every existing entry's rate too.
pollPoe2Now().catch((err) => logger.error('poe2', `Immediate poll failed: ${err.message}`));
return reply.code(201).send(created);
});
app.delete('/api/admin/poe2/watchlist/:id', async (req, reply) => {
const { id } = req.params as { id: string };
poe2Db.removeWatchlistEntry(id);
return reply.code(204).send();
});
},
uninstall(db: DatabaseSync) {
db.exec('DROP TABLE IF EXISTS widget_poe2_rate_history;');
db.exec('DROP TABLE IF EXISTS widget_poe2_watchlist;');
deleteAllKv('poe2');
}
};
backend/src/widgets/poe2/poll.ts
+74
View File
@@ -0,0 +1,74 @@
import * as poe2Db from './db.js';
import { getKv, setKv } from '../../storage/db/widgetKv.js';
import { logger } from '../../storage/db/logs.js';
import { fetchCurrentLeague, fetchCurrencyValues } from './client.js';
const DAY_MS = 24 * 60 * 60_000;
const WIDGET_ID = 'poe2';
interface LeagueCache {
leagueId: string | null;
leagueName: string | null;
updatedAt: string | null;
}
// Cache of what the last poll learned about the current league — no admin-set config
// (unlike weather): the league is always auto-detected, so this is purely a cache. Lives
// in the generic widget_kv store (see storage/db/widgetKv.ts) so it's prunable via the
// same deleteAllKv('poe2') path as everything else this widget owns.
export function getLeagueCache(): LeagueCache {
return getKv<LeagueCache>(WIDGET_ID, 'leagueCache') ?? { leagueId: null, leagueName: null, updatedAt: null };
}
function pctChange(current: number, past: number | null): number | null {
if (past === null || past === 0) return null;
return ((current - past) / past) * 100;
}
// Called on a schedule (see queue/scheduler.ts, via plugin.poll) and immediately after the
// admin adds a pair (see plugin.ts's admin routes) — always re-detects the current
// challenge league fresh (cheap, guarantees correctness across league rotations with no
// separate staleness logic), then one overview request covers the whole watchlist. A pair
// whose base or quote currency is no longer traded this league gets its own lastError, it
// never aborts the rest of the batch.
export async function pollPoe2Now(): Promise<void> {
let league;
try {
league = await fetchCurrentLeague();
} catch (err) {
logger.error('poe2', `League lookup failed: ${(err as Error).message}`);
return;
}
const entries = poe2Db.listWatchlist();
if (entries.length === 0) {
setKv(WIDGET_ID, 'leagueCache', { leagueId: league.id, leagueName: league.name, updatedAt: new Date().toISOString() });
return;
}
try {
const valuesById = await fetchCurrencyValues(league.id);
const now = new Date();
const nowIso = now.toISOString();
const cutoff24h = new Date(now.getTime() - DAY_MS).toISOString();
for (const entry of entries) {
const baseValue = valuesById.get(entry.baseCurrencyId);
const quoteValue = valuesById.get(entry.quoteCurrencyId);
if (baseValue === undefined || quoteValue === undefined) {
poe2Db.markPolled(entry.id, null, null, 'One or both currencies no longer traded in this league');
continue;
}
const rate = baseValue / quoteValue;
const change24h = pctChange(rate, poe2Db.rateAtOrBefore(entry.id, cutoff24h));
poe2Db.recordRate(entry.id, rate, nowIso);
poe2Db.markPolled(entry.id, rate, change24h, null);
}
poe2Db.pruneOldHistory();
setKv(WIDGET_ID, 'leagueCache', { leagueId: league.id, leagueName: league.name, updatedAt: nowIso });
} catch (err) {
logger.error('poe2', `Watchlist poll failed: ${(err as Error).message}`);
}
}
backend/src/widgets/registry.ts
+69
View File
@@ -0,0 +1,69 @@
import fs from 'node:fs';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { db } from '../storage/db/index.js';
import * as installedWidgetsDb from '../storage/db/installedWidgets.js';
import { logger } from '../storage/db/logs.js';
import type { WidgetPlugin } from './types.js';
import { sweepOrphanedWidgetData } from './sweep.js';
import { weatherPlugin } from '../weather/plugin.js';
import { stocksPlugin } from '../stocks/plugin.js';
import { bookmarksPlugin } from '../bookmarks/plugin.js';
import { poe2Plugin } from './poe2/plugin.js';
// Both built-in and uploaded widgets funnel into this one map — scheduler dispatch and
// route registration (see index.ts, queue/scheduler.ts) iterate it generically instead of
// hardcoding a block per widget, mirroring ingestion/poller.ts's
// Record<Source['type'], SourceAdapter> dispatch, just keyed by a dynamic string id.
export const loadedWidgets = new Map<string, WidgetPlugin>();
function registerBuiltinWidget(plugin: WidgetPlugin) {
plugin.migrate?.(db);
loadedWidgets.set(plugin.id, plugin);
}
// Called once at startup (see index.ts, after migrate() and before route registration /
// startScheduler()) and again after a live upload (see widgets/install.ts) to pick up just
// the newly-installed one without reloading everything else.
export async function loadAllWidgets(): Promise<void> {
loadedWidgets.clear();
registerBuiltinWidget(weatherPlugin);
registerBuiltinWidget(stocksPlugin);
registerBuiltinWidget(bookmarksPlugin);
registerBuiltinWidget(poe2Plugin);
for (const row of installedWidgetsDb.listInstalled().filter((w) => w.source === 'uploaded')) {
await loadUploadedWidget(row.id, row.codePath);
}
sweepOrphanedWidgetData(
db,
installedWidgetsDb.listInstalled().map((w) => w.id)
);
}
// A widget whose code fails to load logs an error and is simply absent from
// loadedWidgets — scheduler/route registration skip an id with no entry. Its
// installed_widgets row stays, so it's still visible and deletable from the admin side
// (see widgets/uninstall.ts, which doesn't require the widget's own code to be loadable).
export async function loadUploadedWidget(id: string, codePath: string): Promise<WidgetPlugin | null> {
try {
const manifest = JSON.parse(fs.readFileSync(path.join(codePath, 'manifest.json'), 'utf8')) as { entry: string };
const entryUrl = pathToFileURL(path.join(codePath, manifest.entry)).href;
const mod = await import(entryUrl);
const plugin: WidgetPlugin = mod.default;
if (!plugin || typeof plugin.id !== 'string') {
throw new Error('module has no default-exported WidgetPlugin');
}
if (plugin.id !== id) {
throw new Error(`plugin id "${plugin.id}" does not match installed id "${id}"`);
}
plugin.migrate?.(db);
loadedWidgets.set(id, plugin);
return plugin;
} catch (err) {
logger.error('widgets', `Failed to load uploaded widget "${id}": ${(err as Error).message}`);
return null;
}
}
backend/src/widgets/sweep.ts
+48
View File
@@ -0,0 +1,48 @@
import type { DatabaseSync } from 'node:sqlite';
// Host-side safety net for widget data pruning (see widgets/types.ts's uninstall contract
// and the plan's "data isolation strategy"). This alone is sufficient to fully clean a
// widget's data even with zero cooperation from its own code — a widget's own uninstall()
// hook (if present) is an optimization/extension point, not a requirement for correctness.
function ownedTablesForId(db: DatabaseSync, widgetId: string, extraOwnedTables: string[] = []): string[] {
const prefix = `widget_${widgetId}_`;
const rows = db
.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? ESCAPE '\\'`)
.all(`${prefix.replace(/_/g, '\\_')}%`) as { name: string }[];
return Array.from(new Set([...rows.map((r) => r.name), ...extraOwnedTables]));
}
// Drops every table owned by a single widget (by the widget_<id>_ naming convention, plus
// any self-reported extraOwnedTables for a grandfathered name), and clears its widget_kv
// rows. Safe to call even if the widget's own code is missing/broken/never uninstall()ed.
export function sweepWidgetData(db: DatabaseSync, widgetId: string, extraOwnedTables: string[] = []) {
for (const table of ownedTablesForId(db, widgetId, extraOwnedTables)) {
db.exec(`DROP TABLE IF EXISTS "${table}";`);
}
db.prepare('DELETE FROM widget_kv WHERE widget_id = ?').run(widgetId);
}
// Full-registry sweep, run once at every startup after the registry loads (see
// widgets/registry.ts) — catches anything left behind by a process crash mid-uninstall, or
// a manually-edited installed_widgets table, that the per-widget sweep above never got a
// chance to run for.
export function sweepOrphanedWidgetData(db: DatabaseSync, knownWidgetIds: string[]) {
const known = new Set(knownWidgetIds);
const tables = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'widget\\_%' ESCAPE '\\'`).all() as {
name: string;
}[];
for (const { name } of tables) {
if (name === 'widget_kv') continue; // the shared kv table itself, not a per-widget owned table
const id = name.slice('widget_'.length).split('_')[0];
if (!known.has(id)) {
db.exec(`DROP TABLE IF EXISTS "${name}";`);
}
}
const kvWidgetIds = db.prepare('SELECT DISTINCT widget_id FROM widget_kv').all() as { widget_id: string }[];
const orphanedKvIds = kvWidgetIds.map((r) => r.widget_id).filter((id) => !known.has(id));
if (orphanedKvIds.length > 0) {
const placeholders = orphanedKvIds.map(() => '?').join(',');
db.prepare(`DELETE FROM widget_kv WHERE widget_id IN (${placeholders})`).run(...orphanedKvIds);
}
}
backend/src/widgets/types.ts
+31
View File
@@ -0,0 +1,31 @@
import type { DatabaseSync } from 'node:sqlite';
import type { FastifyInstance } from 'fastify';
/**
* A widget's own lifecycle hooks — deliberately all-optional (unlike
* ingestion/adapters/base.ts's SourceAdapter, which has one mandatory fetch()) because
* unlike source adapters, widgets don't share a common downstream table or data shape
* to normalize into; each is a self-contained module that opts into only what it needs.
*/
export interface WidgetPlugin {
/** Stable id — also the table-name (`widget_<id>_*`) and directory-name namespace. Validated on install: /^[a-z0-9-]{1,40}$/ */
id: string;
displayName: string;
version: string;
/** Widget-owned schema. Must be idempotent (CREATE TABLE IF NOT EXISTS style), same contract as the host's own migrate(). */
migrate?(db: DatabaseSync): void;
/** Self-reported table names, for the uninstall safety-net sweep — belt-and-suspenders alongside the widget_<id>_ naming convention (see widgets/sweep.ts). */
ownedTables?: string[];
/** Host owns the setInterval/enable-gating (see queue/scheduler.ts); the plugin just does the fetch-and-persist work. */
poll?: { intervalMs: number; run(): Promise<void> };
/** Called once at load time (startup, or immediately after a live upload). Admin routes registered here are auto-gated by the existing X-Api-Key preHandler (see api/auth.ts), since it matches on any /api/admin/* path. */
registerPublicRoutes?(app: FastifyInstance): void;
registerAdminRoutes?(app: FastifyInstance): void;
/** Best-effort cleanup, called before the host's safety-net sweep on delete. Should not assume its own migrate() succeeded or that external APIs are reachable — the sweep is the real guarantee, this is just an extension point. */
uninstall?(db: DatabaseSync): void;
}
backend/src/widgets/uninstall.ts
+38
View File
@@ -0,0 +1,38 @@
import fs from 'node:fs';
import path from 'node:path';
import { db } from '../storage/db/index.js';
import * as installedWidgetsDb from '../storage/db/installedWidgets.js';
import { logger } from '../storage/db/logs.js';
import { loadedWidgets } from './registry.js';
import { sweepWidgetData } from './sweep.js';
import { stopWidgetPolling } from '../queue/scheduler.js';
const WIDGETS_INSTALLED_DIR = process.env.WIDGETS_INSTALLED_DIR || './data/widgets-installed';
const WIDGETS_DATA_DIR = process.env.WIDGETS_DATA_DIR || './data/widgets-data';
// Fully removes an uploaded widget — safe to call even if its code is already
// broken/missing (the plugin.uninstall() step is best-effort; sweepWidgetData is the real
// guarantee, matching every table/kv row/on-disk file regardless of the widget's own
// cooperation). Callers (see api/admin.ts's DELETE /api/admin/widgets/:id) are
// responsible for rejecting built-in widgets before calling this.
export async function uninstallWidget(id: string): Promise<void> {
stopWidgetPolling(id);
const plugin = loadedWidgets.get(id);
if (plugin?.uninstall) {
try {
plugin.uninstall(db);
} catch (err) {
logger.error('widgets', `uninstall() hook failed for "${id}": ${(err as Error).message}`);
}
}
const row = installedWidgetsDb.getInstalled(id);
sweepWidgetData(db, id, row?.ownedTables ?? []);
fs.rmSync(path.join(WIDGETS_INSTALLED_DIR, id), { recursive: true, force: true });
fs.rmSync(path.join(WIDGETS_DATA_DIR, id), { recursive: true, force: true });
loadedWidgets.delete(id);
installedWidgetsDb.deleteInstalled(id);
}