diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 4bd71b8..c52bf4c 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -4,15 +4,23 @@ import * as sourcesDb from '../storage/db/sources.js'; import * as eventsDb from '../storage/db/events.js'; import * as categoriesDb from '../storage/db/categories.js'; import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; +import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; import * as telegramClient from '../telegram/client.js'; +// Not part of GlobalSettings itself (nothing to persist) — computed fresh on every +// settings read/write so the Retention tab's "currently using" line and usage bar +// always reflect the real total, not whatever was true when the row was last saved. +function withStorageUsed(settings: ReturnType) { + return { ...settings, retention: { ...settings.retention, storageUsedMB: Math.round(totalStorageBytes() / (1024 * 1024)) } }; +} + export async function registerAdminRoutes(app: FastifyInstance) { // --- Settings --- app.get('/api/admin/settings', async () => { - const settings = settingsDb.getSettings(); + const settings = withStorageUsed(settingsDb.getSettings()); return { ...settings, categoryPriority: categoriesDb.listCategories() }; }); @@ -22,7 +30,7 @@ export async function registerAdminRoutes(app: FastifyInstance) { categoriesDb.setCategoryOrder(body.categoryPriority); delete body.categoryPriority; } - const settings = settingsDb.updateSettings(body); + const settings = withStorageUsed(settingsDb.updateSettings(body)); return { ...settings, categoryPriority: categoriesDb.listCategories() }; }); diff --git a/backend/src/queue/eventsRecap.ts b/backend/src/queue/eventsRecap.ts index 703f6da..329e581 100644 --- a/backend/src/queue/eventsRecap.ts +++ b/backend/src/queue/eventsRecap.ts @@ -37,7 +37,9 @@ export async function runEventRecaps(provider: InferenceProvider, settings: Glob if (event.sourceIds.length === 0 || !isDue(event)) continue; const since = event.lastRecapAt ?? new Date(Date.now() - 24 * 3600_000).toISOString(); - const items = contentItemsDb.unclusteredItemsForSources(event.sourceIds, since); + const items = contentItemsDb + .unclusteredItemsForSources(event.sourceIds, since) + .filter((item) => eventsDb.itemMatchesEventKeywords(item, event.keywords)); if (items.length === 0) continue; const embedded = await embedPendingItems(provider, settings.selectedModels.embedding, items); diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 00a0d15..68b2355 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -7,7 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js'; import { clusterItems } from '../pipeline/clustering.js'; import { publishCluster, publishDirect } from '../pipeline/publish.js'; import { logger } from '../storage/db/logs.js'; -import type { GlobalSettings, ContentItem } from '../storage/db/types.js'; +import type { GlobalSettings, ContentItem, TrackedEvent } from '../storage/db/types.js'; function partition(items: T[], predicate: (item: T) => boolean): [T[], T[]] { const matches: T[] = []; @@ -16,6 +16,18 @@ function partition(items: T[], predicate: (item: T) => boolean): [T[], T[]] { return [matches, rest]; } +/** + * An item is "claimed" by a tracked event — and so left for eventsRecap.ts to handle + * instead of normal synthesis — only if it belongs to one of the event's sources AND + * matches its keyword filter. An item from an event-linked source that doesn't match + * (e.g. a general Middle-East feed assigned to an "Iran war" event, but this particular + * item doesn't mention Iran) falls through to normal synthesis rather than being + * silently dropped — it just isn't part of that event's recap. + */ +function isClaimedByEvent(item: ContentItem, events: TrackedEvent[]): boolean { + return events.some((e) => e.sourceIds.includes(item.sourceId) && eventsDb.itemMatchesEventKeywords(item, e.keywords)); +} + function primaryCategoryRank(item: ContentItem, rankByName: Map): number { const source = sourcesDb.getSource(item.sourceId); const cats = source?.category ?? []; @@ -39,8 +51,8 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map) * Still respects category priority. */ export async function runPassthroughCycle(settings: GlobalSettings): Promise { - const eventSourceIds = eventsDb.listActiveEvents().flatMap((e) => e.sourceIds); - const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds); + const activeEvents = eventsDb.listActiveEvents(); + const items = contentItemsDb.unclusteredItemsExcludingSources([]).filter((item) => !isClaimedByEvent(item, activeEvents)); if (items.length === 0) return 0; const categories = categoriesDb.listCategories(); @@ -68,13 +80,14 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { - const eventSourceIds = eventsDb.listActiveEvents().flatMap((e) => e.sourceIds); - const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds); + const activeEvents = eventsDb.listActiveEvents(); + const items = contentItemsDb.unclusteredItemsExcludingSources([]).filter((item) => !isClaimedByEvent(item, activeEvents)); if (items.length === 0) return 0; // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with diff --git a/backend/src/storage/db/events.ts b/backend/src/storage/db/events.ts index 5225cfe..70ab9f4 100644 --- a/backend/src/storage/db/events.ts +++ b/backend/src/storage/db/events.ts @@ -8,6 +8,7 @@ function rowToEvent(row: any): TrackedEvent { name: row.name, description: row.description, sourceIds: JSON.parse(row.source_ids), + keywords: JSON.parse(row.keywords), cadence: row.cadence, cadenceTime: row.cadence_time, active: !!row.active, @@ -17,6 +18,21 @@ function rowToEvent(row: any): TrackedEvent { }; } +/** + * Whether an item qualifies for a tracked event's keyword filter — empty keywords + * means "match everything" (the original, filter-less behavior). Matching is a plain + * case-insensitive substring check against title+summary+body; works the same for a + * word/phrase ("Tehran") or an emoji ("🇮🇷", which has no case to fold). + */ +export function itemMatchesEventKeywords( + item: { title: string; summary: string; body: string | null }, + keywords: string[] +): boolean { + if (keywords.length === 0) return true; + const haystack = `${item.title} ${item.summary} ${item.body ?? ''}`.toLowerCase(); + return keywords.some((k) => haystack.includes(k.toLowerCase())); +} + export function listEvents(): TrackedEvent[] { const rows = db.prepare('SELECT * FROM tracked_events ORDER BY created_at').all(); return rows.map(rowToEvent); @@ -36,13 +52,14 @@ export function createEvent(input: Partial): TrackedEvent { const id = `evt-${randomUUID()}`; const now = new Date().toISOString(); db.prepare( - `INSERT INTO tracked_events (id, name, description, source_ids, cadence, cadence_time, active, retention_override_days, last_recap_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)` + `INSERT INTO tracked_events (id, name, description, source_ids, keywords, cadence, cadence_time, active, retention_override_days, last_recap_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)` ).run( id, input.name ?? 'Untitled event', input.description ?? '', JSON.stringify(input.sourceIds ?? []), + JSON.stringify(input.keywords ?? []), input.cadence ?? 'continuous', input.cadenceTime ?? null, input.active === false ? 0 : 1, @@ -57,11 +74,12 @@ export function updateEvent(id: string, patch: Partial): TrackedEv if (!existing) return null; const merged = { ...existing, ...patch }; db.prepare( - `UPDATE tracked_events SET name=?, description=?, source_ids=?, cadence=?, cadence_time=?, active=?, retention_override_days=?, last_recap_at=? WHERE id=?` + `UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, cadence=?, cadence_time=?, active=?, retention_override_days=?, last_recap_at=? WHERE id=?` ).run( merged.name, merged.description, JSON.stringify(merged.sourceIds), + JSON.stringify(merged.keywords), merged.cadence, merged.cadenceTime, merged.active ? 1 : 0, diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index c6c25d6..e05746f 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -120,6 +120,7 @@ export function migrate() { name TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', source_ids TEXT NOT NULL DEFAULT '[]', -- JSON + keywords TEXT NOT NULL DEFAULT '[]', -- JSON string array — empty means "match everything from source_ids" cadence TEXT NOT NULL DEFAULT 'continuous', cadence_time TEXT, active INTEGER NOT NULL DEFAULT 1, @@ -236,6 +237,9 @@ export function migrate() { if (!hasColumn('global_settings', 'telegram_media_mode')) { db.exec("ALTER TABLE global_settings ADD COLUMN telegram_media_mode TEXT NOT NULL DEFAULT 'self-host'"); } + if (!hasColumn('tracked_events', 'keywords')) { + db.exec("ALTER TABLE tracked_events ADD COLUMN keywords TEXT NOT NULL DEFAULT '[]'"); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 47483dc..152c3f2 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -174,6 +174,8 @@ export interface TrackedEvent { name: string; description: string; sourceIds: string[]; + /** Only items whose title/summary/body contain at least one of these (case-insensitive) qualify for this event — empty means "match everything from sourceIds", the original behavior. */ + keywords: string[]; cadence: 'continuous' | 'daily' | 'hourly' | 'custom'; cadenceTime: string | null; active: boolean; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index ee0289d..0ee3e7d 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -52,6 +52,8 @@ export interface AdminTrackedEvent { name: string; description: string; sourceIds: string[]; + /** Only items whose title/summary/body contain at least one of these (case-insensitive) qualify for this event — empty means "match everything from sourceIds". */ + keywords: string[]; cadence: 'continuous' | 'daily' | 'hourly' | 'custom'; cadenceTime: string | null; active: boolean; diff --git a/frontend/src/lib/components/admin/EventsTab.svelte b/frontend/src/lib/components/admin/EventsTab.svelte index ebcf81b..db7c18d 100644 --- a/frontend/src/lib/components/admin/EventsTab.svelte +++ b/frontend/src/lib/components/admin/EventsTab.svelte @@ -7,6 +7,20 @@ let showAdd = $state(false); let newEvent = $state({ name: '', cadence: 'daily' as AdminTrackedEvent['cadence'], cadenceTime: '18:00' }); + let editingId = $state(null); + function emptyEditForm() { + return { + name: '', + description: '', + sourceIdSet: new Set(), + keywordsText: '', + cadence: 'daily' as AdminTrackedEvent['cadence'], + cadenceTime: '18:00', + retentionOverrideDays: null as number | null + }; + } + let editForm = $state(emptyEditForm()); + function sourceNames(ids: string[]) { return ids.map((id) => sources.find((s) => s.id === id)?.name).filter(Boolean).join(', ') || 'No sources assigned'; } @@ -17,6 +31,7 @@ name: newEvent.name, description: '', sourceIds: [], + keywords: [], cadence: newEvent.cadence, cadenceTime: newEvent.cadence === 'daily' ? newEvent.cadenceTime : null, retentionOverrideDays: null @@ -35,6 +50,49 @@ await deleteEvent(id); events = events.filter((e) => e.id !== id); } + + function startEdit(event: AdminTrackedEvent) { + editingId = event.id; + editForm = { + name: event.name, + description: event.description, + sourceIdSet: new Set(event.sourceIds), + keywordsText: event.keywords.join(', '), + cadence: event.cadence, + cadenceTime: event.cadenceTime ?? '18:00', + retentionOverrideDays: event.retentionOverrideDays + }; + } + + function cancelEdit() { + editingId = null; + } + + function toggleEditSource(id: string) { + const next = new Set(editForm.sourceIdSet); + if (next.has(id)) next.delete(id); + else next.add(id); + editForm.sourceIdSet = next; + } + + async function saveEdit() { + if (!editingId || !editForm.name) return; + const keywords = editForm.keywordsText + .split(',') + .map((k) => k.trim()) + .filter(Boolean); + const updated = await updateEvent(editingId, { + name: editForm.name, + description: editForm.description, + sourceIds: [...editForm.sourceIdSet], + keywords, + cadence: editForm.cadence, + cadenceTime: editForm.cadence === 'daily' ? editForm.cadenceTime : null, + retentionOverrideDays: editForm.retentionOverrideDays + }); + events = events.map((e) => (e.id === editingId ? updated : e)); + editingId = null; + }
@@ -59,25 +117,74 @@
-

Assign sources to this event from the Sources tab once created.

+

Assign sources and a keyword filter via Edit once created.

{/if}
{#each events as event (event.id)} -
-
-
{event.name}
-
- {sourceNames(event.sourceIds)} · - {event.cadence === 'daily' ? `daily recap at ${event.cadenceTime}` : event.cadence} + {#if editingId === event.id} +
+
+ + + {#if editForm.cadence === 'daily'} + + {/if} +
+ + +
Sources
+
+ {#each sources as source (source.id)} + + {/each} +
+ +
Keyword filter
+ +

+ Comma-separated words, phrases, or emoji — only items from the sources above whose + title/summary/body contain at least one qualify for this event's recap. Leave blank to + include everything from the assigned sources. +

+ +
+ +
- toggleActive(event)} role="button" tabindex="0"> - {event.active ? 'Active' : 'Paused'} - - -
+ {:else} +
+
+
{event.name}
+
+ {sourceNames(event.sourceIds)} + {#if event.keywords.length > 0} + · matching {event.keywords.map((k) => `"${k}"`).join(', ')} + {/if} + · + {event.cadence === 'daily' ? `daily recap at ${event.cadenceTime}` : event.cadence} +
+
+ toggleActive(event)} role="button" tabindex="0"> + {event.active ? 'Active' : 'Paused'} + + + +
+ {/if} {/each}
@@ -166,7 +273,45 @@ border: none; color: var(--text-secondary); } + .icon-btn:hover { + color: var(--text-accent); + } .icon-btn.danger:hover { color: var(--text-danger); } + .edit-panel { + background: var(--surface-1); + border-radius: 12px; + padding: 14px; + } + .edit-panel textarea { + width: 100%; + resize: vertical; + font: inherit; + margin-bottom: 10px; + } + .field-label { + font-size: 11px; + color: var(--text-muted); + margin-bottom: 6px; + } + .source-checks { + display: flex; + flex-wrap: wrap; + gap: 10px; + margin-bottom: 12px; + } + .source-check { + display: flex; + align-items: center; + gap: 5px; + font-size: 12px; + color: var(--text-secondary); + } + .source-check input { + width: auto; + } + .edit-panel .hint { + margin: 6px 0 10px; + }