Salastil/homefeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix missing storage-used display; add keyword filter for tracked events

Browse Source 
Retention panel's "currently using" line and usage bar were always blank
— totalStorageBytes() existed but was never wired into the settings
response, so retention.storageUsedMB was undefined on every load. Now
computed fresh on every GET/PATCH /api/admin/settings.

Tracked events gain a keywords field: only items whose title/summary/
body contain at least one of them (word, phrase, or emoji — e.g. 🇮🇷 for
an "Iran war" event) qualify for that event's recap, instead of every
item from its assigned sources. An item from an event-linked source that
doesn't match now falls through to normal synthesis rather than being
silently dropped. Also added the source-assignment + keyword-filter edit
UI to EventsTab.svelte, which had no way to populate sourceIds at all
before this (the "assign from the Sources tab" comment referenced a
feature that was never built).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8
This commit is contained in:
Claude
2026-07-23 23:07:50 +00:00
parent a2b9d3bb0a
commit 97d3ed366c
8 changed files with 220 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/api/admin.ts
+10 -2
View File
@@ -4,15 +4,23 @@ import * as sourcesDb from '../storage/db/sources.js';
import * as eventsDb from '../storage/db/events.js';
import * as categoriesDb from '../storage/db/categories.js';
import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
import { totalStorageBytes } from '../storage/media/index.js';
import { OllamaProvider } from '../inference/ollama-provider.js';
import { pollSourceNow } from '../ingestion/poller.js';
import { logger, listLogs } from '../storage/db/logs.js';
import * as telegramClient from '../telegram/client.js';
// Not part of GlobalSettings itself (nothing to persist) — computed fresh on every
// settings read/write so the Retention tab's "currently using" line and usage bar
// always reflect the real total, not whatever was true when the row was last saved.
function withStorageUsed(settings: ReturnType<typeof settingsDb.getSettings>) {
return { ...settings, retention: { ...settings.retention, storageUsedMB: Math.round(totalStorageBytes() / (1024 * 1024)) } };
}
export async function registerAdminRoutes(app: FastifyInstance) {
// --- Settings ---
app.get('/api/admin/settings', async () => {
const settings = settingsDb.getSettings();
const settings = withStorageUsed(settingsDb.getSettings());
return { ...settings, categoryPriority: categoriesDb.listCategories() };
});
@@ -22,7 +30,7 @@ export async function registerAdminRoutes(app: FastifyInstance) {
categoriesDb.setCategoryOrder(body.categoryPriority);
delete body.categoryPriority;
}
const settings = settingsDb.updateSettings(body);
const settings = withStorageUsed(settingsDb.updateSettings(body));
return { ...settings, categoryPriority: categoriesDb.listCategories() };
});
backend/src/queue/eventsRecap.ts
+3 -1
View File
@@ -37,7 +37,9 @@ export async function runEventRecaps(provider: InferenceProvider, settings: Glob
if (event.sourceIds.length === 0 || !isDue(event)) continue;
const since = event.lastRecapAt ?? new Date(Date.now() - 24 * 3600_000).toISOString();
const items = contentItemsDb.unclusteredItemsForSources(event.sourceIds, since);
const items = contentItemsDb
.unclusteredItemsForSources(event.sourceIds, since)
.filter((item) => eventsDb.itemMatchesEventKeywords(item, event.keywords));
if (items.length === 0) continue;
const embedded = await embedPendingItems(provider, settings.selectedModels.embedding, items);
backend/src/queue/priorityQueue.ts
+21 -8
View File
@@ -7,7 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js';
import { clusterItems } from '../pipeline/clustering.js';
import { publishCluster, publishDirect } from '../pipeline/publish.js';
import { logger } from '../storage/db/logs.js';
import type { GlobalSettings, ContentItem } from '../storage/db/types.js';
import type { GlobalSettings, ContentItem, TrackedEvent } from '../storage/db/types.js';
function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
const matches: T[] = [];
@@ -16,6 +16,18 @@ function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
return [matches, rest];
}
/**
* An item is "claimed" by a tracked event — and so left for eventsRecap.ts to handle
* instead of normal synthesis — only if it belongs to one of the event's sources AND
* matches its keyword filter. An item from an event-linked source that doesn't match
* (e.g. a general Middle-East feed assigned to an "Iran war" event, but this particular
* item doesn't mention Iran) falls through to normal synthesis rather than being
* silently dropped — it just isn't part of that event's recap.
*/
function isClaimedByEvent(item: ContentItem, events: TrackedEvent[]): boolean {
return events.some((e) => e.sourceIds.includes(item.sourceId) && eventsDb.itemMatchesEventKeywords(item, e.keywords));
}
function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>): number {
const source = sourcesDb.getSource(item.sourceId);
const cats = source?.category ?? [];
@@ -39,8 +51,8 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>)
* Still respects category priority.
*/
export async function runPassthroughCycle(settings: GlobalSettings): Promise<number> {
const eventSourceIds = eventsDb.listActiveEvents().flatMap((e) => e.sourceIds);
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
const activeEvents = eventsDb.listActiveEvents();
const items = contentItemsDb.unclusteredItemsExcludingSources([]).filter((item) => !isClaimedByEvent(item, activeEvents));
if (items.length === 0) return 0;
const categories = categoriesDb.listCategories();
@@ -68,13 +80,14 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
/**
* One pass of the synthesis queue: cluster whatever's unclustered (excluding items
* belonging to tracked-event sources, which are handled by eventsRecap.ts instead),
* ordered by admin-defined category priority, and publish clusters that have cleared
* the hold-before-publish window.
* claimed by a tracked event — belonging to one of its sources AND matching its
* keyword filter, if any — which are handled by eventsRecap.ts instead), ordered by
* admin-defined category priority, and publish clusters that have cleared the
* hold-before-publish window.
*/
export async function runSynthesisCycle(provider: InferenceProvider, settings: GlobalSettings): Promise<number> {
const eventSourceIds = eventsDb.listActiveEvents().flatMap((e) => e.sourceIds);
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
const activeEvents = eventsDb.listActiveEvents();
const items = contentItemsDb.unclusteredItemsExcludingSources([]).filter((item) => !isClaimedByEvent(item, activeEvents));
if (items.length === 0) return 0;
// YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with
backend/src/storage/db/events.ts
+21 -3
View File
@@ -8,6 +8,7 @@ function rowToEvent(row: any): TrackedEvent {
name: row.name,
description: row.description,
sourceIds: JSON.parse(row.source_ids),
keywords: JSON.parse(row.keywords),
cadence: row.cadence,
cadenceTime: row.cadence_time,
active: !!row.active,
@@ -17,6 +18,21 @@ function rowToEvent(row: any): TrackedEvent {
};
}
/**
* Whether an item qualifies for a tracked event's keyword filter — empty keywords
* means "match everything" (the original, filter-less behavior). Matching is a plain
* case-insensitive substring check against title+summary+body; works the same for a
* word/phrase ("Tehran") or an emoji ("🇮🇷", which has no case to fold).
*/
export function itemMatchesEventKeywords(
item: { title: string; summary: string; body: string | null },
keywords: string[]
): boolean {
if (keywords.length === 0) return true;
const haystack = `${item.title} ${item.summary} ${item.body ?? ''}`.toLowerCase();
return keywords.some((k) => haystack.includes(k.toLowerCase()));
}
export function listEvents(): TrackedEvent[] {
const rows = db.prepare('SELECT * FROM tracked_events ORDER BY created_at').all();
return rows.map(rowToEvent);
@@ -36,13 +52,14 @@ export function createEvent(input: Partial<TrackedEvent>): TrackedEvent {
const id = `evt-${randomUUID()}`;
const now = new Date().toISOString();
db.prepare(
`INSERT INTO tracked_events (id, name, description, source_ids, cadence, cadence_time, active, retention_override_days, last_recap_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`
`INSERT INTO tracked_events (id, name, description, source_ids, keywords, cadence, cadence_time, active, retention_override_days, last_recap_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`
).run(
id,
input.name ?? 'Untitled event',
input.description ?? '',
JSON.stringify(input.sourceIds ?? []),
JSON.stringify(input.keywords ?? []),
input.cadence ?? 'continuous',
input.cadenceTime ?? null,
input.active === false ? 0 : 1,
@@ -57,11 +74,12 @@ export function updateEvent(id: string, patch: Partial<TrackedEvent>): TrackedEv
if (!existing) return null;
const merged = { ...existing, ...patch };
db.prepare(
`UPDATE tracked_events SET name=?, description=?, source_ids=?, cadence=?, cadence_time=?, active=?, retention_override_days=?, last_recap_at=? WHERE id=?`
`UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, cadence=?, cadence_time=?, active=?, retention_override_days=?, last_recap_at=? WHERE id=?`
).run(
merged.name,
merged.description,
JSON.stringify(merged.sourceIds),
JSON.stringify(merged.keywords),
merged.cadence,
merged.cadenceTime,
merged.active ? 1 : 0,
backend/src/storage/db/index.ts
+4
View File
@@ -120,6 +120,7 @@ export function migrate() {
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
source_ids TEXT NOT NULL DEFAULT '[]', -- JSON
keywords TEXT NOT NULL DEFAULT '[]', -- JSON string array — empty means "match everything from source_ids"
cadence TEXT NOT NULL DEFAULT 'continuous',
cadence_time TEXT,
active INTEGER NOT NULL DEFAULT 1,
@@ -236,6 +237,9 @@ export function migrate() {
if (!hasColumn('global_settings', 'telegram_media_mode')) {
db.exec("ALTER TABLE global_settings ADD COLUMN telegram_media_mode TEXT NOT NULL DEFAULT 'self-host'");
}
if (!hasColumn('tracked_events', 'keywords')) {
db.exec("ALTER TABLE tracked_events ADD COLUMN keywords TEXT NOT NULL DEFAULT '[]'");
}
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
// general news sources belong here, not on "Top stories" itself, which isn't a real
backend/src/storage/db/types.ts
+2
View File
@@ -174,6 +174,8 @@ export interface TrackedEvent {
name: string;
description: string;
sourceIds: string[];
/** Only items whose title/summary/body contain at least one of these (case-insensitive) qualify for this event — empty means "match everything from sourceIds", the original behavior. */
keywords: string[];
cadence: 'continuous' | 'daily' | 'hourly' | 'custom';
cadenceTime: string | null;
active: boolean;