diff --git a/backend/src/api/public.ts b/backend/src/api/public.ts index e2a0898..1dc876b 100644 --- a/backend/src/api/public.ts +++ b/backend/src/api/public.ts @@ -45,11 +45,25 @@ export async function registerPublicRoutes(app: FastifyInstance) { return tagsDb.listActiveTags(); }); + app.get('/api/tag/:slug', async (req, reply) => { + const { slug } = req.params as { slug: string }; + const tag = tagsDb.getTagBySlug(slug); + if (!tag) return reply.code(404).send({ error: 'not found' }); + return tag; + }); + app.get('/api/events', async () => { - // Public fields only — sourceIds, keywords etc. stay admin-only. - return eventsDb - .listEvents() - .map((e) => ({ id: e.id, name: e.name, active: e.active, recapIntervalHours: e.recapIntervalHours, isSpillover: e.isSpillover })); + // Public fields only — sourceIds, keywords etc. stay admin-only. lastRecapAt is + // safe to expose (just a timestamp, no source/keyword detail) and lets the + // tracked-event page show when the next AI recap is due. + return eventsDb.listEvents().map((e) => ({ + id: e.id, + name: e.name, + active: e.active, + recapIntervalHours: e.recapIntervalHours, + lastRecapAt: e.lastRecapAt, + isSpillover: e.isSpillover + })); }); // Drives the site nav — admin-editable (add/remove/reorder) via /api/admin/categories, diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 0c18fcf..9ff908e 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import type { InferenceProvider } from '../inference/provider.js'; import type { Cluster } from './clustering.js'; -import { synthesizeArticle, synthesizeRecap } from './synthesis.js'; +import { synthesizeArticle, synthesizeRecap, extractTags } from './synthesis.js'; import { selectBestImage, faviconUrlFor } from './image-selection.js'; import { downloadAndStore, promoteToPublished, storeMediaBuffer } from '../storage/media/index.js'; import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js'; @@ -36,6 +36,25 @@ function anyPushesToTopStories(items: ContentItem[]): boolean { return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false); } +/** Embeds each label and resolves/dedupes it against existing tags — shared by every publish path that has tagLabels in hand (from a full synthesis call or the lightweight extractTags), so the dedup behavior stays identical regardless of how the labels were produced. */ +async function resolveTagIds( + provider: InferenceProvider, + tagLabels: string[], + settings: GlobalSettings, + logSource: string +): Promise { + const tagIds: string[] = []; + for (const label of tagLabels) { + try { + const embedding = await provider.embed(label, { model: settings.selectedModels.embedding }); + tagIds.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold).id); + } catch (err) { + logger.error(logSource, `Tag embedding failed for "${label}": ${(err as Error).message}`); + } + } + return tagIds; +} + /** * Resolves the hero image for a regular (non-tweet) article: try the best candidate * from the source items, download and locally host it; if there isn't one, fall back @@ -219,19 +238,19 @@ async function resolveQuotedTweet( } /** - * Publishes a single item as-is, with no AI calls at all — used when the AI service - * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives - * after the backend launches" requirement). No rewriting, no tag extraction, no - * embedding. This is deliberately a lesser version of the real pipeline: once Ollama - * is available, newly-ingested items get the full embed/cluster/synthesize treatment - * and can be linked as follow-ups to these passthrough articles via the normal - * tag-based thread detection — but these earlier articles aren't retroactively - * rewritten or merged with anything after the fact. + * Publishes a single item as-is — the title/body are never rewritten or merged (see + * priorityQueue.ts for why: single-source clusters, YouTube/Nitter/Telegram items, and + * AI-disabled categories all route here specifically to avoid that risk). When a + * provider is given (AI is actually reachable and this item isn't in an AI-disabled + * category), it still gets tags via a lightweight standalone extraction call — every + * published article should be taggable/discoverable via /tag/[slug], not just the + * AI-merged ones. Passing no provider (Ollama unreachable, or the item's category has + * AI turned off entirely) skips tagging too — same as the old "no tags yet" behavior. */ export async function publishDirect( item: ContentItem, settings: GlobalSettings, - opts: { eventId?: string } = {} + opts: { eventId?: string; provider?: InferenceProvider } = {} ): Promise { const category = uniqueCategories([item]); const storedMediaIds: string[] = []; @@ -305,6 +324,16 @@ export async function publishDirect( }; } + let tagIds: string[] = []; + if (opts.provider) { + try { + const tagLabels = await extractTags(opts.provider, settings.selectedModels.synthesis, item); + tagIds = await resolveTagIds(opts.provider, tagLabels, settings, 'synthesis'); + } catch (err) { + logger.error('synthesis', `Tag extraction failed for "${item.title}": ${(err as Error).message}`); + } + } + const article = await articles.insertArticle({ title: item.title, body: item.body || item.summary, @@ -329,7 +358,7 @@ export async function publishDirect( publishedAt: item.publishedAt, updatedAt: item.publishedAt, mergeConfidence: 1.0, - tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement + tags: tagIds, threadId: randomUUID(), previousArticleId: null, nextArticleId: null, @@ -358,16 +387,7 @@ export async function publishCluster( const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); const { title, body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); - const resolvedTags = []; - for (const label of tagLabels) { - try { - const embedding = await provider.embed(label, { model: settings.selectedModels.embedding }); - resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold)); - } catch (err) { - logger.error('synthesis', `Tag embedding failed for "${label}": ${(err as Error).message}`); - } - } - const tagIds = resolvedTags.map((t) => t.id); + const tagIds = await resolveTagIds(provider, tagLabels, settings, 'synthesis'); const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? ''); const videoItem = items.find((i) => i.videos.length > 0); @@ -456,16 +476,7 @@ export async function publishEventRecap( constituents: MergedArticle[] ): Promise { const { title, body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); - - const resolvedTags = []; - for (const label of tagLabels) { - try { - const embedding = await provider.embed(label, { model: settings.selectedModels.embedding }); - resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold)); - } catch (err) { - logger.error('events', `Tag embedding failed for "${label}": ${(err as Error).message}`); - } - } + const tagIds = await resolveTagIds(provider, tagLabels, settings, 'events'); const category = [...new Set(constituents.flatMap((a) => a.category))]; const heroImage = constituents.find((a) => a.heroImage)?.heroImage ?? null; @@ -486,7 +497,7 @@ export async function publishEventRecap( publishedAt: now, updatedAt: now, mergeConfidence: 1.0, - tags: resolvedTags.map((t) => t.id), + tags: tagIds, threadId: randomUUID(), previousArticleId: null, nextArticleId: null, diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 6006132..dfdd5be 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -32,6 +32,44 @@ function capEntryText(text: string, budgetChars: number): string { return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; } +const TAG_EXTRACTION_SYSTEM_PROMPT = `You are a tagging assistant. Given a news item's title and summary, respond with ONLY 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named people, places, organizations, or named events) that this item is about — nothing else, no commentary, no leading text. If nothing salient qualifies, respond with an empty line.`; + +/** Short response — a handful of tags, not prose — so this doesn't need DEFAULT_NUM_PREDICT's full budget. */ +const TAG_EXTRACTION_NUM_PREDICT = 40; + +function parseTagLabels(raw: string): string[] { + return raw + .split(',') + .map((t) => t.trim()) + .filter((t) => t.length > 0 && t.length < 60); +} + +/** + * Lightweight standalone tag extraction for a single item — unlike synthesizeArticle, + * this doesn't rewrite or attribute anything, so it's safe to run even for items that + * publish verbatim via publishDirect (single-source clusters, or format-based direct + * publishes like YouTube/Nitter/Telegram — see priorityQueue.ts). Every published + * article should end up with tags regardless of whether it went through a full AI + * merge, and this is the minimal AI call that makes that possible without triggering + * the rewrite/attribution risk a full synthesizeArticle call would add for no benefit. + */ +export async function extractTags( + provider: InferenceProvider, + model: string, + item: Pick +): Promise { + const summary = capEntryText(item.body || item.summary, MAX_INPUT_CHARS); + const prompt = `Title: ${item.title}\nSummary: ${summary}`; + const raw = await provider.generate(prompt, { + model, + system: TAG_EXTRACTION_SYSTEM_PROMPT, + numCtx: DEFAULT_NUM_CTX, + numPredict: TAG_EXTRACTION_NUM_PREDICT, + label: `Extracting tags: "${item.title.slice(0, 60)}"` + }); + return parseTagLabels(raw); +} + const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write your response in exactly three parts, in this order: 1. A short, specific headline for this recap (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period). @@ -109,10 +147,7 @@ function buildPrompt(items: ContentItem[], sourceNames: Map): st function parseResult(raw: string): SynthesisResult { const [beforeTags, tagSection] = raw.split(TAG_DELIMITER_RE); - const tagLabels = (tagSection ?? '') - .split(',') - .map((t) => t.trim()) - .filter((t) => t.length > 0 && t.length < 60); + const tagLabels = parseTagLabels(tagSection ?? ''); const titleSplit = (beforeTags ?? raw).split(TITLE_DELIMITER_RE); const titlePart = titleSplit[0]; diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index e832241..a1cc0ec 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -59,20 +59,24 @@ function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sou /** * Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same * publish-then-tag-then-log/error shape, differing only in how the success/failure - * message describes why the item skipped merging. + * message describes why the item skipped merging. `provider`, when given, still gets + * these articles tagged (via publishDirect's lightweight extraction) without rewriting + * anything — omit it entirely for items whose category has AI turned off, or when + * Ollama isn't reachable at all (see call sites). */ async function publishItemsDirect( items: ContentItem[], settings: GlobalSettings, activeEvents: TrackedEvent[], describeSuccess: (item: ContentItem) => string, - failureLabel: string + failureLabel: string, + provider?: InferenceProvider ): Promise { let published = 0; for (const item of items) { try { const eventId = claimedEventId(item, activeEvents) ?? undefined; - const article = await publishDirect(item, settings, { eventId }); + const article = await publishDirect(item, settings, { eventId, provider }); contentItemsDb.assignCluster([item.id], article.id); published++; logger.info('synthesis', `Published "${article.title}" directly (${describeSuccess(item)})`); @@ -119,9 +123,13 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { +export async function runDirectPublishCycle(settings: GlobalSettings, provider?: InferenceProvider): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); if (items.length === 0) { @@ -145,7 +153,8 @@ export async function runDirectPublishCycle(settings: GlobalSettings): Promise sourcesById.get(item.sourceId)?.type ?? 'unknown', - 'Direct publish failed' + 'Direct publish failed', + provider ); const publishedCategoryDirect = await publishItemsDirect( @@ -230,7 +239,7 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // actual synthesis to justify the risk. const article = cluster.items.length === 1 - ? await publishDirect(cluster.items[0], settings, { eventId }) + ? await publishDirect(cluster.items[0], settings, { eventId, provider }) : await publishCluster(provider, settings, cluster, { eventId }); contentItemsDb.assignCluster( cluster.items.map((i) => i.id), diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index 28074a2..b790bfe 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -64,7 +64,14 @@ export function startScheduler() { everyTickSkippingOverlap(DIRECT_PUBLISH_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); - const published = await runDirectPublishCycle(settings); + const p = provider(); + // This tick runs regardless of Ollama's reachability (nothing here rewrites or + // merges), but tagging direct-published items (see runDirectPublishCycle/ + // publishDirect) does need a working AI service — only offer the provider + // through when it's actually reachable, so an unconfigured Ollama doesn't spam + // the log with a failed tag-extraction attempt on every single item, every tick. + const reachable = await p.isReachable(); + const published = await runDirectPublishCycle(settings, reachable ? p : undefined); if (published > 0) { logger.info('scheduler', `Direct-publish tick: published ${published} article(s)`); } diff --git a/backend/src/storage/db/tags.ts b/backend/src/storage/db/tags.ts index 455f47a..728e89a 100644 --- a/backend/src/storage/db/tags.ts +++ b/backend/src/storage/db/tags.ts @@ -28,6 +28,12 @@ export function listActiveTags(): Tag[] { return rows.map(rowToTag); } +/** By slug rather than id — that's what tag chips link by (see publish.ts/frontend tag pages). Matches regardless of active/expired status: an old article's tag chip should still resolve to its (now possibly expired) tag rather than 404 just because nothing new has used it lately. */ +export function getTagBySlug(slug: string): Tag | null { + const row = db.prepare('SELECT * FROM tags WHERE slug = ?').get(slug); + return row ? rowToTag(row) : null; +} + function cosineSimilarity(a: number[], b: number[]): number { if (a.length === 0 || b.length === 0 || a.length !== b.length) return 0; let dot = 0, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index dba92e5..387dd23 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -34,6 +34,10 @@ export function getTags(fetchFn?: typeof fetch): Promise { return get('/api/tags', fetchFn); } +export function getTagBySlug(slug: string, fetchFn?: typeof fetch): Promise { + return get(`/api/tag/${slug}`, fetchFn); +} + export function getEvents(fetchFn?: typeof fetch): Promise { return get('/api/events', fetchFn); } diff --git a/frontend/src/lib/format.ts b/frontend/src/lib/format.ts index bccf078..e325d61 100644 --- a/frontend/src/lib/format.ts +++ b/frontend/src/lib/format.ts @@ -1,3 +1,15 @@ +/** Inverse of timeAgo — "in 3h", "in 45m" for a future timestamp; "any moment now" once it's passed. */ +export function timeUntil(iso: string): string { + const diffMs = new Date(iso).getTime() - Date.now(); + if (diffMs <= 0) return 'any moment now'; + const mins = Math.round(diffMs / 60000); + if (mins < 60) return `in ${mins}m`; + const hours = Math.round(mins / 60); + if (hours < 24) return `in ${hours}h`; + const days = Math.round(hours / 24); + return `in ${days}d`; +} + export function timeAgo(iso: string): string { const diffMs = Date.now() - new Date(iso).getTime(); const mins = Math.round(diffMs / 60000); diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 7a9d9b1..0f3c62a 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -81,6 +81,8 @@ export interface TrackedEventPublic { name: string; active: boolean; recapIntervalHours: number | null; + /** Timestamp of the last AI recap, or null if none has run yet — used to compute "next recap" on the /event/[id] page. */ + lastRecapAt: string | null; isSpillover: boolean; } diff --git a/frontend/src/routes/event/[id]/+page.svelte b/frontend/src/routes/event/[id]/+page.svelte index 57f14bb..380b635 100644 --- a/frontend/src/routes/event/[id]/+page.svelte +++ b/frontend/src/routes/event/[id]/+page.svelte @@ -1,13 +1,29 @@
{data.name} - Tracked item — periodically recapped by AI + + Tracked item — periodically recapped by AI + {#if nextRecapText} + · {nextRecapText} + {/if} +
diff --git a/frontend/src/routes/event/[id]/+page.ts b/frontend/src/routes/event/[id]/+page.ts index 29bac4b..8b4a4a9 100644 --- a/frontend/src/routes/event/[id]/+page.ts +++ b/frontend/src/routes/event/[id]/+page.ts @@ -14,5 +14,12 @@ export const load: PageLoad = async ({ params, fetch, parent }) => { const filters = { eventId: params.id }; const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); - return { initial, filters, name: match?.name ?? 'Tracked event', pageSize: PAGE_SIZE }; + return { + initial, + filters, + name: match?.name ?? 'Tracked event', + recapIntervalHours: match?.recapIntervalHours ?? null, + lastRecapAt: match?.lastRecapAt ?? null, + pageSize: PAGE_SIZE + }; }; diff --git a/frontend/src/routes/tag/[slug]/+page.svelte b/frontend/src/routes/tag/[slug]/+page.svelte new file mode 100644 index 0000000..47dc9a1 --- /dev/null +++ b/frontend/src/routes/tag/[slug]/+page.svelte @@ -0,0 +1,23 @@ + + +
+ #{data.tag.label} +
+ + + + diff --git a/frontend/src/routes/tag/[slug]/+page.ts b/frontend/src/routes/tag/[slug]/+page.ts new file mode 100644 index 0000000..04f9471 --- /dev/null +++ b/frontend/src/routes/tag/[slug]/+page.ts @@ -0,0 +1,22 @@ +import { error } from '@sveltejs/kit'; +import type { PageLoad } from './$types'; +import { getFeed, getTagBySlug } from '$lib/api'; + +const PAGE_SIZE = 15; + +// Mirrors /category/[name] and /event/[id] — a tag chip links by slug, so the slug is +// resolved to the real tag (id + label) via a dedicated backend lookup (GET +// /api/tag/:slug) rather than a preloaded list, since tags aren't loaded by the root +// layout the way categories/events are. +export const load: PageLoad = async ({ params, fetch }) => { + let tag; + try { + tag = await getTagBySlug(params.slug, fetch); + } catch { + throw error(404, 'Tag not found'); + } + + const filters = { tag: tag.id }; + const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); + return { initial, filters, tag, pageSize: PAGE_SIZE }; +};