From c45d7d5acf03038dde148bbe1381d1b1cc7887c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 03:31:31 +0000 Subject: [PATCH 01/18] Fix silent Ollama prompt truncation in synthesis pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ollama was defaulting to a 4096-token context (vs. the model's 32768 training context) and silently truncating any oversized prompt by dropping content from the middle, with no error surfaced anywhere — observed losing ~53% of a merge-cluster prompt in production. Two prompt builders (buildPrompt/buildRecapPrompt) concatenated all source summaries/article bodies with no size cap, so a cluster with enough sources (or a recap spanning enough articles) could easily exceed the window. Fix: OllamaProvider.generate() now always sends explicit num_ctx/ num_predict options (sized for CPU-only inference — i5-6600K, no GPU, ~17 tok/s prompt processing) instead of leaving Ollama to pick a default. synthesis.ts now caps prompt size itself before it ever reaches Ollama, giving each source/article an equal character budget and trimming individual entries rather than dropping whole ones off the end — every source stays at least partially represented and attributable. Trims are logged via the existing admin log stream instead of failing silently. --- backend/src/inference/ollama-provider.ts | 31 ++++++++++++- backend/src/inference/provider.ts | 2 +- backend/src/pipeline/synthesis.ts | 58 ++++++++++++++++++++---- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index b735c5f..6817232 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,25 @@ import type { InferenceProvider } from './provider.js'; +/** + * Default context window / max-generation length requested from Ollama when a caller + * doesn't specify its own. Ollama otherwise falls back to whatever the model's + * Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct + * here, well under that model's 32768-token training context) and SILENTLY truncates + * any prompt that doesn't fit — dropping the middle of the prompt with no error + * surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the + * limit deliberate and stable instead of whatever Ollama happens to pick. + * + * 8192 is sized for CPU-only inference (the reference box is an i5-6600K running + * Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the + * constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but + * prompt-processing time scales with context, so this trades headroom against + * per-request latency rather than maxing out the model's full 32768-token capacity. + * Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit + * within this budget up front, rather than relying on Ollama to truncate for them. + */ +export const DEFAULT_NUM_CTX = 8192; +export const DEFAULT_NUM_PREDICT = 700; + /** * Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend * setting (GlobalSettings.aiServiceHost/Port), editable via the admin panel — @@ -15,7 +35,10 @@ export class OllamaProvider implements InferenceProvider { return `${this.host}:${this.port}`; } - async generate(prompt: string, opts: { model?: string; system?: string } = {}): Promise { + async generate( + prompt: string, + opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {} + ): Promise { const res = await fetch(`${this.base()}/api/generate`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -23,7 +46,11 @@ export class OllamaProvider implements InferenceProvider { model: opts.model, prompt, system: opts.system, - stream: false + stream: false, + options: { + num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, + num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT + } }) }); if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); diff --git a/backend/src/inference/provider.ts b/backend/src/inference/provider.ts index d994b1d..aaa61f8 100644 --- a/backend/src/inference/provider.ts +++ b/backend/src/inference/provider.ts @@ -1,5 +1,5 @@ export interface InferenceProvider { - generate(prompt: string, opts?: { model?: string; system?: string }): Promise; + generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise; embed(text: string, opts?: { model?: string }): Promise; listModels(): Promise; isReachable(): Promise; diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index d52a539..2cbac86 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -1,8 +1,28 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { ContentItem, MergedArticle } from '../storage/db/types.js'; +import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; +import { logger } from '../storage/db/logs.js'; const TAG_DELIMITER = '---TAGS---'; +// Ollama truncates prompts that don't fit its context window by keeping a small prefix +// and dropping everything else in the middle — silently, with no error, and with no +// regard for which sources end up cut (see ollama-provider.ts for the incident that +// prompted this). Rather than relying on that, prompts here are sized to fit +// DEFAULT_NUM_CTX up front: each source/article gets an equal character budget, cut only +// when the whole prompt would otherwise overflow, so every source stays at least +// partially represented (and attributable) instead of some being dropped outright. +// ~4 chars/token is a rough heuristic (no tokenizer available here) — good enough for a +// safety margin, not meant to be exact. +const CHARS_PER_TOKEN = 4; +const RESERVED_OVERHEAD_TOKENS = 300; // system prompt + per-entry headers/formatting +const MAX_INPUT_CHARS = (DEFAULT_NUM_CTX - DEFAULT_NUM_PREDICT - RESERVED_OVERHEAD_TOKENS) * CHARS_PER_TOKEN; +const MIN_ENTRY_CHARS = 300; // floor so a huge cluster/recap doesn't shrink every entry to nothing + +function capEntryText(text: string, budgetChars: number): string { + return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; +} + const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: - Summarizes what has happened across the period covered, in chronological order - Highlights the most significant developments rather than restating every article @@ -27,9 +47,17 @@ export interface SynthesisResult { } function buildPrompt(items: ContentItem[]): string { - return items - .map((item, i) => `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${item.summary}`) - .join('\n\n'); + const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); + let truncated = 0; + const entries = items.map((item, i) => { + const summary = capEntryText(item.summary, budgetPerItem); + if (summary !== item.summary) truncated++; + return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`; + }); + if (truncated > 0) { + logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return entries.join('\n\n'); } function parseResult(raw: string): SynthesisResult { @@ -48,15 +76,22 @@ export async function synthesizeArticle( items: ContentItem[] ): Promise { const prompt = buildPrompt(items); - const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string { - const entries = articles - .map((article, i) => `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${article.body}`) - .join('\n\n'); - return `Tracked event: ${eventName}\n\n${entries}`; + const budgetPerArticle = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / articles.length)); + let truncated = 0; + const entries = articles.map((article, i) => { + const body = capEntryText(article.body, budgetPerArticle); + if (body !== article.body) truncated++; + return `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${body}`; + }); + if (truncated > 0) { + logger.warn('events', `Trimmed ${truncated}/${articles.length} recap article bod${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return `Tracked event: ${eventName}\n\n${entries.join('\n\n')}`; } /** @@ -74,6 +109,11 @@ export async function synthesizeRecap( articles: MergedArticle[] ): Promise { const prompt = buildRecapPrompt(eventName, articles); - const raw = await provider.generate(prompt, { model, system: RECAP_SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { + model, + system: RECAP_SYSTEM_PROMPT, + numCtx: DEFAULT_NUM_CTX, + numPredict: DEFAULT_NUM_PREDICT + }); return parseResult(raw); } From 63d510df572d69fa9bcd54a082d74bdcc6772c16 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 03:52:42 +0000 Subject: [PATCH 02/18] Use full article body, not just the RSS blurb, in synthesis prompts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildPrompt() only ever sent ContentItem.summary (a ~500-char RSS description) to the model, never .body (the full article text when the feed provides ) — even though publishDirect already preferred body over summary for the no-AI-merge path. A single-source cluster was effectively asking the model to "lightly rewrite" a one-paragraph blurb, which it did almost verbatim, producing a short repeated synopsis instead of an actual article. Now mirrors publishDirect's item.body || item.summary fallback. Body is already HTML-stripped at ingestion (ingestion/adapters/base.ts), so no new sanitization needed. The per-entry character budget added in the previous truncation fix now does real work here, since full bodies can be much longer than summaries. --- backend/src/pipeline/synthesis.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 2cbac86..1405563 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -50,12 +50,17 @@ function buildPrompt(items: ContentItem[]): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; const entries = items.map((item, i) => { - const summary = capEntryText(item.summary, budgetPerItem); - if (summary !== item.summary) truncated++; - return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`; + // Same fallback publishDirect uses (publish.ts) — body is the full article text + // when the feed supplies it (e.g. RSS ), summary is a ~500-char + // blurb. Using summary alone starved the model of real content to synthesize + // from, so a single-source cluster just echoed the blurb back nearly verbatim. + const full = item.body || item.summary; + const text = capEntryText(full, budgetPerItem); + if (text !== full) truncated++; + return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`; }); if (truncated > 0) { - logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`); } return entries.join('\n\n'); } From 53b11124d503084344f11f2f936c76e183d29bae Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 04:10:01 +0000 Subject: [PATCH 03/18] Add per-category "No AI" toggle to skip clustering/synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category priority admin pane gains a "No AI" checkbox alongside Private/More. When set, items whose source falls under that category skip embedding, clustering, and LLM synthesis entirely — each publishes on its own, verbatim from its source (title + body/summary), the same direct-publish path YouTube/Nitter/Telegram items always use. Backend: new categories.disable_ai column (default off, migrated in for existing installs), threaded through categories.ts CRUD and the POST /api/admin/categories + PATCH /api/admin/settings routes. priorityQueue.ts's runSynthesisCycle now partitions items three ways before clustering: source-type direct (youtube/nitter/telegram), category-disabled direct (new), then whatever's left goes through the normal embed/cluster/synthesize pipeline. Tracked-event recaps are a separate, already-existing per-event toggle (TrackedEvent.recapIntervalHours) since events aren't tied to a single category — unaffected by this change. --- backend/src/api/admin.ts | 9 ++++- backend/src/queue/priorityQueue.ts | 37 ++++++++++++++++--- backend/src/storage/db/categories.ts | 19 ++++++---- backend/src/storage/db/index.ts | 6 ++- backend/src/storage/db/types.ts | 2 + frontend/src/lib/adminApi.ts | 10 ++++- frontend/src/lib/adminTypes.ts | 1 + .../src/lib/components/admin/MergeTab.svelte | 21 ++++++++++- 8 files changed, 84 insertions(+), 21 deletions(-) diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index cf2b1f3..947b6f0 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -65,9 +65,14 @@ export async function registerAdminRoutes(app: FastifyInstance) { // --- Categories (add/remove — reordering/privacy is via PATCH /settings above) --- app.post('/api/admin/categories', async (req, reply) => { - const { name, isPrivate, isSpillover } = req.body as { name?: string; isPrivate?: boolean; isSpillover?: boolean }; + const { name, isPrivate, isSpillover, disableAi } = req.body as { + name?: string; + isPrivate?: boolean; + isSpillover?: boolean; + disableAi?: boolean; + }; if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' }); - const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover); + const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover, !!disableAi); return reply.code(201).send(created); }); diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 1eded36..3894bc8 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -45,6 +45,16 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map, return best; } +/** True if any of the item's source's categories (same leading-segment match as primaryCategoryRank) has AI disabled. */ +function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sourcesById: Map): boolean { + const source = sourcesById.get(item.sourceId); + for (const cat of source?.category ?? []) { + const leading = cat.split(':')[0].trim().toLowerCase(); + if (disabledNames.has(leading)) return true; + } + return false; +} + /** * Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same * publish-then-tag-then-log/error shape, differing only in how the success/failure @@ -114,6 +124,8 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // direct-publish partition and each item's category/type lookups — avoids a // separate sourcesDb.getSource() round-trip per item. const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with // anything else — each is always its own article, same shape whether the AI service @@ -121,18 +133,31 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const directPublishSourceIds = new Set( [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); - const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); + const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - const publishedDirect = await publishItemsDirect( - directItems, + // A category with disableAi set (see the Category priority admin pane) opts its + // items out of clustering/synthesis entirely — each publishes on its own, using its + // own source's text, same as the source-type-driven direct items above. + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + const [categoryDirectItems, mergeableItems] = partition(remaining, (item) => + inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) + ); + + const publishedTypeDirect = await publishItemsDirect( + typeDirectItems, settings, activeEvents, (item) => sourcesById.get(item.sourceId)?.type ?? 'unknown', 'Direct publish failed' ); - const categories = categoriesDb.listCategories(); - const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); + const publishedCategoryDirect = await publishItemsDirect( + categoryDirectItems, + settings, + activeEvents, + () => 'AI disabled for category', + 'Direct publish failed' + ); const ranked = mergeableItems .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) @@ -184,5 +209,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } - return published + publishedDirect; + return published + publishedTypeDirect + publishedCategoryDirect; } diff --git a/backend/src/storage/db/categories.ts b/backend/src/storage/db/categories.ts index 2397610..42fb7d3 100644 --- a/backend/src/storage/db/categories.ts +++ b/backend/src/storage/db/categories.ts @@ -9,7 +9,8 @@ function rowToCategory(row: any): Category { priorityRank: row.priority_rank, isDefault: !!row.is_default, isPrivate: !!row.is_private, - isSpillover: !!row.is_spillover + isSpillover: !!row.is_spillover, + disableAi: !!row.disable_ai }; } @@ -24,18 +25,20 @@ export function listPrivateCategoryNames(): string[] { return rows.map((r) => r.name); } -export function setCategoryOrder(order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean }[]) { - const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ? WHERE id = ?'); - for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.id); +export function setCategoryOrder( + order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean; disableAi: boolean }[] +) { + const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ?, disable_ai = ? WHERE id = ?'); + for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.disableAi ? 1 : 0, c.id); } -export function createCategory(name: string, isPrivate = false, isSpillover = false): Category { +export function createCategory(name: string, isPrivate = false, isSpillover = false, disableAi = false): Category { const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`; const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number }; db.prepare( - 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover) VALUES (?, ?, ?, 0, ?, ?)' - ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0); - return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover }; + 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover, disable_ai) VALUES (?, ?, ?, 0, ?, ?, ?)' + ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0, disableAi ? 1 : 0); + return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover, disableAi }; } export function deleteCategory(id: string) { diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 2de9caa..0058dab 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -153,7 +153,8 @@ export function migrate() { priority_rank INTEGER NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, is_private INTEGER NOT NULL DEFAULT 0, - is_spillover INTEGER NOT NULL DEFAULT 0 -- collapsed into the nav's "More »" overflow page instead of its own tab + is_spillover INTEGER NOT NULL DEFAULT 0, -- collapsed into the nav's "More »" overflow page instead of its own tab + disable_ai INTEGER NOT NULL DEFAULT 0 -- skip clustering/synthesis for this category's items; publish each one directly ); CREATE TABLE IF NOT EXISTS logs ( @@ -321,6 +322,9 @@ export function migrate() { if (!hasColumn('categories', 'is_spillover')) { db.exec('ALTER TABLE categories ADD COLUMN is_spillover INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('categories', 'disable_ai')) { + db.exec('ALTER TABLE categories ADD COLUMN disable_ai INTEGER NOT NULL DEFAULT 0'); + } if (!hasColumn('content_items', 'telegram_message')) { db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT'); } diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 82836f4..66c1af9 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -206,6 +206,8 @@ export interface Category { isPrivate: boolean; /** Grouped into the nav's "More »" overflow page instead of getting its own top-level tab — see +layout.svelte and /more. */ isSpillover: boolean; + /** Skips clustering/AI synthesis for this category's items — each one publishes directly (own article, own source's text), same as YouTube/Nitter/Telegram items always do. See priorityQueue.ts's runSynthesisCycle. */ + disableAi: boolean; } export interface WeatherHourEntry { diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 304d197..b2f0479 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -66,10 +66,16 @@ export const updateSettings = (patch: Partial, fetchFn?: typeof f request('/api/admin/settings', { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); // Categories -export const createCategory = (name: string, isPrivate = false, isSpillover = false, fetchFn?: typeof fetch) => +export const createCategory = ( + name: string, + isPrivate = false, + isSpillover = false, + disableAi = false, + fetchFn?: typeof fetch +) => request( '/api/admin/categories', - { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover }) }, + { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover, disableAi }) }, fetchFn ); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index a703596..891a856 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -14,6 +14,7 @@ export interface CategoryPriority { isDefault: boolean; isPrivate: boolean; isSpillover: boolean; + disableAi: boolean; } export interface WeatherHourEntry { diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index ab4d0fa..731f143 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -13,6 +13,7 @@ let newCategoryName = $state(''); let newCategoryPrivate = $state(false); let newCategorySpillover = $state(false); + let newCategoryDisableAi = $state(false); let addingCategory = $state(false); // Advisory only — the nav starts getting too wide / wrapping past ~10 tabs, so this @@ -48,11 +49,12 @@ if (!name) return; addingCategory = true; try { - const created = await createCategory(name, newCategoryPrivate, newCategorySpillover); + const created = await createCategory(name, newCategoryPrivate, newCategorySpillover, newCategoryDisableAi); local.categoryPriority = [...local.categoryPriority, created]; newCategoryName = ''; newCategoryPrivate = false; newCategorySpillover = false; + newCategoryDisableAi = false; } finally { addingCategory = false; } @@ -68,6 +70,11 @@ scheduleSave(); } + function toggleDisableAi(id: string) { + local.categoryPriority = local.categoryPriority.map((c) => (c.id === id ? { ...c, disableAi: !c.disableAi } : c)); + scheduleSave(); + } + async function removeCategory(id: string, isDefault: boolean, name: string) { if (isDefault) { // Sensible-default categories can still be removed — e.g. a fresh install's @@ -98,7 +105,9 @@ private category (and everything in it) is hidden from the public site until a visitor logs in with the lock icon in the masthead. A "More" category is collapsed into a single "More »" nav tab instead of getting its own, and shows up on that overflow page with its - latest few articles. + latest few articles. "No AI" skips clustering and synthesis for that category — each item + publishes on its own, using its own source's text, instead of being merged/rewritten by the + model.

{#if primaryCategoryCount > 10}

@@ -120,6 +129,10 @@ toggleSpillover(cat.id)} /> More + {/if} From bfd7967bbbe7a63503399bb3a1d90fdb6188f366 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:10:50 +0000 Subject: [PATCH 04/18] Fix synthesis fetch failures from Node's default 5-minute HTTP timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing published for hours, every cluster failing with "fetch failed". Ollama's own log showed the real story: requests being cancelled at exactly 5m0s with a 500, not a model or server error. Node's global fetch (undici) defaults to a 5-minute headers/body timeout, and CPU-only prompt processing on the reference hardware (i5-6600K, no GPU, ~17 tok/s) legitimately takes longer than that once prompts carry full article bodies instead of short blurbs (the previous fix in this same line of work) — every generate() call past a few thousand tokens got killed client-side before Ollama could finish. OllamaProvider.generate() now passes a dedicated undici Agent with headersTimeout/bodyTimeout disabled as the fetch dispatcher, so the request runs as long as it actually needs to. Verified the failure mode and the fix directly: a short-timeout dispatcher against a deliberately slow server reproduces the exact same "fetch failed" / UND_ERR_HEADERS_TIMEOUT error seen in production, and a zero-timeout dispatcher completes the same slow request without issue. undici was already a transitive dependency (via jsdom); added directly since ollama-provider.ts now imports from it. --- backend/package-lock.json | 3 ++- backend/package.json | 3 ++- backend/src/inference/ollama-provider.ts | 21 +++++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 3bff8b6..120652a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,7 +15,8 @@ "fastify": "^5.10.0", "jsdom": "^29.1.1", "rss-parser": "^3.13.0", - "telegram": "^2.26.22" + "telegram": "^2.26.22", + "undici": "^7.28.0" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/package.json b/backend/package.json index 9eca3ab..fb0285e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -18,7 +18,8 @@ "fastify": "^5.10.0", "jsdom": "^29.1.1", "rss-parser": "^3.13.0", - "telegram": "^2.26.22" + "telegram": "^2.26.22", + "undici": "^7.28.0" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 6817232..7173922 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,19 @@ +import { Agent } from 'undici'; import type { InferenceProvider } from './provider.js'; +/** + * Node's global fetch (undici) defaults to a 5-minute headers/body timeout — fine for + * ordinary HTTP calls, but a real problem for /api/generate on CPU-only inference: a + * near-full context window can legitimately take longer than that just for prompt + * processing on the reference hardware (i5-6600K, no GPU, ~17 tokens/sec). Once + * synthesis prompts started carrying full article bodies instead of short blurbs, every + * generate() call past a few thousand tokens got killed at exactly 5m0s — visible in + * Ollama's own log as the request being cancelled, not a genuine model/server error — + * so no cluster could ever finish synthesizing. No timeout at all here; Ollama's own + * process is the natural backstop, not a clock tuned for hardware this doesn't run on. + */ +const noTimeoutDispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0 }); + /** * Default context window / max-generation length requested from Ollama when a caller * doesn't specify its own. Ollama otherwise falls back to whatever the model's @@ -51,8 +65,11 @@ export class OllamaProvider implements InferenceProvider { num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT } - }) - }); + }), + // Not in the ambient RequestInit type this project resolves to, but Node's global + // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. + dispatcher: noTimeoutDispatcher + } as RequestInit); if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); const data = (await res.json()) as { response: string }; return data.response; From 683ca6c880eac75d7b360267b08ab39cad7ce2fc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:22:23 +0000 Subject: [PATCH 05/18] Fix scheduler racing itself into publishing duplicate articles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthesis tick fires every 60 seconds via setInterval with no reentrancy guard. An item only gets marked "clustered" after its article finishes synthesizing and publishing — so once generate() calls started legitimately taking longer than 60 seconds (bigger prompts + no client timeout, both from earlier fixes in this line of work), the next tick would fire mid-generation, see the same item still "unclustered", and synthesize + publish it again as a fresh, differently-worded article. Repeated overlaps produced a run of near-identical articles from the same single source item, seconds apart. everyTickSkippingOverlap() now guards all three scheduler intervals (poll, synthesis, retention): a tick is skipped outright if the previous invocation hasn't finished, rather than overlapping it. Verified in isolation — a task slower than its own tick interval never overlaps itself (measured max concurrency of 1). --- backend/src/queue/scheduler.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index 76dd754..f75670d 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -16,22 +16,44 @@ const WEATHER_TICK_MS = 45 * 60_000; const STOCKS_TICK_MS = 15 * 60_000; // per admin spec — stock prices move faster than weather const POE2_TICK_MS = 60 * 60_000; // poe.ninja's own overview data doesn't refresh faster than hourly, so polling more often than this just re-fetches the same numbers +/** + * Runs fn on every tick, but skips a tick outright if the previous one is still in + * flight instead of overlapping it. Matters most for the synthesis tick: an item stays + * "unclustered" (cluster_id IS NULL — see contentItems.unclusteredItemsExcludingSources) + * until AFTER its cluster finishes synthesizing and publishing, so a generate() call + * that runs past the next tick (easily minutes, on CPU-only inference — see + * ollama-provider.ts) used to let the same item get picked up and republished as a + * fresh, differently-worded article by an overlapping cycle, repeatedly, until the + * first cycle's assignCluster() finally landed. Node is single-threaded, so the only + * source of "concurrent" runs here is exactly this interval overlap. + */ +function everyTickSkippingOverlap(ms: number, fn: () => Promise) { + let running = false; + setInterval(() => { + if (running) return; + running = true; + fn().finally(() => { + running = false; + }); + }, ms); +} + export function startScheduler() { const provider = () => { const s = settingsDb.getSettings(); return new OllamaProvider(s.aiServiceHost, s.aiServicePort); }; - setInterval(async () => { + everyTickSkippingOverlap(POLL_TICK_MS, async () => { try { const ingested = await pollDueSources(); if (ingested > 0) logger.info('scheduler', `Poll tick: ingested ${ingested} new item(s)`); } catch (err) { logger.error('scheduler', `Poll tick failed: ${(err as Error).message}`); } - }, POLL_TICK_MS); + }); - setInterval(async () => { + everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); const p = provider(); @@ -55,16 +77,16 @@ export function startScheduler() { } catch (err) { logger.error('scheduler', `Synthesis tick failed: ${(err as Error).message}`); } - }, SYNTHESIS_TICK_MS); + }); - setInterval(() => { + everyTickSkippingOverlap(RETENTION_TICK_MS, async () => { try { runRetentionSweep(settingsDb.getSettings()); logger.info('retention', 'Retention sweep completed'); } catch (err) { logger.error('retention', `Retention tick failed: ${(err as Error).message}`); } - }, RETENTION_TICK_MS); + }); // Immediate first call for all three — unlike RSS sources (whose "due" check makes a // brand-new source eligible on the very next 1-minute tick), weather/stocks/poe2 have From a46ba5f4b026c2be6bf65f91a565c86d19fece97 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:26:43 +0000 Subject: [PATCH 06/18] Add 15-minute and 1-hour options to Hold before publish setting --- frontend/src/lib/components/admin/MergeTab.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index 731f143..7439a51 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -204,7 +204,9 @@

Wait window to gather more sources before finalizing a story.

From 80fc31c8446ca228ca982c347914c81a8127e984 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:37:37 +0000 Subject: [PATCH 07/18] Fix synthesis prompt labeling sources by opaque ID, causing hallucinated attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildPrompt() labeled each source with item.sourceId — an internal DB foreign key like "src-e8dbf745-..." — never the outlet's actual name. The model had no real outlet to attribute to, so on a single-source item it fell back to copying the illustrative example names straight out of its own system prompt ("Reuters reported...", "AP notes...") and fabricated a two-outlet merge out of one real 6abc article. The article's sources metadata (built separately from real DB records) was correct the whole time; only the AI-written body text invented sources that were never in the input. synthesizeArticle now takes a sourceId->name map (built in publish.ts via the same sources.getSource() lookup already used for the sources metadata) and buildPrompt labels each entry with the real name. SYSTEM_PROMPT no longer gives concrete example outlet names to copy — it references "each source's exact name as given below" and explicitly forbids attributing to any outlet not actually provided. Verified directly: captured the exact prompt text sent to a mock provider and confirmed it now contains the real source name and never the raw internal id. --- backend/src/pipeline/publish.ts | 3 ++- backend/src/pipeline/synthesis.ts | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index ab19350..d09a7b4 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -360,7 +360,8 @@ export async function publishCluster( ): Promise { const items = cluster.items; - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items); + const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); + const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames); const resolvedTags = []; for (const label of tagLabels) { diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 1405563..fd1898f 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -32,12 +32,12 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: -- Attributes specific claims to the outlet that reported them (e.g. "Reuters reported...", "AP notes...") +- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. - Does not copy phrasing verbatim from any source - Stays neutral and factual, without editorializing - Is 2-4 short paragraphs -If only one source is provided, lightly rewrite it in your own words rather than merging. +If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source. After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; @@ -46,7 +46,7 @@ export interface SynthesisResult { tagLabels: string[]; } -function buildPrompt(items: ContentItem[]): string { +function buildPrompt(items: ContentItem[], sourceNames: Map): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; const entries = items.map((item, i) => { @@ -57,7 +57,13 @@ function buildPrompt(items: ContentItem[]): string { const full = item.body || item.summary; const text = capEntryText(full, budgetPerItem); if (text !== full) truncated++; - return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`; + // The label here (not item.sourceId, an opaque internal id the model can't use) + // is the only real outlet name the model ever sees — without it, a small model + // has nothing to attribute to and falls back to copying the illustrative outlet + // names out of its own system prompt instructions instead (seen in production: + // a single-source item fabricating "Reuters reported..."/"AP notes..." wholesale). + const name = sourceNames.get(item.sourceId) ?? 'Unknown source'; + return `Source ${i + 1} (${name}):\nTitle: ${item.title}\nSummary: ${text}`; }); if (truncated > 0) { logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`); @@ -78,9 +84,10 @@ function parseResult(raw: string): SynthesisResult { export async function synthesizeArticle( provider: InferenceProvider, model: string, - items: ContentItem[] + items: ContentItem[], + sourceNames: Map ): Promise { - const prompt = buildPrompt(items); + const prompt = buildPrompt(items, sourceNames); const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } From dae6a51db03f0888ce50ca99f6a5b65641eda7bc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:41:11 +0000 Subject: [PATCH 08/18] Skip the AI rewrite entirely for single-source clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cluster of one item still went through synthesizeArticle to be "lightly rewritten" — the only recent real-world example fabricated a fake two-outlet merge out of one genuine article (see the opaque-sourceId attribution fix). There's no actual synthesis to do with one source, so the rewrite step only added risk (hallucinated attribution, subtly altered facts) for no benefit. priorityQueue.ts's runSynthesisCycle now routes a 1-item cluster to publishDirect instead of publishCluster — same verbatim-text path already used for youtube/nitter/telegram items and AI-disabled categories. publishCluster is now only ever called with 2+ items, so its doc comment and synthesis.ts's system prompt no longer reference the single-source case. Verified directly: a 1-item cluster now publishes with the original body untouched and zero calls to the model, while a 2-item cluster still goes through the AI merge path unchanged. --- backend/src/pipeline/publish.ts | 5 +++-- backend/src/pipeline/synthesis.ts | 2 -- backend/src/queue/priorityQueue.ts | 9 ++++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index d09a7b4..27ac515 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -349,8 +349,9 @@ export async function publishDirect( /** * Publishing is always automatic — there's no draft/review state (see schema doc). - * A cluster of size 1 publishes as-is via the same path; synthesizeArticle lightly - * rewrites rather than merges when there's only one source. + * Callers should route a size-1 cluster to publishDirect instead — there's nothing to + * merge, so an LLM rewrite would only add risk (hallucinated attribution, altered + * facts) for no synthesis benefit. See priorityQueue.ts's runSynthesisCycle. */ export async function publishCluster( provider: InferenceProvider, diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index fd1898f..21bacbf 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -37,8 +37,6 @@ const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summari - Stays neutral and factual, without editorializing - Is 2-4 short paragraphs -If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source. - After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; export interface SynthesisResult { diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 3894bc8..15323e3 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -186,7 +186,14 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // in practice a cluster's items are all near-duplicate coverage of the same // story, so they'd all match the same event's filter anyway when they match at all. const eventId = cluster.items.map((i) => claimedEventId(i, activeEvents)).find((id) => id !== null) ?? undefined; - const article = await publishCluster(provider, settings, cluster, { eventId }); + // A single-item cluster has nothing to merge — publish the source's own text + // verbatim instead of asking the LLM to "lightly rewrite" it, which only risked + // introducing errors (or fabricated attribution — see synthesis.ts) with no + // actual synthesis to justify the risk. + const article = + cluster.items.length === 1 + ? await publishDirect(cluster.items[0], settings, { eventId }) + : await publishCluster(provider, settings, cluster, { eventId }); contentItemsDb.assignCluster( cluster.items.map((i) => i.id), cluster.id From 1176bb44256ff0c67149d6aacff2469b049c7c7f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:55:26 +0000 Subject: [PATCH 09/18] Add admin-configurable writing style for AI synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synthesis system prompts were previously the only "instructions" the AI ever got, hardcoded and invisible from the admin panel — no way to control tone, and no way to know what was actually being sent without reading the source. Adds a "Writing style" panel to the Merge tab: a preset dropdown (Default/Casual/Formal) plus a free-text field for arbitrary additional instructions (e.g. "keep paragraphs under 3 sentences"). Both are appended as an addendum to the existing base system prompts in synthesis.ts — the structural rules (attribution, paragraph count, tag format) are never overridden, only style on top of them. Applies to AI-merged articles and event recaps; single-source items still publish verbatim with no AI involved either way. Backend: new global_settings.synthesis_style_preset (default) and .synthesis_custom_instructions ('') columns, migrated in for existing installs, threaded through settings.ts and into synthesizeArticle/ synthesizeRecap's system prompt construction. Verified: settings round-trip through GET/PATCH /api/admin/settings with correct defaults; a captured prompt confirms 'default' with no custom text produces the exact original prompt unchanged, while 'casual' + custom text appends both correctly; migration against an old-schema global_settings table adds both columns with correct defaults. --- backend/src/pipeline/publish.ts | 4 +-- backend/src/pipeline/synthesis.ts | 35 +++++++++++++++---- backend/src/storage/db/index.ts | 8 +++++ backend/src/storage/db/settings.ts | 5 +++ backend/src/storage/db/types.ts | 4 +++ frontend/src/lib/adminTypes.ts | 2 ++ .../src/lib/components/admin/MergeTab.svelte | 33 ++++++++++++++++- 7 files changed, 81 insertions(+), 10 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 27ac515..b5348ef 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -362,7 +362,7 @@ export async function publishCluster( const items = cluster.items; const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames); + const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -461,7 +461,7 @@ export async function publishEventRecap( event: TrackedEvent, constituents: MergedArticle[] ): Promise { - const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents); + const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); const resolvedTags = []; for (const label of tagLabels) { diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 21bacbf..a3be66c 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -1,5 +1,5 @@ import type { InferenceProvider } from '../inference/provider.js'; -import type { ContentItem, MergedArticle } from '../storage/db/types.js'; +import type { ContentItem, GlobalSettings, MergedArticle } from '../storage/db/types.js'; import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; import { logger } from '../storage/db/logs.js'; @@ -23,7 +23,7 @@ function capEntryText(text: string, budgetChars: number): string { return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; } -const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: +const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: - Summarizes what has happened across the period covered, in chronological order - Highlights the most significant developments rather than restating every article - Stays neutral and factual, without editorializing @@ -31,7 +31,7 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; -const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: +const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: - Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. - Does not copy phrasing verbatim from any source - Stays neutral and factual, without editorializing @@ -39,6 +39,24 @@ const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summari After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; +// Admin-selectable presets (Merge tab, "Writing style") — appended to whichever base +// prompt applies. 'default' adds nothing: the base prompts above already describe the +// original neutral wire-service tone this pipeline shipped with. +const STYLE_PRESETS: Record = { + default: '', + casual: 'Write in a casual, conversational tone, like a knowledgeable friend catching you up on what happened — contractions and plain language are fine. Still stay factual and keep outlet attribution accurate.', + formal: 'Write in a formal, measured register — precise language, no contractions, no colloquialisms.' +}; + +/** Admin-configurable tone: a preset plus optional free-text instructions, both from GlobalSettings — the only two knobs that affect HOW the model writes, as opposed to WHAT gets clustered/published. Appended to the base prompt, never replacing its structural rules (attribution, paragraph count, tag format). */ +function styleAddendum(settings: GlobalSettings): string { + const preset = STYLE_PRESETS[settings.synthesisStylePreset] ?? ''; + const custom = settings.synthesisCustomInstructions.trim(); + const lines = [preset, custom].filter(Boolean); + if (lines.length === 0) return ''; + return `\n\nAdditional style instructions from the site admin (follow these without breaking the rules above):\n${lines.join('\n')}`; +} + export interface SynthesisResult { body: string; tagLabels: string[]; @@ -83,10 +101,12 @@ export async function synthesizeArticle( provider: InferenceProvider, model: string, items: ContentItem[], - sourceNames: Map + sourceNames: Map, + settings: GlobalSettings ): Promise { const prompt = buildPrompt(items, sourceNames); - const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); + const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); + const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } @@ -116,12 +136,13 @@ export async function synthesizeRecap( provider: InferenceProvider, model: string, eventName: string, - articles: MergedArticle[] + articles: MergedArticle[], + settings: GlobalSettings ): Promise { const prompt = buildRecapPrompt(eventName, articles); const raw = await provider.generate(prompt, { model, - system: RECAP_SYSTEM_PROMPT, + system: RECAP_SYSTEM_PROMPT_BASE + styleAddendum(settings), numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 0058dab..b4541c0 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -186,6 +186,8 @@ export function migrate() { fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com', nitter_instance_url TEXT NOT NULL DEFAULT 'https://nitter.net', -- admin's preferred instance, prefills new Nitter sources (Connections tab) telegram_media_mode TEXT NOT NULL DEFAULT 'self-host', -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL) + synthesis_style_preset TEXT NOT NULL DEFAULT 'default', -- default | casual | formal — see pipeline/synthesis.ts's STYLE_PRESETS + synthesis_custom_instructions TEXT NOT NULL DEFAULT '', -- free-text addendum appended to the synthesis system prompt, on top of the preset widget_weather_enabled INTEGER NOT NULL DEFAULT 1, widget_stocks_enabled INTEGER NOT NULL DEFAULT 1, widget_bookmarks_enabled INTEGER NOT NULL DEFAULT 1, @@ -394,6 +396,12 @@ export function migrate() { stmt.run(`stk-${symbol.replace(/[^a-z0-9]+/gi, '-')}`, label, symbol, i + 1, new Date().toISOString()); }); } + if (!hasColumn('global_settings', 'synthesis_style_preset')) { + db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_style_preset TEXT NOT NULL DEFAULT 'default'"); + } + if (!hasColumn('global_settings', 'synthesis_custom_instructions')) { + db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_custom_instructions TEXT NOT NULL DEFAULT ''"); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index 604422e..6b49290 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -16,6 +16,8 @@ function rowToSettings(row: any): GlobalSettings { fxtwitterBaseUrl: row.fxtwitter_base_url, nitterInstanceUrl: row.nitter_instance_url, telegramMediaMode: row.telegram_media_mode, + synthesisStylePreset: row.synthesis_style_preset, + synthesisCustomInstructions: row.synthesis_custom_instructions, widgets: { weather: !!row.widget_weather_enabled, stocks: !!row.widget_stocks_enabled, @@ -80,6 +82,7 @@ export function updateSettings(patch: Partial): GlobalSettings { ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models, nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url, telegram_media_mode=$telegram_media_mode, + synthesis_style_preset=$synthesis_style_preset, synthesis_custom_instructions=$synthesis_custom_instructions, widget_weather_enabled=$widget_weather_enabled, widget_stocks_enabled=$widget_stocks_enabled, widget_bookmarks_enabled=$widget_bookmarks_enabled, widget_poe2_enabled=$widget_poe2_enabled, widget_order=$widget_order, @@ -105,6 +108,8 @@ export function updateSettings(patch: Partial): GlobalSettings { $fxtwitter_base_url: merged.fxtwitterBaseUrl, $nitter_instance_url: merged.nitterInstanceUrl, $telegram_media_mode: merged.telegramMediaMode, + $synthesis_style_preset: merged.synthesisStylePreset, + $synthesis_custom_instructions: merged.synthesisCustomInstructions, $widget_weather_enabled: merged.widgets.weather ? 1 : 0, $widget_stocks_enabled: merged.widgets.stocks ? 1 : 0, $widget_bookmarks_enabled: merged.widgets.bookmarks ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 66c1af9..3b7f69c 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -285,6 +285,10 @@ export interface GlobalSettings { nitterInstanceUrl: string; /** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */ telegramMediaMode: 'self-host' | 'proxy'; + /** Tone preset applied to every AI-synthesized article/recap (see pipeline/synthesis.ts's STYLE_PRESETS) — 'default' is the original neutral wire-service tone with no addendum. Never applies to single-source items, which always publish verbatim without going through the AI at all. */ + synthesisStylePreset: 'default' | 'casual' | 'formal'; + /** Free-text instructions appended to the synthesis system prompt alongside the style preset — e.g. "keep it under 3 sentences per paragraph". Empty string means no addendum. */ + synthesisCustomInstructions: string; /** Per-widget enable flags — see admin/settings' consolidated "Widgets" tab. Weather/Stocks/PoE2's backend pollers (scheduler.ts) are gated on these too, not just sidebar visibility; Bookmarks has no poller so its flag only affects the sidebar. */ widgets: { weather: boolean; diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 891a856..20b00ee 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -140,6 +140,8 @@ export interface AdminSettings { fxtwitterBaseUrl: string; nitterInstanceUrl: string; telegramMediaMode: 'self-host' | 'proxy'; + synthesisStylePreset: 'default' | 'casual' | 'formal'; + synthesisCustomInstructions: string; widgets: AdminWidgetsEnabled; widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[]; retention: RetentionSettings; diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index 7439a51..137e32f 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -34,7 +34,9 @@ followUpMinNewSources: local.followUpMinNewSources, tagDedupThreshold: local.tagDedupThreshold, tagExpiryDays: local.tagExpiryDays, - categoryPriority: local.categoryPriority + categoryPriority: local.categoryPriority, + synthesisStylePreset: local.synthesisStylePreset, + synthesisCustomInstructions: local.synthesisCustomInstructions }); status = 'saved'; setTimeout(() => (status = 'idle'), 1500); @@ -199,6 +201,29 @@ +
+ Writing style +

+ Applies to AI-merged articles and event recaps only — a story with just one source + publishes with its original text untouched, no AI involved. +

+ + + +
+
Hold before publish

Wait window to gather more sources before finalizing a story.

@@ -328,6 +353,12 @@ select { width: 100%; } + textarea { + width: 100%; + margin-top: 6px; + font: inherit; + resize: vertical; + } .priority-list { display: flex; flex-direction: column; From ecfcce9eb17e1a92dc15a6dc89e8a110d8a1273e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:29:45 +0000 Subject: [PATCH 10/18] Have the model synthesize a real title instead of truncating the body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every article title ended in "…" because there was never an actual title — deriveTitle() just took the body's first paragraph and cut it at 97 characters. The AI was never asked for a headline at all. Both system prompts now ask for a response in three parts (headline, then the article/recap, then tags), each separated by a delimiter. parseResult() extracts all three; if the model doesn't follow the format at all, it falls back to the old truncated-first-line heuristic rather than breaking. Delimiter matching is now a loose regex instead of an exact string — production had already shown a small model reproducing "---TAGS---" inexactly (e.g. "---\n\nTAGS---"), which the old exact-string split missed entirely and leaked into the published body. Same tolerance now applies to the new title delimiter. publishCluster uses the synthesized title directly; publishEventRecap uses it too, falling back to the previous ": recap" format only if the model returns an empty title. Verified: exact-format output, sloppy-delimiter output, and no-delimiter-at-all output all parse into sensible {title, body, tags}; a full runSynthesisCycle pass against a mock provider publishes an article with the real synthesized headline as its title. --- backend/src/pipeline/publish.ts | 14 +++----- backend/src/pipeline/synthesis.ts | 59 +++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index b5348ef..0c18fcf 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -36,12 +36,6 @@ function anyPushesToTopStories(items: ContentItem[]): boolean { return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false); } -/** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */ -function deriveTitle(body: string): string { - const firstLine = body.split('\n')[0]; - return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine; -} - /** * Resolves the hero image for a regular (non-tweet) article: try the best candidate * from the source items, download and locally host it; if there isn't one, fall back @@ -362,7 +356,7 @@ export async function publishCluster( const items = cluster.items; const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); + const { title, body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -418,7 +412,7 @@ export async function publishCluster( const now = new Date().toISOString(); const article = articles.insertArticle({ - title: deriveTitle(body), + title, body, heroImage, video, @@ -461,7 +455,7 @@ export async function publishEventRecap( event: TrackedEvent, constituents: MergedArticle[] ): Promise { - const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); + const { title, body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings); const resolvedTags = []; for (const label of tagLabels) { @@ -478,7 +472,7 @@ export async function publishEventRecap( const now = new Date().toISOString(); return articles.insertArticle({ - title: `${event.name}: recap`, + title: title || `${event.name}: recap`, body, heroImage, video: null, diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index a3be66c..3c04413 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -3,8 +3,17 @@ import type { ContentItem, GlobalSettings, MergedArticle } from '../storage/db/t import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; import { logger } from '../storage/db/logs.js'; +const TITLE_DELIMITER = '---TITLE---'; const TAG_DELIMITER = '---TAGS---'; +// Small/quantized models don't always reproduce a literal delimiter exactly — extra +// dashes, an inserted blank line, different case (seen in production with the tag +// delimiter: "---\n\nTAGS---" instead of "---TAGS---", which an exact-string split +// missed entirely, leaking the raw delimiter text into the published body). Splitting +// on a loose regex instead tolerates that variance. +const TITLE_DELIMITER_RE = /-{2,}\s*TITLE\s*-{2,}/i; +const TAG_DELIMITER_RE = /-{2,}\s*TAGS\s*-{2,}/i; + // Ollama truncates prompts that don't fit its context window by keeping a small prefix // and dropping everything else in the middle — silently, with no error, and with no // regard for which sources end up cut (see ollama-provider.ts for the incident that @@ -23,21 +32,25 @@ function capEntryText(text: string, budgetChars: number): string { return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; } -const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: -- Summarizes what has happened across the period covered, in chronological order -- Highlights the most significant developments rather than restating every article -- Stays neutral and factual, without editorializing -- Is 3-5 short paragraphs +const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write your response in exactly three parts, in this order: -After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; +1. A short, specific headline for this recap (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period). +2. On a new line, write exactly "${TITLE_DELIMITER}", then the recap article: + - Summarizes what has happened across the period covered, in chronological order + - Highlights the most significant developments rather than restating every article + - Stays neutral and factual, without editorializing + - Is 3-5 short paragraphs +3. On a new line after the recap, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; -const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: -- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. -- Does not copy phrasing verbatim from any source -- Stays neutral and factual, without editorializing -- Is 2-4 short paragraphs +const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write your response in exactly three parts, in this order: -After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; +1. A short, specific headline for this story (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period, no site/outlet name). +2. On a new line, write exactly "${TITLE_DELIMITER}", then the article: + - Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. + - Does not copy phrasing verbatim from any source + - Stays neutral and factual, without editorializing + - Is 2-4 short paragraphs +3. On a new line after the article, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; // Admin-selectable presets (Merge tab, "Writing style") — appended to whichever base // prompt applies. 'default' adds nothing: the base prompts above already describe the @@ -58,10 +71,17 @@ function styleAddendum(settings: GlobalSettings): string { } export interface SynthesisResult { + title: string; body: string; tagLabels: string[]; } +/** Only used when the model doesn't follow the requested title/delimiter format at all — a real headline beats a truncated sentence fragment, but publishing with no title at all is worse than either. */ +function fallbackTitle(body: string): string { + const firstLine = body.split('\n')[0]; + return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine; +} + function buildPrompt(items: ContentItem[], sourceNames: Map): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; @@ -88,13 +108,24 @@ function buildPrompt(items: ContentItem[], sourceNames: Map): st } function parseResult(raw: string): SynthesisResult { - const [body, tagSection] = raw.split(TAG_DELIMITER); + const [beforeTags, tagSection] = raw.split(TAG_DELIMITER_RE); const tagLabels = (tagSection ?? '') .split(',') .map((t) => t.trim()) .filter((t) => t.length > 0 && t.length < 60); - return { body: body.trim(), tagLabels }; + const titleSplit = (beforeTags ?? raw).split(TITLE_DELIMITER_RE); + const titlePart = titleSplit[0]; + // join() rather than titleSplit[1] in case the delimiter text somehow appears again + // inside the body itself — keeps that content rather than silently dropping it. + const bodyPart = titleSplit.length > 1 ? titleSplit.slice(1).join('') : undefined; + // If the title delimiter never showed up, the model didn't follow the requested + // format — treat the whole thing as body rather than mistaking the article itself + // for a "title", and fall back to the old truncated-first-line heuristic. + const body = (bodyPart ?? titlePart).trim(); + const title = bodyPart !== undefined ? titlePart.trim() : fallbackTitle(body); + + return { title, body, tagLabels }; } export async function synthesizeArticle( From afdb5ad0366f060cd004539139e84585a59430b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:36:27 +0000 Subject: [PATCH 11/18] Fix embed() calls silently timing out, dropping single-source items forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported symptom: articles that never got AI-merged (single source, nothing else to combine with) simply never published at all. Root cause: the same default-5-minute-fetch-timeout bug fixed for generate() earlier was never applied to embed(). Ollama serves one inference request at a time (n_slots = 1) — an embed() call issued while a slow generate() call is in flight has to wait in queue for that same slot, and on this CPU-only hardware a generate() call can easily run past 5 minutes. That wait alone was enough to trip Node's default fetch timeout on the embed request. embedPendingItems() catches that failure and just drops the item from its result (logged, not thrown) — clusterItems() only ever sees items that already have an embedding, so a dropped item never joins a cluster, never gets assignCluster() called, and stays "unclustered" forever, retried every cycle with the same failure for as long as Ollama stays busy. An item that happened to embed during an idle window still merges or publishes fine — which is exactly the split reported: synthesized articles show up, standalone ones don't. Fix: embed() now uses the same noTimeoutDispatcher already wired into generate(). Verified the request completes correctly end-to-end against a real HTTP server that delays its response. --- backend/src/inference/ollama-provider.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 7173922..002f68b 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -79,8 +79,17 @@ export class OllamaProvider implements InferenceProvider { const res = await fetch(`${this.base()}/api/embeddings`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: opts.model, prompt: text }) - }); + body: JSON.stringify({ model: opts.model, prompt: text }), + // Ollama serves one inference request at a time (n_slots = 1) — an embed call + // queued behind a slow generate() call waits for that same slot, and on this + // CPU-only hardware a generate() call can easily run past 5 minutes. Without + // this, that wait alone was enough to trip the same default fetch timeout + // generate() had (see noTimeoutDispatcher above), silently dropping the item + // from embedPendingItems — it never got clustered, so a single-source item + // unlucky enough to be embedded while Ollama was busy never published at all, + // retried every cycle with the same result for as long as Ollama stayed busy. + dispatcher: noTimeoutDispatcher + } as RequestInit); if (!res.ok) throw new Error(`Ollama embed failed: ${res.status} ${await res.text()}`); const data = (await res.json()) as { embedding: number[] }; return data.embedding; From 2bb8463c09b56c637a2d883474dd969bde7ed7d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:48:38 +0000 Subject: [PATCH 12/18] Give direct-publish items their own tick so a slow AI backlog can't block them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported: after clearing all articles/media and rescanning every source, items in "No AI" categories weren't publishing instantly like they should. Root cause: runSynthesisCycle bundled three unrelated jobs into one function, all guarded by a single reentrancy lock (added earlier this session to stop the AI-merge path from racing itself into duplicate articles): (1) YouTube/Nitter/Telegram direct-publish, (2) "No AI" category direct-publish, (3) embed/cluster/AI-merge. A mass rescan produces a big backlog of slow generate() calls for (3) — each one can run minutes on this CPU-only hardware — and since the whole function shared one guard, a newly-ingested "No AI" item had to wait for that entire backlog to drain before its own (fast, no-AI-needed) publish step even got a turn. Split into two independently-scheduled, independently-guarded ticks: runDirectPublishCycle (source-type-driven + "No AI"-category items, regardless of Ollama's reachability) and runSynthesisCycle (now only the embed/cluster/merge path). They operate on disjoint item sets, so running them "concurrently" is safe — no risk of the duplicate-publish race the shared guard was originally added to prevent. Verified directly: with a mock provider whose generate() call takes 3 seconds (standing in for a multi-minute real one), a "No AI" category item published in 68ms — before the slow merge was even close to finishing — while the merge itself still completed correctly on its own schedule. --- backend/src/queue/priorityQueue.ts | 75 +++++++++++++++++++++--------- backend/src/queue/scheduler.ts | 27 ++++++++++- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 15323e3..679f594 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -88,7 +88,10 @@ async function publishItemsDirect( * pipeline, this doesn't wait out the hold-before-publish window: that window exists to * give corroborating sources time to arrive before an AI merge locks in, which doesn't * apply here since there's no merging happening at all — each item is just itself. - * Still respects category priority. + * Still respects category priority. Harmless overlap with runDirectPublishCycle (which + * runs regardless of reachability) — an item already published by one is simply gone + * from the other's next "unclustered" query, since assignCluster lands before either + * moves on to its next item. */ export async function runPassthroughCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); @@ -107,41 +110,31 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { +export async function runDirectPublishCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); if (items.length === 0) return 0; - // One fetch of the full source list per cycle, reused below for both the - // direct-publish partition and each item's category/type lookups — avoids a - // separate sourcesDb.getSource() round-trip per item. const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); const categories = categoriesDb.listCategories(); - const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); - // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with - // anything else — each is always its own article, same shape whether the AI service - // is up or not. Route them straight to publishDirect, same as the no-AI passthrough path. const directPublishSourceIds = new Set( [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - // A category with disableAi set (see the Category priority admin pane) opts its - // items out of clustering/synthesis entirely — each publishes on its own, using its - // own source's text, same as the source-type-driven direct items above. const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); - const [categoryDirectItems, mergeableItems] = partition(remaining, (item) => - inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) - ); + const [categoryDirectItems] = partition(remaining, (item) => inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)); const publishedTypeDirect = await publishItemsDirect( typeDirectItems, @@ -159,6 +152,42 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G 'Direct publish failed' ); + return publishedTypeDirect + publishedCategoryDirect; +} + +/** + * One pass of the synthesis queue: cluster whatever's unclustered (excluding items + * runDirectPublishCycle already owns — see there), ordered by admin-defined category + * priority, and publish clusters that have cleared the hold-before-publish window. + * Items claimed by an active tracked event (belonging to one of its sources and + * matching its keyword filter, if any) publish exactly like everything else — + * individually or merged with same-story coverage — just tagged with the event's id so + * they're browsable under it and eligible for eventsRecap.ts's periodic AI wrap-up. + */ +export async function runSynthesisCycle(provider: InferenceProvider, settings: GlobalSettings): Promise { + const activeEvents = eventsDb.listActiveEvents(); + const items = contentItemsDb.unclusteredItemsExcludingSources([]); + if (items.length === 0) return 0; + + // One fetch of the full source list per cycle, reused below for both the + // direct-publish exclusion and each item's category/rank lookups — avoids a + // separate sourcesDb.getSource() round-trip per item. + const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); + + // YouTube/Nitter/Telegram items and AI-disabled-category items are runDirectPublishCycle's + // job (its own guarded tick, so a slow merge backlog here never blocks them) — excluded + // here too since a batch just ingested this instant may still be unclustered when this + // runs before that cycle's own pass gets to it. + const directPublishSourceIds = new Set( + [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) + ); + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + const mergeableItems = items.filter( + (item) => !directPublishSourceIds.has(item.sourceId) && !inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) + ); + const ranked = mergeableItems .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) .sort((a, b) => a.rank - b.rank) @@ -216,5 +245,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } - return published + publishedTypeDirect + publishedCategoryDirect; + return published; } diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index f75670d..28074a2 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -1,5 +1,5 @@ import { pollDueSources } from '../ingestion/poller.js'; -import { runSynthesisCycle, runPassthroughCycle } from './priorityQueue.js'; +import { runSynthesisCycle, runPassthroughCycle, runDirectPublishCycle } from './priorityQueue.js'; import { runEventRecaps } from './eventsRecap.js'; import { runRetentionSweep } from './retention.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; @@ -10,6 +10,7 @@ import { pollStocksNow } from '../stocks/poller.js'; import { pollPoe2Now } from '../poe2/poller.js'; const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency +const DIRECT_PUBLISH_TICK_MS = 60_000; const SYNTHESIS_TICK_MS = 60_000; const RETENTION_TICK_MS = 60 * 60_000; // hourly const WEATHER_TICK_MS = 45 * 60_000; @@ -26,6 +27,13 @@ const POE2_TICK_MS = 60 * 60_000; // poe.ninja's own overview data doesn't refre * fresh, differently-worded article by an overlapping cycle, repeatedly, until the * first cycle's assignCluster() finally landed. Node is single-threaded, so the only * source of "concurrent" runs here is exactly this interval overlap. + * + * Each call gets its own independent `running` flag/timer — the direct-publish and + * synthesis ticks are deliberately two separate calls to this (not one shared guard) + * precisely so a slow AI-merge backlog on one never blocks the other's fast, + * no-AI-needed items from publishing on schedule. They operate on disjoint item sets + * (see priorityQueue.ts), so there's no risk of the two racing each other into a + * duplicate publish the way an overlapping call to the *same* fn would. */ function everyTickSkippingOverlap(ms: number, fn: () => Promise) { let running = false; @@ -53,6 +61,18 @@ export function startScheduler() { } }); + everyTickSkippingOverlap(DIRECT_PUBLISH_TICK_MS, async () => { + try { + const settings = settingsDb.getSettings(); + const published = await runDirectPublishCycle(settings); + if (published > 0) { + logger.info('scheduler', `Direct-publish tick: published ${published} article(s)`); + } + } catch (err) { + logger.error('scheduler', `Direct-publish tick failed: ${(err as Error).message}`); + } + }); + everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); @@ -120,5 +140,8 @@ export function startScheduler() { pollPoe2Now().catch((err) => logger.error('poe2', `Poll tick failed: ${err.message}`)); }, POE2_TICK_MS); - logger.info('scheduler', 'Started: poll every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h'); + logger.info( + 'scheduler', + 'Started: poll every 1m, direct-publish every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h' + ); } From 9f4f1d1b7132621f967d03bc2e36ee41ed5d7ee1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:27:01 +0000 Subject: [PATCH 13/18] Add pipeline backlog/throughput dashboard to admin Logs tab Admins had no visibility into how many articles were queued for AI synthesis or waiting out the hold-before-publish window, nor how fast Ollama could clear that backlog. GET /api/admin/pipeline-stats reports a live backlog snapshot (items awaiting embedding, clusters on hold vs. ready, items still held) computed straight from the DB with no AI calls, plus real Ollama generate() throughput (tokens/sec, in-flight call) tracked from actual requests, and estimates minutes-to-clear from recent generate() call durations. Surfaced as a stat-tile dashboard atop the Logs tab. --- backend/src/api/admin.ts | 35 +++++ backend/src/inference/ollama-provider.ts | 67 ++++++--- backend/src/inference/provider.ts | 5 +- backend/src/inference/stats.ts | 57 ++++++++ backend/src/pipeline/synthesis.ts | 6 +- backend/src/queue/backlogStats.ts | 114 +++++++++++++++ backend/src/queue/priorityQueue.ts | 17 ++- frontend/src/lib/adminApi.ts | 5 +- frontend/src/lib/adminTypes.ts | 27 ++++ .../src/lib/components/admin/LogsTab.svelte | 138 +++++++++++++++++- 10 files changed, 440 insertions(+), 31 deletions(-) create mode 100644 backend/src/inference/stats.ts create mode 100644 backend/src/queue/backlogStats.ts diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 947b6f0..77521e0 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -11,6 +11,8 @@ import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; +import * as backlogStats from '../queue/backlogStats.js'; +import * as ollamaStats from '../inference/stats.js'; import * as telegramClient from '../telegram/client.js'; import { geocodeLocation } from '../weather/client.js'; import { pollWeatherNow } from '../weather/poller.js'; @@ -347,4 +349,37 @@ export async function registerAdminRoutes(app: FastifyInstance) { limit: limit ? Number(limit) : undefined }); }); + + // Backlog/throughput dashboard for the Logs tab — backlog counts are recomputed live + // from the DB on every request (cheap: no AI calls, see backlogStats.ts), while Ollama + // throughput/in-flight status comes from a rolling in-memory sample of recent + // generate() calls (see inference/stats.ts) since that can only be observed as calls + // actually happen, not recomputed on demand. + app.get('/api/admin/pipeline-stats', async () => { + const settings = settingsDb.getSettings(); + const backlog = backlogStats.getBacklogSnapshot(settings); + const throughput = ollamaStats.getThroughput(); + const inFlight = ollamaStats.getInFlight(); + const { lastDirectCycle, lastSynthesisCycle } = backlogStats.getLastCycles(); + + // Estimate is deliberately conservative: only clusters that actually need an LLM + // call (2+ items — see backlogStats.ts) count toward it, and it's null (rather than + // a misleading guess) until at least one real generate() call has completed, since + // there's no token-speed data to estimate from yet. + const estimatedMinutesToClear = + backlog.clusters.readyNowNeedingSynthesis === 0 + ? 0 + : throughput.avgGenerateDurationMs !== null + ? Math.ceil((backlog.clusters.readyNowNeedingSynthesis * throughput.avgGenerateDurationMs) / 60_000) + : null; + + return { + timestamp: new Date().toISOString(), + ollama: { inFlight, ...throughput }, + backlog, + estimatedMinutesToClear, + lastDirectCycle, + lastSynthesisCycle + }; + }); } diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index 002f68b..df32f58 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,6 @@ import { Agent } from 'undici'; import type { InferenceProvider } from './provider.js'; +import * as stats from './stats.js'; /** * Node's global fetch (undici) defaults to a 5-minute headers/body timeout — fine for @@ -51,28 +52,52 @@ export class OllamaProvider implements InferenceProvider { async generate( prompt: string, - opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {} + opts: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string } = {} ): Promise { - const res = await fetch(`${this.base()}/api/generate`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: opts.model, - prompt, - system: opts.system, - stream: false, - options: { - num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, - num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT - } - }), - // Not in the ambient RequestInit type this project resolves to, but Node's global - // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. - dispatcher: noTimeoutDispatcher - } as RequestInit); - if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); - const data = (await res.json()) as { response: string }; - return data.response; + const startedAt = Date.now(); + stats.recordGenerateStart(opts.label ?? 'synthesis'); + try { + const res = await fetch(`${this.base()}/api/generate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: opts.model, + prompt, + system: opts.system, + stream: false, + options: { + num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, + num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT + } + }), + // Not in the ambient RequestInit type this project resolves to, but Node's global + // fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above. + dispatcher: noTimeoutDispatcher + } as RequestInit); + if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); + const data = (await res.json()) as { + response: string; + eval_count?: number; + eval_duration?: number; + prompt_eval_count?: number; + prompt_eval_duration?: number; + total_duration?: number; + }; + // Ollama reports these *_duration fields in nanoseconds — dividing eval_count by + // (eval_duration/1e9) gives generation tokens/sec, and total_duration/1e6 gives + // wall-clock milliseconds (falling back to a local measurement if a given Ollama + // version's response ever omits it). + stats.recordGenerateEnd({ + genTokensPerSec: data.eval_count && data.eval_duration ? data.eval_count / (data.eval_duration / 1e9) : null, + promptTokensPerSec: + data.prompt_eval_count && data.prompt_eval_duration ? data.prompt_eval_count / (data.prompt_eval_duration / 1e9) : null, + totalDurationMs: data.total_duration ? data.total_duration / 1e6 : Date.now() - startedAt + }); + return data.response; + } catch (err) { + stats.recordGenerateEnd(null); + throw err; + } } async embed(text: string, opts: { model?: string } = {}): Promise { diff --git a/backend/src/inference/provider.ts b/backend/src/inference/provider.ts index aaa61f8..a542722 100644 --- a/backend/src/inference/provider.ts +++ b/backend/src/inference/provider.ts @@ -1,5 +1,8 @@ export interface InferenceProvider { - generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise; + generate( + prompt: string, + opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string } + ): Promise; embed(text: string, opts?: { model?: string }): Promise; listModels(): Promise; isReachable(): Promise; diff --git a/backend/src/inference/stats.ts b/backend/src/inference/stats.ts new file mode 100644 index 0000000..4f1d68a --- /dev/null +++ b/backend/src/inference/stats.ts @@ -0,0 +1,57 @@ +/** + * In-memory-only tracking of Ollama generate() throughput and in-flight status, for the + * admin "Logs" dashboard (see queue/backlogStats.ts, api/admin.ts's GET + * /api/admin/pipeline-stats). Deliberately not persisted to disk — a restart losing a + * few minutes of rolling samples is fine, since the next few generate() calls rebuild it. + */ + +const MAX_SAMPLES = 20; + +export interface GenerateSample { + /** Generation speed (tokens/sec) from Ollama's eval_count/eval_duration — null if the response omitted them. */ + genTokensPerSec: number | null; + /** Prompt-processing speed (tokens/sec) from prompt_eval_count/prompt_eval_duration — usually the dominant cost on CPU-only inference. */ + promptTokensPerSec: number | null; + totalDurationMs: number; +} + +const samples: GenerateSample[] = []; +let inFlight: { label: string; startedAt: number } | null = null; + +/** Call immediately before issuing a generate() request. */ +export function recordGenerateStart(label: string): void { + inFlight = { label, startedAt: Date.now() }; +} + +/** Call in a finally block after the request settles — pass null on failure/abort. */ +export function recordGenerateEnd(sample: GenerateSample | null): void { + inFlight = null; + if (!sample) return; + samples.push(sample); + if (samples.length > MAX_SAMPLES) samples.shift(); +} + +export function getInFlight(): { label: string; elapsedMs: number } | null { + return inFlight ? { label: inFlight.label, elapsedMs: Date.now() - inFlight.startedAt } : null; +} + +function average(nums: number[]): number | null { + if (nums.length === 0) return null; + return nums.reduce((a, b) => a + b, 0) / nums.length; +} + +export interface ThroughputStats { + sampleCount: number; + avgGenTokensPerSec: number | null; + avgPromptTokensPerSec: number | null; + avgGenerateDurationMs: number | null; +} + +export function getThroughput(): ThroughputStats { + return { + sampleCount: samples.length, + avgGenTokensPerSec: average(samples.map((s) => s.genTokensPerSec).filter((n): n is number => n !== null)), + avgPromptTokensPerSec: average(samples.map((s) => s.promptTokensPerSec).filter((n): n is number => n !== null)), + avgGenerateDurationMs: average(samples.map((s) => s.totalDurationMs)) + }; +} diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 3c04413..6006132 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -137,7 +137,8 @@ export async function synthesizeArticle( ): Promise { const prompt = buildPrompt(items, sourceNames); const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); - const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); + const label = `Merging ${items.length} source${items.length === 1 ? '' : 's'}: "${items[0]?.title.slice(0, 60) ?? ''}"`; + const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT, label }); return parseResult(raw); } @@ -175,7 +176,8 @@ export async function synthesizeRecap( model, system: RECAP_SYSTEM_PROMPT_BASE + styleAddendum(settings), numCtx: DEFAULT_NUM_CTX, - numPredict: DEFAULT_NUM_PREDICT + numPredict: DEFAULT_NUM_PREDICT, + label: `Recapping event: "${eventName.slice(0, 60)}"` }); return parseResult(raw); } diff --git a/backend/src/queue/backlogStats.ts b/backend/src/queue/backlogStats.ts new file mode 100644 index 0000000..ff69d4f --- /dev/null +++ b/backend/src/queue/backlogStats.ts @@ -0,0 +1,114 @@ +import * as contentItemsDb from '../storage/db/contentItems.js'; +import * as sourcesDb from '../storage/db/sources.js'; +import * as categoriesDb from '../storage/db/categories.js'; +import { clusterItems } from '../pipeline/clustering.js'; +import type { ContentItem, GlobalSettings, Source } from '../storage/db/types.js'; + +interface CycleRecord { + at: string; + published: number; +} + +let lastDirectCycle: CycleRecord | null = null; +let lastSynthesisCycle: CycleRecord | null = null; + +/** Called by priorityQueue.ts at the end of runDirectPublishCycle. */ +export function recordDirectPublishCycle(published: number): void { + lastDirectCycle = { at: new Date().toISOString(), published }; +} + +/** Called by priorityQueue.ts at the end of runSynthesisCycle. */ +export function recordSynthesisCycle(published: number): void { + lastSynthesisCycle = { at: new Date().toISOString(), published }; +} + +export function getLastCycles(): { lastDirectCycle: CycleRecord | null; lastSynthesisCycle: CycleRecord | null } { + return { lastDirectCycle, lastSynthesisCycle }; +} + +function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sourcesById: Map): boolean { + const source = sourcesById.get(item.sourceId); + for (const cat of source?.category ?? []) { + if (disabledNames.has(cat.split(':')[0].trim().toLowerCase())) return true; + } + return false; +} + +export interface BacklogSnapshot { + totalUnclusteredItems: number; + /** Items that need no AI at all (YouTube/Nitter/Telegram sources, or "No AI" categories) — publish on the next direct-publish tick. */ + directEligibleItems: number; + /** Mergeable items that haven't been embedded yet (embed() failed/pending, or just ingested since the last synthesis tick). */ + awaitingEmbeddingItems: number; + clusters: { + total: number; + /** Cleared the hold-before-publish window — will publish on the next synthesis tick. */ + readyNow: number; + /** Of readyNow, clusters with 2+ items — these are the ones that actually need an LLM generate() call (single-item clusters publish verbatim, no AI). */ + readyNowNeedingSynthesis: number; + /** Still waiting out the hold-before-publish window. */ + onHold: number; + itemsOnHold: number; + earliestHoldRemainingMs: number | null; + }; +} + +/** + * Read-only snapshot of the current backlog for the admin dashboard — mirrors the same + * categorization runDirectPublishCycle/runSynthesisCycle use (priorityQueue.ts), but never + * calls the AI itself: items with no embedding yet are just counted, not embedded, and + * clustering only runs over items that already have one (cosine similarity over stored + * vectors — no network call). Cheap enough to call on every dashboard refresh. + */ +export function getBacklogSnapshot(settings: GlobalSettings): BacklogSnapshot { + const items = contentItemsDb.unclusteredItemsExcludingSources([]); + const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + + const directPublishSourceIds = new Set( + [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) + ); + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + + const directEligible: ContentItem[] = []; + const mergeable: ContentItem[] = []; + for (const item of items) { + if (directPublishSourceIds.has(item.sourceId) || inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)) { + directEligible.push(item); + } else { + mergeable.push(item); + } + } + + const awaitingEmbedding = mergeable.filter((item) => !item.embedding); + const embedded = mergeable.filter((item) => item.embedding); + + const clusters = clusterItems(embedded, settings.mergeStrictness); + const holdMs = settings.holdBeforePublishMinutes * 60_000; + + let readyNow = 0; + let readyNowNeedingSynthesis = 0; + let onHold = 0; + let itemsOnHold = 0; + let earliestHoldRemainingMs: number | null = null; + + for (const cluster of clusters) { + const earliestFetch = Math.min(...cluster.items.map((i) => new Date(i.fetchedAt).getTime())); + const remaining = holdMs - (Date.now() - earliestFetch); + if (remaining > 0) { + onHold++; + itemsOnHold += cluster.items.length; + earliestHoldRemainingMs = earliestHoldRemainingMs === null ? remaining : Math.min(earliestHoldRemainingMs, remaining); + } else { + readyNow++; + if (cluster.items.length > 1) readyNowNeedingSynthesis++; + } + } + + return { + totalUnclusteredItems: items.length, + directEligibleItems: directEligible.length, + awaitingEmbeddingItems: awaitingEmbedding.length, + clusters: { total: clusters.length, readyNow, readyNowNeedingSynthesis, onHold, itemsOnHold, earliestHoldRemainingMs } + }; +} diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 679f594..e832241 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -7,6 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js'; import { clusterItems } from '../pipeline/clustering.js'; import { publishCluster, publishDirect } from '../pipeline/publish.js'; import { logger } from '../storage/db/logs.js'; +import * as backlogStats from './backlogStats.js'; import type { GlobalSettings, ContentItem, TrackedEvent, Source } from '../storage/db/types.js'; function partition(items: T[], predicate: (item: T) => boolean): [T[], T[]] { @@ -123,7 +124,10 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); - if (items.length === 0) return 0; + if (items.length === 0) { + backlogStats.recordDirectPublishCycle(0); + return 0; + } const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); const categories = categoriesDb.listCategories(); @@ -152,7 +156,9 @@ export async function runDirectPublishCycle(settings: GlobalSettings): Promise { const activeEvents = eventsDb.listActiveEvents(); const items = contentItemsDb.unclusteredItemsExcludingSources([]); - if (items.length === 0) return 0; + if (items.length === 0) { + backlogStats.recordSynthesisCycle(0); + return 0; + } // One fetch of the full source list per cycle, reused below for both the // direct-publish exclusion and each item's category/rank lookups — avoids a @@ -245,5 +254,7 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } + backlogStats.recordSynthesisCycle(published); + return published; } diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index b2f0479..53caa3b 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -13,7 +13,8 @@ import type { AdminStockTicker, AdminBookmark, Poe2BrowseEntry, - AdminPoe2Entry + AdminPoe2Entry, + PipelineStats } from './adminTypes'; async function request(path: string, options: RequestInit = {}, fetchFn: typeof fetch = fetch): Promise { @@ -176,6 +177,8 @@ export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: nu return request(`/api/admin/logs${qs ? `?${qs}` : ''}`, {}, fetchFn); }; +export const getPipelineStats = (fetchFn?: typeof fetch) => request('/api/admin/pipeline-stats', {}, fetchFn); + // Weather — config/cache lives on AdminSettings.weather (see updateSettings above); this // is just the geocoding lookup used to resolve a typed city name to lat/lon. export const geocodeLocation = (query: string, fetchFn?: typeof fetch) => diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 20b00ee..927e3ca 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -198,6 +198,33 @@ export interface TelegramStatus { phone: string | null; } +export interface PipelineStats { + timestamp: string; + ollama: { + inFlight: { label: string; elapsedMs: number } | null; + sampleCount: number; + avgGenTokensPerSec: number | null; + avgPromptTokensPerSec: number | null; + avgGenerateDurationMs: number | null; + }; + backlog: { + totalUnclusteredItems: number; + directEligibleItems: number; + awaitingEmbeddingItems: number; + clusters: { + total: number; + readyNow: number; + readyNowNeedingSynthesis: number; + onHold: number; + itemsOnHold: number; + earliestHoldRemainingMs: number | null; + }; + }; + estimatedMinutesToClear: number | null; + lastDirectCycle: { at: string; published: number } | null; + lastSynthesisCycle: { at: string; published: number } | null; +} + export interface LogEntry { id: number; timestamp: string; diff --git a/frontend/src/lib/components/admin/LogsTab.svelte b/frontend/src/lib/components/admin/LogsTab.svelte index 8406d9e..30dc117 100644 --- a/frontend/src/lib/components/admin/LogsTab.svelte +++ b/frontend/src/lib/components/admin/LogsTab.svelte @@ -1,10 +1,11 @@ +{#if stats} +
+
+ Backlog + {stats.backlog.totalUnclusteredItems} + item{stats.backlog.totalUnclusteredItems === 1 ? '' : 's'} not yet published +
+
+ Awaiting embedding + {stats.backlog.awaitingEmbeddingItems} + need an embed() call before they can cluster +
+
+ Held for publishing + {stats.backlog.clusters.itemsOnHold} + + {stats.backlog.clusters.onHold} cluster{stats.backlog.clusters.onHold === 1 ? '' : 's'} on hold-before-publish + {#if stats.backlog.clusters.earliestHoldRemainingMs !== null} + · earliest clears in {formatDuration(stats.backlog.clusters.earliestHoldRemainingMs)} + {/if} + +
+
+ Awaiting synthesis + {stats.backlog.clusters.readyNowNeedingSynthesis} + multi-source cluster{stats.backlog.clusters.readyNowNeedingSynthesis === 1 ? '' : 's'} ready, needs an AI merge +
+
+ Estimated to clear + {formatEta(stats.estimatedMinutesToClear)} + + {#if stats.ollama.avgGenerateDurationMs !== null} + based on {stats.ollama.sampleCount} recent generate call{stats.ollama.sampleCount === 1 ? '' : 's'}, avg {formatDuration(stats.ollama.avgGenerateDurationMs)} each + {:else} + no completed generate calls yet + {/if} + +
+
+ Ollama right now + {#if stats.ollama.inFlight} + Synthesizing + {stats.ollama.inFlight.label} · {formatDuration(stats.ollama.inFlight.elapsedMs)} elapsed + {:else} + Idle + + {#if stats.ollama.avgGenTokensPerSec !== null} + ~{stats.ollama.avgGenTokensPerSec.toFixed(1)} gen tok/s, ~{stats.ollama.avgPromptTokensPerSec?.toFixed(1) ?? '?'} prompt tok/s + {:else} + no throughput data yet + {/if} + + {/if} +
+
+ Last direct-publish tick + {stats.lastDirectCycle ? stats.lastDirectCycle.published : '—'} + {formatAgo(stats.lastDirectCycle?.at ?? null)} +
+
+ Last synthesis tick + {stats.lastSynthesisCycle ? stats.lastSynthesisCycle.published : '—'} + {formatAgo(stats.lastSynthesisCycle?.at ?? null)} +
+
+{/if} +
@@ -70,6 +167,41 @@
diff --git a/frontend/src/routes/tag/[slug]/+page.ts b/frontend/src/routes/tag/[slug]/+page.ts new file mode 100644 index 0000000..04f9471 --- /dev/null +++ b/frontend/src/routes/tag/[slug]/+page.ts @@ -0,0 +1,22 @@ +import { error } from '@sveltejs/kit'; +import type { PageLoad } from './$types'; +import { getFeed, getTagBySlug } from '$lib/api'; + +const PAGE_SIZE = 15; + +// Mirrors /category/[name] and /event/[id] — a tag chip links by slug, so the slug is +// resolved to the real tag (id + label) via a dedicated backend lookup (GET +// /api/tag/:slug) rather than a preloaded list, since tags aren't loaded by the root +// layout the way categories/events are. +export const load: PageLoad = async ({ params, fetch }) => { + let tag; + try { + tag = await getTagBySlug(params.slug, fetch); + } catch { + throw error(404, 'Tag not found'); + } + + const filters = { tag: tag.id }; + const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); + return { initial, filters, tag, pageSize: PAGE_SIZE }; +}; From 23be5086c56e11f053a80b878a5a161165d5fda2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 19:14:50 +0000 Subject: [PATCH 15/18] Fix blank-article publishing bug + add per-article reissue tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A quantized model can occasionally return just the delimiter scaffold ("---TITLE---" / "---TAGS---") with no real headline or article text in between — parseResult treated that as a structurally valid response and published a blank article with empty title/body but real sources and a hero image attached. synthesizeArticle/synthesizeRecap now throw on an empty parsed body instead, so the existing catch-and-retry logic in runSynthesisCycle leaves the cluster unclustered for the next tick rather than ever inserting one of these. Also adds POST /api/admin/articles/:id/reissue to fix articles already published this way: the existing per-source reissue tool explicitly refuses to touch a multi-source article, which this failure mode always produces (an empty synthesis only happens on an actual multi-item merge — a single-item cluster publishes verbatim with no AI call at all), so there was no way to recover one without this. --- backend/src/api/admin.ts | 13 ++++++++++++- backend/src/pipeline/synthesis.ts | 21 +++++++++++++++++++-- backend/src/storage/contentCascade.ts | 23 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 77521e0..30f3034 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -6,7 +6,7 @@ import * as categoriesDb from '../storage/db/categories.js'; import * as stocksDb from '../storage/db/stocks.js'; import * as bookmarksDb from '../storage/db/bookmarks.js'; import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js'; -import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; +import { clearSourceContent, reissueSourceContent, reissueArticle, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js'; import { totalStorageBytes } from '../storage/media/index.js'; import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; @@ -151,6 +151,17 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reissueSourceContent(id); }); + // Fixes one specific bad article (e.g. a degenerate/empty AI synthesis — see + // synthesis.ts's assertNonEmpty) by deleting it and requeuing every item it merged, + // regardless of how many different sources contributed — reissueSourceContent above + // deliberately won't touch a multi-source article at all. + app.post('/api/admin/articles/:id/reissue', async (req, reply) => { + const { id } = req.params as { id: string }; + const result = reissueArticle(id); + if (!result) return reply.code(404).send({ error: 'not found' }); + return result; + }); + // --- Tracked events --- app.get('/api/admin/events', async () => eventsDb.listEvents()); diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index dfdd5be..d9bcf83 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -163,6 +163,23 @@ function parseResult(raw: string): SynthesisResult { return { title, body, tagLabels }; } +/** + * A quantized/small model occasionally reproduces just the requested delimiter + * scaffold ("---TITLE---\n\n---TAGS---") with no real headline or article text in + * between — a structurally "valid" response by parseResult's own logic (delimiters + * found, nothing crashed) but empty in substance. Left unchecked this published a + * blank article (empty title/body, still with real sources/hero image attached) once + * in production. Treating an empty body as a hard failure lets the caller's existing + * catch-and-retry logic (see priorityQueue.ts's runSynthesisCycle) leave the cluster + * unclustered for the next cycle instead of ever inserting one of these. + */ +function assertNonEmpty(result: SynthesisResult, context: string): SynthesisResult { + if (!result.body.trim()) { + throw new Error(`Model returned an empty article body for ${context}`); + } + return result; +} + export async function synthesizeArticle( provider: InferenceProvider, model: string, @@ -174,7 +191,7 @@ export async function synthesizeArticle( const system = SYSTEM_PROMPT_BASE + styleAddendum(settings); const label = `Merging ${items.length} source${items.length === 1 ? '' : 's'}: "${items[0]?.title.slice(0, 60) ?? ''}"`; const raw = await provider.generate(prompt, { model, system, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT, label }); - return parseResult(raw); + return assertNonEmpty(parseResult(raw), `"${items[0]?.title.slice(0, 60) ?? ''}"`); } function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string { @@ -214,5 +231,5 @@ export async function synthesizeRecap( numPredict: DEFAULT_NUM_PREDICT, label: `Recapping event: "${eventName.slice(0, 60)}"` }); - return parseResult(raw); + return assertNonEmpty(parseResult(raw), `event recap "${eventName.slice(0, 60)}"`); } diff --git a/backend/src/storage/contentCascade.ts b/backend/src/storage/contentCascade.ts index 3757245..15ef491 100644 --- a/backend/src/storage/contentCascade.ts +++ b/backend/src/storage/contentCascade.ts @@ -86,6 +86,29 @@ export function reissueSourceContent(sourceId: string): ReissueResult { return { articlesDeleted, itemsRequeued: requeueIds.size }; } +/** + * Deletes one specific article (and its media) and requeues every content item that + * contributed to it — unlike reissueSourceContent, this works regardless of how many + * different sources the article merged together, since it's scoped to the article + * itself rather than "everything from source X". Exists for exactly the failure mode + * synthesis.ts's assertNonEmpty guards against going forward: a bad synthesis call + * that already made it into a published (garbage) article before that guard existed, + * where the source-scoped reissue tools can't help because the article spans sources. + * Returns null if the article doesn't exist. + */ +export function reissueArticle(articleId: string): ReissueResult | null { + const article = articlesDb.getArticle(articleId); + if (!article) return null; + + const itemIds = article.sources.map((s) => s.itemId); + deleteMediaByArticleId(article.id); + articlesDb.deleteArticle(article.id); + contentItemsDb.resetClusterForItems(itemIds); + + logger.info('admin', `Reissuing article ${articleId}: deleted, ${itemIds.length} item(s) requeued`); + return { articlesDeleted: 1, itemsRequeued: itemIds.length }; +} + /** Wipes every published article and its media, keeping raw ingested items intact so they can be re-synthesized fresh. */ export function clearAllArticles(): number { const articles = articlesDb.allArticlesNewestFirst(); From 12f8e2b525255fecd204632fe775b11243ca259a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 19:21:28 +0000 Subject: [PATCH 16/18] Add "Reissue an article" panel to the Retention admin tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires up POST /api/admin/articles/:id/reissue (added alongside the blank-article fix) as a UI panel instead of requiring curl: paste an article ID, it deletes the article and requeues its source items for re-publish. Verified live in a browser against a real backend/DB — both the success path and the "no article with that ID" 404 case. --- frontend/src/lib/adminApi.ts | 5 ++ .../lib/components/admin/RetentionTab.svelte | 61 ++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 53caa3b..a7fb9f6 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -104,6 +104,11 @@ export const pollSourceNow = (id: string, fetchFn?: typeof fetch) => export const reissueSourceContent = (id: string, fetchFn?: typeof fetch) => request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/sources/${id}/reissue`, { method: 'POST' }, fetchFn); +// Fixes one specific bad article regardless of how many sources it merged — unlike +// reissueSourceContent above, which deliberately won't touch a multi-source article. +export const reissueArticle = (id: string, fetchFn?: typeof fetch) => + request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/articles/${id}/reissue`, { method: 'POST' }, fetchFn); + // Content clearing — wipe articles/media/a source's raw items so they can be repopulated fresh. export const clearSourceContent = (id: string, fetchFn?: typeof fetch) => request<{ itemsDeleted: number; articlesDeleted: number }>(`/api/admin/content/sources/${id}`, { method: 'DELETE' }, fetchFn); diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index f2a1058..b583620 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -1,6 +1,6 @@
@@ -84,6 +141,57 @@
+
+
+ Context window + +
+

+ How much text the synthesis model can take in (context window) and how long its response + can be (max response length). Too low a response limit is why an article or event recap + sometimes cuts off mid-sentence instead of finishing. + {#if detecting} + Detecting {selected.synthesis}'s limit… + {:else if detectedMax} + Detected max for {selected.synthesis}: {detectedMax.toLocaleString()} tokens. + {:else} + Couldn't detect a limit for {selected.synthesis} — defaulting the slider's ceiling to + {FALLBACK_MAX_CTX.toLocaleString()}. Setting num_ctx above what the model actually + supports will make Ollama reject or silently degrade requests. + {/if} +

+ + +
+ +
+ + +
+ +
+
+