From c45d7d5acf03038dde148bbe1381d1b1cc7887c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 03:31:31 +0000 Subject: [PATCH] Fix silent Ollama prompt truncation in synthesis pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ollama was defaulting to a 4096-token context (vs. the model's 32768 training context) and silently truncating any oversized prompt by dropping content from the middle, with no error surfaced anywhere — observed losing ~53% of a merge-cluster prompt in production. Two prompt builders (buildPrompt/buildRecapPrompt) concatenated all source summaries/article bodies with no size cap, so a cluster with enough sources (or a recap spanning enough articles) could easily exceed the window. Fix: OllamaProvider.generate() now always sends explicit num_ctx/ num_predict options (sized for CPU-only inference — i5-6600K, no GPU, ~17 tok/s prompt processing) instead of leaving Ollama to pick a default. synthesis.ts now caps prompt size itself before it ever reaches Ollama, giving each source/article an equal character budget and trimming individual entries rather than dropping whole ones off the end — every source stays at least partially represented and attributable. Trims are logged via the existing admin log stream instead of failing silently. --- backend/src/inference/ollama-provider.ts | 31 ++++++++++++- backend/src/inference/provider.ts | 2 +- backend/src/pipeline/synthesis.ts | 58 ++++++++++++++++++++---- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/backend/src/inference/ollama-provider.ts b/backend/src/inference/ollama-provider.ts index b735c5f..6817232 100644 --- a/backend/src/inference/ollama-provider.ts +++ b/backend/src/inference/ollama-provider.ts @@ -1,5 +1,25 @@ import type { InferenceProvider } from './provider.js'; +/** + * Default context window / max-generation length requested from Ollama when a caller + * doesn't specify its own. Ollama otherwise falls back to whatever the model's + * Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct + * here, well under that model's 32768-token training context) and SILENTLY truncates + * any prompt that doesn't fit — dropping the middle of the prompt with no error + * surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the + * limit deliberate and stable instead of whatever Ollama happens to pick. + * + * 8192 is sized for CPU-only inference (the reference box is an i5-6600K running + * Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the + * constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but + * prompt-processing time scales with context, so this trades headroom against + * per-request latency rather than maxing out the model's full 32768-token capacity. + * Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit + * within this budget up front, rather than relying on Ollama to truncate for them. + */ +export const DEFAULT_NUM_CTX = 8192; +export const DEFAULT_NUM_PREDICT = 700; + /** * Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend * setting (GlobalSettings.aiServiceHost/Port), editable via the admin panel — @@ -15,7 +35,10 @@ export class OllamaProvider implements InferenceProvider { return `${this.host}:${this.port}`; } - async generate(prompt: string, opts: { model?: string; system?: string } = {}): Promise { + async generate( + prompt: string, + opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {} + ): Promise { const res = await fetch(`${this.base()}/api/generate`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -23,7 +46,11 @@ export class OllamaProvider implements InferenceProvider { model: opts.model, prompt, system: opts.system, - stream: false + stream: false, + options: { + num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX, + num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT + } }) }); if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`); diff --git a/backend/src/inference/provider.ts b/backend/src/inference/provider.ts index d994b1d..aaa61f8 100644 --- a/backend/src/inference/provider.ts +++ b/backend/src/inference/provider.ts @@ -1,5 +1,5 @@ export interface InferenceProvider { - generate(prompt: string, opts?: { model?: string; system?: string }): Promise; + generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise; embed(text: string, opts?: { model?: string }): Promise; listModels(): Promise; isReachable(): Promise; diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index d52a539..2cbac86 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -1,8 +1,28 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { ContentItem, MergedArticle } from '../storage/db/types.js'; +import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js'; +import { logger } from '../storage/db/logs.js'; const TAG_DELIMITER = '---TAGS---'; +// Ollama truncates prompts that don't fit its context window by keeping a small prefix +// and dropping everything else in the middle — silently, with no error, and with no +// regard for which sources end up cut (see ollama-provider.ts for the incident that +// prompted this). Rather than relying on that, prompts here are sized to fit +// DEFAULT_NUM_CTX up front: each source/article gets an equal character budget, cut only +// when the whole prompt would otherwise overflow, so every source stays at least +// partially represented (and attributable) instead of some being dropped outright. +// ~4 chars/token is a rough heuristic (no tokenizer available here) — good enough for a +// safety margin, not meant to be exact. +const CHARS_PER_TOKEN = 4; +const RESERVED_OVERHEAD_TOKENS = 300; // system prompt + per-entry headers/formatting +const MAX_INPUT_CHARS = (DEFAULT_NUM_CTX - DEFAULT_NUM_PREDICT - RESERVED_OVERHEAD_TOKENS) * CHARS_PER_TOKEN; +const MIN_ENTRY_CHARS = 300; // floor so a huge cluster/recap doesn't shrink every entry to nothing + +function capEntryText(text: string, budgetChars: number): string { + return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text; +} + const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that: - Summarizes what has happened across the period covered, in chronological order - Highlights the most significant developments rather than restating every article @@ -27,9 +47,17 @@ export interface SynthesisResult { } function buildPrompt(items: ContentItem[]): string { - return items - .map((item, i) => `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${item.summary}`) - .join('\n\n'); + const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); + let truncated = 0; + const entries = items.map((item, i) => { + const summary = capEntryText(item.summary, budgetPerItem); + if (summary !== item.summary) truncated++; + return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`; + }); + if (truncated > 0) { + logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return entries.join('\n\n'); } function parseResult(raw: string): SynthesisResult { @@ -48,15 +76,22 @@ export async function synthesizeArticle( items: ContentItem[] ): Promise { const prompt = buildPrompt(items); - const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); } function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string { - const entries = articles - .map((article, i) => `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${article.body}`) - .join('\n\n'); - return `Tracked event: ${eventName}\n\n${entries}`; + const budgetPerArticle = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / articles.length)); + let truncated = 0; + const entries = articles.map((article, i) => { + const body = capEntryText(article.body, budgetPerArticle); + if (body !== article.body) truncated++; + return `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${body}`; + }); + if (truncated > 0) { + logger.warn('events', `Trimmed ${truncated}/${articles.length} recap article bod${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`); + } + return `Tracked event: ${eventName}\n\n${entries.join('\n\n')}`; } /** @@ -74,6 +109,11 @@ export async function synthesizeRecap( articles: MergedArticle[] ): Promise { const prompt = buildRecapPrompt(eventName, articles); - const raw = await provider.generate(prompt, { model, system: RECAP_SYSTEM_PROMPT }); + const raw = await provider.generate(prompt, { + model, + system: RECAP_SYSTEM_PROMPT, + numCtx: DEFAULT_NUM_CTX, + numPredict: DEFAULT_NUM_PREDICT + }); return parseResult(raw); }