Salastil/homefeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix silent Ollama prompt truncation in synthesis pipeline

Browse Source 
Ollama was defaulting to a 4096-token context (vs. the model's 32768
training context) and silently truncating any oversized prompt by
dropping content from the middle, with no error surfaced anywhere —
observed losing ~53% of a merge-cluster prompt in production. Two
prompt builders (buildPrompt/buildRecapPrompt) concatenated all
source summaries/article bodies with no size cap, so a cluster with
enough sources (or a recap spanning enough articles) could easily
exceed the window.

Fix: OllamaProvider.generate() now always sends explicit num_ctx/
num_predict options (sized for CPU-only inference — i5-6600K, no GPU,
~17 tok/s prompt processing) instead of leaving Ollama to pick a
default. synthesis.ts now caps prompt size itself before it ever
reaches Ollama, giving each source/article an equal character budget
and trimming individual entries rather than dropping whole ones off
the end — every source stays at least partially represented and
attributable. Trims are logged via the existing admin log stream
instead of failing silently.
This commit is contained in:
Claude
2026-07-27 03:31:31 +00:00
parent b30657a161
commit c45d7d5acf
3 changed files with 79 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/inference/ollama-provider.ts
+29 -2
View File
@@ -1,5 +1,25 @@
import type { InferenceProvider } from './provider.js';
/**
* Default context window / max-generation length requested from Ollama when a caller
* doesn't specify its own. Ollama otherwise falls back to whatever the model's
* Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct
* here, well under that model's 32768-token training context) and SILENTLY truncates
* any prompt that doesn't fit — dropping the middle of the prompt with no error
* surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the
* limit deliberate and stable instead of whatever Ollama happens to pick.
*
* 8192 is sized for CPU-only inference (the reference box is an i5-6600K running
* Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the
* constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but
* prompt-processing time scales with context, so this trades headroom against
* per-request latency rather than maxing out the model's full 32768-token capacity.
* Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit
* within this budget up front, rather than relying on Ollama to truncate for them.
*/
export const DEFAULT_NUM_CTX = 8192;
export const DEFAULT_NUM_PREDICT = 700;
/**
* Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend
* setting (GlobalSettings.aiServiceHost/Port), editable via the admin panel —
@@ -15,7 +35,10 @@ export class OllamaProvider implements InferenceProvider {
return `${this.host}:${this.port}`;
}
async generate(prompt: string, opts: { model?: string; system?: string } = {}): Promise<string> {
async generate(
prompt: string,
opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {}
): Promise<string> {
const res = await fetch(`${this.base()}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -23,7 +46,11 @@ export class OllamaProvider implements InferenceProvider {
model: opts.model,
prompt,
system: opts.system,
stream: false
stream: false,
options: {
num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX,
num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT
}
})
});
if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`);
backend/src/inference/provider.ts
+1 -1
View File
@@ -1,5 +1,5 @@
export interface InferenceProvider {
generate(prompt: string, opts?: { model?: string; system?: string }): Promise<string>;
generate(prompt: string, opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number }): Promise<string>;
embed(text: string, opts?: { model?: string }): Promise<number[]>;
listModels(): Promise<string[]>;
isReachable(): Promise<boolean>;
backend/src/pipeline/synthesis.ts
+49 -9
View File
@@ -1,8 +1,28 @@
import type { InferenceProvider } from '../inference/provider.js';
import type { ContentItem, MergedArticle } from '../storage/db/types.js';
import { DEFAULT_NUM_CTX, DEFAULT_NUM_PREDICT } from '../inference/ollama-provider.js';
import { logger } from '../storage/db/logs.js';
const TAG_DELIMITER = '---TAGS---';
// Ollama truncates prompts that don't fit its context window by keeping a small prefix
// and dropping everything else in the middle — silently, with no error, and with no
// regard for which sources end up cut (see ollama-provider.ts for the incident that
// prompted this). Rather than relying on that, prompts here are sized to fit
// DEFAULT_NUM_CTX up front: each source/article gets an equal character budget, cut only
// when the whole prompt would otherwise overflow, so every source stays at least
// partially represented (and attributable) instead of some being dropped outright.
// ~4 chars/token is a rough heuristic (no tokenizer available here) — good enough for a
// safety margin, not meant to be exact.
const CHARS_PER_TOKEN = 4;
const RESERVED_OVERHEAD_TOKENS = 300; // system prompt + per-entry headers/formatting
const MAX_INPUT_CHARS = (DEFAULT_NUM_CTX - DEFAULT_NUM_PREDICT - RESERVED_OVERHEAD_TOKENS) * CHARS_PER_TOKEN;
const MIN_ENTRY_CHARS = 300; // floor so a huge cluster/recap doesn't shrink every entry to nothing
function capEntryText(text: string, budgetChars: number): string {
return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text;
}
const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that:
- Summarizes what has happened across the period covered, in chronological order
- Highlights the most significant developments rather than restating every article
@@ -27,9 +47,17 @@ export interface SynthesisResult {
}
function buildPrompt(items: ContentItem[]): string {
return items
.map((item, i) => `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${item.summary}`)
.join('\n\n');
const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length));
let truncated = 0;
const entries = items.map((item, i) => {
const summary = capEntryText(item.summary, budgetPerItem);
if (summary !== item.summary) truncated++;
return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${summary}`;
});
if (truncated > 0) {
logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source summar${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`);
}
return entries.join('\n\n');
}
function parseResult(raw: string): SynthesisResult {
@@ -48,15 +76,22 @@ export async function synthesizeArticle(
items: ContentItem[]
): Promise<SynthesisResult> {
const prompt = buildPrompt(items);
const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT });
const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT });
return parseResult(raw);
}
function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string {
const entries = articles
.map((article, i) => `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${article.body}`)
.join('\n\n');
return `Tracked event: ${eventName}\n\n${entries}`;
const budgetPerArticle = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / articles.length));
let truncated = 0;
const entries = articles.map((article, i) => {
const body = capEntryText(article.body, budgetPerArticle);
if (body !== article.body) truncated++;
return `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${body}`;
});
if (truncated > 0) {
logger.warn('events', `Trimmed ${truncated}/${articles.length} recap article bod${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`);
}
return `Tracked event: ${eventName}\n\n${entries.join('\n\n')}`;
}
/**
@@ -74,6 +109,11 @@ export async function synthesizeRecap(
articles: MergedArticle[]
): Promise<SynthesisResult> {
const prompt = buildRecapPrompt(eventName, articles);
const raw = await provider.generate(prompt, { model, system: RECAP_SYSTEM_PROMPT });
const raw = await provider.generate(prompt, {
model,
system: RECAP_SYSTEM_PROMPT,
numCtx: DEFAULT_NUM_CTX,
numPredict: DEFAULT_NUM_PREDICT
});
return parseResult(raw);
}