Claude
ee585ea65c
Ollama was defaulting to a 4096-token context (vs. the model's 32768 training context) and silently truncating any oversized prompt by dropping content from the middle, with no error surfaced anywhere — observed losing ~53% of a merge-cluster prompt in production. Two prompt builders (buildPrompt/buildRecapPrompt) concatenated all source summaries/article bodies with no size cap, so a cluster with enough sources (or a recap spanning enough articles) could easily exceed the window. Fix: OllamaProvider.generate() now always sends explicit num_ctx/ num_predict options (sized for CPU-only inference — i5-6600K, no GPU, ~17 tok/s prompt processing) instead of leaving Ollama to pick a default. synthesis.ts now caps prompt size itself before it ever reaches Ollama, giving each source/article an equal character budget and trimming individual entries rather than dropping whole ones off the end — every source stays at least partially represented and attributable. Trims are logged via the existing admin log stream instead of failing silently.
88 lines
3.3 KiB
TypeScript
88 lines
3.3 KiB
TypeScript
import type { InferenceProvider } from './provider.js';
|
|
|
|
/**
|
|
* Default context window / max-generation length requested from Ollama when a caller
|
|
* doesn't specify its own. Ollama otherwise falls back to whatever the model's
|
|
* Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct
|
|
* here, well under that model's 32768-token training context) and SILENTLY truncates
|
|
* any prompt that doesn't fit — dropping the middle of the prompt with no error
|
|
* surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the
|
|
* limit deliberate and stable instead of whatever Ollama happens to pick.
|
|
*
|
|
* 8192 is sized for CPU-only inference (the reference box is an i5-6600K running
|
|
* Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the
|
|
* constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but
|
|
* prompt-processing time scales with context, so this trades headroom against
|
|
* per-request latency rather than maxing out the model's full 32768-token capacity.
|
|
* Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit
|
|
* within this budget up front, rather than relying on Ollama to truncate for them.
|
|
*/
|
|
export const DEFAULT_NUM_CTX = 8192;
|
|
export const DEFAULT_NUM_PREDICT = 700;
|
|
|
|
/**
|
|
* Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend
|
|
* setting (GlobalSettings.aiServiceHost/Port), editable via the admin panel —
|
|
* see the Connections tab and "AI service connection" in the schema doc.
|
|
*/
|
|
export class OllamaProvider implements InferenceProvider {
|
|
constructor(
|
|
private host: string,
|
|
private port: number
|
|
) {}
|
|
|
|
private base(): string {
|
|
return `${this.host}:${this.port}`;
|
|
}
|
|
|
|
async generate(
|
|
prompt: string,
|
|
opts: { model?: string; system?: string; numCtx?: number; numPredict?: number } = {}
|
|
): Promise<string> {
|
|
const res = await fetch(`${this.base()}/api/generate`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
model: opts.model,
|
|
prompt,
|
|
system: opts.system,
|
|
stream: false,
|
|
options: {
|
|
num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX,
|
|
num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT
|
|
}
|
|
})
|
|
});
|
|
if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`);
|
|
const data = (await res.json()) as { response: string };
|
|
return data.response;
|
|
}
|
|
|
|
async embed(text: string, opts: { model?: string } = {}): Promise<number[]> {
|
|
const res = await fetch(`${this.base()}/api/embeddings`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ model: opts.model, prompt: text })
|
|
});
|
|
if (!res.ok) throw new Error(`Ollama embed failed: ${res.status} ${await res.text()}`);
|
|
const data = (await res.json()) as { embedding: number[] };
|
|
return data.embedding;
|
|
}
|
|
|
|
async listModels(): Promise<string[]> {
|
|
const res = await fetch(`${this.base()}/api/tags`);
|
|
if (!res.ok) throw new Error(`Ollama listModels failed: ${res.status}`);
|
|
const data = (await res.json()) as { models: { name: string }[] };
|
|
return data.models.map((m) => m.name);
|
|
}
|
|
|
|
async isReachable(): Promise<boolean> {
|
|
try {
|
|
const res = await fetch(`${this.base()}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
|
return res.ok;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
}
|