From 4b2def21511829efcbc0df4dc5019aa3156d038f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:37:37 +0000 Subject: [PATCH] Fix synthesis prompt labeling sources by opaque ID, causing hallucinated attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildPrompt() labeled each source with item.sourceId — an internal DB foreign key like "src-e8dbf745-..." — never the outlet's actual name. The model had no real outlet to attribute to, so on a single-source item it fell back to copying the illustrative example names straight out of its own system prompt ("Reuters reported...", "AP notes...") and fabricated a two-outlet merge out of one real 6abc article. The article's sources metadata (built separately from real DB records) was correct the whole time; only the AI-written body text invented sources that were never in the input. synthesizeArticle now takes a sourceId->name map (built in publish.ts via the same sources.getSource() lookup already used for the sources metadata) and buildPrompt labels each entry with the real name. SYSTEM_PROMPT no longer gives concrete example outlet names to copy — it references "each source's exact name as given below" and explicitly forbids attributing to any outlet not actually provided. Verified directly: captured the exact prompt text sent to a mock provider and confirmed it now contains the real source name and never the raw internal id. --- backend/src/pipeline/publish.ts | 3 ++- backend/src/pipeline/synthesis.ts | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index ab19350..d09a7b4 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -360,7 +360,8 @@ export async function publishCluster( ): Promise { const items = cluster.items; - const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items); + const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source'])); + const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames); const resolvedTags = []; for (const label of tagLabels) { diff --git a/backend/src/pipeline/synthesis.ts b/backend/src/pipeline/synthesis.ts index 1405563..fd1898f 100644 --- a/backend/src/pipeline/synthesis.ts +++ b/backend/src/pipeline/synthesis.ts @@ -32,12 +32,12 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`; const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that: -- Attributes specific claims to the outlet that reported them (e.g. "Reuters reported...", "AP notes...") +- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below. - Does not copy phrasing verbatim from any source - Stays neutral and factual, without editorializing - Is 2-4 short paragraphs -If only one source is provided, lightly rewrite it in your own words rather than merging. +If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source. After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`; @@ -46,7 +46,7 @@ export interface SynthesisResult { tagLabels: string[]; } -function buildPrompt(items: ContentItem[]): string { +function buildPrompt(items: ContentItem[], sourceNames: Map): string { const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length)); let truncated = 0; const entries = items.map((item, i) => { @@ -57,7 +57,13 @@ function buildPrompt(items: ContentItem[]): string { const full = item.body || item.summary; const text = capEntryText(full, budgetPerItem); if (text !== full) truncated++; - return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`; + // The label here (not item.sourceId, an opaque internal id the model can't use) + // is the only real outlet name the model ever sees — without it, a small model + // has nothing to attribute to and falls back to copying the illustrative outlet + // names out of its own system prompt instructions instead (seen in production: + // a single-source item fabricating "Reuters reported..."/"AP notes..." wholesale). + const name = sourceNames.get(item.sourceId) ?? 'Unknown source'; + return `Source ${i + 1} (${name}):\nTitle: ${item.title}\nSummary: ${text}`; }); if (truncated > 0) { logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`); @@ -78,9 +84,10 @@ function parseResult(raw: string): SynthesisResult { export async function synthesizeArticle( provider: InferenceProvider, model: string, - items: ContentItem[] + items: ContentItem[], + sourceNames: Map ): Promise { - const prompt = buildPrompt(items); + const prompt = buildPrompt(items, sourceNames); const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT }); return parseResult(raw); }