Fix synthesis prompt labeling sources by opaque ID, causing hallucinated attribution
buildPrompt() labeled each source with item.sourceId — an internal
DB foreign key like "src-e8dbf745-..." — never the outlet's actual
name. The model had no real outlet to attribute to, so on a
single-source item it fell back to copying the illustrative example
names straight out of its own system prompt ("Reuters reported...",
"AP notes...") and fabricated a two-outlet merge out of one real
6abc article. The article's sources metadata (built separately from
real DB records) was correct the whole time; only the AI-written body
text invented sources that were never in the input.
synthesizeArticle now takes a sourceId->name map (built in publish.ts
via the same sources.getSource() lookup already used for the sources
metadata) and buildPrompt labels each entry with the real name.
SYSTEM_PROMPT no longer gives concrete example outlet names to copy —
it references "each source's exact name as given below" and
explicitly forbids attributing to any outlet not actually provided.
Verified directly: captured the exact prompt text sent to a mock
provider and confirmed it now contains the real source name and never
the raw internal id.
This commit is contained in:
@@ -360,7 +360,8 @@ export async function publishCluster(
|
|||||||
): Promise<MergedArticle> {
|
): Promise<MergedArticle> {
|
||||||
const items = cluster.items;
|
const items = cluster.items;
|
||||||
|
|
||||||
const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items);
|
const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source']));
|
||||||
|
const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames);
|
||||||
|
|
||||||
const resolvedTags = [];
|
const resolvedTags = [];
|
||||||
for (const label of tagLabels) {
|
for (const label of tagLabels) {
|
||||||
|
|||||||
@@ -32,12 +32,12 @@ const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a
|
|||||||
After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`;
|
After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||||
|
|
||||||
const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that:
|
const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that:
|
||||||
- Attributes specific claims to the outlet that reported them (e.g. "Reuters reported...", "AP notes...")
|
- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below.
|
||||||
- Does not copy phrasing verbatim from any source
|
- Does not copy phrasing verbatim from any source
|
||||||
- Stays neutral and factual, without editorializing
|
- Stays neutral and factual, without editorializing
|
||||||
- Is 2-4 short paragraphs
|
- Is 2-4 short paragraphs
|
||||||
|
|
||||||
If only one source is provided, lightly rewrite it in your own words rather than merging.
|
If only one source is provided, lightly rewrite it in your own words rather than merging, and do not attribute it to any outlet other than that single given source.
|
||||||
|
|
||||||
After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`;
|
After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ export interface SynthesisResult {
|
|||||||
tagLabels: string[];
|
tagLabels: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildPrompt(items: ContentItem[]): string {
|
function buildPrompt(items: ContentItem[], sourceNames: Map<string, string>): string {
|
||||||
const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length));
|
const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(MAX_INPUT_CHARS / items.length));
|
||||||
let truncated = 0;
|
let truncated = 0;
|
||||||
const entries = items.map((item, i) => {
|
const entries = items.map((item, i) => {
|
||||||
@@ -57,7 +57,13 @@ function buildPrompt(items: ContentItem[]): string {
|
|||||||
const full = item.body || item.summary;
|
const full = item.body || item.summary;
|
||||||
const text = capEntryText(full, budgetPerItem);
|
const text = capEntryText(full, budgetPerItem);
|
||||||
if (text !== full) truncated++;
|
if (text !== full) truncated++;
|
||||||
return `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${text}`;
|
// The label here (not item.sourceId, an opaque internal id the model can't use)
|
||||||
|
// is the only real outlet name the model ever sees — without it, a small model
|
||||||
|
// has nothing to attribute to and falls back to copying the illustrative outlet
|
||||||
|
// names out of its own system prompt instructions instead (seen in production:
|
||||||
|
// a single-source item fabricating "Reuters reported..."/"AP notes..." wholesale).
|
||||||
|
const name = sourceNames.get(item.sourceId) ?? 'Unknown source';
|
||||||
|
return `Source ${i + 1} (${name}):\nTitle: ${item.title}\nSummary: ${text}`;
|
||||||
});
|
});
|
||||||
if (truncated > 0) {
|
if (truncated > 0) {
|
||||||
logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`);
|
logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`);
|
||||||
@@ -78,9 +84,10 @@ function parseResult(raw: string): SynthesisResult {
|
|||||||
export async function synthesizeArticle(
|
export async function synthesizeArticle(
|
||||||
provider: InferenceProvider,
|
provider: InferenceProvider,
|
||||||
model: string,
|
model: string,
|
||||||
items: ContentItem[]
|
items: ContentItem[],
|
||||||
|
sourceNames: Map<string, string>
|
||||||
): Promise<SynthesisResult> {
|
): Promise<SynthesisResult> {
|
||||||
const prompt = buildPrompt(items);
|
const prompt = buildPrompt(items, sourceNames);
|
||||||
const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT });
|
const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT, numCtx: DEFAULT_NUM_CTX, numPredict: DEFAULT_NUM_PREDICT });
|
||||||
return parseResult(raw);
|
return parseResult(raw);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user