homefeed/backend/src/pipeline/publish.ts

import { randomUUID } from 'node:crypto';
import type { InferenceProvider } from '../inference/provider.js';
import type { Cluster } from './clustering.js';
import { synthesizeArticle, synthesizeRecap } from './synthesis.js';
import { selectBestImage, faviconUrlFor } from './image-selection.js';
import { downloadAndStore, promoteToPublished, storeMediaBuffer } from '../storage/media/index.js';
import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js';
import { logger } from '../storage/db/logs.js';
import * as articles from '../storage/db/articles.js';
import * as tags from '../storage/db/tags.js';
import * as sources from '../storage/db/sources.js';
import type {
	GlobalSettings,
	MergedArticle,
	ContentItem,
	TweetMediaItem,
	QuotedTweet,
	TelegramMediaItem,
	TelegramMediaRef,
	TrackedEvent
} from '../storage/db/types.js';

const FOLLOW_UP_LOOKBACK_DAYS = 3;

function uniqueCategories(items: ContentItem[]): string[] {
	const cats = new Set<string>();
	for (const item of items) {
		const source = sources.getSource(item.sourceId);
		for (const cat of source?.category ?? []) cats.add(cat);
	}
	return [...cats];
}

/** An article shows on the homepage if any of its contributing sources opted into "Push to Top Stories?". */
function anyPushesToTopStories(items: ContentItem[]): boolean {
	return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false);
}

/** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */
function deriveTitle(body: string): string {
	const firstLine = body.split('\n')[0];
	return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine;
}

/**
 * Resolves the hero image for a regular (non-tweet) article: try the best candidate
 * from the source items, download and locally host it; if there isn't one, fall back
 * to the site's favicon rather than leaving the article with no art at all. Tweets
 * never reach this function — see resolveTweetMediaUrl below, which applies the
 * admin-configured Nitter media mode instead of always downloading, and skips the
 * favicon fallback entirely (a Nitter instance's own favicon slapped onto an
 * image-less tweet reads as a mistake, not a placeholder; TweetCard.svelte already
 * handles no-image tweets gracefully with no image at all).
 */
async function resolveHeroImage(
	items: ContentItem[],
	primaryLink: string
): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> {
	const selected = selectBestImage(items);

	if (selected) {
		const stored = await downloadAndStore(selected.url, 'published', {});
		if (stored) {
			return {
				heroImage: { url: stored.servedPath, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason },
				storedMediaId: stored.id
			};
		}
		// Download failed — fall back to the hotlinked URL rather than losing the image entirely.
		return { heroImage: selected, storedMediaId: null };
	}

	const favicon = faviconUrlFor(primaryLink);
	if (favicon) {
		const stored = await downloadAndStore(favicon, 'published', {});
		if (stored) {
			return {
				heroImage: { url: stored.servedPath, sourceItemId: items[0]?.id ?? '', selectionReason: 'Site favicon — no article image available' },
				storedMediaId: stored.id
			};
		}
	}

	return { heroImage: null, storedMediaId: null };
}

/**
 * Applies the admin-configured Nitter media mode (Retention tab) to a single
 * externally-hosted tweet media URL — an attached photo or the author's avatar.
 * 'self-host' downloads and serves it locally like any other article image;
 * 'proxy' routes it through this server's own /media/proxy route so only this
 * server's IP is ever exposed to Twitter/the Nitter instance's CDN (the
 * "anonymity" the admin asked for) without persisting anything to disk; 'direct'
 * hotlinks the original URL unchanged, the cheapest option with no server involvement.
 */
async function resolveTweetMediaUrl(
	url: string,
	mode: GlobalSettings['nitterMediaMode']
): Promise<{ url: string; storedMediaId: string | null }> {
	if (mode === 'direct') return { url, storedMediaId: null };

	if (mode === 'proxy') {
		return { url: `/media/proxy?url=${encodeURIComponent(url)}`, storedMediaId: null };
	}

	const stored = await downloadAndStore(url, 'published', {});
	if (stored) return { url: stored.servedPath, storedMediaId: stored.id };
	// Download failed — fall back to hotlinking rather than losing the media entirely.
	return { url, storedMediaId: null };
}

/**
 * Resolves every attached photo/video/gif's url — and, for video/gif, its poster
 * thumbnail — through the admin's chosen Nitter media mode. Order and item count are
 * preserved; TweetCard.svelte renders this array directly, there's no separate
 * "hero image" concept for tweets the way there is for regular articles.
 */
async function resolveTweetMedia(
	media: TweetMediaItem[],
	mode: GlobalSettings['nitterMediaMode']
): Promise<{ media: TweetMediaItem[]; storedMediaIds: string[] }> {
	const storedMediaIds: string[] = [];
	const resolved: TweetMediaItem[] = [];

	for (const item of media) {
		const url = await resolveTweetMediaUrl(item.url, mode);
		if (url.storedMediaId) storedMediaIds.push(url.storedMediaId);

		let thumbnailUrl = item.thumbnailUrl;
		if (thumbnailUrl) {
			const resolvedThumb = await resolveTweetMediaUrl(thumbnailUrl, mode);
			thumbnailUrl = resolvedThumb.url;
			if (resolvedThumb.storedMediaId) storedMediaIds.push(resolvedThumb.storedMediaId);
		}

		resolved.push({ ...item, url: url.url, thumbnailUrl });
	}

	return { media: resolved, storedMediaIds };
}

function guessTelegramExtension(mimeType: string | null, kind: 'photo' | 'video' | 'gif'): string {
	if (mimeType?.includes('png')) return '.png';
	if (mimeType?.includes('webp')) return '.webp';
	if (mimeType?.includes('jpeg') || mimeType?.includes('jpg')) return '.jpg';
	if (mimeType?.includes('mp4')) return '.mp4';
	return kind === 'photo' ? '.jpg' : '.mp4';
}

/**
 * Resolves a single Telegram media reference into a servable url per the admin's
 * chosen telegramMediaMode (Retention tab). Unlike Nitter — which starts from an
 * already-public CDN URL — Telegram media only exists behind the authenticated MTProto
 * session, so there's no 'direct' hotlink option: 'self-host' downloads it now (via the
 * live session) and stores it locally exactly like every other self-hosted image;
 * 'proxy' doesn't touch Telegram at all here, it just builds a url the live
 * telegram-proxy route resolves (via that same session) on each view.
 */
async function resolveTelegramMediaUrl(
	channelUsername: string,
	ref: TelegramMediaRef,
	mode: GlobalSettings['telegramMediaMode']
): Promise<{ item: TelegramMediaItem; storedMediaId: string | null } | null> {
	if (mode === 'proxy') {
		const url = `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&message=${encodeURIComponent(ref.messageId)}&type=${ref.type}`;
		return { item: { type: ref.type, url, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: null };
	}

	try {
		const buffer = await downloadMessageMedia(channelUsername, ref.messageId);
		if (!buffer) return null;
		const ext = guessTelegramExtension(ref.mimeType, ref.type);
		const stored = storeMediaBuffer(buffer, ext, `telegram-message:${channelUsername}:${ref.messageId}`, 'published', {});
		return { item: { type: ref.type, url: stored.servedPath, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: stored.id };
	} catch (err) {
		logger.error('telegram', `Failed to self-host media for message ${ref.messageId}: ${(err as Error).message}`);
		return null;
	}
}

/** Resolves every attached photo/video/gif ref for one message — order preserved, failed items dropped rather than leaving a broken entry. */
async function resolveTelegramMedia(
	channelUsername: string,
	refs: TelegramMediaRef[],
	mode: GlobalSettings['telegramMediaMode']
): Promise<{ media: TelegramMediaItem[]; storedMediaIds: string[] }> {
	const media: TelegramMediaItem[] = [];
	const storedMediaIds: string[] = [];
	for (const ref of refs) {
		const resolved = await resolveTelegramMediaUrl(channelUsername, ref, mode);
		if (resolved) {
			media.push(resolved.item);
			if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
		}
	}
	return { media, storedMediaIds };
}

async function resolveTelegramAvatarUrl(
	channelUsername: string,
	mode: GlobalSettings['telegramMediaMode']
): Promise<{ url: string | null; storedMediaId: string | null }> {
	if (mode === 'proxy') {
		return { url: `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&avatar=1`, storedMediaId: null };
	}
	try {
		const buffer = await downloadChannelAvatar(channelUsername);
		if (!buffer) return { url: null, storedMediaId: null };
		const stored = storeMediaBuffer(buffer, '.jpg', `telegram-avatar:${channelUsername}`, 'published', {});
		return { url: stored.servedPath, storedMediaId: stored.id };
	} catch (err) {
		logger.error('telegram', `Failed to self-host avatar for "${channelUsername}": ${(err as Error).message}`);
		return { url: null, storedMediaId: null };
	}
}

/** Resolves a quote-tweet's embedded image (if any) through the Nitter media mode, same as any other tweet media. */
async function resolveQuotedTweet(
	quoted: QuotedTweet,
	mode: GlobalSettings['nitterMediaMode']
): Promise<{ quotedTweet: QuotedTweet; storedMediaId: string | null }> {
	if (!quoted.imageUrl) return { quotedTweet: quoted, storedMediaId: null };
	const resolved = await resolveTweetMediaUrl(quoted.imageUrl, mode);
	return { quotedTweet: { ...quoted, imageUrl: resolved.url }, storedMediaId: resolved.storedMediaId };
}

/**
 * Publishes a single item as-is, with no AI calls at all — used when the AI service
 * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
 * after the backend launches" requirement). No rewriting, no tag extraction, no
 * embedding. This is deliberately a lesser version of the real pipeline: once Ollama
 * is available, newly-ingested items get the full embed/cluster/synthesize treatment
 * and can be linked as follow-ups to these passthrough articles via the normal
 * tag-based thread detection — but these earlier articles aren't retroactively
 * rewritten or merged with anything after the fact.
 */
export async function publishDirect(
	item: ContentItem,
	settings: GlobalSettings,
	opts: { eventId?: string } = {}
): Promise<MergedArticle> {
	const category = uniqueCategories([item]);
	const storedMediaIds: string[] = [];

	// Tweets and Telegram messages never get a "hero image" — their own card components
	// render tweet.media / telegramMessage.media directly.
	let heroImage: MergedArticle['heroImage'] = null;
	if (!item.tweet && !item.telegramMessage) {
		const resolved = await resolveHeroImage([item], item.link);
		heroImage = resolved.heroImage;
		if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
	}

	const video = item.videos[0]
		? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
		: null;

	let tweet: MergedArticle['tweet'] = null;
	if (item.tweet) {
		let avatarUrl: string | null = null;
		if (item.tweet.avatarUrl) {
			const resolved = await resolveTweetMediaUrl(item.tweet.avatarUrl, settings.nitterMediaMode);
			avatarUrl = resolved.url;
			if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId);
		}
		const resolvedMedia = await resolveTweetMedia(item.tweet.media, settings.nitterMediaMode);
		storedMediaIds.push(...resolvedMedia.storedMediaIds);

		let quotedTweet: QuotedTweet | null = null;
		if (item.tweet.quotedTweet) {
			const resolvedQuoted = await resolveQuotedTweet(item.tweet.quotedTweet, settings.nitterMediaMode);
			quotedTweet = resolvedQuoted.quotedTweet;
			if (resolvedQuoted.storedMediaId) storedMediaIds.push(resolvedQuoted.storedMediaId);
		}

		tweet = {
			authorName: item.tweet.authorName,
			authorHandle: item.tweet.authorHandle,
			avatarUrl,
			sourceItemId: item.id,
			media: resolvedMedia.media,
			repostedByHandle: item.tweet.repostedByHandle,
			quotedTweet
		};
	}

	let telegramMessage: MergedArticle['telegramMessage'] = null;
	if (item.telegramMessage) {
		const { channelUsername, sourceChannelUsername } = item.telegramMessage;

		// The avatar shown is whoever is DISPLAYED (the origin channel on a forward) — if
		// that has no public handle, there's no avatar to fetch at all, regardless of mode.
		const avatar = channelUsername
			? await resolveTelegramAvatarUrl(channelUsername, settings.telegramMediaMode)
			: { url: null, storedMediaId: null };
		if (avatar.storedMediaId) storedMediaIds.push(avatar.storedMediaId);

		// Attached media always lives on the polled channel's own copy of the message
		// (forward or not), so media resolution uses sourceChannelUsername, never the
		// (possibly different, possibly null) displayed channelUsername.
		const resolvedMedia = await resolveTelegramMedia(sourceChannelUsername, item.telegramMessage.media, settings.telegramMediaMode);
		storedMediaIds.push(...resolvedMedia.storedMediaIds);

		telegramMessage = {
			channelName: item.telegramMessage.channelName,
			channelUsername,
			channelAvatarUrl: avatar.url,
			sourceItemId: item.id,
			media: resolvedMedia.media,
			repostedByHandle: item.telegramMessage.repostedByHandle
		};
	}

	const article = await articles.insertArticle({
		title: item.title,
		body: item.body || item.summary,
		heroImage,
		video,
		tweet,
		telegramMessage,
		category,
		geo: item.geo,
		eventId: opts.eventId ?? item.eventId,
		sourceCount: 1,
		sources: [
			{
				itemId: item.id,
				sourceName: sources.getSource(item.sourceId)?.name ?? 'Unknown source',
				link: item.link,
				publishedAt: item.publishedAt
			}
		],
		// Passthrough articles show the original feed date — only AI-synthesized/merged
		// articles get a "modified" publish timeframe reflecting when the merge happened.
		publishedAt: item.publishedAt,
		updatedAt: item.publishedAt,
		mergeConfidence: 1.0,
		tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement
		threadId: randomUUID(),
		previousArticleId: null,
		nextArticleId: null,
		topStories: anyPushesToTopStories([item]),
		isRecap: false
	});

	for (const id of storedMediaIds) promoteToPublished(id, article.id);
	return article;
}

/**
 * Publishing is always automatic — there's no draft/review state (see schema doc).
 * Callers should route a size-1 cluster to publishDirect instead — there's nothing to
 * merge, so an LLM rewrite would only add risk (hallucinated attribution, altered
 * facts) for no synthesis benefit. See priorityQueue.ts's runSynthesisCycle.
 */
export async function publishCluster(
	provider: InferenceProvider,
	settings: GlobalSettings,
	cluster: Cluster,
	opts: { eventId?: string } = {}
): Promise<MergedArticle> {
	const items = cluster.items;

	const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source']));
	const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings);

	const resolvedTags = [];
	for (const label of tagLabels) {
		try {
			const embedding = await provider.embed(label, { model: settings.selectedModels.embedding });
			resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold));
		} catch (err) {
			logger.error('synthesis', `Tag embedding failed for "${label}": ${(err as Error).message}`);
		}
	}
	const tagIds = resolvedTags.map((t) => t.id);

	const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? '');
	const videoItem = items.find((i) => i.videos.length > 0);
	const video = videoItem
		? {
				url: videoItem.videos[0].url,
				provider: videoItem.videos[0].provider,
				embedUrl: videoItem.videos[0].embedHtml,
				sourceItemId: videoItem.id
			}
		: null;

	const category = uniqueCategories(items);
	const geo = items.find((i) => i.geo)?.geo ?? null;

	const itemSources = items.map((item) => ({
		itemId: item.id,
		sourceName: sources.getSource(item.sourceId)?.name ?? 'Unknown source',
		link: item.link,
		publishedAt: item.publishedAt
	}));

	// Follow-up detection: only links to a previous article if enough time AND enough
	// new corroborating sources have passed the admin's thresholds (see MergeTab).
	// Otherwise this cluster just becomes its own independent thread — holding items
	// in a "pending, not yet enough to follow up" state is a reasonable future
	// improvement but adds a queue state this MVP doesn't have yet.
	const previous = tagIds.length > 0 ? articles.findRecentArticleByTags(tagIds, FOLLOW_UP_LOOKBACK_DAYS) : null;
	let threadId: string = randomUUID();
	let previousArticleId: string | null = null;

	if (previous) {
		const hoursSince = (Date.now() - new Date(previous.publishedAt).getTime()) / 3600_000;
		const previousLinks = new Set(previous.sources.map((s) => s.link));
		const newSourceCount = itemSources.filter((s) => !previousLinks.has(s.link)).length;

		if (hoursSince >= settings.followUpMinHoursSinceLast && newSourceCount >= settings.followUpMinNewSources) {
			threadId = previous.threadId;
			previousArticleId = previous.id;
		}
	}

	const now = new Date().toISOString();
	const article = articles.insertArticle({
		title: deriveTitle(body),
		body,
		heroImage,
		video,
		tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass
		telegramMessage: null, // telegram messages never reach clustering either — same bypass
		category,
		geo,
		eventId: opts.eventId ?? items[0]?.eventId ?? null,
		sourceCount: items.length,
		sources: itemSources,
		publishedAt: now,
		updatedAt: now,
		mergeConfidence: items.length > 1 ? 0.8 : 1.0,
		tags: tagIds,
		threadId,
		previousArticleId,
		nextArticleId: null,
		topStories: anyPushesToTopStories(items),
		isRecap: false
	});

	if (storedMediaId) {
		promoteToPublished(storedMediaId, article.id);
	}

	return article;
}

/**
 * Builds the periodic AI recap for a tracked event — a standalone summary article of
 * everything published under this event since the last recap, additive alongside those
 * individual articles rather than replacing or consuming them (see eventsRecap.ts).
 * Deliberately much lighter than publishCluster: no embedding/clustering, no follow-up
 * thread detection (each recap stands alone), hero image and sources are just carried
 * over from the constituent articles rather than re-resolved from raw content items.
 */
export async function publishEventRecap(
	provider: InferenceProvider,
	settings: GlobalSettings,
	event: TrackedEvent,
	constituents: MergedArticle[]
): Promise<MergedArticle> {
	const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents, settings);

	const resolvedTags = [];
	for (const label of tagLabels) {
		try {
			const embedding = await provider.embed(label, { model: settings.selectedModels.embedding });
			resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold));
		} catch (err) {
			logger.error('events', `Tag embedding failed for "${label}": ${(err as Error).message}`);
		}
	}

	const category = [...new Set(constituents.flatMap((a) => a.category))];
	const heroImage = constituents.find((a) => a.heroImage)?.heroImage ?? null;
	const now = new Date().toISOString();

	return articles.insertArticle({
		title: `${event.name}: recap`,
		body,
		heroImage,
		video: null,
		tweet: null,
		telegramMessage: null,
		category,
		geo: constituents.find((a) => a.geo)?.geo ?? null,
		eventId: event.id,
		sourceCount: constituents.flatMap((a) => a.sources).length,
		sources: constituents.flatMap((a) => a.sources),
		publishedAt: now,
		updatedAt: now,
		mergeConfidence: 1.0,
		tags: resolvedTags.map((t) => t.id),
		threadId: randomUUID(),
		previousArticleId: null,
		nextArticleId: null,
		topStories: constituents.some((a) => a.topStories),
		isRecap: true
	});
}