diff --git a/backend/src/api/telegramMediaProxy.ts b/backend/src/api/telegramMediaProxy.ts new file mode 100644 index 0000000..2c55d51 --- /dev/null +++ b/backend/src/api/telegramMediaProxy.ts @@ -0,0 +1,81 @@ +// Backs the "proxy" Telegram media mode (see Retention tab / GlobalSettings.telegramMediaMode). +// Unlike media/proxy.ts (which forwards a plain HTTP request to an already-public CDN +// URL), Telegram media has no public URL at all — every request here re-authenticates +// to Telegram via the logged-in account (telegram/client.ts) and streams the result, so +// nothing is written to disk and only this server ever touches Telegram's servers. A +// small time-boxed in-memory cache absorbs repeat views of the same message/avatar +// without hitting Telegram (and its rate limits) on every single page load. + +import type { FastifyInstance } from 'fastify'; +import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js'; +import { logger } from '../storage/db/logs.js'; + +const CACHE_TTL_MS = 10 * 60_000; +const CACHE_MAX_ENTRIES = 200; + +interface CacheEntry { + buffer: Buffer; + contentType: string; + expiresAt: number; +} + +const cache = new Map(); + +function cacheGet(key: string): CacheEntry | null { + const entry = cache.get(key); + if (!entry) return null; + if (entry.expiresAt < Date.now()) { + cache.delete(key); + return null; + } + return entry; +} + +function cacheSet(key: string, entry: CacheEntry) { + if (cache.size >= CACHE_MAX_ENTRIES) { + const oldest = cache.keys().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + cache.set(key, entry); +} + +function contentTypeFor(type: string | undefined): string { + if (type === 'video' || type === 'gif') return 'video/mp4'; + return 'image/jpeg'; +} + +export async function registerTelegramMediaProxy(app: FastifyInstance) { + app.get('/media/telegram-proxy', async (req, reply) => { + const { channel, message, avatar, type } = req.query as { + channel?: string; + message?: string; + avatar?: string; + type?: string; + }; + if (!channel) return reply.code(400).send({ error: 'channel required' }); + if (!avatar && !message) return reply.code(400).send({ error: 'message or avatar required' }); + + const cacheKey = avatar ? `avatar:${channel}` : `message:${channel}:${message}`; + const cached = cacheGet(cacheKey); + if (cached) { + reply.header('content-type', cached.contentType); + reply.header('cache-control', 'private, max-age=300'); + return reply.send(cached.buffer); + } + + try { + const buffer = avatar ? await downloadChannelAvatar(channel) : await downloadMessageMedia(channel, message!); + if (!buffer) return reply.code(404).send(); + + const contentType = contentTypeFor(type); + cacheSet(cacheKey, { buffer, contentType, expiresAt: Date.now() + CACHE_TTL_MS }); + + reply.header('content-type', contentType); + reply.header('cache-control', 'private, max-age=300'); + return reply.send(buffer); + } catch (err) { + logger.error('telegram', `Proxy fetch failed for channel=${channel} message=${message ?? 'avatar'}: ${(err as Error).message}`); + return reply.code(502).send(); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 96c73b9..6fbe6d2 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,6 +9,7 @@ import { registerAuth } from './api/auth.js'; import { registerPublicRoutes } from './api/public.js'; import { registerAdminRoutes } from './api/admin.js'; import { registerMediaProxy } from './api/mediaProxy.js'; +import { registerTelegramMediaProxy } from './api/telegramMediaProxy.js'; import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js'; import { startScheduler } from './queue/scheduler.js'; import { initFromSavedSession } from './telegram/client.js'; @@ -91,10 +92,12 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); - // Static "/media/proxy" takes priority over the "/media/:filename" param route - // above regardless of registration order (find-my-way, Fastify's router, always - // prefers a static segment over a parametric one at the same depth). + // Static "/media/proxy" and "/media/telegram-proxy" take priority over the + // "/media/:filename" param route above regardless of registration order + // (find-my-way, Fastify's router, always prefers a static segment over a parametric + // one at the same depth). await registerMediaProxy(app); + await registerTelegramMediaProxy(app); app.get('/health', async () => ({ ok: true })); diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index bcbe135..cc63dfe 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,4 @@ -import type { Source, ContentItem, TweetMediaItem, TelegramMediaItem } from '../../storage/db/types.js'; +import type { Source, ContentItem, TweetMediaItem, TelegramMediaRef } from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -11,24 +11,13 @@ export interface FetchedItem { publishedAt: string; /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] }; - /** Set by the Telegram adapter only — carries the channel/message info through to ContentItem.telegramMessage. */ + /** Set by the Telegram adapter only — carries the channel/message info through to ContentItem.telegramMessage. Media is unresolved refs; publish.ts resolves them per the admin's configured telegramMediaMode. */ telegramMessage?: { channelName: string; channelUsername: string; - channelAvatarUrl: string | null; messageId: string; - media: TelegramMediaItem[]; + media: TelegramMediaRef[]; }; - /** - * Set by the Telegram adapter only — media_assets row IDs already downloaded and - * self-hosted at fetch time (before this item's ContentItem even exists, since the - * authenticated Telegram access producing them could disappear later). Not part of - * ContentItem itself — poller.ts uses this right after insertContentItem to attach - * these rows to the real content item id (see storage/media/index.ts's - * attachToContentItem), so publishDirect can later find and promote them via - * mediaIdsForContentItem without threading ids through the JSON blob. - */ - telegramMediaAssetIds?: string[]; raw: unknown; } diff --git a/backend/src/ingestion/adapters/telegram.ts b/backend/src/ingestion/adapters/telegram.ts index c81ce5a..4805f0b 100644 --- a/backend/src/ingestion/adapters/telegram.ts +++ b/backend/src/ingestion/adapters/telegram.ts @@ -7,13 +7,19 @@ // reasoning as YouTube/Nitter: merging two unrelated channel posts into one // AI-rewritten story wouldn't make sense the way merging two outlets' coverage of the // same event does. +// +// Media is deliberately NOT downloaded here — Telegram has no public hotlinkable media +// URL, so unlike every field pulled from the message itself, media can only be resolved +// by re-authenticating to Telegram, which the admin's configured telegramMediaMode +// (self-host vs. proxy) governs. This adapter only records *references* (message id, +// kind, mime type, dimensions) — see pipeline/publish.ts's resolveTelegramMedia for +// where those refs turn into an actual servable url. import type { Api } from 'telegram'; -import type { Source, TelegramMediaItem } from '../../storage/db/types.js'; +import type { Source, TelegramMediaRef } from '../../storage/db/types.js'; import type { SourceAdapter, FetchedItem } from './base.js'; import { logger } from '../../storage/db/logs.js'; import { getClient, fetchChannelMessages } from '../../telegram/client.js'; -import { storeMediaBuffer } from '../../storage/media/index.js'; /** Mirrors MAX_TWEET_MEDIA in nitter.ts — the frontend media grid only defines 1/2/3/4-item layouts. */ const MAX_TELEGRAM_MEDIA = 4; @@ -31,37 +37,36 @@ function normalizeChannelIdentifier(raw: string): string { .replace(/\/+$/, ''); } -function guessExtension(mimeType: string, kind: 'photo' | 'video' | 'gif'): string { - if (mimeType.includes('png')) return '.png'; - if (mimeType.includes('webp')) return '.webp'; - if (mimeType.includes('jpeg') || mimeType.includes('jpg')) return '.jpg'; - if (mimeType.includes('mp4')) return '.mp4'; - if (kind === 'photo') return '.jpg'; - return '.mp4'; +function videoDimensions(document: { attributes?: { className?: string; w?: number; h?: number }[] } | undefined) { + const attr = document?.attributes?.find((a) => a.className === 'DocumentAttributeVideo'); + return { width: attr?.w ?? null, height: attr?.h ?? null }; } -/** Downloads and self-hosts a single message's attached media, if any — Telegram has no public hotlinkable media URL, so this happens immediately rather than being deferred to publish time (that authenticated access could disappear later: message deleted, channel left, session revoked). */ -async function extractMedia(message: TgMessage): Promise<{ item: TelegramMediaItem; mediaId: string } | null> { - let kind: 'photo' | 'video' | 'gif'; - let mimeType = ''; +function photoDimensions(photo: { sizes?: { w?: number; h?: number }[] } | undefined) { + const largest = photo?.sizes?.reduce<{ w?: number; h?: number } | undefined>( + (best, size) => (!best || (size.w ?? 0) > (best.w ?? 0) ? size : best), + undefined + ); + return { width: largest?.w ?? null, height: largest?.h ?? null }; +} + +/** A reference to a single message's attached media, if any — no download, just what's needed to resolve it later. */ +function refForMessage(message: TgMessage): TelegramMediaRef | null { if (message.video) { - kind = 'video'; - mimeType = (message.video as unknown as { mimeType?: string }).mimeType ?? ''; - } else if (message.gif) { - kind = 'gif'; - mimeType = (message.gif as unknown as { mimeType?: string }).mimeType ?? ''; - } else if (message.photo) { - kind = 'photo'; - } else { - return null; + const doc = message.video as unknown as { mimeType?: string; attributes?: { className?: string; w?: number; h?: number }[] }; + const { width, height } = videoDimensions(doc); + return { type: 'video', messageId: String(message.id), mimeType: doc.mimeType ?? null, width, height }; } - - const buffer = await message.downloadMedia(); - if (!buffer || typeof buffer === 'string') return null; - - const ext = guessExtension(mimeType, kind); - const stored = storeMediaBuffer(buffer, ext, `telegram-message:${message.id}`, 'candidate', {}); - return { item: { type: kind, url: stored.servedPath, thumbnailUrl: null, width: null, height: null }, mediaId: stored.id }; + if (message.gif) { + const doc = message.gif as unknown as { mimeType?: string; attributes?: { className?: string; w?: number; h?: number }[] }; + const { width, height } = videoDimensions(doc); + return { type: 'gif', messageId: String(message.id), mimeType: doc.mimeType ?? null, width, height }; + } + if (message.photo) { + const { width, height } = photoDimensions(message.photo as unknown as { sizes?: { w?: number; h?: number }[] }); + return { type: 'photo', messageId: String(message.id), mimeType: null, width, height }; + } + return null; } /** Groups consecutive messages sharing a non-null groupedId (Telegram's multi-photo/video "album" concept) into one entry each. */ @@ -111,14 +116,6 @@ export const telegramAdapter: SourceAdapter = { } const channelName: string = entity?.title ?? channelUsername; - let avatarBuffer: Buffer | null = null; - try { - const photo = await client.downloadProfilePhoto(entity); - if (photo && typeof photo !== 'string') avatarBuffer = photo; - } catch (err) { - logger.warn('telegram', `Failed to download avatar for "${channelName}": ${(err as Error).message}`); - } - const items: FetchedItem[] = []; for (const group of groupMessages(messages)) { @@ -126,22 +123,11 @@ export const telegramAdapter: SourceAdapter = { const text = primary.message ?? ''; const firstLine = text.split('\n')[0].trim(); - const media: TelegramMediaItem[] = []; - const mediaAssetIds: string[] = []; + const media: TelegramMediaRef[] = []; for (const message of group) { if (media.length >= MAX_TELEGRAM_MEDIA) break; - const extracted = await extractMedia(message); - if (extracted) { - media.push(extracted.item); - mediaAssetIds.push(extracted.mediaId); - } - } - - let channelAvatarUrl: string | null = null; - if (avatarBuffer) { - const stored = storeMediaBuffer(avatarBuffer, '.jpg', `telegram-avatar:${channelUsername}`, 'candidate', {}); - channelAvatarUrl = stored.servedPath; - mediaAssetIds.push(stored.id); + const ref = refForMessage(message); + if (ref) media.push(ref); } if (!text && media.length === 0) continue; // nothing worth publishing (e.g. a service message) @@ -157,11 +143,9 @@ export const telegramAdapter: SourceAdapter = { telegramMessage: { channelName, channelUsername, - channelAvatarUrl, messageId: String(group[0].id), media }, - telegramMediaAssetIds: mediaAssetIds, raw: { messageIds: group.map((m) => m.id) } }); } diff --git a/backend/src/ingestion/poller.ts b/backend/src/ingestion/poller.ts index a8dda39..5a13c89 100644 --- a/backend/src/ingestion/poller.ts +++ b/backend/src/ingestion/poller.ts @@ -1,6 +1,5 @@ import * as sourcesDb from '../storage/db/sources.js'; import * as contentItemsDb from '../storage/db/contentItems.js'; -import { attachToContentItem } from '../storage/media/index.js'; import { logger } from '../storage/db/logs.js'; import { rssAdapter } from './adapters/rss.js'; import { telegramAdapter } from './adapters/telegram.js'; @@ -48,10 +47,7 @@ async function pollOne(source: Source): Promise { const finalItem = FOLLOWS_LINK_FOR_FULL_ARTICLE.includes(source.type) ? await withFullArticle(item) : item; - const created = contentItemsDb.insertContentItem(toContentItem(source, finalItem)); - if (finalItem.telegramMediaAssetIds?.length) { - attachToContentItem(finalItem.telegramMediaAssetIds, created.id); - } + contentItemsDb.insertContentItem(toContentItem(source, finalItem)); ingested++; } sourcesDb.markPolled(source.id, null); diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 7deb618..9574ea7 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -3,12 +3,13 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { Cluster } from './clustering.js'; import { synthesizeArticle } from './synthesis.js'; import { selectBestImage, faviconUrlFor } from './image-selection.js'; -import { downloadAndStore, promoteToPublished, mediaIdsForContentItem } from '../storage/media/index.js'; +import { downloadAndStore, promoteToPublished, storeMediaBuffer } from '../storage/media/index.js'; +import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js'; import { logger } from '../storage/db/logs.js'; import * as articles from '../storage/db/articles.js'; import * as tags from '../storage/db/tags.js'; import * as sources from '../storage/db/sources.js'; -import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem } from '../storage/db/types.js'; +import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem, TelegramMediaItem, TelegramMediaRef } from '../storage/db/types.js'; const FOLLOW_UP_LOOKBACK_DAYS = 3; @@ -129,6 +130,81 @@ async function resolveTweetMedia( return { media: resolved, storedMediaIds }; } +function guessTelegramExtension(mimeType: string | null, kind: 'photo' | 'video' | 'gif'): string { + if (mimeType?.includes('png')) return '.png'; + if (mimeType?.includes('webp')) return '.webp'; + if (mimeType?.includes('jpeg') || mimeType?.includes('jpg')) return '.jpg'; + if (mimeType?.includes('mp4')) return '.mp4'; + return kind === 'photo' ? '.jpg' : '.mp4'; +} + +/** + * Resolves a single Telegram media reference into a servable url per the admin's + * chosen telegramMediaMode (Retention tab). Unlike Nitter — which starts from an + * already-public CDN URL — Telegram media only exists behind the authenticated MTProto + * session, so there's no 'direct' hotlink option: 'self-host' downloads it now (via the + * live session) and stores it locally exactly like every other self-hosted image; + * 'proxy' doesn't touch Telegram at all here, it just builds a url the live + * telegram-proxy route resolves (via that same session) on each view. + */ +async function resolveTelegramMediaUrl( + channelUsername: string, + ref: TelegramMediaRef, + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ item: TelegramMediaItem; storedMediaId: string | null } | null> { + if (mode === 'proxy') { + const url = `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&message=${encodeURIComponent(ref.messageId)}&type=${ref.type}`; + return { item: { type: ref.type, url, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: null }; + } + + try { + const buffer = await downloadMessageMedia(channelUsername, ref.messageId); + if (!buffer) return null; + const ext = guessTelegramExtension(ref.mimeType, ref.type); + const stored = storeMediaBuffer(buffer, ext, `telegram-message:${channelUsername}:${ref.messageId}`, 'published', {}); + return { item: { type: ref.type, url: stored.servedPath, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: stored.id }; + } catch (err) { + logger.error('telegram', `Failed to self-host media for message ${ref.messageId}: ${(err as Error).message}`); + return null; + } +} + +/** Resolves every attached photo/video/gif ref for one message — order preserved, failed items dropped rather than leaving a broken entry. */ +async function resolveTelegramMedia( + channelUsername: string, + refs: TelegramMediaRef[], + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ media: TelegramMediaItem[]; storedMediaIds: string[] }> { + const media: TelegramMediaItem[] = []; + const storedMediaIds: string[] = []; + for (const ref of refs) { + const resolved = await resolveTelegramMediaUrl(channelUsername, ref, mode); + if (resolved) { + media.push(resolved.item); + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + } + return { media, storedMediaIds }; +} + +async function resolveTelegramAvatarUrl( + channelUsername: string, + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ url: string | null; storedMediaId: string | null }> { + if (mode === 'proxy') { + return { url: `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&avatar=1`, storedMediaId: null }; + } + try { + const buffer = await downloadChannelAvatar(channelUsername); + if (!buffer) return { url: null, storedMediaId: null }; + const stored = storeMediaBuffer(buffer, '.jpg', `telegram-avatar:${channelUsername}`, 'published', {}); + return { url: stored.servedPath, storedMediaId: stored.id }; + } catch (err) { + logger.error('telegram', `Failed to self-host avatar for "${channelUsername}": ${(err as Error).message}`); + return { url: null, storedMediaId: null }; + } +} + /** * Publishes a single item as-is, with no AI calls at all — used when the AI service * isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives @@ -175,19 +251,19 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) }; } - // Telegram media (attached photos/videos and the channel avatar) is already - // downloaded and self-hosted by the adapter at ingestion time — there's no separate - // media-mode resolution step the way Nitter has; just find those already-stored - // assets by content item id and promote them alongside everything else below. let telegramMessage: MergedArticle['telegramMessage'] = null; if (item.telegramMessage) { - storedMediaIds.push(...mediaIdsForContentItem(item.id)); + const { channelUsername } = item.telegramMessage; + const avatar = await resolveTelegramAvatarUrl(channelUsername, settings.telegramMediaMode); + if (avatar.storedMediaId) storedMediaIds.push(avatar.storedMediaId); + const resolvedMedia = await resolveTelegramMedia(channelUsername, item.telegramMessage.media, settings.telegramMediaMode); + storedMediaIds.push(...resolvedMedia.storedMediaIds); telegramMessage = { channelName: item.telegramMessage.channelName, - channelUsername: item.telegramMessage.channelUsername, - channelAvatarUrl: item.telegramMessage.channelAvatarUrl, + channelUsername, + channelAvatarUrl: avatar.url, sourceItemId: item.id, - media: item.telegramMessage.media + media: resolvedMedia.media }; } diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 45bc9e0..c6c25d6 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -172,7 +172,8 @@ export function migrate() { storage_cap_value INTEGER NOT NULL DEFAULT 500, storage_cap_unit TEXT NOT NULL DEFAULT 'GB', nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct - fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com' + fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com', + telegram_media_mode TEXT NOT NULL DEFAULT 'self-host' -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL) ); -- Singleton row (see storage/crypto.ts) — encrypted Telegram API credentials and @@ -232,6 +233,9 @@ export function migrate() { if (!hasColumn('merged_articles', 'telegram_message')) { db.exec('ALTER TABLE merged_articles ADD COLUMN telegram_message TEXT'); } + if (!hasColumn('global_settings', 'telegram_media_mode')) { + db.exec("ALTER TABLE global_settings ADD COLUMN telegram_media_mode TEXT NOT NULL DEFAULT 'self-host'"); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index f9fd366..13cb383 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -15,6 +15,7 @@ function rowToSettings(row: any): GlobalSettings { selectedModels: JSON.parse(row.selected_models), nitterMediaMode: row.nitter_media_mode, fxtwitterBaseUrl: row.fxtwitter_base_url, + telegramMediaMode: row.telegram_media_mode, retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days, @@ -43,7 +44,7 @@ export function updateSettings(patch: Partial): GlobalSettings { merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, ai_service_host=?, ai_service_port=?, selected_models=?, - nitter_media_mode=?, fxtwitter_base_url=?, + nitter_media_mode=?, fxtwitter_base_url=?, telegram_media_mode=?, published_article_max_age_days=?, raw_item_max_age_days=?, storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=? WHERE id = 1` @@ -60,6 +61,7 @@ export function updateSettings(patch: Partial): GlobalSettings { JSON.stringify(merged.selectedModels), merged.nitterMediaMode, merged.fxtwitterBaseUrl, + merged.telegramMediaMode, merged.retention.publishedArticleMaxAgeDays, merged.retention.rawItemMaxAgeDays, merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index dc52707..8f8ef72 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -27,6 +27,24 @@ export interface TweetMediaItem { /** Same shape as TweetMediaItem — distinct name for readability at Telegram call sites. */ export type TelegramMediaItem = TweetMediaItem; +/** + * A raw reference to a single message's attached media, captured at ingestion time — + * deliberately NOT a URL, since Telegram has no public hotlinkable media URL the way + * Twitter does; media bytes only ever come from the authenticated MTProto session. + * publish.ts resolves this into a real TelegramMediaItem (a servable url) according to + * the admin's configured telegramMediaMode, at publish time — mirroring how Nitter's + * tweet media URLs are resolved at publish time too, just starting from a message + * reference here instead of an already-public CDN URL. + */ +export interface TelegramMediaRef { + type: 'photo' | 'video' | 'gif'; + /** This media's own message id (a grouped album's items are separate messages, each individually re-fetchable). */ + messageId: string; + mimeType: string | null; + width: number | null; + height: number | null; +} + export interface ContentItem { id: string; sourceId: string; @@ -46,13 +64,12 @@ export interface ContentItem { clusterId: string | null; /** Nitter-sourced items only — null for everything else. */ tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null; - /** Telegram-sourced items only — null for everything else. */ + /** Telegram-sourced items only — null for everything else. Media is unresolved refs (see TelegramMediaRef); publish.ts resolves them (and the channel avatar) per the admin's configured media mode. */ telegramMessage: { channelName: string; channelUsername: string; - channelAvatarUrl: string | null; messageId: string; - media: TelegramMediaItem[]; + media: TelegramMediaRef[]; } | null; raw: unknown; } @@ -154,6 +171,8 @@ export interface GlobalSettings { nitterMediaMode: 'self-host' | 'proxy' | 'direct'; /** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */ fxtwitterBaseUrl: string; + /** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */ + telegramMediaMode: 'self-host' | 'proxy'; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/backend/src/storage/media/index.ts b/backend/src/storage/media/index.ts index 5f36cc2..8d620e1 100644 --- a/backend/src/storage/media/index.ts +++ b/backend/src/storage/media/index.ts @@ -63,21 +63,6 @@ export function storeMediaBuffer( return { id, localPath, servedPath: `/media/${filename}`, sizeBytes: buffer.length }; } -/** Finds already-downloaded media for a content item (Telegram's adapter self-hosts at ingestion time) so publishDirect can promote them without threading media-asset IDs through the JSON blob. */ -export function mediaIdsForContentItem(contentItemId: string): string[] { - const rows = db.prepare('SELECT id FROM media_assets WHERE content_item_id = ?').all(contentItemId) as { id: string }[]; - return rows.map((r) => r.id); -} - -/** - * Links media rows downloaded before their content item existed (Telegram's adapter — - * see FetchedItem.telegramMediaAssetIds) to the content item id assigned once - * insertContentItem runs. Called by poller.ts right after insertion. - */ -export function attachToContentItem(mediaIds: string[], contentItemId: string): void { - const stmt = db.prepare('UPDATE media_assets SET content_item_id = ? WHERE id = ?'); - for (const id of mediaIds) stmt.run(contentItemId, id); -} /** Promotes a candidate-tier asset to published-tier so raw-item retention can no longer prune it. */ export function promoteToPublished(mediaId: string, articleId: string) { diff --git a/backend/src/telegram/client.ts b/backend/src/telegram/client.ts index ac56a1f..94a909c 100644 --- a/backend/src/telegram/client.ts +++ b/backend/src/telegram/client.ts @@ -153,3 +153,26 @@ export async function fetchChannelMessages(channelIdentifier: string, limit: num const messages = await client.getMessages(entity, { limit }); return { entity, messages }; } + +/** + * Re-fetches a single message by id and downloads its attached media — used by + * pipeline/publish.ts (self-host mode, at publish time) and the live telegram-proxy + * route (proxy mode, on every view). Telegram media has no public URL; this + * authenticated call is the only way to get the bytes. + */ +export async function downloadMessageMedia(channelUsername: string, messageId: string): Promise { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelUsername); + const [message] = await client.getMessages(entity, { ids: [Number(messageId)] }); + if (!message) return null; + const buffer = await message.downloadMedia(); + return buffer && typeof buffer !== 'string' ? buffer : null; +} + +/** Downloads a channel's current avatar — same on-demand, no-public-URL reasoning as downloadMessageMedia. */ +export async function downloadChannelAvatar(channelUsername: string): Promise { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelUsername); + const photo = await client.downloadProfilePhoto(entity); + return photo && typeof photo !== 'string' ? photo : null; +} diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index 6a2f7ef..ee0289d 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -28,6 +28,7 @@ export interface AdminSettings { selectedModels: { embedding: string; image: string; synthesis: string }; nitterMediaMode: 'self-host' | 'proxy' | 'direct'; fxtwitterBaseUrl: string; + telegramMediaMode: 'self-host' | 'proxy'; retention: RetentionSettings; categoryPriority: CategoryPriority[]; } diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index 2215d8d..a00aac9 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -36,6 +36,31 @@ { label: 'Direct', value: 'direct' } ]; + let telegramMediaMode = $state(settings.telegramMediaMode); + let telegramMediaStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle'); + let telegramMediaSaveTimer: ReturnType; + + function scheduleTelegramMediaSave() { + telegramMediaStatus = 'saving'; + clearTimeout(telegramMediaSaveTimer); + telegramMediaSaveTimer = setTimeout(async () => { + try { + await updateSettings({ telegramMediaMode }); + telegramMediaStatus = 'saved'; + setTimeout(() => (telegramMediaStatus = 'idle'), 1500); + } catch { + telegramMediaStatus = 'error'; + } + }, 500); + } + + // No "Direct" option here — unlike Twitter's CDN, Telegram has no public + // hotlinkable media URL, so there's nothing to hotlink straight from. + const telegramMediaModes: { label: string; value: 'self-host' | 'proxy' }[] = [ + { label: 'Self-host', value: 'self-host' }, + { label: 'Proxy', value: 'proxy' } + ]; + async function handleClearArticles() { if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return; clearing = 'articles'; @@ -200,6 +225,36 @@ +
+
+ Telegram (message media) + +
+

+ How photos/videos attached to ingested Telegram messages (and channel avatars) are served + to visitors. Telegram has no public URL for this media the way Twitter's CDN does — bytes + only ever come from the logged-in account (Connections tab), so there's no "Direct" option. + Self-hosting downloads and stores everything locally, same as regular article images. + Proxying re-fetches each request live through the logged-in account and streams it straight + through without persisting anything, so only this server ever touches Telegram's servers — + at the cost of a live round-trip to Telegram on every view (a short cache absorbs repeats). +

+
+ {#each telegramMediaModes as mode} + + {/each} +
+
+
Clear content