diff --git a/backend/src/ingestion/adapters/youtube.ts b/backend/src/ingestion/adapters/youtube.ts index b577acd..05059ba 100644 --- a/backend/src/ingestion/adapters/youtube.ts +++ b/backend/src/ingestion/adapters/youtube.ts @@ -24,14 +24,63 @@ const parser = new Parser, YoutubeEntry>({ } }); -/** Builds the channel's Atom feed URL — YouTube publishes these publicly with no API key required. */ -function feedUrl(source: Source): string | null { - if (source.url) return source.url; +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const CHANNEL_ID_RE = /^UC[\w-]{22}$/; + +/** + * Resolves whatever the admin typed into the source's URL field into the channel's + * Atom feed URL. YouTube's public feed endpoint (no API key needed) only accepts a + * `channel_id` (the UC... hash) or the legacy `user` parameter — it has no equivalent + * for the newer `@handle` format, so a handle (or a /c/ or /user/ vanity URL) has to be + * resolved to its real channel ID first by fetching the channel page and pulling the + * ID out of it. + */ +async function resolveFeedUrl(source: Source): Promise { const channelId = source.config?.channelId as string | undefined; if (channelId) return `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`; + const playlistId = source.config?.playlistId as string | undefined; if (playlistId) return `https://www.youtube.com/feeds/videos.xml?playlist_id=${encodeURIComponent(playlistId)}`; - return null; + + const raw = source.url?.trim(); + if (!raw) return null; + + if (raw.includes('feeds/videos.xml')) return raw; + if (CHANNEL_ID_RE.test(raw)) return `https://www.youtube.com/feeds/videos.xml?channel_id=${raw}`; + + const idInUrl = raw.match(/\/channel\/(UC[\w-]{22})/); + if (idInUrl) return `https://www.youtube.com/feeds/videos.xml?channel_id=${idInUrl[1]}`; + + let pageUrl: string; + if (/^https?:\/\//i.test(raw)) { + pageUrl = raw; + } else if (/youtube\.com|youtu\.be/i.test(raw)) { + pageUrl = `https://${raw}`; + } else { + // A bare handle or legacy username typed on its own, e.g. "PsyopAnime" or "@PsyopAnime". + pageUrl = `https://www.youtube.com/@${raw.replace(/^@/, '')}`; + } + + const resolvedId = await resolveChannelIdFromPage(pageUrl); + if (!resolvedId) { + logger.warn('youtube', `Couldn't resolve a channel ID from "${raw}" — double-check the handle/URL`); + return null; + } + return `https://www.youtube.com/feeds/videos.xml?channel_id=${resolvedId}`; +} + +/** Every YouTube channel page embeds its own canonical channel ID — pulled from either the canonical link tag or the page's inline JSON. */ +async function resolveChannelIdFromPage(url: string): Promise { + try { + const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT }, signal: AbortSignal.timeout(10_000) }); + if (!res.ok) return null; + const html = await res.text(); + const match = html.match(/"channelId":"(UC[\w-]{22})"/) || html.match(/\/channel\/(UC[\w-]{22})/); + return match ? match[1] : null; + } catch (err) { + logger.warn('youtube', `Channel page fetch failed for ${url}: ${(err as Error).message}`); + return null; + } } function extractVideoId(url: string): string | null { @@ -52,9 +101,9 @@ function extractThumbnail(item: YoutubeEntry): string | null { export const youtubeAdapter: SourceAdapter = { async fetch(source: Source): Promise { - const url = feedUrl(source); + const url = await resolveFeedUrl(source); if (!url) { - logger.warn('youtube', `Source "${source.name}" has no url, channelId, or playlistId configured — skipping`); + logger.warn('youtube', `Source "${source.name}" has no usable url, channelId, or playlistId configured — skipping`); return []; } diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index a5051b8..506585d 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -157,7 +157,7 @@ {#if form.type === 'youtube'} - + {:else} {/if}