Fix source deletion, add content clearing, multi-category/editable sources, News category, wider layout, and a YouTube source module
- Fix "Body cannot be empty" error on DELETE by making the JSON content-type parser tolerate empty bodies, and by only sending Content-Type from the frontend when a request actually has one. - Deleting a source now cascades: raw content items and any article composed entirely from that source are removed too, plus their media. - Add admin endpoints/UI to clear all articles, all media, or a single source's content without deleting the source, so things can be repopulated fresh. - Sources can now be assigned multiple categories via checkboxes (instead of free text) and edited in place, not just added/deleted. - Add a "News" default category (seeded fresh, backfilled on existing DBs) so general news sources have a real home instead of the pseudo-category "Top stories", which is just the homepage's all-categories chronological view. - Widen the site's content column 15% (1080px -> 1242px). - Add YouTube as its own source type/ingestion module: pulls a channel's public Atom feed, and each video always publishes directly as its own article (title, embedded video, publish date, description) rather than going through the cross-source clustering/synthesis pipeline.
This commit is contained in:
@@ -3,6 +3,7 @@ import * as settingsDb from '../storage/db/settings.js';
|
||||
import * as sourcesDb from '../storage/db/sources.js';
|
||||
import * as eventsDb from '../storage/db/events.js';
|
||||
import * as categoriesDb from '../storage/db/categories.js';
|
||||
import { clearSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
|
||||
import { OllamaProvider } from '../inference/ollama-provider.js';
|
||||
import { pollSourceNow } from '../ingestion/poller.js';
|
||||
import { logger, listLogs } from '../storage/db/logs.js';
|
||||
@@ -58,10 +59,31 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
|
||||
app.delete('/api/admin/sources/:id', async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
// Deleting a source deletes its raw content and any article composed entirely
|
||||
// from it too — otherwise stale articles from a source the admin just removed
|
||||
// keep showing up on the site pointing at nothing.
|
||||
clearSourceContent(id);
|
||||
sourcesDb.deleteSource(id);
|
||||
return reply.code(204).send();
|
||||
});
|
||||
|
||||
// --- Content clearing (re-populate a source, or the whole site, from scratch) ---
|
||||
app.delete('/api/admin/content/sources/:id', async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const result = clearSourceContent(id);
|
||||
return reply.code(200).send(result);
|
||||
});
|
||||
|
||||
app.delete('/api/admin/content/articles', async (_req, reply) => {
|
||||
const deleted = clearAllArticles();
|
||||
return reply.code(200).send({ deleted });
|
||||
});
|
||||
|
||||
app.delete('/api/admin/content/media', async (_req, reply) => {
|
||||
const deleted = clearAllMedia();
|
||||
return reply.code(200).send({ deleted });
|
||||
});
|
||||
|
||||
// Manual "poll now" — the refresh icon on each source in the admin panel.
|
||||
app.post('/api/admin/sources/:id/poll', async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
|
||||
@@ -37,6 +37,20 @@ async function main() {
|
||||
});
|
||||
await app.register(cookie);
|
||||
|
||||
// Overrides Fastify's default JSON body parser, which throws "Body cannot be empty
|
||||
// when content-type is set to 'application/json'" for any bodyless request (DELETE,
|
||||
// or POST with no payload) that still carries a Content-Type header — exactly what
|
||||
// browsers' fetch() does when a client sets that header unconditionally. An empty
|
||||
// body is just as valid as `{}` for routes that don't read req.body at all.
|
||||
app.addContentTypeParser('application/json', { parseAs: 'string' }, (_req, body, done) => {
|
||||
if (typeof body !== 'string' || body.trim() === '') return done(null, {});
|
||||
try {
|
||||
done(null, JSON.parse(body));
|
||||
} catch (err) {
|
||||
done(err as Error, undefined);
|
||||
}
|
||||
});
|
||||
|
||||
await registerAuth(app);
|
||||
await registerPublicRoutes(app);
|
||||
await registerAdminRoutes(app);
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
// YouTube is its own ingestion module, deliberately separate from the RSS/Telegram
|
||||
// adapters: a channel's public Atom feed (no API key needed) is fetched directly, and
|
||||
// each entry becomes its own single-video article — see pipeline/publish.ts and
|
||||
// queue/priorityQueue.ts, which route youtube-sourced items straight to publishDirect
|
||||
// rather than through the LLM clustering/synthesis pipeline. Merging two unrelated
|
||||
// videos into one AI-rewritten story would make no sense the way merging two outlets'
|
||||
// coverage of the same news event does.
|
||||
|
||||
import Parser from 'rss-parser';
|
||||
import type { Source } from '../../storage/db/types.js';
|
||||
import type { SourceAdapter, FetchedItem } from './base.js';
|
||||
import { logger } from '../../storage/db/logs.js';
|
||||
|
||||
type YoutubeEntry = Parser.Item & {
|
||||
'media:group'?: {
|
||||
'media:description'?: string[];
|
||||
'media:thumbnail'?: { $?: { url?: string } }[];
|
||||
}[];
|
||||
};
|
||||
|
||||
const parser = new Parser<Record<string, unknown>, YoutubeEntry>({
|
||||
customFields: {
|
||||
item: [['media:group', 'media:group']]
|
||||
}
|
||||
});
|
||||
|
||||
/** Builds the channel's Atom feed URL — YouTube publishes these publicly with no API key required. */
|
||||
function feedUrl(source: Source): string | null {
|
||||
if (source.url) return source.url;
|
||||
const channelId = source.config?.channelId as string | undefined;
|
||||
if (channelId) return `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
|
||||
const playlistId = source.config?.playlistId as string | undefined;
|
||||
if (playlistId) return `https://www.youtube.com/feeds/videos.xml?playlist_id=${encodeURIComponent(playlistId)}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractVideoId(url: string): string | null {
|
||||
const match =
|
||||
url.match(/[?&]v=([\w-]{6,})/) || url.match(/youtu\.be\/([\w-]{6,})/) || url.match(/\/embed\/([\w-]{6,})/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function extractDescription(item: YoutubeEntry): string {
|
||||
const group = item['media:group']?.[0];
|
||||
return group?.['media:description']?.[0] ?? '';
|
||||
}
|
||||
|
||||
function extractThumbnail(item: YoutubeEntry): string | null {
|
||||
const group = item['media:group']?.[0];
|
||||
return group?.['media:thumbnail']?.[0]?.$?.url ?? null;
|
||||
}
|
||||
|
||||
export const youtubeAdapter: SourceAdapter = {
|
||||
async fetch(source: Source): Promise<FetchedItem[]> {
|
||||
const url = feedUrl(source);
|
||||
if (!url) {
|
||||
logger.warn('youtube', `Source "${source.name}" has no url, channelId, or playlistId configured — skipping`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const feed = await parser.parseURL(url);
|
||||
const items: FetchedItem[] = [];
|
||||
|
||||
for (const item of feed.items) {
|
||||
if (!item.link || !item.title) continue;
|
||||
const videoId = extractVideoId(item.link);
|
||||
const description = extractDescription(item);
|
||||
const thumbnail = extractThumbnail(item);
|
||||
|
||||
items.push({
|
||||
title: item.title,
|
||||
summary: description.slice(0, 500),
|
||||
body: description || null,
|
||||
images: thumbnail ? [{ url: thumbnail }] : [],
|
||||
videos: [
|
||||
{
|
||||
url: item.link,
|
||||
provider: 'youtube',
|
||||
embedHtml: videoId ? `https://www.youtube.com/embed/${videoId}` : undefined
|
||||
}
|
||||
],
|
||||
link: item.link,
|
||||
publishedAt: item.isoDate ?? item.pubDate ?? new Date().toISOString(),
|
||||
raw: item
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
};
|
||||
@@ -4,6 +4,7 @@ import { logger } from '../storage/db/logs.js';
|
||||
import { rssAdapter } from './adapters/rss.js';
|
||||
import { telegramAdapter } from './adapters/telegram.js';
|
||||
import { apiAdapter } from './adapters/api.js';
|
||||
import { youtubeAdapter } from './adapters/youtube.js';
|
||||
import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
|
||||
import { fetchFullArticle } from './articleFetcher.js';
|
||||
import type { Source } from '../storage/db/types.js';
|
||||
@@ -12,6 +13,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
|
||||
rss: rssAdapter,
|
||||
telegram: telegramAdapter,
|
||||
api: apiAdapter,
|
||||
youtube: youtubeAdapter,
|
||||
custom: apiAdapter
|
||||
};
|
||||
|
||||
|
||||
@@ -77,7 +77,9 @@ async function resolveHeroImage(
|
||||
export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
|
||||
const category = uniqueCategories([item]);
|
||||
const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
|
||||
const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, sourceItemId: item.id } : null;
|
||||
const video = item.videos[0]
|
||||
? { url: item.videos[0].url, provider: item.videos[0].provider, embedUrl: item.videos[0].embedHtml, sourceItemId: item.id }
|
||||
: null;
|
||||
|
||||
const article = await articles.insertArticle({
|
||||
title: item.title,
|
||||
@@ -140,7 +142,12 @@ export async function publishCluster(
|
||||
const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? '');
|
||||
const videoItem = items.find((i) => i.videos.length > 0);
|
||||
const video = videoItem
|
||||
? { url: videoItem.videos[0].url, provider: videoItem.videos[0].provider, sourceItemId: videoItem.id }
|
||||
? {
|
||||
url: videoItem.videos[0].url,
|
||||
provider: videoItem.videos[0].provider,
|
||||
embedUrl: videoItem.videos[0].embedHtml,
|
||||
sourceItemId: videoItem.id
|
||||
}
|
||||
: null;
|
||||
|
||||
const category = uniqueCategories(items);
|
||||
|
||||
@@ -9,6 +9,13 @@ import { publishCluster, publishDirect } from '../pipeline/publish.js';
|
||||
import { logger } from '../storage/db/logs.js';
|
||||
import type { GlobalSettings, ContentItem } from '../storage/db/types.js';
|
||||
|
||||
function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
|
||||
const matches: T[] = [];
|
||||
const rest: T[] = [];
|
||||
for (const item of items) (predicate(item) ? matches : rest).push(item);
|
||||
return [matches, rest];
|
||||
}
|
||||
|
||||
function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>): number {
|
||||
const source = sourcesDb.getSource(item.sourceId);
|
||||
const cats = source?.category ?? [];
|
||||
@@ -70,10 +77,28 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
|
||||
const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds);
|
||||
if (items.length === 0) return 0;
|
||||
|
||||
// YouTube videos never get LLM-merged with anything — each is always its own
|
||||
// article (title/video/date/description), same shape whether the AI service is up
|
||||
// or not. Route them straight to publishDirect, same as the no-AI passthrough path.
|
||||
const youtubeSourceIds = new Set(sourcesDb.listSources().filter((s) => s.type === 'youtube').map((s) => s.id));
|
||||
const [youtubeItems, mergeableItems] = partition(items, (item) => youtubeSourceIds.has(item.sourceId));
|
||||
|
||||
let publishedDirect = 0;
|
||||
for (const item of youtubeItems) {
|
||||
try {
|
||||
const article = await publishDirect(item);
|
||||
contentItemsDb.assignCluster([item.id], article.id);
|
||||
publishedDirect++;
|
||||
logger.info('synthesis', `Published "${article.title}" directly (YouTube)`);
|
||||
} catch (err) {
|
||||
logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const categories = categoriesDb.listCategories();
|
||||
const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
|
||||
|
||||
const ranked = items
|
||||
const ranked = mergeableItems
|
||||
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) }))
|
||||
.sort((a, b) => a.rank - b.rank)
|
||||
.map((r) => r.item);
|
||||
@@ -119,5 +144,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
|
||||
);
|
||||
}
|
||||
|
||||
return published;
|
||||
return published + publishedDirect;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
// Deleting or "clearing" a source shouldn't leave orphaned merged articles behind
|
||||
// pointing at raw items that no longer exist — this is the one place that coordinates
|
||||
// content_items, merged_articles, and media_assets together, since none of those three
|
||||
// tables have a single FK chain connecting them all (see storage/db/index.ts schema
|
||||
// comments: sources are copied into merged_articles.sources at publish time, not
|
||||
// referenced live).
|
||||
|
||||
import * as contentItemsDb from './db/contentItems.js';
|
||||
import * as articlesDb from './db/articles.js';
|
||||
import { deleteMediaByArticleId, deleteMediaByContentItemIds, deleteAllMedia } from './media/index.js';
|
||||
import { logger } from './db/logs.js';
|
||||
|
||||
export interface ClearResult {
|
||||
itemsDeleted: number;
|
||||
articlesDeleted: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes every raw content item ingested from a source, plus any merged article that
|
||||
* was composed entirely from that source's items (so it doesn't linger on the site
|
||||
* pointing at deleted raw data). Articles that merged this source's coverage together
|
||||
* with other sources' are left alone — stripping just this source's contribution back
|
||||
* out of an already-published multi-source article isn't something the merge pipeline
|
||||
* supports undoing.
|
||||
*/
|
||||
export function clearSourceContent(sourceId: string): ClearResult {
|
||||
const items = contentItemsDb.itemsForSource(sourceId);
|
||||
const itemIds = new Set(items.map((i) => i.id));
|
||||
|
||||
let articlesDeleted = 0;
|
||||
if (itemIds.size > 0) {
|
||||
for (const article of articlesDb.allArticlesNewestFirst()) {
|
||||
if (article.sources.length > 0 && article.sources.every((s) => itemIds.has(s.itemId))) {
|
||||
deleteMediaByArticleId(article.id);
|
||||
articlesDb.deleteArticle(article.id);
|
||||
articlesDeleted++;
|
||||
}
|
||||
}
|
||||
deleteMediaByContentItemIds([...itemIds]);
|
||||
}
|
||||
|
||||
contentItemsDb.deleteContentItemsForSource(sourceId);
|
||||
logger.info('admin', `Cleared content for source ${sourceId}: ${itemIds.size} item(s), ${articlesDeleted} article(s)`);
|
||||
return { itemsDeleted: itemIds.size, articlesDeleted };
|
||||
}
|
||||
|
||||
/** Wipes every published article and its media, keeping raw ingested items intact so they can be re-synthesized fresh. */
|
||||
export function clearAllArticles(): number {
|
||||
const articles = articlesDb.allArticlesNewestFirst();
|
||||
for (const article of articles) deleteMediaByArticleId(article.id);
|
||||
articlesDb.deleteAllArticles();
|
||||
logger.info('admin', `Cleared all articles: ${articles.length} removed`);
|
||||
return articles.length;
|
||||
}
|
||||
|
||||
/** Wipes every locally-stored media file (both candidate and published tiers). */
|
||||
export function clearAllMedia(): number {
|
||||
const count = deleteAllMedia();
|
||||
logger.info('admin', `Cleared all media: ${count} file(s) removed`);
|
||||
return count;
|
||||
}
|
||||
@@ -131,3 +131,7 @@ export function findRecentArticleByTags(tagIds: string[], sinceDays: number): Me
|
||||
export function deleteArticle(id: string) {
|
||||
db.prepare('DELETE FROM merged_articles WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
export function deleteAllArticles() {
|
||||
db.prepare('DELETE FROM merged_articles').run();
|
||||
}
|
||||
|
||||
@@ -98,3 +98,16 @@ export function deleteContentItems(ids: string[]) {
|
||||
const stmt = db.prepare('DELETE FROM content_items WHERE id = ?');
|
||||
for (const id of ids) stmt.run(id);
|
||||
}
|
||||
|
||||
export function itemsForSource(sourceId: string): ContentItem[] {
|
||||
const rows = db.prepare('SELECT * FROM content_items WHERE source_id = ?').all(sourceId);
|
||||
return rows.map(rowToItem);
|
||||
}
|
||||
|
||||
export function deleteContentItemsForSource(sourceId: string) {
|
||||
db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId);
|
||||
}
|
||||
|
||||
export function deleteAllContentItems() {
|
||||
db.prepare('DELETE FROM content_items').run();
|
||||
}
|
||||
|
||||
@@ -182,15 +182,29 @@ export function migrate() {
|
||||
).run();
|
||||
}
|
||||
|
||||
// Seed default categories if none exist yet.
|
||||
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
|
||||
// general news sources belong here, not on "Top stories" itself, which isn't a real
|
||||
// filterable tag: it's the homepage's all-categories-chronological view (see
|
||||
// +layout.svelte's nav mapping and /api/feed's no-category-filter default).
|
||||
const catCount = db.prepare('SELECT COUNT(*) as c FROM categories').get() as { c: number };
|
||||
if (catCount.c === 0) {
|
||||
const defaults = ['Top stories', 'Local', 'World', 'Business', 'Tech', 'Culture'];
|
||||
const defaults = ['Top stories', 'News', 'Local', 'World', 'Business', 'Tech', 'Culture'];
|
||||
const stmt = db.prepare(
|
||||
'INSERT INTO categories (id, name, priority_rank, is_default) VALUES (?, ?, ?, 1)'
|
||||
);
|
||||
defaults.forEach((name, i) => {
|
||||
stmt.run(`cat-${name.toLowerCase().replace(/\s+/g, '-')}`, name, i + 1);
|
||||
});
|
||||
} else {
|
||||
// Backfill for installs seeded before "News" existed.
|
||||
const hasNews = db.prepare("SELECT id FROM categories WHERE lower(name) = 'news'").get();
|
||||
if (!hasNews) {
|
||||
const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number };
|
||||
db.prepare('INSERT INTO categories (id, name, priority_rank, is_default) VALUES (?, ?, ?, 1)').run(
|
||||
'cat-news',
|
||||
'News',
|
||||
maxRank.m + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
export interface Source {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'rss' | 'api' | 'telegram' | 'custom';
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
|
||||
category: string[];
|
||||
url: string | null;
|
||||
config: Record<string, unknown>;
|
||||
@@ -44,7 +44,7 @@ export interface MergedArticle {
|
||||
title: string;
|
||||
body: string;
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; sourceItemId: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
|
||||
@@ -70,6 +70,39 @@ export function deleteCandidateMediaOlderThan(days: number): number {
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
function deleteRows(rows: { id: string; local_path: string }[]) {
|
||||
for (const row of rows) fs.rm(row.local_path, () => {});
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
export function deleteMediaByArticleId(articleId: string): number {
|
||||
const rows = db.prepare('SELECT id, local_path FROM media_assets WHERE article_id = ?').all(articleId) as {
|
||||
id: string;
|
||||
local_path: string;
|
||||
}[];
|
||||
const count = deleteRows(rows);
|
||||
db.prepare('DELETE FROM media_assets WHERE article_id = ?').run(articleId);
|
||||
return count;
|
||||
}
|
||||
|
||||
export function deleteMediaByContentItemIds(contentItemIds: string[]): number {
|
||||
if (contentItemIds.length === 0) return 0;
|
||||
const placeholders = contentItemIds.map(() => '?').join(',');
|
||||
const rows = db
|
||||
.prepare(`SELECT id, local_path FROM media_assets WHERE content_item_id IN (${placeholders})`)
|
||||
.all(...contentItemIds) as { id: string; local_path: string }[];
|
||||
const count = deleteRows(rows);
|
||||
db.prepare(`DELETE FROM media_assets WHERE content_item_id IN (${placeholders})`).run(...contentItemIds);
|
||||
return count;
|
||||
}
|
||||
|
||||
export function deleteAllMedia(): number {
|
||||
const rows = db.prepare('SELECT id, local_path FROM media_assets').all() as { id: string; local_path: string }[];
|
||||
const count = deleteRows(rows);
|
||||
db.prepare('DELETE FROM media_assets').run();
|
||||
return count;
|
||||
}
|
||||
|
||||
function guessExtension(contentType: string, url: string): string {
|
||||
if (contentType.includes('jpeg')) return '.jpg';
|
||||
if (contentType.includes('png')) return '.png';
|
||||
|
||||
@@ -9,10 +9,15 @@ import type {
|
||||
} from './adminTypes';
|
||||
|
||||
async function request<T>(path: string, options: RequestInit = {}, fetchFn: typeof fetch = fetch): Promise<T> {
|
||||
// Fastify's default JSON body parser rejects an empty body when Content-Type is
|
||||
// application/json ("Body cannot be empty when content-type is set to
|
||||
// 'application/json'") — so this header is only attached when there's actually a
|
||||
// body to send (PATCH/POST with a JSON payload), never for bodyless DELETE/POST calls.
|
||||
const headers = options.body ? { 'Content-Type': 'application/json', ...(options.headers || {}) } : options.headers;
|
||||
const res = await fetchFn(`${getBackendUrl()}${path}`, {
|
||||
...options,
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
|
||||
headers
|
||||
});
|
||||
if (res.status === 401) {
|
||||
const err = new Error('unauthorized') as Error & { status?: number };
|
||||
@@ -76,6 +81,16 @@ export const deleteSource = (id: string, fetchFn?: typeof fetch) =>
|
||||
export const pollSourceNow = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<{ ingested: number; source: AdminSource }>(`/api/admin/sources/${id}/poll`, { method: 'POST' }, fetchFn);
|
||||
|
||||
// Content clearing — wipe articles/media/a source's raw items so they can be repopulated fresh.
|
||||
export const clearSourceContent = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<{ itemsDeleted: number; articlesDeleted: number }>(`/api/admin/content/sources/${id}`, { method: 'DELETE' }, fetchFn);
|
||||
|
||||
export const clearAllArticles = (fetchFn?: typeof fetch) =>
|
||||
request<{ deleted: number }>('/api/admin/content/articles', { method: 'DELETE' }, fetchFn);
|
||||
|
||||
export const clearAllMedia = (fetchFn?: typeof fetch) =>
|
||||
request<{ deleted: number }>('/api/admin/content/media', { method: 'DELETE' }, fetchFn);
|
||||
|
||||
// Tracked events
|
||||
export const getEvents = (fetchFn?: typeof fetch) =>
|
||||
request<AdminTrackedEvent[]>('/api/admin/events', {}, fetchFn);
|
||||
|
||||
@@ -11,6 +11,7 @@ export interface CategoryPriority {
|
||||
id: string;
|
||||
name: string;
|
||||
priorityRank: number;
|
||||
isDefault: boolean;
|
||||
}
|
||||
|
||||
export interface AdminSettings {
|
||||
@@ -31,9 +32,10 @@ export interface AdminSettings {
|
||||
export interface AdminSource {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'rss' | 'api' | 'telegram' | 'custom';
|
||||
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'custom';
|
||||
category: string[];
|
||||
url: string;
|
||||
config?: Record<string, unknown>;
|
||||
pollIntervalMinutes: number;
|
||||
enabled: boolean;
|
||||
lastPolledAt: string | null;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { AdminSettings } from '$lib/adminTypes';
|
||||
import { updateSettings } from '$lib/adminApi';
|
||||
import { updateSettings, clearAllArticles, clearAllMedia } from '$lib/adminApi';
|
||||
import SaveStatus from './SaveStatus.svelte';
|
||||
|
||||
let { settings }: { settings: AdminSettings } = $props();
|
||||
@@ -8,6 +8,32 @@
|
||||
let retention = $state({ ...settings.retention });
|
||||
let status = $state<'idle' | 'saving' | 'saved' | 'error'>('idle');
|
||||
let saveTimer: ReturnType<typeof setTimeout>;
|
||||
let clearing = $state<'articles' | 'media' | null>(null);
|
||||
let clearResult = $state<string | null>(null);
|
||||
|
||||
async function handleClearArticles() {
|
||||
if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return;
|
||||
clearing = 'articles';
|
||||
clearResult = null;
|
||||
try {
|
||||
const { deleted } = await clearAllArticles();
|
||||
clearResult = `${deleted} article(s) deleted`;
|
||||
} finally {
|
||||
clearing = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleClearMedia() {
|
||||
if (!confirm('Delete every locally stored media file? Articles referencing them will show broken images until re-published.')) return;
|
||||
clearing = 'media';
|
||||
clearResult = null;
|
||||
try {
|
||||
const { deleted } = await clearAllMedia();
|
||||
clearResult = `${deleted} media file(s) deleted`;
|
||||
} finally {
|
||||
clearing = null;
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleSave() {
|
||||
status = 'saving';
|
||||
@@ -115,6 +141,26 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<span class="panel-title">Clear content</span>
|
||||
<p class="hint">
|
||||
Wipe everything so a category or the whole site can be repopulated fresh. To clear a single
|
||||
source's content without deleting the source, use the ✕ next to it in the Sources tab's
|
||||
"Clear content" action instead.
|
||||
</p>
|
||||
<div class="clear-row">
|
||||
<button class="danger-btn" onclick={handleClearArticles} disabled={clearing !== null}>
|
||||
{clearing === 'articles' ? 'Clearing…' : 'Clear all articles'}
|
||||
</button>
|
||||
<button class="danger-btn" onclick={handleClearMedia} disabled={clearing !== null}>
|
||||
{clearing === 'media' ? 'Clearing…' : 'Clear all media'}
|
||||
</button>
|
||||
{#if clearResult}
|
||||
<span class="usage-label">{clearResult}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.panel {
|
||||
background: var(--surface-1);
|
||||
@@ -186,4 +232,25 @@
|
||||
height: 100%;
|
||||
background: var(--border-accent);
|
||||
}
|
||||
.clear-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.danger-btn {
|
||||
font-size: 12px;
|
||||
padding: 6px 12px;
|
||||
border-radius: var(--radius);
|
||||
border: 0.5px solid var(--text-danger);
|
||||
background: transparent;
|
||||
color: var(--text-danger);
|
||||
}
|
||||
.danger-btn:hover:not(:disabled) {
|
||||
background: var(--bg-accent);
|
||||
}
|
||||
.danger-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: default;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,33 +1,119 @@
|
||||
<script lang="ts">
|
||||
import type { AdminSource } from '$lib/adminTypes';
|
||||
import { addSource, deleteSource, updateSource, pollSourceNow } from '$lib/adminApi';
|
||||
import type { AdminSource, CategoryPriority } from '$lib/adminTypes';
|
||||
import { addSource, deleteSource, updateSource, pollSourceNow, clearSourceContent } from '$lib/adminApi';
|
||||
|
||||
let { sources: initial }: { sources: AdminSource[] } = $props();
|
||||
let { sources: initial, categories }: { sources: AdminSource[]; categories: CategoryPriority[] } = $props();
|
||||
let sources = $state([...initial]);
|
||||
let showAdd = $state(false);
|
||||
let newSource = $state({ name: '', type: 'rss' as AdminSource['type'], url: '', category: '', pollIntervalMinutes: 15 });
|
||||
let editingId = $state<string | null>(null);
|
||||
let pollingId = $state<string | null>(null);
|
||||
let clearingId = $state<string | null>(null);
|
||||
let justPolled = $state<{ id: string; count: number } | null>(null);
|
||||
let justCleared = $state<{ id: string; items: number; articles: number } | null>(null);
|
||||
|
||||
async function handleAdd() {
|
||||
if (!newSource.name || !newSource.url) return;
|
||||
const created = await addSource({
|
||||
name: newSource.name,
|
||||
type: newSource.type,
|
||||
url: newSource.url,
|
||||
category: newSource.category ? [newSource.category] : [],
|
||||
pollIntervalMinutes: newSource.pollIntervalMinutes
|
||||
});
|
||||
sources = [...sources, created];
|
||||
newSource = { name: '', type: 'rss', url: '', category: '', pollIntervalMinutes: 15 };
|
||||
// "Top stories" isn't a real filterable tag — it's the homepage's all-categories,
|
||||
// chronological view (see /api/feed's no-filter default and +layout.svelte's nav
|
||||
// mapping). Assigning it to a source is the exact miscategorization this list is
|
||||
// meant to prevent — general news belongs under "News" instead.
|
||||
const assignableCategories = $derived(categories.filter((c) => c.name.toLowerCase() !== 'top stories'));
|
||||
|
||||
function emptyForm() {
|
||||
return {
|
||||
name: '',
|
||||
type: 'rss' as AdminSource['type'],
|
||||
url: '',
|
||||
channelId: '',
|
||||
categorySet: new Set<string>(),
|
||||
pollIntervalMinutes: 15
|
||||
};
|
||||
}
|
||||
|
||||
let form = $state(emptyForm());
|
||||
|
||||
function startAdd() {
|
||||
form = emptyForm();
|
||||
editingId = null;
|
||||
showAdd = true;
|
||||
}
|
||||
|
||||
function startEdit(source: AdminSource) {
|
||||
form = {
|
||||
name: source.name,
|
||||
type: source.type,
|
||||
url: source.type === 'youtube' ? '' : source.url,
|
||||
channelId: source.type === 'youtube' ? (source.url || (source.config?.channelId as string) || '') : '',
|
||||
categorySet: new Set(source.category),
|
||||
pollIntervalMinutes: source.pollIntervalMinutes
|
||||
};
|
||||
editingId = source.id;
|
||||
showAdd = true;
|
||||
}
|
||||
|
||||
function cancelForm() {
|
||||
showAdd = false;
|
||||
editingId = null;
|
||||
}
|
||||
|
||||
function toggleCategory(name: string) {
|
||||
const next = new Set(form.categorySet);
|
||||
if (next.has(name)) next.delete(name);
|
||||
else next.add(name);
|
||||
form.categorySet = next;
|
||||
}
|
||||
|
||||
function buildPayload(): Partial<AdminSource> {
|
||||
const category = [...form.categorySet];
|
||||
if (form.type === 'youtube') {
|
||||
return {
|
||||
name: form.name,
|
||||
type: form.type,
|
||||
url: form.channelId,
|
||||
category,
|
||||
pollIntervalMinutes: form.pollIntervalMinutes
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: form.name,
|
||||
type: form.type,
|
||||
url: form.url,
|
||||
category,
|
||||
pollIntervalMinutes: form.pollIntervalMinutes
|
||||
};
|
||||
}
|
||||
|
||||
async function handleSubmit() {
|
||||
if (!form.name || (form.type === 'youtube' ? !form.channelId : !form.url)) return;
|
||||
if (editingId) {
|
||||
const updated = await updateSource(editingId, buildPayload());
|
||||
sources = sources.map((s) => (s.id === editingId ? updated : s));
|
||||
} else {
|
||||
const created = await addSource(buildPayload());
|
||||
sources = [...sources, created];
|
||||
}
|
||||
cancelForm();
|
||||
}
|
||||
|
||||
async function handleDelete(id: string) {
|
||||
if (!confirm('Delete this source? All of its ingested content (and any article made up entirely of it) will be deleted too.')) return;
|
||||
await deleteSource(id);
|
||||
sources = sources.filter((s) => s.id !== id);
|
||||
}
|
||||
|
||||
async function handleClearContent(source: AdminSource) {
|
||||
if (!confirm(`Clear all ingested content for "${source.name}" so it can be repopulated fresh? The source itself stays.`)) return;
|
||||
clearingId = source.id;
|
||||
justCleared = null;
|
||||
try {
|
||||
const { itemsDeleted, articlesDeleted } = await clearSourceContent(source.id);
|
||||
justCleared = { id: source.id, items: itemsDeleted, articles: articlesDeleted };
|
||||
setTimeout(() => {
|
||||
if (justCleared?.id === source.id) justCleared = null;
|
||||
}, 4000);
|
||||
} finally {
|
||||
clearingId = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleEnabled(source: AdminSource) {
|
||||
const updated = await updateSource(source.id, { enabled: !source.enabled });
|
||||
sources = sources.map((s) => (s.id === source.id ? updated : s));
|
||||
@@ -48,30 +134,51 @@
|
||||
}
|
||||
}
|
||||
|
||||
const typeIcon = (type: string) => (type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'api' ? '⇄' : '•');
|
||||
const typeIcon = (type: string) =>
|
||||
type === 'rss' ? '⟳' : type === 'telegram' ? '✈' : type === 'youtube' ? '▶' : type === 'api' ? '⇄' : '•';
|
||||
</script>
|
||||
|
||||
<div class="toolbar">
|
||||
<span class="count">{sources.length} sources</span>
|
||||
<button class="add-btn" onclick={() => (showAdd = !showAdd)}>+ Add source</button>
|
||||
<button class="add-btn" onclick={() => (showAdd ? cancelForm() : startAdd())}>
|
||||
{showAdd ? 'Cancel' : '+ Add source'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if showAdd}
|
||||
<div class="add-panel">
|
||||
<div class="add-grid">
|
||||
<input placeholder="Name" bind:value={newSource.name} />
|
||||
<select bind:value={newSource.type}>
|
||||
<input placeholder="Name" bind:value={form.name} />
|
||||
<select bind:value={form.type}>
|
||||
<option value="rss">RSS</option>
|
||||
<option value="api">API</option>
|
||||
<option value="telegram">Telegram</option>
|
||||
<option value="youtube">YouTube</option>
|
||||
<option value="custom">Custom</option>
|
||||
</select>
|
||||
<input placeholder="URL or channel" bind:value={newSource.url} />
|
||||
<input placeholder="Category" bind:value={newSource.category} />
|
||||
{#if form.type === 'youtube'}
|
||||
<input placeholder="Channel ID, playlist ID, or full feed URL" bind:value={form.channelId} />
|
||||
{:else}
|
||||
<input placeholder="URL or channel" bind:value={form.url} />
|
||||
{/if}
|
||||
<select bind:value={form.pollIntervalMinutes}>
|
||||
<option value={5}>Every 5 minutes</option>
|
||||
<option value={15}>Every 15 minutes</option>
|
||||
<option value={60}>Every hour</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="categories-label">Categories</div>
|
||||
<div class="category-checks">
|
||||
{#each assignableCategories as cat (cat.id)}
|
||||
<label class="category-check">
|
||||
<input type="checkbox" checked={form.categorySet.has(cat.name)} onchange={() => toggleCategory(cat.name)} />
|
||||
{cat.name}
|
||||
</label>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="add-actions">
|
||||
<button onclick={() => (showAdd = false)}>Cancel</button>
|
||||
<button class="primary" onclick={handleAdd}>Add</button>
|
||||
<button onclick={cancelForm}>Cancel</button>
|
||||
<button class="primary" onclick={handleSubmit}>{editingId ? 'Save' : 'Add'}</button>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
@@ -99,8 +206,14 @@
|
||||
</button>
|
||||
<div>
|
||||
<div class="name">{source.name}</div>
|
||||
<div class="sub" class:error={source.lastError && !justPolled} class:success={justPolled?.id === source.id}>
|
||||
{#if justPolled?.id === source.id}
|
||||
<div
|
||||
class="sub"
|
||||
class:error={source.lastError && !justPolled && !justCleared}
|
||||
class:success={justPolled?.id === source.id || justCleared?.id === source.id}
|
||||
>
|
||||
{#if justCleared?.id === source.id}
|
||||
✓ cleared {justCleared.items} item(s), {justCleared.articles} article(s)
|
||||
{:else if justPolled?.id === source.id}
|
||||
{justPolled.count > 0 ? `✓ ${justPolled.count} new item(s)` : '✓ up to date, nothing new'}
|
||||
{:else if source.lastError}
|
||||
last poll failed · {source.lastError}
|
||||
@@ -113,9 +226,18 @@
|
||||
<span class="cat">{source.category.join(', ')}</span>
|
||||
<span class="cat">{source.pollIntervalMinutes} min</span>
|
||||
<div class="actions">
|
||||
<button class="icon-btn" onclick={() => startEdit(source)} title="Edit">✎</button>
|
||||
<button class="icon-btn" onclick={() => toggleEnabled(source)} title={source.enabled ? 'Disable' : 'Enable'}>
|
||||
{source.enabled ? '⏸' : '▶'}
|
||||
</button>
|
||||
<button
|
||||
class="icon-btn"
|
||||
onclick={() => handleClearContent(source)}
|
||||
disabled={clearingId === source.id}
|
||||
title="Clear content (keep source)"
|
||||
>
|
||||
⟲
|
||||
</button>
|
||||
<button class="icon-btn danger" onclick={() => handleDelete(source.id)} title="Delete">✕</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -149,6 +271,27 @@
|
||||
gap: 8px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.categories-label {
|
||||
font-size: 11px;
|
||||
color: var(--text-muted);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.category-checks {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.category-check {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 5px;
|
||||
font-size: 12px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.category-check input {
|
||||
width: auto;
|
||||
}
|
||||
.add-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
@@ -165,7 +308,7 @@
|
||||
}
|
||||
.row {
|
||||
display: grid;
|
||||
grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 60px;
|
||||
grid-template-columns: 20px 1.4fr 0.7fr 0.9fr 0.7fr 96px;
|
||||
gap: 10px;
|
||||
padding: 10px;
|
||||
align-items: center;
|
||||
@@ -231,7 +374,7 @@
|
||||
}
|
||||
.actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
gap: 4px;
|
||||
}
|
||||
.icon-btn {
|
||||
font-size: 12px;
|
||||
|
||||
@@ -77,7 +77,7 @@ button {
|
||||
}
|
||||
|
||||
.page {
|
||||
max-width: 1080px;
|
||||
max-width: 1242px; /* 1080px + 15% */
|
||||
margin: 0 auto;
|
||||
padding: 0 24px 60px;
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ export interface MergedArticle {
|
||||
title: string;
|
||||
body: string;
|
||||
heroImage: { url: string; sourceItemId: string; selectionReason: string } | null;
|
||||
video: { url: string; provider?: string; sourceItemId: string } | null;
|
||||
video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null;
|
||||
category: string[];
|
||||
geo: string | null;
|
||||
eventId: string | null;
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
{#if active === 'merge'}
|
||||
<MergeTab settings={data.settings} />
|
||||
{:else if active === 'sources'}
|
||||
<SourcesTab sources={data.sources} />
|
||||
<SourcesTab sources={data.sources} categories={data.settings.categoryPriority} />
|
||||
{:else if active === 'models'}
|
||||
<ModelsTab settings={data.settings} models={data.models} aiStatus={data.aiStatus} />
|
||||
{:else if active === 'retention'}
|
||||
|
||||
@@ -24,26 +24,49 @@
|
||||
|
||||
<h1>{a.title}</h1>
|
||||
|
||||
<div class="dates">
|
||||
<span>Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)}</span>
|
||||
{#if a.updatedAt !== a.publishedAt}
|
||||
<span>· Updated {timeAgo(a.updatedAt)} · {exactTime(a.updatedAt)}</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if a.heroImage}
|
||||
<img class="hero-img" src={resolveMediaUrl(a.heroImage.url)} alt="" />
|
||||
<div class="img-caption">
|
||||
Image via <span class="accent">{a.sources[0]?.sourceName ?? 'source'}</span>
|
||||
{#if a.video?.provider === 'youtube'}
|
||||
<!-- YouTube's own layout: title, then the video itself, then when it was published
|
||||
and its description — no hero image (the embed already shows the thumbnail) and
|
||||
no cross-source merging (see backend priorityQueue.ts — YouTube items never merge). -->
|
||||
<div class="video-frame">
|
||||
<iframe
|
||||
src={a.video.embedUrl ?? a.video.url}
|
||||
title={a.title}
|
||||
loading="lazy"
|
||||
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
|
||||
allowfullscreen
|
||||
></iframe>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#each a.body.split('\n\n') as paragraph}
|
||||
<p>{paragraph}</p>
|
||||
{/each}
|
||||
<div class="dates">
|
||||
<span>Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)}</span>
|
||||
</div>
|
||||
|
||||
{#if a.video}
|
||||
<div class="video-embed">▶ video embed · via {a.video.provider}</div>
|
||||
{#each a.body.split('\n\n') as paragraph}
|
||||
<p>{paragraph}</p>
|
||||
{/each}
|
||||
{:else}
|
||||
<div class="dates">
|
||||
<span>Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)}</span>
|
||||
{#if a.updatedAt !== a.publishedAt}
|
||||
<span>· Updated {timeAgo(a.updatedAt)} · {exactTime(a.updatedAt)}</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if a.heroImage}
|
||||
<img class="hero-img" src={resolveMediaUrl(a.heroImage.url)} alt="" />
|
||||
<div class="img-caption">
|
||||
Image via <span class="accent">{a.sources[0]?.sourceName ?? 'source'}</span>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#each a.body.split('\n\n') as paragraph}
|
||||
<p>{paragraph}</p>
|
||||
{/each}
|
||||
|
||||
{#if a.video}
|
||||
<div class="video-embed">▶ video embed · via {a.video.provider}</div>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
{#if data.tagLabels.length}
|
||||
@@ -141,6 +164,22 @@
|
||||
font-size: 12px;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
.video-frame {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
aspect-ratio: 16 / 9;
|
||||
border-radius: 12px;
|
||||
overflow: hidden;
|
||||
background: var(--surface-1);
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.video-frame iframe {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: none;
|
||||
}
|
||||
.tags {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
|
||||
Reference in New Issue
Block a user