From 53b11124d503084344f11f2f936c76e183d29bae Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 04:10:01 +0000 Subject: [PATCH] Add per-category "No AI" toggle to skip clustering/synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Category priority admin pane gains a "No AI" checkbox alongside Private/More. When set, items whose source falls under that category skip embedding, clustering, and LLM synthesis entirely — each publishes on its own, verbatim from its source (title + body/summary), the same direct-publish path YouTube/Nitter/Telegram items always use. Backend: new categories.disable_ai column (default off, migrated in for existing installs), threaded through categories.ts CRUD and the POST /api/admin/categories + PATCH /api/admin/settings routes. priorityQueue.ts's runSynthesisCycle now partitions items three ways before clustering: source-type direct (youtube/nitter/telegram), category-disabled direct (new), then whatever's left goes through the normal embed/cluster/synthesize pipeline. Tracked-event recaps are a separate, already-existing per-event toggle (TrackedEvent.recapIntervalHours) since events aren't tied to a single category — unaffected by this change. --- backend/src/api/admin.ts | 9 ++++- backend/src/queue/priorityQueue.ts | 37 ++++++++++++++++--- backend/src/storage/db/categories.ts | 19 ++++++---- backend/src/storage/db/index.ts | 6 ++- backend/src/storage/db/types.ts | 2 + frontend/src/lib/adminApi.ts | 10 ++++- frontend/src/lib/adminTypes.ts | 1 + .../src/lib/components/admin/MergeTab.svelte | 21 ++++++++++- 8 files changed, 84 insertions(+), 21 deletions(-) diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index cf2b1f3..947b6f0 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -65,9 +65,14 @@ export async function registerAdminRoutes(app: FastifyInstance) { // --- Categories (add/remove — reordering/privacy is via PATCH /settings above) --- app.post('/api/admin/categories', async (req, reply) => { - const { name, isPrivate, isSpillover } = req.body as { name?: string; isPrivate?: boolean; isSpillover?: boolean }; + const { name, isPrivate, isSpillover, disableAi } = req.body as { + name?: string; + isPrivate?: boolean; + isSpillover?: boolean; + disableAi?: boolean; + }; if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' }); - const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover); + const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover, !!disableAi); return reply.code(201).send(created); }); diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 1eded36..3894bc8 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -45,6 +45,16 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map, return best; } +/** True if any of the item's source's categories (same leading-segment match as primaryCategoryRank) has AI disabled. */ +function inAiDisabledCategory(item: ContentItem, disabledNames: Set, sourcesById: Map): boolean { + const source = sourcesById.get(item.sourceId); + for (const cat of source?.category ?? []) { + const leading = cat.split(':')[0].trim().toLowerCase(); + if (disabledNames.has(leading)) return true; + } + return false; +} + /** * Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same * publish-then-tag-then-log/error shape, differing only in how the success/failure @@ -114,6 +124,8 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G // direct-publish partition and each item's category/type lookups — avoids a // separate sourcesDb.getSource() round-trip per item. const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s])); + const categories = categoriesDb.listCategories(); + const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with // anything else — each is always its own article, same shape whether the AI service @@ -121,18 +133,31 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const directPublishSourceIds = new Set( [...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id) ); - const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); + const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); - const publishedDirect = await publishItemsDirect( - directItems, + // A category with disableAi set (see the Category priority admin pane) opts its + // items out of clustering/synthesis entirely — each publishes on its own, using its + // own source's text, same as the source-type-driven direct items above. + const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase())); + const [categoryDirectItems, mergeableItems] = partition(remaining, (item) => + inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById) + ); + + const publishedTypeDirect = await publishItemsDirect( + typeDirectItems, settings, activeEvents, (item) => sourcesById.get(item.sourceId)?.type ?? 'unknown', 'Direct publish failed' ); - const categories = categoriesDb.listCategories(); - const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank])); + const publishedCategoryDirect = await publishItemsDirect( + categoryDirectItems, + settings, + activeEvents, + () => 'AI disabled for category', + 'Direct publish failed' + ); const ranked = mergeableItems .map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) })) @@ -184,5 +209,5 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G ); } - return published + publishedDirect; + return published + publishedTypeDirect + publishedCategoryDirect; } diff --git a/backend/src/storage/db/categories.ts b/backend/src/storage/db/categories.ts index 2397610..42fb7d3 100644 --- a/backend/src/storage/db/categories.ts +++ b/backend/src/storage/db/categories.ts @@ -9,7 +9,8 @@ function rowToCategory(row: any): Category { priorityRank: row.priority_rank, isDefault: !!row.is_default, isPrivate: !!row.is_private, - isSpillover: !!row.is_spillover + isSpillover: !!row.is_spillover, + disableAi: !!row.disable_ai }; } @@ -24,18 +25,20 @@ export function listPrivateCategoryNames(): string[] { return rows.map((r) => r.name); } -export function setCategoryOrder(order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean }[]) { - const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ? WHERE id = ?'); - for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.id); +export function setCategoryOrder( + order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean; disableAi: boolean }[] +) { + const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ?, disable_ai = ? WHERE id = ?'); + for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.disableAi ? 1 : 0, c.id); } -export function createCategory(name: string, isPrivate = false, isSpillover = false): Category { +export function createCategory(name: string, isPrivate = false, isSpillover = false, disableAi = false): Category { const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`; const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number }; db.prepare( - 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover) VALUES (?, ?, ?, 0, ?, ?)' - ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0); - return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover }; + 'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover, disable_ai) VALUES (?, ?, ?, 0, ?, ?, ?)' + ).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0, disableAi ? 1 : 0); + return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover, disableAi }; } export function deleteCategory(id: string) { diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 2de9caa..0058dab 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -153,7 +153,8 @@ export function migrate() { priority_rank INTEGER NOT NULL, is_default INTEGER NOT NULL DEFAULT 0, is_private INTEGER NOT NULL DEFAULT 0, - is_spillover INTEGER NOT NULL DEFAULT 0 -- collapsed into the nav's "More »" overflow page instead of its own tab + is_spillover INTEGER NOT NULL DEFAULT 0, -- collapsed into the nav's "More »" overflow page instead of its own tab + disable_ai INTEGER NOT NULL DEFAULT 0 -- skip clustering/synthesis for this category's items; publish each one directly ); CREATE TABLE IF NOT EXISTS logs ( @@ -321,6 +322,9 @@ export function migrate() { if (!hasColumn('categories', 'is_spillover')) { db.exec('ALTER TABLE categories ADD COLUMN is_spillover INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('categories', 'disable_ai')) { + db.exec('ALTER TABLE categories ADD COLUMN disable_ai INTEGER NOT NULL DEFAULT 0'); + } if (!hasColumn('content_items', 'telegram_message')) { db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT'); } diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index 82836f4..66c1af9 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -206,6 +206,8 @@ export interface Category { isPrivate: boolean; /** Grouped into the nav's "More »" overflow page instead of getting its own top-level tab — see +layout.svelte and /more. */ isSpillover: boolean; + /** Skips clustering/AI synthesis for this category's items — each one publishes directly (own article, own source's text), same as YouTube/Nitter/Telegram items always do. See priorityQueue.ts's runSynthesisCycle. */ + disableAi: boolean; } export interface WeatherHourEntry { diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 304d197..b2f0479 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -66,10 +66,16 @@ export const updateSettings = (patch: Partial, fetchFn?: typeof f request('/api/admin/settings', { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn); // Categories -export const createCategory = (name: string, isPrivate = false, isSpillover = false, fetchFn?: typeof fetch) => +export const createCategory = ( + name: string, + isPrivate = false, + isSpillover = false, + disableAi = false, + fetchFn?: typeof fetch +) => request( '/api/admin/categories', - { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover }) }, + { method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover, disableAi }) }, fetchFn ); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index a703596..891a856 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -14,6 +14,7 @@ export interface CategoryPriority { isDefault: boolean; isPrivate: boolean; isSpillover: boolean; + disableAi: boolean; } export interface WeatherHourEntry { diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index ab4d0fa..731f143 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -13,6 +13,7 @@ let newCategoryName = $state(''); let newCategoryPrivate = $state(false); let newCategorySpillover = $state(false); + let newCategoryDisableAi = $state(false); let addingCategory = $state(false); // Advisory only — the nav starts getting too wide / wrapping past ~10 tabs, so this @@ -48,11 +49,12 @@ if (!name) return; addingCategory = true; try { - const created = await createCategory(name, newCategoryPrivate, newCategorySpillover); + const created = await createCategory(name, newCategoryPrivate, newCategorySpillover, newCategoryDisableAi); local.categoryPriority = [...local.categoryPriority, created]; newCategoryName = ''; newCategoryPrivate = false; newCategorySpillover = false; + newCategoryDisableAi = false; } finally { addingCategory = false; } @@ -68,6 +70,11 @@ scheduleSave(); } + function toggleDisableAi(id: string) { + local.categoryPriority = local.categoryPriority.map((c) => (c.id === id ? { ...c, disableAi: !c.disableAi } : c)); + scheduleSave(); + } + async function removeCategory(id: string, isDefault: boolean, name: string) { if (isDefault) { // Sensible-default categories can still be removed — e.g. a fresh install's @@ -98,7 +105,9 @@ private category (and everything in it) is hidden from the public site until a visitor logs in with the lock icon in the masthead. A "More" category is collapsed into a single "More »" nav tab instead of getting its own, and shows up on that overflow page with its - latest few articles. + latest few articles. "No AI" skips clustering and synthesis for that category — each item + publishes on its own, using its own source's text, instead of being merged/rewritten by the + model.

{#if primaryCategoryCount > 10}

@@ -120,6 +129,10 @@ toggleSpillover(cat.id)} /> More + {/if}