Generated
+2
-1
@@ -15,7 +15,8 @@
|
||||
"fastify": "^5.10.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"rss-parser": "^3.13.0",
|
||||
"telegram": "^2.26.22"
|
||||
"telegram": "^2.26.22",
|
||||
"undici": "^7.28.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jsdom": "^28.0.3",
|
||||
|
||||
@@ -18,7 +18,8 @@
|
||||
"fastify": "^5.10.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"rss-parser": "^3.13.0",
|
||||
"telegram": "^2.26.22"
|
||||
"telegram": "^2.26.22",
|
||||
"undici": "^7.28.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jsdom": "^28.0.3",
|
||||
|
||||
+103
-3
@@ -2,15 +2,19 @@ import type { FastifyInstance } from 'fastify';
|
||||
import * as settingsDb from '../storage/db/settings.js';
|
||||
import * as sourcesDb from '../storage/db/sources.js';
|
||||
import * as eventsDb from '../storage/db/events.js';
|
||||
import * as articlesDb from '../storage/db/articles.js';
|
||||
import * as categoriesDb from '../storage/db/categories.js';
|
||||
import * as stocksDb from '../storage/db/stocks.js';
|
||||
import * as bookmarksDb from '../storage/db/bookmarks.js';
|
||||
import * as poe2WatchlistDb from '../storage/db/poe2Watchlist.js';
|
||||
import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
|
||||
import { clearSourceContent, reissueSourceContent, reissueArticle, clearAllArticles, clearAllMedia } from '../storage/contentCascade.js';
|
||||
import { totalStorageBytes } from '../storage/media/index.js';
|
||||
import { OllamaProvider } from '../inference/ollama-provider.js';
|
||||
import { publishEventRecap } from '../pipeline/publish.js';
|
||||
import { pollSourceNow } from '../ingestion/poller.js';
|
||||
import { logger, listLogs } from '../storage/db/logs.js';
|
||||
import * as backlogStats from '../queue/backlogStats.js';
|
||||
import * as ollamaStats from '../inference/stats.js';
|
||||
import * as telegramClient from '../telegram/client.js';
|
||||
import { geocodeLocation } from '../weather/client.js';
|
||||
import { pollWeatherNow } from '../weather/poller.js';
|
||||
@@ -65,9 +69,14 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
|
||||
// --- Categories (add/remove — reordering/privacy is via PATCH /settings above) ---
|
||||
app.post('/api/admin/categories', async (req, reply) => {
|
||||
const { name, isPrivate, isSpillover } = req.body as { name?: string; isPrivate?: boolean; isSpillover?: boolean };
|
||||
const { name, isPrivate, isSpillover, disableAi } = req.body as {
|
||||
name?: string;
|
||||
isPrivate?: boolean;
|
||||
isSpillover?: boolean;
|
||||
disableAi?: boolean;
|
||||
};
|
||||
if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' });
|
||||
const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover);
|
||||
const created = categoriesDb.createCategory(name.trim(), !!isPrivate, !!isSpillover, !!disableAi);
|
||||
return reply.code(201).send(created);
|
||||
});
|
||||
|
||||
@@ -144,6 +153,17 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
return reissueSourceContent(id);
|
||||
});
|
||||
|
||||
// Fixes one specific bad article (e.g. a degenerate/empty AI synthesis — see
|
||||
// synthesis.ts's assertNonEmpty) by deleting it and requeuing every item it merged,
|
||||
// regardless of how many different sources contributed — reissueSourceContent above
|
||||
// deliberately won't touch a multi-source article at all.
|
||||
app.post('/api/admin/articles/:id/reissue', async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const result = reissueArticle(id);
|
||||
if (!result) return reply.code(404).send({ error: 'not found' });
|
||||
return result;
|
||||
});
|
||||
|
||||
// --- Tracked events ---
|
||||
app.get('/api/admin/events', async () => eventsDb.listEvents());
|
||||
|
||||
@@ -165,6 +185,39 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
return reply.code(204).send();
|
||||
});
|
||||
|
||||
// Forces one tracked item's recap to run right now, ignoring its recapIntervalHours
|
||||
// cadence entirely (even if recaps are turned off for it) — for "I want a wrap-up
|
||||
// right now" rather than waiting out the timer. Still summarizes the same real
|
||||
// window eventsRecap.ts would (everything published since lastRecapAt, or the last
|
||||
// 24h if it's never recapped) rather than some arbitrary admin-chosen range, and
|
||||
// still requires that window to actually contain something — an AI call with zero
|
||||
// source material to summarize would just hallucinate content it wasn't given.
|
||||
app.post('/api/admin/events/:id/recap-now', async (req, reply) => {
|
||||
const { id } = req.params as { id: string };
|
||||
const event = eventsDb.getEvent(id);
|
||||
if (!event) return reply.code(404).send({ error: 'not found' });
|
||||
if (event.sourceIds.length === 0) {
|
||||
return { published: false, reason: 'No sources assigned to this item yet.' };
|
||||
}
|
||||
|
||||
const since = event.lastRecapAt ?? new Date(Date.now() - 24 * 3600_000).toISOString();
|
||||
const constituents = articlesDb.articlesForEventSince(event.id, since);
|
||||
if (constituents.length === 0) {
|
||||
return { published: false, reason: 'Nothing new published under this item since its last recap.' };
|
||||
}
|
||||
|
||||
const settings = settingsDb.getSettings();
|
||||
const provider = new OllamaProvider(settings.aiServiceHost, settings.aiServicePort);
|
||||
try {
|
||||
const article = await publishEventRecap(provider, settings, event, constituents);
|
||||
eventsDb.markRecapped(event.id);
|
||||
logger.info('events', `Manually forced recap for "${event.name}" from ${constituents.length} article(s)`);
|
||||
return { published: true, title: article.title };
|
||||
} catch (err) {
|
||||
return reply.code(502).send({ error: `Recap failed: ${(err as Error).message}` });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Models / AI service (fetched live from the configured Ollama host) ---
|
||||
app.get('/api/admin/models', async (_req, reply) => {
|
||||
const settings = settingsDb.getSettings();
|
||||
@@ -186,6 +239,20 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
return { connected, host: settings.aiServiceHost, port: settings.aiServicePort, ramGB: null, gpu: null };
|
||||
});
|
||||
|
||||
// Detects the selected synthesis model's own max context length (when Ollama exposes
|
||||
// it) so the Models tab's num_ctx/num_predict sliders can be bounded by what the
|
||||
// model actually supports, instead of an arbitrary fixed cap. contextLength is null
|
||||
// when undetectable (older Ollama version, unusual model format, unreachable) — the
|
||||
// frontend falls back to a generous default range in that case rather than blocking.
|
||||
app.get('/api/admin/model-context', async (req, reply) => {
|
||||
const { model } = req.query as { model?: string };
|
||||
if (!model) return reply.code(400).send({ error: 'model query param required' });
|
||||
const settings = settingsDb.getSettings();
|
||||
const provider = new OllamaProvider(settings.aiServiceHost, settings.aiServicePort);
|
||||
const contextLength = await provider.getModelContextLength(model);
|
||||
return { contextLength };
|
||||
});
|
||||
|
||||
// --- Telegram account (Connections tab — see telegram/client.ts and credentials.ts.
|
||||
// API ID/hash and the resulting login session are stored encrypted at rest; none of
|
||||
// these routes ever echo them back to the client.) ---
|
||||
@@ -342,4 +409,37 @@ export async function registerAdminRoutes(app: FastifyInstance) {
|
||||
limit: limit ? Number(limit) : undefined
|
||||
});
|
||||
});
|
||||
|
||||
// Backlog/throughput dashboard for the Logs tab — backlog counts are recomputed live
|
||||
// from the DB on every request (cheap: no AI calls, see backlogStats.ts), while Ollama
|
||||
// throughput/in-flight status comes from a rolling in-memory sample of recent
|
||||
// generate() calls (see inference/stats.ts) since that can only be observed as calls
|
||||
// actually happen, not recomputed on demand.
|
||||
app.get('/api/admin/pipeline-stats', async () => {
|
||||
const settings = settingsDb.getSettings();
|
||||
const backlog = backlogStats.getBacklogSnapshot(settings);
|
||||
const throughput = ollamaStats.getThroughput();
|
||||
const inFlight = ollamaStats.getInFlight();
|
||||
const { lastDirectCycle, lastSynthesisCycle } = backlogStats.getLastCycles();
|
||||
|
||||
// Estimate is deliberately conservative: only clusters that actually need an LLM
|
||||
// call (2+ items — see backlogStats.ts) count toward it, and it's null (rather than
|
||||
// a misleading guess) until at least one real generate() call has completed, since
|
||||
// there's no token-speed data to estimate from yet.
|
||||
const estimatedMinutesToClear =
|
||||
backlog.clusters.readyNowNeedingSynthesis === 0
|
||||
? 0
|
||||
: throughput.avgGenerateDurationMs !== null
|
||||
? Math.ceil((backlog.clusters.readyNowNeedingSynthesis * throughput.avgGenerateDurationMs) / 60_000)
|
||||
: null;
|
||||
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
ollama: { inFlight, ...throughput },
|
||||
backlog,
|
||||
estimatedMinutesToClear,
|
||||
lastDirectCycle,
|
||||
lastSynthesisCycle
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -45,11 +45,25 @@ export async function registerPublicRoutes(app: FastifyInstance) {
|
||||
return tagsDb.listActiveTags();
|
||||
});
|
||||
|
||||
app.get('/api/tag/:slug', async (req, reply) => {
|
||||
const { slug } = req.params as { slug: string };
|
||||
const tag = tagsDb.getTagBySlug(slug);
|
||||
if (!tag) return reply.code(404).send({ error: 'not found' });
|
||||
return tag;
|
||||
});
|
||||
|
||||
app.get('/api/events', async () => {
|
||||
// Public fields only — sourceIds, keywords etc. stay admin-only.
|
||||
return eventsDb
|
||||
.listEvents()
|
||||
.map((e) => ({ id: e.id, name: e.name, active: e.active, recapIntervalHours: e.recapIntervalHours, isSpillover: e.isSpillover }));
|
||||
// Public fields only — sourceIds, keywords etc. stay admin-only. lastRecapAt is
|
||||
// safe to expose (just a timestamp, no source/keyword detail) and lets the
|
||||
// tracked-event page show when the next AI recap is due.
|
||||
return eventsDb.listEvents().map((e) => ({
|
||||
id: e.id,
|
||||
name: e.name,
|
||||
active: e.active,
|
||||
recapIntervalHours: e.recapIntervalHours,
|
||||
lastRecapAt: e.lastRecapAt,
|
||||
isSpillover: e.isSpillover
|
||||
}));
|
||||
});
|
||||
|
||||
// Drives the site nav — admin-editable (add/remove/reorder) via /api/admin/categories,
|
||||
|
||||
@@ -1,4 +1,39 @@
|
||||
import { Agent } from 'undici';
|
||||
import type { InferenceProvider } from './provider.js';
|
||||
import * as stats from './stats.js';
|
||||
|
||||
/**
|
||||
* Node's global fetch (undici) defaults to a 5-minute headers/body timeout — fine for
|
||||
* ordinary HTTP calls, but a real problem for /api/generate on CPU-only inference: a
|
||||
* near-full context window can legitimately take longer than that just for prompt
|
||||
* processing on the reference hardware (i5-6600K, no GPU, ~17 tokens/sec). Once
|
||||
* synthesis prompts started carrying full article bodies instead of short blurbs, every
|
||||
* generate() call past a few thousand tokens got killed at exactly 5m0s — visible in
|
||||
* Ollama's own log as the request being cancelled, not a genuine model/server error —
|
||||
* so no cluster could ever finish synthesizing. No timeout at all here; Ollama's own
|
||||
* process is the natural backstop, not a clock tuned for hardware this doesn't run on.
|
||||
*/
|
||||
const noTimeoutDispatcher = new Agent({ headersTimeout: 0, bodyTimeout: 0 });
|
||||
|
||||
/**
|
||||
* Default context window / max-generation length requested from Ollama when a caller
|
||||
* doesn't specify its own. Ollama otherwise falls back to whatever the model's
|
||||
* Modelfile/runner defaults to (observed as low as 4096 tokens for qwen2.5:7b-instruct
|
||||
* here, well under that model's 32768-token training context) and SILENTLY truncates
|
||||
* any prompt that doesn't fit — dropping the middle of the prompt with no error
|
||||
* surfaced anywhere. Explicitly setting num_ctx/num_predict on every request makes the
|
||||
* limit deliberate and stable instead of whatever Ollama happens to pick.
|
||||
*
|
||||
* 8192 is sized for CPU-only inference (the reference box is an i5-6600K running
|
||||
* Ollama in Docker, no GPU, ~17 tokens/sec prompt processing) — RAM is not the
|
||||
* constraint (48GB available; the KV cache for 8192 tokens is well under 1GB), but
|
||||
* prompt-processing time scales with context, so this trades headroom against
|
||||
* per-request latency rather than maxing out the model's full 32768-token capacity.
|
||||
* Callers that build prompts (see pipeline/synthesis.ts) size their own content to fit
|
||||
* within this budget up front, rather than relying on Ollama to truncate for them.
|
||||
*/
|
||||
export const DEFAULT_NUM_CTX = 8192;
|
||||
export const DEFAULT_NUM_PREDICT = 700;
|
||||
|
||||
/**
|
||||
* Talks to a self-hosted Ollama instance over HTTP. Address is a normal backend
|
||||
@@ -15,28 +50,71 @@ export class OllamaProvider implements InferenceProvider {
|
||||
return `${this.host}:${this.port}`;
|
||||
}
|
||||
|
||||
async generate(prompt: string, opts: { model?: string; system?: string } = {}): Promise<string> {
|
||||
const res = await fetch(`${this.base()}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: opts.model,
|
||||
prompt,
|
||||
system: opts.system,
|
||||
stream: false
|
||||
})
|
||||
});
|
||||
if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`);
|
||||
const data = (await res.json()) as { response: string };
|
||||
return data.response;
|
||||
async generate(
|
||||
prompt: string,
|
||||
opts: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string } = {}
|
||||
): Promise<string> {
|
||||
const startedAt = Date.now();
|
||||
stats.recordGenerateStart(opts.label ?? 'synthesis');
|
||||
try {
|
||||
const res = await fetch(`${this.base()}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: opts.model,
|
||||
prompt,
|
||||
system: opts.system,
|
||||
stream: false,
|
||||
options: {
|
||||
num_ctx: opts.numCtx ?? DEFAULT_NUM_CTX,
|
||||
num_predict: opts.numPredict ?? DEFAULT_NUM_PREDICT
|
||||
}
|
||||
}),
|
||||
// Not in the ambient RequestInit type this project resolves to, but Node's global
|
||||
// fetch (built on undici) honors it at runtime — see noTimeoutDispatcher above.
|
||||
dispatcher: noTimeoutDispatcher
|
||||
} as RequestInit);
|
||||
if (!res.ok) throw new Error(`Ollama generate failed: ${res.status} ${await res.text()}`);
|
||||
const data = (await res.json()) as {
|
||||
response: string;
|
||||
eval_count?: number;
|
||||
eval_duration?: number;
|
||||
prompt_eval_count?: number;
|
||||
prompt_eval_duration?: number;
|
||||
total_duration?: number;
|
||||
};
|
||||
// Ollama reports these *_duration fields in nanoseconds — dividing eval_count by
|
||||
// (eval_duration/1e9) gives generation tokens/sec, and total_duration/1e6 gives
|
||||
// wall-clock milliseconds (falling back to a local measurement if a given Ollama
|
||||
// version's response ever omits it).
|
||||
stats.recordGenerateEnd({
|
||||
genTokensPerSec: data.eval_count && data.eval_duration ? data.eval_count / (data.eval_duration / 1e9) : null,
|
||||
promptTokensPerSec:
|
||||
data.prompt_eval_count && data.prompt_eval_duration ? data.prompt_eval_count / (data.prompt_eval_duration / 1e9) : null,
|
||||
totalDurationMs: data.total_duration ? data.total_duration / 1e6 : Date.now() - startedAt
|
||||
});
|
||||
return data.response;
|
||||
} catch (err) {
|
||||
stats.recordGenerateEnd(null);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async embed(text: string, opts: { model?: string } = {}): Promise<number[]> {
|
||||
const res = await fetch(`${this.base()}/api/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model: opts.model, prompt: text })
|
||||
});
|
||||
body: JSON.stringify({ model: opts.model, prompt: text }),
|
||||
// Ollama serves one inference request at a time (n_slots = 1) — an embed call
|
||||
// queued behind a slow generate() call waits for that same slot, and on this
|
||||
// CPU-only hardware a generate() call can easily run past 5 minutes. Without
|
||||
// this, that wait alone was enough to trip the same default fetch timeout
|
||||
// generate() had (see noTimeoutDispatcher above), silently dropping the item
|
||||
// from embedPendingItems — it never got clustered, so a single-source item
|
||||
// unlucky enough to be embedded while Ollama was busy never published at all,
|
||||
// retried every cycle with the same result for as long as Ollama stayed busy.
|
||||
dispatcher: noTimeoutDispatcher
|
||||
} as RequestInit);
|
||||
if (!res.ok) throw new Error(`Ollama embed failed: ${res.status} ${await res.text()}`);
|
||||
const data = (await res.json()) as { embedding: number[] };
|
||||
return data.embedding;
|
||||
@@ -57,4 +135,32 @@ export class OllamaProvider implements InferenceProvider {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ollama's /api/show returns a model_info object whose keys are prefixed by the
|
||||
* model's own architecture name (e.g. "qwen2.context_length", "llama.context_length")
|
||||
* rather than one fixed field — there's no single stable key across model families.
|
||||
* Scanning for whichever key ends in ".context_length" avoids hardcoding a list of
|
||||
* known architectures that will inevitably miss a future/uncommon one. Returns null
|
||||
* (rather than throwing) on any failure — the admin-facing slider falls back to a
|
||||
* generous default cap when this can't be determined, rather than blocking the whole
|
||||
* Models tab on one unreliable, best-effort lookup.
|
||||
*/
|
||||
async getModelContextLength(model: string): Promise<number | null> {
|
||||
try {
|
||||
const res = await fetch(`${this.base()}/api/show`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, name: model }),
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as { model_info?: Record<string, unknown> };
|
||||
const entry = Object.entries(data.model_info ?? {}).find(([key]) => key.endsWith('.context_length'));
|
||||
const value = entry?.[1];
|
||||
return typeof value === 'number' && value > 0 ? value : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
export interface InferenceProvider {
|
||||
generate(prompt: string, opts?: { model?: string; system?: string }): Promise<string>;
|
||||
generate(
|
||||
prompt: string,
|
||||
opts?: { model?: string; system?: string; numCtx?: number; numPredict?: number; label?: string }
|
||||
): Promise<string>;
|
||||
embed(text: string, opts?: { model?: string }): Promise<number[]>;
|
||||
listModels(): Promise<string[]>;
|
||||
isReachable(): Promise<boolean>;
|
||||
/** The model's own reported max context length (training/architecture limit), or null if the server doesn't expose it — used to bound the admin-facing num_ctx slider (Models tab) so it can't be set past what the model actually supports. */
|
||||
getModelContextLength(model: string): Promise<number | null>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* In-memory-only tracking of Ollama generate() throughput and in-flight status, for the
|
||||
* admin "Logs" dashboard (see queue/backlogStats.ts, api/admin.ts's GET
|
||||
* /api/admin/pipeline-stats). Deliberately not persisted to disk — a restart losing a
|
||||
* few minutes of rolling samples is fine, since the next few generate() calls rebuild it.
|
||||
*/
|
||||
|
||||
const MAX_SAMPLES = 20;
|
||||
|
||||
export interface GenerateSample {
|
||||
/** Generation speed (tokens/sec) from Ollama's eval_count/eval_duration — null if the response omitted them. */
|
||||
genTokensPerSec: number | null;
|
||||
/** Prompt-processing speed (tokens/sec) from prompt_eval_count/prompt_eval_duration — usually the dominant cost on CPU-only inference. */
|
||||
promptTokensPerSec: number | null;
|
||||
totalDurationMs: number;
|
||||
}
|
||||
|
||||
const samples: GenerateSample[] = [];
|
||||
let inFlight: { label: string; startedAt: number } | null = null;
|
||||
|
||||
/** Call immediately before issuing a generate() request. */
|
||||
export function recordGenerateStart(label: string): void {
|
||||
inFlight = { label, startedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Call in a finally block after the request settles — pass null on failure/abort. */
|
||||
export function recordGenerateEnd(sample: GenerateSample | null): void {
|
||||
inFlight = null;
|
||||
if (!sample) return;
|
||||
samples.push(sample);
|
||||
if (samples.length > MAX_SAMPLES) samples.shift();
|
||||
}
|
||||
|
||||
export function getInFlight(): { label: string; elapsedMs: number } | null {
|
||||
return inFlight ? { label: inFlight.label, elapsedMs: Date.now() - inFlight.startedAt } : null;
|
||||
}
|
||||
|
||||
function average(nums: number[]): number | null {
|
||||
if (nums.length === 0) return null;
|
||||
return nums.reduce((a, b) => a + b, 0) / nums.length;
|
||||
}
|
||||
|
||||
export interface ThroughputStats {
|
||||
sampleCount: number;
|
||||
avgGenTokensPerSec: number | null;
|
||||
avgPromptTokensPerSec: number | null;
|
||||
avgGenerateDurationMs: number | null;
|
||||
}
|
||||
|
||||
export function getThroughput(): ThroughputStats {
|
||||
return {
|
||||
sampleCount: samples.length,
|
||||
avgGenTokensPerSec: average(samples.map((s) => s.genTokensPerSec).filter((n): n is number => n !== null)),
|
||||
avgPromptTokensPerSec: average(samples.map((s) => s.promptTokensPerSec).filter((n): n is number => n !== null)),
|
||||
avgGenerateDurationMs: average(samples.map((s) => s.totalDurationMs))
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { InferenceProvider } from '../inference/provider.js';
|
||||
import type { Cluster } from './clustering.js';
|
||||
import { synthesizeArticle, synthesizeRecap } from './synthesis.js';
|
||||
import { synthesizeArticle, synthesizeRecap, extractTags } from './synthesis.js';
|
||||
import { selectBestImage, faviconUrlFor } from './image-selection.js';
|
||||
import { downloadAndStore, promoteToPublished, storeMediaBuffer } from '../storage/media/index.js';
|
||||
import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js';
|
||||
@@ -36,10 +36,23 @@ function anyPushesToTopStories(items: ContentItem[]): boolean {
|
||||
return items.some((item) => sources.getSource(item.sourceId)?.pushToTopStories ?? false);
|
||||
}
|
||||
|
||||
/** Takes the first line of the synthesized body as a working title until a dedicated title-generation step exists. */
|
||||
function deriveTitle(body: string): string {
|
||||
const firstLine = body.split('\n')[0];
|
||||
return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine;
|
||||
/** Embeds each label and resolves/dedupes it against existing tags — shared by every publish path that has tagLabels in hand (from a full synthesis call or the lightweight extractTags), so the dedup behavior stays identical regardless of how the labels were produced. */
|
||||
async function resolveTagIds(
|
||||
provider: InferenceProvider,
|
||||
tagLabels: string[],
|
||||
settings: GlobalSettings,
|
||||
logSource: string
|
||||
): Promise<string[]> {
|
||||
const tagIds: string[] = [];
|
||||
for (const label of tagLabels) {
|
||||
try {
|
||||
const embedding = await provider.embed(label, { model: settings.selectedModels.embedding });
|
||||
tagIds.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold).id);
|
||||
} catch (err) {
|
||||
logger.error(logSource, `Tag embedding failed for "${label}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return tagIds;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -225,19 +238,19 @@ async function resolveQuotedTweet(
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes a single item as-is, with no AI calls at all — used when the AI service
|
||||
* isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
|
||||
* after the backend launches" requirement). No rewriting, no tag extraction, no
|
||||
* embedding. This is deliberately a lesser version of the real pipeline: once Ollama
|
||||
* is available, newly-ingested items get the full embed/cluster/synthesize treatment
|
||||
* and can be linked as follow-ups to these passthrough articles via the normal
|
||||
* tag-based thread detection — but these earlier articles aren't retroactively
|
||||
* rewritten or merged with anything after the fact.
|
||||
* Publishes a single item as-is — the title/body are never rewritten or merged (see
|
||||
* priorityQueue.ts for why: single-source clusters, YouTube/Nitter/Telegram items, and
|
||||
* AI-disabled categories all route here specifically to avoid that risk). When a
|
||||
* provider is given (AI is actually reachable and this item isn't in an AI-disabled
|
||||
* category), it still gets tags via a lightweight standalone extraction call — every
|
||||
* published article should be taggable/discoverable via /tag/[slug], not just the
|
||||
* AI-merged ones. Passing no provider (Ollama unreachable, or the item's category has
|
||||
* AI turned off entirely) skips tagging too — same as the old "no tags yet" behavior.
|
||||
*/
|
||||
export async function publishDirect(
|
||||
item: ContentItem,
|
||||
settings: GlobalSettings,
|
||||
opts: { eventId?: string } = {}
|
||||
opts: { eventId?: string; provider?: InferenceProvider } = {}
|
||||
): Promise<MergedArticle> {
|
||||
const category = uniqueCategories([item]);
|
||||
const storedMediaIds: string[] = [];
|
||||
@@ -311,6 +324,16 @@ export async function publishDirect(
|
||||
};
|
||||
}
|
||||
|
||||
let tagIds: string[] = [];
|
||||
if (opts.provider) {
|
||||
try {
|
||||
const tagLabels = await extractTags(opts.provider, settings.selectedModels.synthesis, item, settings);
|
||||
tagIds = await resolveTagIds(opts.provider, tagLabels, settings, 'synthesis');
|
||||
} catch (err) {
|
||||
logger.error('synthesis', `Tag extraction failed for "${item.title}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const article = await articles.insertArticle({
|
||||
title: item.title,
|
||||
body: item.body || item.summary,
|
||||
@@ -335,7 +358,7 @@ export async function publishDirect(
|
||||
publishedAt: item.publishedAt,
|
||||
updatedAt: item.publishedAt,
|
||||
mergeConfidence: 1.0,
|
||||
tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement
|
||||
tags: tagIds,
|
||||
threadId: randomUUID(),
|
||||
previousArticleId: null,
|
||||
nextArticleId: null,
|
||||
@@ -349,8 +372,9 @@ export async function publishDirect(
|
||||
|
||||
/**
|
||||
* Publishing is always automatic — there's no draft/review state (see schema doc).
|
||||
* A cluster of size 1 publishes as-is via the same path; synthesizeArticle lightly
|
||||
* rewrites rather than merges when there's only one source.
|
||||
* Callers should route a size-1 cluster to publishDirect instead — there's nothing to
|
||||
* merge, so an LLM rewrite would only add risk (hallucinated attribution, altered
|
||||
* facts) for no synthesis benefit. See priorityQueue.ts's runSynthesisCycle.
|
||||
*/
|
||||
export async function publishCluster(
|
||||
provider: InferenceProvider,
|
||||
@@ -360,18 +384,10 @@ export async function publishCluster(
|
||||
): Promise<MergedArticle> {
|
||||
const items = cluster.items;
|
||||
|
||||
const { body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items);
|
||||
const sourceNames = new Map(items.map((item) => [item.sourceId, sources.getSource(item.sourceId)?.name ?? 'Unknown source']));
|
||||
const { title, body, tagLabels } = await synthesizeArticle(provider, settings.selectedModels.synthesis, items, sourceNames, settings);
|
||||
|
||||
const resolvedTags = [];
|
||||
for (const label of tagLabels) {
|
||||
try {
|
||||
const embedding = await provider.embed(label, { model: settings.selectedModels.embedding });
|
||||
resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold));
|
||||
} catch (err) {
|
||||
logger.error('synthesis', `Tag embedding failed for "${label}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
const tagIds = resolvedTags.map((t) => t.id);
|
||||
const tagIds = await resolveTagIds(provider, tagLabels, settings, 'synthesis');
|
||||
|
||||
const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? '');
|
||||
const videoItem = items.find((i) => i.videos.length > 0);
|
||||
@@ -416,7 +432,7 @@ export async function publishCluster(
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const article = articles.insertArticle({
|
||||
title: deriveTitle(body),
|
||||
title,
|
||||
body,
|
||||
heroImage,
|
||||
video,
|
||||
@@ -459,24 +475,15 @@ export async function publishEventRecap(
|
||||
event: TrackedEvent,
|
||||
constituents: MergedArticle[]
|
||||
): Promise<MergedArticle> {
|
||||
const { body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event.name, constituents);
|
||||
|
||||
const resolvedTags = [];
|
||||
for (const label of tagLabels) {
|
||||
try {
|
||||
const embedding = await provider.embed(label, { model: settings.selectedModels.embedding });
|
||||
resolvedTags.push(tags.resolveOrCreateTag(label, embedding, settings.tagDedupThreshold));
|
||||
} catch (err) {
|
||||
logger.error('events', `Tag embedding failed for "${label}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
const { title, body, tagLabels } = await synthesizeRecap(provider, settings.selectedModels.synthesis, event, constituents, settings);
|
||||
const tagIds = await resolveTagIds(provider, tagLabels, settings, 'events');
|
||||
|
||||
const category = [...new Set(constituents.flatMap((a) => a.category))];
|
||||
const heroImage = constituents.find((a) => a.heroImage)?.heroImage ?? null;
|
||||
const now = new Date().toISOString();
|
||||
|
||||
return articles.insertArticle({
|
||||
title: `${event.name}: recap`,
|
||||
title: title || `${event.name}: recap`,
|
||||
body,
|
||||
heroImage,
|
||||
video: null,
|
||||
@@ -490,7 +497,7 @@ export async function publishEventRecap(
|
||||
publishedAt: now,
|
||||
updatedAt: now,
|
||||
mergeConfidence: 1.0,
|
||||
tags: resolvedTags.map((t) => t.id),
|
||||
tags: tagIds,
|
||||
threadId: randomUUID(),
|
||||
previousArticleId: null,
|
||||
nextArticleId: null,
|
||||
|
||||
@@ -1,62 +1,231 @@
|
||||
import type { InferenceProvider } from '../inference/provider.js';
|
||||
import type { ContentItem, MergedArticle } from '../storage/db/types.js';
|
||||
import type { ContentItem, GlobalSettings, MergedArticle, TrackedEvent } from '../storage/db/types.js';
|
||||
import { logger } from '../storage/db/logs.js';
|
||||
|
||||
const TITLE_DELIMITER = '---TITLE---';
|
||||
const TAG_DELIMITER = '---TAGS---';
|
||||
|
||||
const RECAP_SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write a single recap article that:
|
||||
- Summarizes what has happened across the period covered, in chronological order
|
||||
- Highlights the most significant developments rather than restating every article
|
||||
- Stays neutral and factual, without editorializing
|
||||
- Is 3-5 short paragraphs
|
||||
// Small/quantized models don't always reproduce a literal delimiter exactly — extra
|
||||
// dashes, an inserted blank line, different case (seen in production with the tag
|
||||
// delimiter: "---\n\nTAGS---" instead of "---TAGS---", which an exact-string split
|
||||
// missed entirely, leaking the raw delimiter text into the published body). Splitting
|
||||
// on a loose regex instead tolerates that variance.
|
||||
const TITLE_DELIMITER_RE = /-{2,}\s*TITLE\s*-{2,}/i;
|
||||
const TAG_DELIMITER_RE = /-{2,}\s*TAGS\s*-{2,}/i;
|
||||
|
||||
After the recap, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||
// Ollama truncates prompts that don't fit its context window by keeping a small prefix
|
||||
// and dropping everything else in the middle — silently, with no error, and with no
|
||||
// regard for which sources end up cut (see ollama-provider.ts for the incident that
|
||||
// prompted this). Rather than relying on that, prompts here are sized to fit the
|
||||
// admin-configured num_ctx/num_predict (Models tab) up front: each source/article gets
|
||||
// an equal character budget, cut only when the whole prompt would otherwise overflow, so
|
||||
// every source stays at least partially represented (and attributable) instead of some
|
||||
// being dropped outright. ~4 chars/token is a rough heuristic (no tokenizer available
|
||||
// here) — good enough for a safety margin, not meant to be exact.
|
||||
const CHARS_PER_TOKEN = 4;
|
||||
const RESERVED_OVERHEAD_TOKENS = 300; // system prompt + per-entry headers/formatting
|
||||
const MIN_ENTRY_CHARS = 300; // floor so a huge cluster/recap doesn't shrink every entry to nothing
|
||||
|
||||
const SYSTEM_PROMPT = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write a single original article that:
|
||||
- Attributes specific claims to the outlet that reported them (e.g. "Reuters reported...", "AP notes...")
|
||||
- Does not copy phrasing verbatim from any source
|
||||
- Stays neutral and factual, without editorializing
|
||||
- Is 2-4 short paragraphs
|
||||
/** Character budget for prompt *input* — leaves numPredict's worth of the context window free for the model's own response, per the admin's configured num_ctx/num_predict (GlobalSettings.synthesisNumCtx/synthesisNumPredict). */
|
||||
function maxInputChars(numCtx: number, numPredict: number): number {
|
||||
return Math.max(0, (numCtx - numPredict - RESERVED_OVERHEAD_TOKENS) * CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
If only one source is provided, lightly rewrite it in your own words rather than merging.
|
||||
function capEntryText(text: string, budgetChars: number): string {
|
||||
return text.length > budgetChars ? text.slice(0, budgetChars) + '…' : text;
|
||||
}
|
||||
|
||||
After the article, on a new line, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||
const TAG_EXTRACTION_SYSTEM_PROMPT = `You are a tagging assistant. Given a news item's title and summary, respond with ONLY 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named people, places, organizations, or named events) that this item is about — nothing else, no commentary, no leading text. If nothing salient qualifies, respond with an empty line.`;
|
||||
|
||||
/** Short response — a handful of tags, not prose — so this doesn't need DEFAULT_NUM_PREDICT's full budget. */
|
||||
const TAG_EXTRACTION_NUM_PREDICT = 40;
|
||||
|
||||
function parseTagLabels(raw: string): string[] {
|
||||
return raw
|
||||
.split(',')
|
||||
.map((t) => t.trim())
|
||||
.filter((t) => t.length > 0 && t.length < 60);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight standalone tag extraction for a single item — unlike synthesizeArticle,
|
||||
* this doesn't rewrite or attribute anything, so it's safe to run even for items that
|
||||
* publish verbatim via publishDirect (single-source clusters, or format-based direct
|
||||
* publishes like YouTube/Nitter/Telegram — see priorityQueue.ts). Every published
|
||||
* article should end up with tags regardless of whether it went through a full AI
|
||||
* merge, and this is the minimal AI call that makes that possible without triggering
|
||||
* the rewrite/attribution risk a full synthesizeArticle call would add for no benefit.
|
||||
*/
|
||||
export async function extractTags(
|
||||
provider: InferenceProvider,
|
||||
model: string,
|
||||
item: Pick<ContentItem, 'title' | 'summary' | 'body'>,
|
||||
settings: GlobalSettings
|
||||
): Promise<string[]> {
|
||||
const summary = capEntryText(item.body || item.summary, maxInputChars(settings.synthesisNumCtx, TAG_EXTRACTION_NUM_PREDICT));
|
||||
const prompt = `Title: ${item.title}\nSummary: ${summary}`;
|
||||
const raw = await provider.generate(prompt, {
|
||||
model,
|
||||
system: TAG_EXTRACTION_SYSTEM_PROMPT,
|
||||
numCtx: settings.synthesisNumCtx,
|
||||
numPredict: TAG_EXTRACTION_NUM_PREDICT,
|
||||
label: `Extracting tags: "${item.title.slice(0, 60)}"`
|
||||
});
|
||||
return parseTagLabels(raw);
|
||||
}
|
||||
|
||||
const RECAP_SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given a chronological list of articles already published about an ongoing tracked event, write your response in exactly three parts, in this order:
|
||||
|
||||
1. A short, specific headline for this recap (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period).
|
||||
2. On a new line, write exactly "${TITLE_DELIMITER}", then the recap:
|
||||
- Write a full, comprehensive news article covering the period — not a short summary or a bare list of bullet points. Use as many paragraphs and as much length as the material actually warrants; do not artificially cut it short.
|
||||
- Organize it in chronological order, but group and connect related developments into a coherent narrative rather than restating each source article one at a time
|
||||
- Give real weight and detail to the most significant developments; minor ones can be covered more briefly, but nothing significant should be dropped for the sake of brevity
|
||||
- Stays neutral and factual, without editorializing
|
||||
3. On a new line after the recap, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this recap is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||
|
||||
const SYSTEM_PROMPT_BASE = `You are a neutral news synthesis assistant. Given summaries from multiple news sources describing the same event, write your response in exactly three parts, in this order:
|
||||
|
||||
1. A short, specific headline for this story (a single line, ideally under 12 words, no surrounding quotation marks, no trailing period, no site/outlet name).
|
||||
2. On a new line, write exactly "${TITLE_DELIMITER}", then the article:
|
||||
- Attributes specific claims to the outlet that reported them, using each source's exact name as given below (e.g. if a source is labeled "Source 1 (Reuters)", write "Reuters reported..."). Never invent, guess, or substitute an outlet name that isn't one of the source names actually given below.
|
||||
- Does not copy phrasing verbatim from any source
|
||||
- Stays neutral and factual, without editorializing
|
||||
- Is 2-4 short paragraphs
|
||||
3. On a new line after the article, write exactly "${TAG_DELIMITER}" followed by 2-4 short comma-separated topic/entity tags (e.g. proper nouns, named events) that this article is about. If nothing salient qualifies, leave the tag line empty.`;
|
||||
|
||||
// Admin-selectable presets (Merge tab, "Writing style") — appended to whichever base
|
||||
// prompt applies. 'default' adds nothing: the base prompts above already describe the
|
||||
// original neutral wire-service tone this pipeline shipped with.
|
||||
const STYLE_PRESETS: Record<GlobalSettings['synthesisStylePreset'], string> = {
|
||||
default: '',
|
||||
casual: 'Write in a casual, conversational tone, like a knowledgeable friend catching you up on what happened — contractions and plain language are fine. Still stay factual and keep outlet attribution accurate.',
|
||||
formal: 'Write in a formal, measured register — precise language, no contractions, no colloquialisms.'
|
||||
};
|
||||
|
||||
/** Admin-configurable tone: a preset plus optional free-text instructions, both from GlobalSettings — the only two knobs that affect HOW the model writes, as opposed to WHAT gets clustered/published. Appended to the base prompt, never replacing its structural rules (attribution, paragraph count, tag format). Applies only to regular same-story merges — recaps have their own independent style knob, see recapStyleAddendum below. */
|
||||
function styleAddendum(settings: GlobalSettings): string {
|
||||
const preset = STYLE_PRESETS[settings.synthesisStylePreset] ?? '';
|
||||
const custom = settings.synthesisCustomInstructions.trim();
|
||||
const lines = [preset, custom].filter(Boolean);
|
||||
if (lines.length === 0) return '';
|
||||
return `\n\nAdditional style instructions from the site admin (follow these without breaking the rules above):\n${lines.join('\n')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-tracked-item recap style — deliberately independent of the global synthesisStylePreset
|
||||
* above (set in the Merge tab), since a recap's tone/scope is a very different kind of
|
||||
* knob: it's set once per tracked item (the "More" section on its own edit panel, next to
|
||||
* its recap cadence), not globally for every merge on the site. Reuses the same preset
|
||||
* strings for consistency, but reads from the event's own fields instead of GlobalSettings.
|
||||
*/
|
||||
function recapStyleAddendum(event: TrackedEvent): string {
|
||||
const preset = STYLE_PRESETS[event.recapStylePreset] ?? '';
|
||||
const custom = event.recapCustomInstructions.trim();
|
||||
const lines = [preset, custom].filter(Boolean);
|
||||
if (lines.length === 0) return '';
|
||||
return `\n\nAdditional style instructions from the site admin for this recap (follow these without breaking the rules above):\n${lines.join('\n')}`;
|
||||
}
|
||||
|
||||
export interface SynthesisResult {
|
||||
title: string;
|
||||
body: string;
|
||||
tagLabels: string[];
|
||||
}
|
||||
|
||||
function buildPrompt(items: ContentItem[]): string {
|
||||
return items
|
||||
.map((item, i) => `Source ${i + 1} (${item.sourceId}):\nTitle: ${item.title}\nSummary: ${item.summary}`)
|
||||
.join('\n\n');
|
||||
/** Only used when the model doesn't follow the requested title/delimiter format at all — a real headline beats a truncated sentence fragment, but publishing with no title at all is worse than either. */
|
||||
function fallbackTitle(body: string): string {
|
||||
const firstLine = body.split('\n')[0];
|
||||
return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine;
|
||||
}
|
||||
|
||||
function buildPrompt(items: ContentItem[], sourceNames: Map<string, string>, numCtx: number, numPredict: number): string {
|
||||
const budgetPerItem = Math.max(MIN_ENTRY_CHARS, Math.floor(maxInputChars(numCtx, numPredict) / items.length));
|
||||
let truncated = 0;
|
||||
const entries = items.map((item, i) => {
|
||||
// Same fallback publishDirect uses (publish.ts) — body is the full article text
|
||||
// when the feed supplies it (e.g. RSS <content:encoded>), summary is a ~500-char
|
||||
// blurb. Using summary alone starved the model of real content to synthesize
|
||||
// from, so a single-source cluster just echoed the blurb back nearly verbatim.
|
||||
const full = item.body || item.summary;
|
||||
const text = capEntryText(full, budgetPerItem);
|
||||
if (text !== full) truncated++;
|
||||
// The label here (not item.sourceId, an opaque internal id the model can't use)
|
||||
// is the only real outlet name the model ever sees — without it, a small model
|
||||
// has nothing to attribute to and falls back to copying the illustrative outlet
|
||||
// names out of its own system prompt instructions instead (seen in production:
|
||||
// a single-source item fabricating "Reuters reported..."/"AP notes..." wholesale).
|
||||
const name = sourceNames.get(item.sourceId) ?? 'Unknown source';
|
||||
return `Source ${i + 1} (${name}):\nTitle: ${item.title}\nSummary: ${text}`;
|
||||
});
|
||||
if (truncated > 0) {
|
||||
logger.warn('synthesis', `Trimmed ${truncated}/${items.length} source article${truncated === 1 ? '' : 's'} to fit the model's context window`);
|
||||
}
|
||||
return entries.join('\n\n');
|
||||
}
|
||||
|
||||
function parseResult(raw: string): SynthesisResult {
|
||||
const [body, tagSection] = raw.split(TAG_DELIMITER);
|
||||
const tagLabels = (tagSection ?? '')
|
||||
.split(',')
|
||||
.map((t) => t.trim())
|
||||
.filter((t) => t.length > 0 && t.length < 60);
|
||||
const [beforeTags, tagSection] = raw.split(TAG_DELIMITER_RE);
|
||||
const tagLabels = parseTagLabels(tagSection ?? '');
|
||||
|
||||
return { body: body.trim(), tagLabels };
|
||||
const titleSplit = (beforeTags ?? raw).split(TITLE_DELIMITER_RE);
|
||||
const titlePart = titleSplit[0];
|
||||
// join() rather than titleSplit[1] in case the delimiter text somehow appears again
|
||||
// inside the body itself — keeps that content rather than silently dropping it.
|
||||
const bodyPart = titleSplit.length > 1 ? titleSplit.slice(1).join('') : undefined;
|
||||
// If the title delimiter never showed up, the model didn't follow the requested
|
||||
// format — treat the whole thing as body rather than mistaking the article itself
|
||||
// for a "title", and fall back to the old truncated-first-line heuristic.
|
||||
const body = (bodyPart ?? titlePart).trim();
|
||||
const title = bodyPart !== undefined ? titlePart.trim() : fallbackTitle(body);
|
||||
|
||||
return { title, body, tagLabels };
|
||||
}
|
||||
|
||||
/**
|
||||
* A quantized/small model occasionally reproduces just the requested delimiter
|
||||
* scaffold ("---TITLE---\n\n---TAGS---") with no real headline or article text in
|
||||
* between — a structurally "valid" response by parseResult's own logic (delimiters
|
||||
* found, nothing crashed) but empty in substance. Left unchecked this published a
|
||||
* blank article (empty title/body, still with real sources/hero image attached) once
|
||||
* in production. Treating an empty body as a hard failure lets the caller's existing
|
||||
* catch-and-retry logic (see priorityQueue.ts's runSynthesisCycle) leave the cluster
|
||||
* unclustered for the next cycle instead of ever inserting one of these.
|
||||
*/
|
||||
function assertNonEmpty(result: SynthesisResult, context: string): SynthesisResult {
|
||||
if (!result.body.trim()) {
|
||||
throw new Error(`Model returned an empty article body for ${context}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function synthesizeArticle(
|
||||
provider: InferenceProvider,
|
||||
model: string,
|
||||
items: ContentItem[]
|
||||
items: ContentItem[],
|
||||
sourceNames: Map<string, string>,
|
||||
settings: GlobalSettings
|
||||
): Promise<SynthesisResult> {
|
||||
const prompt = buildPrompt(items);
|
||||
const raw = await provider.generate(prompt, { model, system: SYSTEM_PROMPT });
|
||||
return parseResult(raw);
|
||||
const { synthesisNumCtx: numCtx, synthesisNumPredict: numPredict } = settings;
|
||||
const prompt = buildPrompt(items, sourceNames, numCtx, numPredict);
|
||||
const system = SYSTEM_PROMPT_BASE + styleAddendum(settings);
|
||||
const label = `Merging ${items.length} source${items.length === 1 ? '' : 's'}: "${items[0]?.title.slice(0, 60) ?? ''}"`;
|
||||
const raw = await provider.generate(prompt, { model, system, numCtx, numPredict, label });
|
||||
return assertNonEmpty(parseResult(raw), `"${items[0]?.title.slice(0, 60) ?? ''}"`);
|
||||
}
|
||||
|
||||
function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string {
|
||||
const entries = articles
|
||||
.map((article, i) => `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${article.body}`)
|
||||
.join('\n\n');
|
||||
return `Tracked event: ${eventName}\n\n${entries}`;
|
||||
function buildRecapPrompt(eventName: string, articles: MergedArticle[], numCtx: number, numPredict: number): string {
|
||||
const budgetPerArticle = Math.max(MIN_ENTRY_CHARS, Math.floor(maxInputChars(numCtx, numPredict) / articles.length));
|
||||
let truncated = 0;
|
||||
const entries = articles.map((article, i) => {
|
||||
const body = capEntryText(article.body, budgetPerArticle);
|
||||
if (body !== article.body) truncated++;
|
||||
return `Article ${i + 1} (published ${article.publishedAt}):\nTitle: ${article.title}\n${body}`;
|
||||
});
|
||||
if (truncated > 0) {
|
||||
logger.warn('events', `Trimmed ${truncated}/${articles.length} recap article bod${truncated === 1 ? 'y' : 'ies'} to fit the model's context window`);
|
||||
}
|
||||
return `Tracked event: ${eventName}\n\n${entries.join('\n\n')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,10 +239,18 @@ function buildRecapPrompt(eventName: string, articles: MergedArticle[]): string
|
||||
export async function synthesizeRecap(
|
||||
provider: InferenceProvider,
|
||||
model: string,
|
||||
eventName: string,
|
||||
articles: MergedArticle[]
|
||||
event: TrackedEvent,
|
||||
articles: MergedArticle[],
|
||||
settings: GlobalSettings
|
||||
): Promise<SynthesisResult> {
|
||||
const prompt = buildRecapPrompt(eventName, articles);
|
||||
const raw = await provider.generate(prompt, { model, system: RECAP_SYSTEM_PROMPT });
|
||||
return parseResult(raw);
|
||||
const { synthesisNumCtx: numCtx, synthesisNumPredict: numPredict } = settings;
|
||||
const prompt = buildRecapPrompt(event.name, articles, numCtx, numPredict);
|
||||
const raw = await provider.generate(prompt, {
|
||||
model,
|
||||
system: RECAP_SYSTEM_PROMPT_BASE + recapStyleAddendum(event),
|
||||
numCtx,
|
||||
numPredict,
|
||||
label: `Recapping event: "${event.name.slice(0, 60)}"`
|
||||
});
|
||||
return assertNonEmpty(parseResult(raw), `event recap "${event.name.slice(0, 60)}"`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import * as contentItemsDb from '../storage/db/contentItems.js';
|
||||
import * as sourcesDb from '../storage/db/sources.js';
|
||||
import * as categoriesDb from '../storage/db/categories.js';
|
||||
import { clusterItems } from '../pipeline/clustering.js';
|
||||
import type { ContentItem, GlobalSettings, Source } from '../storage/db/types.js';
|
||||
|
||||
interface CycleRecord {
|
||||
at: string;
|
||||
published: number;
|
||||
}
|
||||
|
||||
let lastDirectCycle: CycleRecord | null = null;
|
||||
let lastSynthesisCycle: CycleRecord | null = null;
|
||||
|
||||
/** Called by priorityQueue.ts at the end of runDirectPublishCycle. */
|
||||
export function recordDirectPublishCycle(published: number): void {
|
||||
lastDirectCycle = { at: new Date().toISOString(), published };
|
||||
}
|
||||
|
||||
/** Called by priorityQueue.ts at the end of runSynthesisCycle. */
|
||||
export function recordSynthesisCycle(published: number): void {
|
||||
lastSynthesisCycle = { at: new Date().toISOString(), published };
|
||||
}
|
||||
|
||||
export function getLastCycles(): { lastDirectCycle: CycleRecord | null; lastSynthesisCycle: CycleRecord | null } {
|
||||
return { lastDirectCycle, lastSynthesisCycle };
|
||||
}
|
||||
|
||||
function inAiDisabledCategory(item: ContentItem, disabledNames: Set<string>, sourcesById: Map<string, Source>): boolean {
|
||||
const source = sourcesById.get(item.sourceId);
|
||||
for (const cat of source?.category ?? []) {
|
||||
if (disabledNames.has(cat.split(':')[0].trim().toLowerCase())) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export interface BacklogSnapshot {
|
||||
totalUnclusteredItems: number;
|
||||
/** Items that need no AI at all (YouTube/Nitter/Telegram sources, or "No AI" categories) — publish on the next direct-publish tick. */
|
||||
directEligibleItems: number;
|
||||
/** Mergeable items that haven't been embedded yet (embed() failed/pending, or just ingested since the last synthesis tick). */
|
||||
awaitingEmbeddingItems: number;
|
||||
clusters: {
|
||||
total: number;
|
||||
/** Cleared the hold-before-publish window — will publish on the next synthesis tick. */
|
||||
readyNow: number;
|
||||
/** Of readyNow, clusters with 2+ items — these are the ones that actually need an LLM generate() call (single-item clusters publish verbatim, no AI). */
|
||||
readyNowNeedingSynthesis: number;
|
||||
/** Still waiting out the hold-before-publish window. */
|
||||
onHold: number;
|
||||
itemsOnHold: number;
|
||||
earliestHoldRemainingMs: number | null;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only snapshot of the current backlog for the admin dashboard — mirrors the same
|
||||
* categorization runDirectPublishCycle/runSynthesisCycle use (priorityQueue.ts), but never
|
||||
* calls the AI itself: items with no embedding yet are just counted, not embedded, and
|
||||
* clustering only runs over items that already have one (cosine similarity over stored
|
||||
* vectors — no network call). Cheap enough to call on every dashboard refresh.
|
||||
*/
|
||||
export function getBacklogSnapshot(settings: GlobalSettings): BacklogSnapshot {
|
||||
const items = contentItemsDb.unclusteredItemsExcludingSources([]);
|
||||
const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s]));
|
||||
const categories = categoriesDb.listCategories();
|
||||
|
||||
const directPublishSourceIds = new Set(
|
||||
[...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id)
|
||||
);
|
||||
const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase()));
|
||||
|
||||
const directEligible: ContentItem[] = [];
|
||||
const mergeable: ContentItem[] = [];
|
||||
for (const item of items) {
|
||||
if (directPublishSourceIds.has(item.sourceId) || inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)) {
|
||||
directEligible.push(item);
|
||||
} else {
|
||||
mergeable.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
const awaitingEmbedding = mergeable.filter((item) => !item.embedding);
|
||||
const embedded = mergeable.filter((item) => item.embedding);
|
||||
|
||||
const clusters = clusterItems(embedded, settings.mergeStrictness);
|
||||
const holdMs = settings.holdBeforePublishMinutes * 60_000;
|
||||
|
||||
let readyNow = 0;
|
||||
let readyNowNeedingSynthesis = 0;
|
||||
let onHold = 0;
|
||||
let itemsOnHold = 0;
|
||||
let earliestHoldRemainingMs: number | null = null;
|
||||
|
||||
for (const cluster of clusters) {
|
||||
const earliestFetch = Math.min(...cluster.items.map((i) => new Date(i.fetchedAt).getTime()));
|
||||
const remaining = holdMs - (Date.now() - earliestFetch);
|
||||
if (remaining > 0) {
|
||||
onHold++;
|
||||
itemsOnHold += cluster.items.length;
|
||||
earliestHoldRemainingMs = earliestHoldRemainingMs === null ? remaining : Math.min(earliestHoldRemainingMs, remaining);
|
||||
} else {
|
||||
readyNow++;
|
||||
if (cluster.items.length > 1) readyNowNeedingSynthesis++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalUnclusteredItems: items.length,
|
||||
directEligibleItems: directEligible.length,
|
||||
awaitingEmbeddingItems: awaitingEmbedding.length,
|
||||
clusters: { total: clusters.length, readyNow, readyNowNeedingSynthesis, onHold, itemsOnHold, earliestHoldRemainingMs }
|
||||
};
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js';
|
||||
import { clusterItems } from '../pipeline/clustering.js';
|
||||
import { publishCluster, publishDirect } from '../pipeline/publish.js';
|
||||
import { logger } from '../storage/db/logs.js';
|
||||
import * as backlogStats from './backlogStats.js';
|
||||
import type { GlobalSettings, ContentItem, TrackedEvent, Source } from '../storage/db/types.js';
|
||||
|
||||
function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
|
||||
@@ -45,23 +46,37 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>,
|
||||
return best;
|
||||
}
|
||||
|
||||
/** True if any of the item's source's categories (same leading-segment match as primaryCategoryRank) has AI disabled. */
|
||||
function inAiDisabledCategory(item: ContentItem, disabledNames: Set<string>, sourcesById: Map<string, Source>): boolean {
|
||||
const source = sourcesById.get(item.sourceId);
|
||||
for (const cat of source?.category ?? []) {
|
||||
const leading = cat.split(':')[0].trim().toLowerCase();
|
||||
if (disabledNames.has(leading)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same
|
||||
* publish-then-tag-then-log/error shape, differing only in how the success/failure
|
||||
* message describes why the item skipped merging.
|
||||
* message describes why the item skipped merging. `provider`, when given, still gets
|
||||
* these articles tagged (via publishDirect's lightweight extraction) without rewriting
|
||||
* anything — omit it entirely for items whose category has AI turned off, or when
|
||||
* Ollama isn't reachable at all (see call sites).
|
||||
*/
|
||||
async function publishItemsDirect(
|
||||
items: ContentItem[],
|
||||
settings: GlobalSettings,
|
||||
activeEvents: TrackedEvent[],
|
||||
describeSuccess: (item: ContentItem) => string,
|
||||
failureLabel: string
|
||||
failureLabel: string,
|
||||
provider?: InferenceProvider
|
||||
): Promise<number> {
|
||||
let published = 0;
|
||||
for (const item of items) {
|
||||
try {
|
||||
const eventId = claimedEventId(item, activeEvents) ?? undefined;
|
||||
const article = await publishDirect(item, settings, { eventId });
|
||||
const article = await publishDirect(item, settings, { eventId, provider });
|
||||
contentItemsDb.assignCluster([item.id], article.id);
|
||||
published++;
|
||||
logger.info('synthesis', `Published "${article.title}" directly (${describeSuccess(item)})`);
|
||||
@@ -78,7 +93,10 @@ async function publishItemsDirect(
|
||||
* pipeline, this doesn't wait out the hold-before-publish window: that window exists to
|
||||
* give corroborating sources time to arrive before an AI merge locks in, which doesn't
|
||||
* apply here since there's no merging happening at all — each item is just itself.
|
||||
* Still respects category priority.
|
||||
* Still respects category priority. Harmless overlap with runDirectPublishCycle (which
|
||||
* runs regardless of reachability) — an item already published by one is simply gone
|
||||
* from the other's next "unclustered" query, since assignCluster lands before either
|
||||
* moves on to its next item.
|
||||
*/
|
||||
export async function runPassthroughCycle(settings: GlobalSettings): Promise<number> {
|
||||
const activeEvents = eventsDb.listActiveEvents();
|
||||
@@ -97,43 +115,97 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass of the synthesis queue: cluster whatever's unclustered, ordered by
|
||||
* admin-defined category priority, and publish clusters that have cleared the
|
||||
* hold-before-publish window. Items claimed by an active tracked event (belonging to
|
||||
* one of its sources and matching its keyword filter, if any) publish exactly like
|
||||
* everything else — individually or merged with same-story coverage — just tagged with
|
||||
* the event's id so they're browsable under it and eligible for eventsRecap.ts's
|
||||
* periodic AI wrap-up.
|
||||
* Publishes every item that never needs the AI at all: YouTube/Nitter/Telegram items
|
||||
* (always their own article, merge or no merge) and items whose category has "No AI"
|
||||
* set (Category priority admin pane). Deliberately its own guarded tick in scheduler.ts,
|
||||
* separate from runSynthesisCycle — these items have no reason to wait behind a slow AI
|
||||
* merge backlog (a generate() call can run minutes on CPU-only inference; see
|
||||
* ollama-provider.ts), and runSynthesisCycle's own reentrancy guard used to make them
|
||||
* do exactly that: stuck for however long the current cycle's clustering/synthesis
|
||||
* portion took, since both used to run under one guarded function. Runs regardless of
|
||||
* Ollama's reachability — merging/rewriting never happens here either way. `provider`
|
||||
* is optional and only used for tagging (see publishItemsDirect): scheduler.ts passes
|
||||
* one only when Ollama is actually reachable, and even then only type-direct items
|
||||
* (YouTube/Nitter/Telegram — direct-published purely because of format) get tagged,
|
||||
* never category-direct items (AI turned off for that category entirely, on purpose).
|
||||
*/
|
||||
export async function runDirectPublishCycle(settings: GlobalSettings, provider?: InferenceProvider): Promise<number> {
|
||||
const activeEvents = eventsDb.listActiveEvents();
|
||||
const items = contentItemsDb.unclusteredItemsExcludingSources([]);
|
||||
if (items.length === 0) {
|
||||
backlogStats.recordDirectPublishCycle(0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s]));
|
||||
const categories = categoriesDb.listCategories();
|
||||
|
||||
const directPublishSourceIds = new Set(
|
||||
[...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id)
|
||||
);
|
||||
const [typeDirectItems, remaining] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
|
||||
|
||||
const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase()));
|
||||
const [categoryDirectItems] = partition(remaining, (item) => inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById));
|
||||
|
||||
const publishedTypeDirect = await publishItemsDirect(
|
||||
typeDirectItems,
|
||||
settings,
|
||||
activeEvents,
|
||||
(item) => sourcesById.get(item.sourceId)?.type ?? 'unknown',
|
||||
'Direct publish failed',
|
||||
provider
|
||||
);
|
||||
|
||||
const publishedCategoryDirect = await publishItemsDirect(
|
||||
categoryDirectItems,
|
||||
settings,
|
||||
activeEvents,
|
||||
() => 'AI disabled for category',
|
||||
'Direct publish failed'
|
||||
);
|
||||
|
||||
const total = publishedTypeDirect + publishedCategoryDirect;
|
||||
backlogStats.recordDirectPublishCycle(total);
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass of the synthesis queue: cluster whatever's unclustered (excluding items
|
||||
* runDirectPublishCycle already owns — see there), ordered by admin-defined category
|
||||
* priority, and publish clusters that have cleared the hold-before-publish window.
|
||||
* Items claimed by an active tracked event (belonging to one of its sources and
|
||||
* matching its keyword filter, if any) publish exactly like everything else —
|
||||
* individually or merged with same-story coverage — just tagged with the event's id so
|
||||
* they're browsable under it and eligible for eventsRecap.ts's periodic AI wrap-up.
|
||||
*/
|
||||
export async function runSynthesisCycle(provider: InferenceProvider, settings: GlobalSettings): Promise<number> {
|
||||
const activeEvents = eventsDb.listActiveEvents();
|
||||
const items = contentItemsDb.unclusteredItemsExcludingSources([]);
|
||||
if (items.length === 0) return 0;
|
||||
if (items.length === 0) {
|
||||
backlogStats.recordSynthesisCycle(0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// One fetch of the full source list per cycle, reused below for both the
|
||||
// direct-publish partition and each item's category/type lookups — avoids a
|
||||
// direct-publish exclusion and each item's category/rank lookups — avoids a
|
||||
// separate sourcesDb.getSource() round-trip per item.
|
||||
const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s]));
|
||||
const categories = categoriesDb.listCategories();
|
||||
const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
|
||||
|
||||
// YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with
|
||||
// anything else — each is always its own article, same shape whether the AI service
|
||||
// is up or not. Route them straight to publishDirect, same as the no-AI passthrough path.
|
||||
// YouTube/Nitter/Telegram items and AI-disabled-category items are runDirectPublishCycle's
|
||||
// job (its own guarded tick, so a slow merge backlog here never blocks them) — excluded
|
||||
// here too since a batch just ingested this instant may still be unclustered when this
|
||||
// runs before that cycle's own pass gets to it.
|
||||
const directPublishSourceIds = new Set(
|
||||
[...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id)
|
||||
);
|
||||
const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
|
||||
|
||||
const publishedDirect = await publishItemsDirect(
|
||||
directItems,
|
||||
settings,
|
||||
activeEvents,
|
||||
(item) => sourcesById.get(item.sourceId)?.type ?? 'unknown',
|
||||
'Direct publish failed'
|
||||
const aiDisabledCategoryNames = new Set(categories.filter((c) => c.disableAi).map((c) => c.name.toLowerCase()));
|
||||
const mergeableItems = items.filter(
|
||||
(item) => !directPublishSourceIds.has(item.sourceId) && !inAiDisabledCategory(item, aiDisabledCategoryNames, sourcesById)
|
||||
);
|
||||
|
||||
const categories = categoriesDb.listCategories();
|
||||
const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
|
||||
|
||||
const ranked = mergeableItems
|
||||
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) }))
|
||||
.sort((a, b) => a.rank - b.rank)
|
||||
@@ -161,7 +233,14 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
|
||||
// in practice a cluster's items are all near-duplicate coverage of the same
|
||||
// story, so they'd all match the same event's filter anyway when they match at all.
|
||||
const eventId = cluster.items.map((i) => claimedEventId(i, activeEvents)).find((id) => id !== null) ?? undefined;
|
||||
const article = await publishCluster(provider, settings, cluster, { eventId });
|
||||
// A single-item cluster has nothing to merge — publish the source's own text
|
||||
// verbatim instead of asking the LLM to "lightly rewrite" it, which only risked
|
||||
// introducing errors (or fabricated attribution — see synthesis.ts) with no
|
||||
// actual synthesis to justify the risk.
|
||||
const article =
|
||||
cluster.items.length === 1
|
||||
? await publishDirect(cluster.items[0], settings, { eventId, provider })
|
||||
: await publishCluster(provider, settings, cluster, { eventId });
|
||||
contentItemsDb.assignCluster(
|
||||
cluster.items.map((i) => i.id),
|
||||
cluster.id
|
||||
@@ -184,5 +263,7 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
|
||||
);
|
||||
}
|
||||
|
||||
return published + publishedDirect;
|
||||
backlogStats.recordSynthesisCycle(published);
|
||||
|
||||
return published;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { pollDueSources } from '../ingestion/poller.js';
|
||||
import { runSynthesisCycle, runPassthroughCycle } from './priorityQueue.js';
|
||||
import { runSynthesisCycle, runPassthroughCycle, runDirectPublishCycle } from './priorityQueue.js';
|
||||
import { runEventRecaps } from './eventsRecap.js';
|
||||
import { runRetentionSweep } from './retention.js';
|
||||
import { OllamaProvider } from '../inference/ollama-provider.js';
|
||||
@@ -10,28 +10,77 @@ import { pollStocksNow } from '../stocks/poller.js';
|
||||
import { pollPoe2Now } from '../poe2/poller.js';
|
||||
|
||||
const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each source's own interval governs actual fetch frequency
|
||||
const DIRECT_PUBLISH_TICK_MS = 60_000;
|
||||
const SYNTHESIS_TICK_MS = 60_000;
|
||||
const RETENTION_TICK_MS = 60 * 60_000; // hourly
|
||||
const WEATHER_TICK_MS = 45 * 60_000;
|
||||
const STOCKS_TICK_MS = 15 * 60_000; // per admin spec — stock prices move faster than weather
|
||||
const POE2_TICK_MS = 60 * 60_000; // poe.ninja's own overview data doesn't refresh faster than hourly, so polling more often than this just re-fetches the same numbers
|
||||
|
||||
/**
|
||||
* Runs fn on every tick, but skips a tick outright if the previous one is still in
|
||||
* flight instead of overlapping it. Matters most for the synthesis tick: an item stays
|
||||
* "unclustered" (cluster_id IS NULL — see contentItems.unclusteredItemsExcludingSources)
|
||||
* until AFTER its cluster finishes synthesizing and publishing, so a generate() call
|
||||
* that runs past the next tick (easily minutes, on CPU-only inference — see
|
||||
* ollama-provider.ts) used to let the same item get picked up and republished as a
|
||||
* fresh, differently-worded article by an overlapping cycle, repeatedly, until the
|
||||
* first cycle's assignCluster() finally landed. Node is single-threaded, so the only
|
||||
* source of "concurrent" runs here is exactly this interval overlap.
|
||||
*
|
||||
* Each call gets its own independent `running` flag/timer — the direct-publish and
|
||||
* synthesis ticks are deliberately two separate calls to this (not one shared guard)
|
||||
* precisely so a slow AI-merge backlog on one never blocks the other's fast,
|
||||
* no-AI-needed items from publishing on schedule. They operate on disjoint item sets
|
||||
* (see priorityQueue.ts), so there's no risk of the two racing each other into a
|
||||
* duplicate publish the way an overlapping call to the *same* fn would.
|
||||
*/
|
||||
function everyTickSkippingOverlap(ms: number, fn: () => Promise<void>) {
|
||||
let running = false;
|
||||
setInterval(() => {
|
||||
if (running) return;
|
||||
running = true;
|
||||
fn().finally(() => {
|
||||
running = false;
|
||||
});
|
||||
}, ms);
|
||||
}
|
||||
|
||||
export function startScheduler() {
|
||||
const provider = () => {
|
||||
const s = settingsDb.getSettings();
|
||||
return new OllamaProvider(s.aiServiceHost, s.aiServicePort);
|
||||
};
|
||||
|
||||
setInterval(async () => {
|
||||
everyTickSkippingOverlap(POLL_TICK_MS, async () => {
|
||||
try {
|
||||
const ingested = await pollDueSources();
|
||||
if (ingested > 0) logger.info('scheduler', `Poll tick: ingested ${ingested} new item(s)`);
|
||||
} catch (err) {
|
||||
logger.error('scheduler', `Poll tick failed: ${(err as Error).message}`);
|
||||
}
|
||||
}, POLL_TICK_MS);
|
||||
});
|
||||
|
||||
setInterval(async () => {
|
||||
everyTickSkippingOverlap(DIRECT_PUBLISH_TICK_MS, async () => {
|
||||
try {
|
||||
const settings = settingsDb.getSettings();
|
||||
const p = provider();
|
||||
// This tick runs regardless of Ollama's reachability (nothing here rewrites or
|
||||
// merges), but tagging direct-published items (see runDirectPublishCycle/
|
||||
// publishDirect) does need a working AI service — only offer the provider
|
||||
// through when it's actually reachable, so an unconfigured Ollama doesn't spam
|
||||
// the log with a failed tag-extraction attempt on every single item, every tick.
|
||||
const reachable = await p.isReachable();
|
||||
const published = await runDirectPublishCycle(settings, reachable ? p : undefined);
|
||||
if (published > 0) {
|
||||
logger.info('scheduler', `Direct-publish tick: published ${published} article(s)`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('scheduler', `Direct-publish tick failed: ${(err as Error).message}`);
|
||||
}
|
||||
});
|
||||
|
||||
everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => {
|
||||
try {
|
||||
const settings = settingsDb.getSettings();
|
||||
const p = provider();
|
||||
@@ -55,16 +104,16 @@ export function startScheduler() {
|
||||
} catch (err) {
|
||||
logger.error('scheduler', `Synthesis tick failed: ${(err as Error).message}`);
|
||||
}
|
||||
}, SYNTHESIS_TICK_MS);
|
||||
});
|
||||
|
||||
setInterval(() => {
|
||||
everyTickSkippingOverlap(RETENTION_TICK_MS, async () => {
|
||||
try {
|
||||
runRetentionSweep(settingsDb.getSettings());
|
||||
logger.info('retention', 'Retention sweep completed');
|
||||
} catch (err) {
|
||||
logger.error('retention', `Retention tick failed: ${(err as Error).message}`);
|
||||
}
|
||||
}, RETENTION_TICK_MS);
|
||||
});
|
||||
|
||||
// Immediate first call for all three — unlike RSS sources (whose "due" check makes a
|
||||
// brand-new source eligible on the very next 1-minute tick), weather/stocks/poe2 have
|
||||
@@ -98,5 +147,8 @@ export function startScheduler() {
|
||||
pollPoe2Now().catch((err) => logger.error('poe2', `Poll tick failed: ${err.message}`));
|
||||
}, POE2_TICK_MS);
|
||||
|
||||
logger.info('scheduler', 'Started: poll every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h');
|
||||
logger.info(
|
||||
'scheduler',
|
||||
'Started: poll every 1m, direct-publish every 1m, synthesis every 1m, retention every 1h, weather every 45m, stocks every 15m, poe2 every 1h'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -86,6 +86,29 @@ export function reissueSourceContent(sourceId: string): ReissueResult {
|
||||
return { articlesDeleted, itemsRequeued: requeueIds.size };
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes one specific article (and its media) and requeues every content item that
|
||||
* contributed to it — unlike reissueSourceContent, this works regardless of how many
|
||||
* different sources the article merged together, since it's scoped to the article
|
||||
* itself rather than "everything from source X". Exists for exactly the failure mode
|
||||
* synthesis.ts's assertNonEmpty guards against going forward: a bad synthesis call
|
||||
* that already made it into a published (garbage) article before that guard existed,
|
||||
* where the source-scoped reissue tools can't help because the article spans sources.
|
||||
* Returns null if the article doesn't exist.
|
||||
*/
|
||||
export function reissueArticle(articleId: string): ReissueResult | null {
|
||||
const article = articlesDb.getArticle(articleId);
|
||||
if (!article) return null;
|
||||
|
||||
const itemIds = article.sources.map((s) => s.itemId);
|
||||
deleteMediaByArticleId(article.id);
|
||||
articlesDb.deleteArticle(article.id);
|
||||
contentItemsDb.resetClusterForItems(itemIds);
|
||||
|
||||
logger.info('admin', `Reissuing article ${articleId}: deleted, ${itemIds.length} item(s) requeued`);
|
||||
return { articlesDeleted: 1, itemsRequeued: itemIds.length };
|
||||
}
|
||||
|
||||
/** Wipes every published article and its media, keeping raw ingested items intact so they can be re-synthesized fresh. */
|
||||
export function clearAllArticles(): number {
|
||||
const articles = articlesDb.allArticlesNewestFirst();
|
||||
|
||||
@@ -9,7 +9,8 @@ function rowToCategory(row: any): Category {
|
||||
priorityRank: row.priority_rank,
|
||||
isDefault: !!row.is_default,
|
||||
isPrivate: !!row.is_private,
|
||||
isSpillover: !!row.is_spillover
|
||||
isSpillover: !!row.is_spillover,
|
||||
disableAi: !!row.disable_ai
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,18 +25,20 @@ export function listPrivateCategoryNames(): string[] {
|
||||
return rows.map((r) => r.name);
|
||||
}
|
||||
|
||||
export function setCategoryOrder(order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean }[]) {
|
||||
const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ? WHERE id = ?');
|
||||
for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.id);
|
||||
export function setCategoryOrder(
|
||||
order: { id: string; priorityRank: number; isPrivate: boolean; isSpillover: boolean; disableAi: boolean }[]
|
||||
) {
|
||||
const stmt = db.prepare('UPDATE categories SET priority_rank = ?, is_private = ?, is_spillover = ?, disable_ai = ? WHERE id = ?');
|
||||
for (const c of order) stmt.run(c.priorityRank, c.isPrivate ? 1 : 0, c.isSpillover ? 1 : 0, c.disableAi ? 1 : 0, c.id);
|
||||
}
|
||||
|
||||
export function createCategory(name: string, isPrivate = false, isSpillover = false): Category {
|
||||
export function createCategory(name: string, isPrivate = false, isSpillover = false, disableAi = false): Category {
|
||||
const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`;
|
||||
const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number };
|
||||
db.prepare(
|
||||
'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover) VALUES (?, ?, ?, 0, ?, ?)'
|
||||
).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0);
|
||||
return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover };
|
||||
'INSERT INTO categories (id, name, priority_rank, is_default, is_private, is_spillover, disable_ai) VALUES (?, ?, ?, 0, ?, ?, ?)'
|
||||
).run(id, name, maxRank.m + 1, isPrivate ? 1 : 0, isSpillover ? 1 : 0, disableAi ? 1 : 0);
|
||||
return { id, name, priorityRank: maxRank.m + 1, isDefault: false, isPrivate, isSpillover, disableAi };
|
||||
}
|
||||
|
||||
export function deleteCategory(id: string) {
|
||||
|
||||
@@ -14,6 +14,8 @@ function rowToEvent(row: any): TrackedEvent {
|
||||
isSpillover: !!row.is_spillover,
|
||||
retentionOverrideDays: row.retention_override_days,
|
||||
lastRecapAt: row.last_recap_at,
|
||||
recapStylePreset: row.recap_style_preset,
|
||||
recapCustomInstructions: row.recap_custom_instructions,
|
||||
createdAt: row.created_at
|
||||
};
|
||||
}
|
||||
@@ -52,8 +54,8 @@ export function createEvent(input: Partial<TrackedEvent>): TrackedEvent {
|
||||
const id = `evt-${randomUUID()}`;
|
||||
const now = new Date().toISOString();
|
||||
db.prepare(
|
||||
`INSERT INTO tracked_events (id, name, description, source_ids, keywords, recap_interval_hours, active, is_spillover, retention_override_days, last_recap_at, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`
|
||||
`INSERT INTO tracked_events (id, name, description, source_ids, keywords, recap_interval_hours, active, is_spillover, retention_override_days, last_recap_at, recap_style_preset, recap_custom_instructions, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)`
|
||||
).run(
|
||||
id,
|
||||
input.name ?? 'Untitled event',
|
||||
@@ -64,6 +66,8 @@ export function createEvent(input: Partial<TrackedEvent>): TrackedEvent {
|
||||
input.active === false ? 0 : 1,
|
||||
input.isSpillover ? 1 : 0,
|
||||
input.retentionOverrideDays ?? null,
|
||||
input.recapStylePreset ?? 'default',
|
||||
input.recapCustomInstructions ?? '',
|
||||
now
|
||||
);
|
||||
return getEvent(id)!;
|
||||
@@ -74,7 +78,7 @@ export function updateEvent(id: string, patch: Partial<TrackedEvent>): TrackedEv
|
||||
if (!existing) return null;
|
||||
const merged = { ...existing, ...patch };
|
||||
db.prepare(
|
||||
`UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, recap_interval_hours=?, active=?, is_spillover=?, retention_override_days=?, last_recap_at=? WHERE id=?`
|
||||
`UPDATE tracked_events SET name=?, description=?, source_ids=?, keywords=?, recap_interval_hours=?, active=?, is_spillover=?, retention_override_days=?, last_recap_at=?, recap_style_preset=?, recap_custom_instructions=? WHERE id=?`
|
||||
).run(
|
||||
merged.name,
|
||||
merged.description,
|
||||
@@ -85,6 +89,8 @@ export function updateEvent(id: string, patch: Partial<TrackedEvent>): TrackedEv
|
||||
merged.isSpillover ? 1 : 0,
|
||||
merged.retentionOverrideDays,
|
||||
merged.lastRecapAt,
|
||||
merged.recapStylePreset,
|
||||
merged.recapCustomInstructions,
|
||||
id
|
||||
);
|
||||
return getEvent(id);
|
||||
|
||||
@@ -135,6 +135,8 @@ export function migrate() {
|
||||
is_spillover INTEGER NOT NULL DEFAULT 0,
|
||||
retention_override_days INTEGER,
|
||||
last_recap_at TEXT,
|
||||
recap_style_preset TEXT NOT NULL DEFAULT 'default', -- default | casual | formal — this item's own recap tone, independent of the global Merge-tab style (see pipeline/synthesis.ts)
|
||||
recap_custom_instructions TEXT NOT NULL DEFAULT '', -- free-text addendum for this item's recap prompt specifically
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
@@ -153,7 +155,8 @@ export function migrate() {
|
||||
priority_rank INTEGER NOT NULL,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
is_private INTEGER NOT NULL DEFAULT 0,
|
||||
is_spillover INTEGER NOT NULL DEFAULT 0 -- collapsed into the nav's "More »" overflow page instead of its own tab
|
||||
is_spillover INTEGER NOT NULL DEFAULT 0, -- collapsed into the nav's "More »" overflow page instead of its own tab
|
||||
disable_ai INTEGER NOT NULL DEFAULT 0 -- skip clustering/synthesis for this category's items; publish each one directly
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs (
|
||||
@@ -185,6 +188,10 @@ export function migrate() {
|
||||
fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com',
|
||||
nitter_instance_url TEXT NOT NULL DEFAULT 'https://nitter.net', -- admin's preferred instance, prefills new Nitter sources (Connections tab)
|
||||
telegram_media_mode TEXT NOT NULL DEFAULT 'self-host', -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL)
|
||||
synthesis_style_preset TEXT NOT NULL DEFAULT 'default', -- default | casual | formal — see pipeline/synthesis.ts's STYLE_PRESETS
|
||||
synthesis_custom_instructions TEXT NOT NULL DEFAULT '', -- free-text addendum appended to the synthesis system prompt, on top of the preset
|
||||
synthesis_num_ctx INTEGER NOT NULL DEFAULT 8192, -- admin-tunable context window (Models tab) — see inference/ollama-provider.ts's DEFAULT_NUM_CTX
|
||||
synthesis_num_predict INTEGER NOT NULL DEFAULT 700, -- admin-tunable max response length — too low silently truncates output mid-sentence
|
||||
widget_weather_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
widget_stocks_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
widget_bookmarks_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
@@ -321,6 +328,9 @@ export function migrate() {
|
||||
if (!hasColumn('categories', 'is_spillover')) {
|
||||
db.exec('ALTER TABLE categories ADD COLUMN is_spillover INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
if (!hasColumn('categories', 'disable_ai')) {
|
||||
db.exec('ALTER TABLE categories ADD COLUMN disable_ai INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
if (!hasColumn('content_items', 'telegram_message')) {
|
||||
db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT');
|
||||
}
|
||||
@@ -339,6 +349,12 @@ export function migrate() {
|
||||
if (!hasColumn('tracked_events', 'recap_interval_hours')) {
|
||||
db.exec('ALTER TABLE tracked_events ADD COLUMN recap_interval_hours INTEGER');
|
||||
}
|
||||
if (!hasColumn('tracked_events', 'recap_style_preset')) {
|
||||
db.exec("ALTER TABLE tracked_events ADD COLUMN recap_style_preset TEXT NOT NULL DEFAULT 'default'");
|
||||
}
|
||||
if (!hasColumn('tracked_events', 'recap_custom_instructions')) {
|
||||
db.exec("ALTER TABLE tracked_events ADD COLUMN recap_custom_instructions TEXT NOT NULL DEFAULT ''");
|
||||
}
|
||||
if (!hasColumn('merged_articles', 'is_recap')) {
|
||||
db.exec('ALTER TABLE merged_articles ADD COLUMN is_recap INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
@@ -390,6 +406,18 @@ export function migrate() {
|
||||
stmt.run(`stk-${symbol.replace(/[^a-z0-9]+/gi, '-')}`, label, symbol, i + 1, new Date().toISOString());
|
||||
});
|
||||
}
|
||||
if (!hasColumn('global_settings', 'synthesis_style_preset')) {
|
||||
db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_style_preset TEXT NOT NULL DEFAULT 'default'");
|
||||
}
|
||||
if (!hasColumn('global_settings', 'synthesis_custom_instructions')) {
|
||||
db.exec("ALTER TABLE global_settings ADD COLUMN synthesis_custom_instructions TEXT NOT NULL DEFAULT ''");
|
||||
}
|
||||
if (!hasColumn('global_settings', 'synthesis_num_ctx')) {
|
||||
db.exec('ALTER TABLE global_settings ADD COLUMN synthesis_num_ctx INTEGER NOT NULL DEFAULT 8192');
|
||||
}
|
||||
if (!hasColumn('global_settings', 'synthesis_num_predict')) {
|
||||
db.exec('ALTER TABLE global_settings ADD COLUMN synthesis_num_predict INTEGER NOT NULL DEFAULT 700');
|
||||
}
|
||||
|
||||
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
|
||||
// general news sources belong here, not on "Top stories" itself, which isn't a real
|
||||
|
||||
@@ -16,6 +16,10 @@ function rowToSettings(row: any): GlobalSettings {
|
||||
fxtwitterBaseUrl: row.fxtwitter_base_url,
|
||||
nitterInstanceUrl: row.nitter_instance_url,
|
||||
telegramMediaMode: row.telegram_media_mode,
|
||||
synthesisStylePreset: row.synthesis_style_preset,
|
||||
synthesisCustomInstructions: row.synthesis_custom_instructions,
|
||||
synthesisNumCtx: row.synthesis_num_ctx,
|
||||
synthesisNumPredict: row.synthesis_num_predict,
|
||||
widgets: {
|
||||
weather: !!row.widget_weather_enabled,
|
||||
stocks: !!row.widget_stocks_enabled,
|
||||
@@ -80,6 +84,8 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
|
||||
ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models,
|
||||
nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, nitter_instance_url=$nitter_instance_url,
|
||||
telegram_media_mode=$telegram_media_mode,
|
||||
synthesis_style_preset=$synthesis_style_preset, synthesis_custom_instructions=$synthesis_custom_instructions,
|
||||
synthesis_num_ctx=$synthesis_num_ctx, synthesis_num_predict=$synthesis_num_predict,
|
||||
widget_weather_enabled=$widget_weather_enabled, widget_stocks_enabled=$widget_stocks_enabled,
|
||||
widget_bookmarks_enabled=$widget_bookmarks_enabled, widget_poe2_enabled=$widget_poe2_enabled,
|
||||
widget_order=$widget_order,
|
||||
@@ -105,6 +111,10 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
|
||||
$fxtwitter_base_url: merged.fxtwitterBaseUrl,
|
||||
$nitter_instance_url: merged.nitterInstanceUrl,
|
||||
$telegram_media_mode: merged.telegramMediaMode,
|
||||
$synthesis_style_preset: merged.synthesisStylePreset,
|
||||
$synthesis_custom_instructions: merged.synthesisCustomInstructions,
|
||||
$synthesis_num_ctx: merged.synthesisNumCtx,
|
||||
$synthesis_num_predict: merged.synthesisNumPredict,
|
||||
$widget_weather_enabled: merged.widgets.weather ? 1 : 0,
|
||||
$widget_stocks_enabled: merged.widgets.stocks ? 1 : 0,
|
||||
$widget_bookmarks_enabled: merged.widgets.bookmarks ? 1 : 0,
|
||||
|
||||
@@ -28,6 +28,12 @@ export function listActiveTags(): Tag[] {
|
||||
return rows.map(rowToTag);
|
||||
}
|
||||
|
||||
/** By slug rather than id — that's what tag chips link by (see publish.ts/frontend tag pages). Matches regardless of active/expired status: an old article's tag chip should still resolve to its (now possibly expired) tag rather than 404 just because nothing new has used it lately. */
|
||||
export function getTagBySlug(slug: string): Tag | null {
|
||||
const row = db.prepare('SELECT * FROM tags WHERE slug = ?').get(slug);
|
||||
return row ? rowToTag(row) : null;
|
||||
}
|
||||
|
||||
function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length === 0 || b.length === 0 || a.length !== b.length) return 0;
|
||||
let dot = 0,
|
||||
|
||||
@@ -185,6 +185,10 @@ export interface TrackedEvent {
|
||||
isSpillover: boolean;
|
||||
retentionOverrideDays: number | null;
|
||||
lastRecapAt: string | null;
|
||||
/** Tone preset for this item's own recap, independent of the global Merge-tab synthesis style — see pipeline/synthesis.ts's STYLE_PRESETS. 'default' adds nothing on top of the base recap prompt. */
|
||||
recapStylePreset: 'default' | 'casual' | 'formal';
|
||||
/** Free-text instructions appended to the recap system prompt for this item specifically — e.g. "focus on military developments", "write as a full narrative, not bullet points". Empty string means no addendum. */
|
||||
recapCustomInstructions: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
@@ -206,6 +210,8 @@ export interface Category {
|
||||
isPrivate: boolean;
|
||||
/** Grouped into the nav's "More »" overflow page instead of getting its own top-level tab — see +layout.svelte and /more. */
|
||||
isSpillover: boolean;
|
||||
/** Skips clustering/AI synthesis for this category's items — each one publishes directly (own article, own source's text), same as YouTube/Nitter/Telegram items always do. See priorityQueue.ts's runSynthesisCycle. */
|
||||
disableAi: boolean;
|
||||
}
|
||||
|
||||
export interface WeatherHourEntry {
|
||||
@@ -283,6 +289,14 @@ export interface GlobalSettings {
|
||||
nitterInstanceUrl: string;
|
||||
/** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */
|
||||
telegramMediaMode: 'self-host' | 'proxy';
|
||||
/** Tone preset applied to every AI-synthesized article/recap (see pipeline/synthesis.ts's STYLE_PRESETS) — 'default' is the original neutral wire-service tone with no addendum. Never applies to single-source items, which always publish verbatim without going through the AI at all. */
|
||||
synthesisStylePreset: 'default' | 'casual' | 'formal';
|
||||
/** Free-text instructions appended to the synthesis system prompt alongside the style preset — e.g. "keep it under 3 sentences per paragraph". Empty string means no addendum. */
|
||||
synthesisCustomInstructions: string;
|
||||
/** Total context window (prompt + response) requested from Ollama for every synthesis/recap/tag-extraction call — see inference/ollama-provider.ts's DEFAULT_NUM_CTX for why this is ever explicit at all, and the Models tab for the admin-facing slider (bounded by the selected synthesis model's own reported max, when Ollama exposes it). */
|
||||
synthesisNumCtx: number;
|
||||
/** Max tokens the model is allowed to generate per synthesis/recap call — too low silently truncates the output mid-sentence rather than erroring (this is what a "cut off" recap/article means). Recaps in particular need real headroom: they're asked to summarize many source articles into several paragraphs, unlike a same-story merge. */
|
||||
synthesisNumPredict: number;
|
||||
/** Per-widget enable flags — see admin/settings' consolidated "Widgets" tab. Weather/Stocks/PoE2's backend pollers (scheduler.ts) are gated on these too, not just sidebar visibility; Bookmarks has no poller so its flag only affects the sidebar. */
|
||||
widgets: {
|
||||
weather: boolean;
|
||||
|
||||
@@ -13,7 +13,10 @@ import type {
|
||||
AdminStockTicker,
|
||||
AdminBookmark,
|
||||
Poe2BrowseEntry,
|
||||
AdminPoe2Entry
|
||||
AdminPoe2Entry,
|
||||
PipelineStats,
|
||||
ModelContextInfo,
|
||||
ForceRecapResult
|
||||
} from './adminTypes';
|
||||
|
||||
async function request<T>(path: string, options: RequestInit = {}, fetchFn: typeof fetch = fetch): Promise<T> {
|
||||
@@ -66,10 +69,16 @@ export const updateSettings = (patch: Partial<AdminSettings>, fetchFn?: typeof f
|
||||
request<AdminSettings>('/api/admin/settings', { method: 'PATCH', body: JSON.stringify(patch) }, fetchFn);
|
||||
|
||||
// Categories
|
||||
export const createCategory = (name: string, isPrivate = false, isSpillover = false, fetchFn?: typeof fetch) =>
|
||||
export const createCategory = (
|
||||
name: string,
|
||||
isPrivate = false,
|
||||
isSpillover = false,
|
||||
disableAi = false,
|
||||
fetchFn?: typeof fetch
|
||||
) =>
|
||||
request<CategoryPriority>(
|
||||
'/api/admin/categories',
|
||||
{ method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover }) },
|
||||
{ method: 'POST', body: JSON.stringify({ name, isPrivate, isSpillover, disableAi }) },
|
||||
fetchFn
|
||||
);
|
||||
|
||||
@@ -97,6 +106,11 @@ export const pollSourceNow = (id: string, fetchFn?: typeof fetch) =>
|
||||
export const reissueSourceContent = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/sources/${id}/reissue`, { method: 'POST' }, fetchFn);
|
||||
|
||||
// Fixes one specific bad article regardless of how many sources it merged — unlike
|
||||
// reissueSourceContent above, which deliberately won't touch a multi-source article.
|
||||
export const reissueArticle = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<{ articlesDeleted: number; itemsRequeued: number }>(`/api/admin/articles/${id}/reissue`, { method: 'POST' }, fetchFn);
|
||||
|
||||
// Content clearing — wipe articles/media/a source's raw items so they can be repopulated fresh.
|
||||
export const clearSourceContent = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<{ itemsDeleted: number; articlesDeleted: number }>(`/api/admin/content/sources/${id}`, { method: 'DELETE' }, fetchFn);
|
||||
@@ -120,6 +134,12 @@ export const updateEvent = (id: string, patch: Partial<AdminTrackedEvent>, fetch
|
||||
export const deleteEvent = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<void>(`/api/admin/events/${id}`, { method: 'DELETE' }, fetchFn);
|
||||
|
||||
// Runs this item's recap immediately, ignoring its recapIntervalHours cadence — still
|
||||
// summarizes the same real window (everything since lastRecapAt) and still requires
|
||||
// there to actually be something new to summarize (see the backend route).
|
||||
export const forceRecap = (id: string, fetchFn?: typeof fetch) =>
|
||||
request<ForceRecapResult>(`/api/admin/events/${id}/recap-now`, { method: 'POST' }, fetchFn);
|
||||
|
||||
// Models / AI service
|
||||
export const getModels = (fetchFn?: typeof fetch) =>
|
||||
request<ModelCatalog>('/api/admin/models', {}, fetchFn);
|
||||
@@ -127,6 +147,9 @@ export const getModels = (fetchFn?: typeof fetch) =>
|
||||
export const getAiStatus = (fetchFn?: typeof fetch) =>
|
||||
request<AiStatus>('/api/admin/ai-status', {}, fetchFn);
|
||||
|
||||
export const getModelContext = (model: string, fetchFn?: typeof fetch) =>
|
||||
request<ModelContextInfo>(`/api/admin/model-context?model=${encodeURIComponent(model)}`, {}, fetchFn);
|
||||
|
||||
// Telegram account (Connections tab) — API ID/hash and the resulting login session are
|
||||
// stored encrypted at rest server-side (see backend telegram/credentials.ts); none of
|
||||
// these ever come back from the server, only status flags.
|
||||
@@ -170,6 +193,8 @@ export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: nu
|
||||
return request<LogEntry[]>(`/api/admin/logs${qs ? `?${qs}` : ''}`, {}, fetchFn);
|
||||
};
|
||||
|
||||
export const getPipelineStats = (fetchFn?: typeof fetch) => request<PipelineStats>('/api/admin/pipeline-stats', {}, fetchFn);
|
||||
|
||||
// Weather — config/cache lives on AdminSettings.weather (see updateSettings above); this
|
||||
// is just the geocoding lookup used to resolve a typed city name to lat/lon.
|
||||
export const geocodeLocation = (query: string, fetchFn?: typeof fetch) =>
|
||||
|
||||
@@ -14,6 +14,7 @@ export interface CategoryPriority {
|
||||
isDefault: boolean;
|
||||
isPrivate: boolean;
|
||||
isSpillover: boolean;
|
||||
disableAi: boolean;
|
||||
}
|
||||
|
||||
export interface WeatherHourEntry {
|
||||
@@ -139,6 +140,12 @@ export interface AdminSettings {
|
||||
fxtwitterBaseUrl: string;
|
||||
nitterInstanceUrl: string;
|
||||
telegramMediaMode: 'self-host' | 'proxy';
|
||||
synthesisStylePreset: 'default' | 'casual' | 'formal';
|
||||
synthesisCustomInstructions: string;
|
||||
/** Total context window (prompt + response) requested from Ollama for every synthesis/recap/tag-extraction call. */
|
||||
synthesisNumCtx: number;
|
||||
/** Max tokens the model may generate per call — too low silently truncates output mid-sentence. */
|
||||
synthesisNumPredict: number;
|
||||
widgets: AdminWidgetsEnabled;
|
||||
widgetOrder: ('weather' | 'stocks' | 'bookmarks' | 'poe2')[];
|
||||
retention: RetentionSettings;
|
||||
@@ -173,6 +180,19 @@ export interface AdminTrackedEvent {
|
||||
active: boolean;
|
||||
isSpillover: boolean;
|
||||
retentionOverrideDays: number | null;
|
||||
/** This item's own recap tone, independent of the global Merge-tab synthesis style. */
|
||||
recapStylePreset: 'default' | 'casual' | 'formal';
|
||||
/** Free-text instructions appended to this item's recap prompt specifically. */
|
||||
recapCustomInstructions: string;
|
||||
}
|
||||
|
||||
/** Response from POST /api/admin/events/:id/recap-now. */
|
||||
export interface ForceRecapResult {
|
||||
published: boolean;
|
||||
/** Set when published is true. */
|
||||
title?: string;
|
||||
/** Set when published is false — why nothing was generated (no sources assigned, nothing new since last recap). */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface ModelCatalog {
|
||||
@@ -181,6 +201,11 @@ export interface ModelCatalog {
|
||||
synthesis: string[];
|
||||
}
|
||||
|
||||
/** Response from GET /api/admin/model-context — the selected model's own reported max context length, or null if Ollama doesn't expose it for this model/version. */
|
||||
export interface ModelContextInfo {
|
||||
contextLength: number | null;
|
||||
}
|
||||
|
||||
export interface AiStatus {
|
||||
connected: boolean;
|
||||
host: string;
|
||||
@@ -195,6 +220,33 @@ export interface TelegramStatus {
|
||||
phone: string | null;
|
||||
}
|
||||
|
||||
export interface PipelineStats {
|
||||
timestamp: string;
|
||||
ollama: {
|
||||
inFlight: { label: string; elapsedMs: number } | null;
|
||||
sampleCount: number;
|
||||
avgGenTokensPerSec: number | null;
|
||||
avgPromptTokensPerSec: number | null;
|
||||
avgGenerateDurationMs: number | null;
|
||||
};
|
||||
backlog: {
|
||||
totalUnclusteredItems: number;
|
||||
directEligibleItems: number;
|
||||
awaitingEmbeddingItems: number;
|
||||
clusters: {
|
||||
total: number;
|
||||
readyNow: number;
|
||||
readyNowNeedingSynthesis: number;
|
||||
onHold: number;
|
||||
itemsOnHold: number;
|
||||
earliestHoldRemainingMs: number | null;
|
||||
};
|
||||
};
|
||||
estimatedMinutesToClear: number | null;
|
||||
lastDirectCycle: { at: string; published: number } | null;
|
||||
lastSynthesisCycle: { at: string; published: number } | null;
|
||||
}
|
||||
|
||||
export interface LogEntry {
|
||||
id: number;
|
||||
timestamp: string;
|
||||
|
||||
@@ -34,6 +34,10 @@ export function getTags(fetchFn?: typeof fetch): Promise<Tag[]> {
|
||||
return get<Tag[]>('/api/tags', fetchFn);
|
||||
}
|
||||
|
||||
export function getTagBySlug(slug: string, fetchFn?: typeof fetch): Promise<Tag> {
|
||||
return get<Tag>(`/api/tag/${slug}`, fetchFn);
|
||||
}
|
||||
|
||||
export function getEvents(fetchFn?: typeof fetch): Promise<TrackedEventPublic[]> {
|
||||
return get<TrackedEventPublic[]>('/api/events', fetchFn);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<script lang="ts">
|
||||
import type { AdminTrackedEvent, AdminSource } from '$lib/adminTypes';
|
||||
import { addEvent, updateEvent, deleteEvent } from '$lib/adminApi';
|
||||
import { addEvent, updateEvent, deleteEvent, forceRecap } from '$lib/adminApi';
|
||||
import CollapsibleSection from './CollapsibleSection.svelte';
|
||||
|
||||
let { events: initial, sources }: { events: AdminTrackedEvent[]; sources: AdminSource[] } = $props();
|
||||
let events = $state([...initial]);
|
||||
@@ -18,11 +19,37 @@
|
||||
keywordsText: '',
|
||||
recapIntervalHours: null as AdminTrackedEvent['recapIntervalHours'],
|
||||
isSpillover: false,
|
||||
retentionOverrideDays: null as number | null
|
||||
retentionOverrideDays: null as number | null,
|
||||
recapStylePreset: 'default' as AdminTrackedEvent['recapStylePreset'],
|
||||
recapCustomInstructions: ''
|
||||
};
|
||||
}
|
||||
let editForm = $state(emptyEditForm());
|
||||
|
||||
let recappingId = $state<string | null>(null);
|
||||
let recapMessage = $state<{ id: string; text: string; isError: boolean } | null>(null);
|
||||
|
||||
// Runs this item's recap right now instead of waiting out its cadence timer — still
|
||||
// summarizes only whatever's genuinely new since the last recap (see the backend
|
||||
// route), so it can come back saying there was nothing to recap rather than always
|
||||
// producing one.
|
||||
async function handleForceRecap(id: string) {
|
||||
recappingId = id;
|
||||
recapMessage = null;
|
||||
try {
|
||||
const result = await forceRecap(id);
|
||||
recapMessage = {
|
||||
id,
|
||||
text: result.published ? `Recap published: "${result.title}"` : (result.reason ?? 'Nothing to recap.'),
|
||||
isError: false
|
||||
};
|
||||
} catch (err) {
|
||||
recapMessage = { id, text: (err as Error).message, isError: true };
|
||||
} finally {
|
||||
recappingId = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleAdd() {
|
||||
if (!newEvent.name) return;
|
||||
const created = await addEvent({
|
||||
@@ -58,7 +85,9 @@
|
||||
keywordsText: event.keywords.join(', '),
|
||||
recapIntervalHours: event.recapIntervalHours,
|
||||
isSpillover: event.isSpillover,
|
||||
retentionOverrideDays: event.retentionOverrideDays
|
||||
retentionOverrideDays: event.retentionOverrideDays,
|
||||
recapStylePreset: event.recapStylePreset,
|
||||
recapCustomInstructions: event.recapCustomInstructions
|
||||
};
|
||||
}
|
||||
|
||||
@@ -86,7 +115,9 @@
|
||||
keywords,
|
||||
recapIntervalHours: editForm.recapIntervalHours,
|
||||
isSpillover: editForm.isSpillover,
|
||||
retentionOverrideDays: editForm.retentionOverrideDays
|
||||
retentionOverrideDays: editForm.retentionOverrideDays,
|
||||
recapStylePreset: editForm.recapStylePreset,
|
||||
recapCustomInstructions: editForm.recapCustomInstructions
|
||||
});
|
||||
events = events.map((e) => (e.id === editingId ? updated : e));
|
||||
editingId = null;
|
||||
@@ -175,6 +206,41 @@
|
||||
it off for something you're just organizing under its own nav entry (a commit feed,
|
||||
a torrent feed) with nothing that needs summarizing.
|
||||
</p>
|
||||
<div class="force-recap-row">
|
||||
<button
|
||||
onclick={() => handleForceRecap(event.id)}
|
||||
disabled={recappingId === event.id}
|
||||
>
|
||||
{recappingId === event.id ? 'Recapping…' : 'Force recap now'}
|
||||
</button>
|
||||
{#if recapMessage && recapMessage.id === event.id}
|
||||
<span class="recap-message" class:error={recapMessage.isError}>{recapMessage.text}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="more-section">
|
||||
<CollapsibleSection title="More">
|
||||
<div class="field-label">Recap writing style</div>
|
||||
<p class="hint">
|
||||
Independent from the global "Writing style" in the Merge tab — that only applies to
|
||||
regular AI-merged articles. This item's own recap uses only what's set here.
|
||||
</p>
|
||||
<select bind:value={editForm.recapStylePreset}>
|
||||
<option value="default">Default (neutral, wire-service tone)</option>
|
||||
<option value="casual">Casual</option>
|
||||
<option value="formal">Formal</option>
|
||||
</select>
|
||||
<label class="field-label" for="recap-custom-instructions" style="margin-top: 10px;">
|
||||
Additional instructions (optional)
|
||||
</label>
|
||||
<textarea
|
||||
id="recap-custom-instructions"
|
||||
rows="3"
|
||||
placeholder={'e.g. "Focus on military developments", "Write as one continuous narrative, not a list"'}
|
||||
bind:value={editForm.recapCustomInstructions}
|
||||
></textarea>
|
||||
</CollapsibleSection>
|
||||
</div>
|
||||
|
||||
<label class="spillover-toggle edit-spillover">
|
||||
@@ -339,6 +405,35 @@
|
||||
.cadence-block .hint {
|
||||
margin-top: 6px;
|
||||
}
|
||||
.force-recap-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-top: 10px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.force-recap-row button {
|
||||
font-size: 12px;
|
||||
padding: 6px 12px;
|
||||
}
|
||||
.recap-message {
|
||||
font-size: 11px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.recap-message.error {
|
||||
color: var(--text-danger);
|
||||
}
|
||||
.more-section {
|
||||
margin-top: 12px;
|
||||
}
|
||||
.more-section select,
|
||||
.more-section textarea {
|
||||
width: 100%;
|
||||
}
|
||||
.more-section textarea {
|
||||
resize: vertical;
|
||||
font: inherit;
|
||||
}
|
||||
.edit-spillover {
|
||||
display: flex;
|
||||
margin-top: 12px;
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import type { LogEntry } from '$lib/adminTypes';
|
||||
import { getLogs } from '$lib/adminApi';
|
||||
import type { LogEntry, PipelineStats } from '$lib/adminTypes';
|
||||
import { getLogs, getPipelineStats } from '$lib/adminApi';
|
||||
|
||||
let { logs: initial }: { logs: LogEntry[] } = $props();
|
||||
let logs = $state([...initial]);
|
||||
let stats = $state<PipelineStats | null>(null);
|
||||
let filter = $state<'all' | 'info' | 'warn' | 'error'>('all');
|
||||
let autoRefresh = $state(true);
|
||||
let loading = $state(false);
|
||||
@@ -13,7 +14,12 @@
|
||||
async function refresh() {
|
||||
loading = true;
|
||||
try {
|
||||
logs = await getLogs(filter === 'all' ? {} : { level: filter });
|
||||
const [nextLogs, nextStats] = await Promise.all([
|
||||
getLogs(filter === 'all' ? {} : { level: filter }),
|
||||
getPipelineStats().catch(() => stats) // stats endpoint failing shouldn't block the log list
|
||||
]);
|
||||
logs = nextLogs;
|
||||
stats = nextStats;
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
@@ -25,6 +31,7 @@
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
refresh();
|
||||
timer = setInterval(() => {
|
||||
if (autoRefresh) refresh();
|
||||
}, 5000);
|
||||
@@ -35,8 +42,98 @@
|
||||
const d = new Date(iso);
|
||||
return d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' });
|
||||
}
|
||||
|
||||
function formatAgo(iso: string | null): string {
|
||||
if (!iso) return 'never';
|
||||
const seconds = Math.round((Date.now() - new Date(iso).getTime()) / 1000);
|
||||
if (seconds < 5) return 'just now';
|
||||
if (seconds < 60) return `${seconds}s ago`;
|
||||
const minutes = Math.round(seconds / 60);
|
||||
if (minutes < 60) return `${minutes}m ago`;
|
||||
return `${Math.round(minutes / 60)}h ago`;
|
||||
}
|
||||
|
||||
function formatDuration(ms: number): string {
|
||||
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
|
||||
return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
|
||||
}
|
||||
|
||||
function formatEta(minutes: number | null): string {
|
||||
if (minutes === null) return 'collecting data…';
|
||||
if (minutes === 0) return 'caught up';
|
||||
if (minutes < 60) return `~${minutes}m`;
|
||||
return `~${Math.floor(minutes / 60)}h ${minutes % 60}m`;
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if stats}
|
||||
<div class="dashboard">
|
||||
<div class="tile">
|
||||
<span class="tile-label">Backlog</span>
|
||||
<span class="tile-value">{stats.backlog.totalUnclusteredItems}</span>
|
||||
<span class="tile-sub">item{stats.backlog.totalUnclusteredItems === 1 ? '' : 's'} not yet published</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Awaiting embedding</span>
|
||||
<span class="tile-value">{stats.backlog.awaitingEmbeddingItems}</span>
|
||||
<span class="tile-sub">need an embed() call before they can cluster</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Held for publishing</span>
|
||||
<span class="tile-value">{stats.backlog.clusters.itemsOnHold}</span>
|
||||
<span class="tile-sub">
|
||||
{stats.backlog.clusters.onHold} cluster{stats.backlog.clusters.onHold === 1 ? '' : 's'} on hold-before-publish
|
||||
{#if stats.backlog.clusters.earliestHoldRemainingMs !== null}
|
||||
· earliest clears in {formatDuration(stats.backlog.clusters.earliestHoldRemainingMs)}
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Awaiting synthesis</span>
|
||||
<span class="tile-value">{stats.backlog.clusters.readyNowNeedingSynthesis}</span>
|
||||
<span class="tile-sub">multi-source cluster{stats.backlog.clusters.readyNowNeedingSynthesis === 1 ? '' : 's'} ready, needs an AI merge</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Estimated to clear</span>
|
||||
<span class="tile-value">{formatEta(stats.estimatedMinutesToClear)}</span>
|
||||
<span class="tile-sub">
|
||||
{#if stats.ollama.avgGenerateDurationMs !== null}
|
||||
based on {stats.ollama.sampleCount} recent generate call{stats.ollama.sampleCount === 1 ? '' : 's'}, avg {formatDuration(stats.ollama.avgGenerateDurationMs)} each
|
||||
{:else}
|
||||
no completed generate calls yet
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Ollama right now</span>
|
||||
{#if stats.ollama.inFlight}
|
||||
<span class="tile-value live">Synthesizing</span>
|
||||
<span class="tile-sub" title={stats.ollama.inFlight.label}>{stats.ollama.inFlight.label} · {formatDuration(stats.ollama.inFlight.elapsedMs)} elapsed</span>
|
||||
{:else}
|
||||
<span class="tile-value">Idle</span>
|
||||
<span class="tile-sub">
|
||||
{#if stats.ollama.avgGenTokensPerSec !== null}
|
||||
~{stats.ollama.avgGenTokensPerSec.toFixed(1)} gen tok/s, ~{stats.ollama.avgPromptTokensPerSec?.toFixed(1) ?? '?'} prompt tok/s
|
||||
{:else}
|
||||
no throughput data yet
|
||||
{/if}
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Last direct-publish tick</span>
|
||||
<span class="tile-value">{stats.lastDirectCycle ? stats.lastDirectCycle.published : '—'}</span>
|
||||
<span class="tile-sub">{formatAgo(stats.lastDirectCycle?.at ?? null)}</span>
|
||||
</div>
|
||||
<div class="tile">
|
||||
<span class="tile-label">Last synthesis tick</span>
|
||||
<span class="tile-value">{stats.lastSynthesisCycle ? stats.lastSynthesisCycle.published : '—'}</span>
|
||||
<span class="tile-sub">{formatAgo(stats.lastSynthesisCycle?.at ?? null)}</span>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="toolbar">
|
||||
<div class="filters">
|
||||
<button class="pill" class:active={filter === 'all'} onclick={() => setFilter('all')}>All</button>
|
||||
@@ -70,6 +167,41 @@
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.dashboard {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
|
||||
gap: 10px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.tile {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
background: var(--surface-1);
|
||||
border: 0.5px solid var(--border);
|
||||
border-radius: 12px;
|
||||
padding: 12px 14px;
|
||||
}
|
||||
.tile-label {
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.02em;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.tile-value {
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
}
|
||||
.tile-value.live {
|
||||
color: var(--text-accent);
|
||||
}
|
||||
.tile-sub {
|
||||
font-size: 11px;
|
||||
color: var(--text-secondary);
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
let newCategoryName = $state('');
|
||||
let newCategoryPrivate = $state(false);
|
||||
let newCategorySpillover = $state(false);
|
||||
let newCategoryDisableAi = $state(false);
|
||||
let addingCategory = $state(false);
|
||||
|
||||
// Advisory only — the nav starts getting too wide / wrapping past ~10 tabs, so this
|
||||
@@ -33,7 +34,9 @@
|
||||
followUpMinNewSources: local.followUpMinNewSources,
|
||||
tagDedupThreshold: local.tagDedupThreshold,
|
||||
tagExpiryDays: local.tagExpiryDays,
|
||||
categoryPriority: local.categoryPriority
|
||||
categoryPriority: local.categoryPriority,
|
||||
synthesisStylePreset: local.synthesisStylePreset,
|
||||
synthesisCustomInstructions: local.synthesisCustomInstructions
|
||||
});
|
||||
status = 'saved';
|
||||
setTimeout(() => (status = 'idle'), 1500);
|
||||
@@ -48,11 +51,12 @@
|
||||
if (!name) return;
|
||||
addingCategory = true;
|
||||
try {
|
||||
const created = await createCategory(name, newCategoryPrivate, newCategorySpillover);
|
||||
const created = await createCategory(name, newCategoryPrivate, newCategorySpillover, newCategoryDisableAi);
|
||||
local.categoryPriority = [...local.categoryPriority, created];
|
||||
newCategoryName = '';
|
||||
newCategoryPrivate = false;
|
||||
newCategorySpillover = false;
|
||||
newCategoryDisableAi = false;
|
||||
} finally {
|
||||
addingCategory = false;
|
||||
}
|
||||
@@ -68,6 +72,11 @@
|
||||
scheduleSave();
|
||||
}
|
||||
|
||||
function toggleDisableAi(id: string) {
|
||||
local.categoryPriority = local.categoryPriority.map((c) => (c.id === id ? { ...c, disableAi: !c.disableAi } : c));
|
||||
scheduleSave();
|
||||
}
|
||||
|
||||
async function removeCategory(id: string, isDefault: boolean, name: string) {
|
||||
if (isDefault) {
|
||||
// Sensible-default categories can still be removed — e.g. a fresh install's
|
||||
@@ -98,7 +107,9 @@
|
||||
private category (and everything in it) is hidden from the public site until a visitor
|
||||
logs in with the lock icon in the masthead. A "More" category is collapsed into a single
|
||||
"More »" nav tab instead of getting its own, and shows up on that overflow page with its
|
||||
latest few articles.
|
||||
latest few articles. "No AI" skips clustering and synthesis for that category — each item
|
||||
publishes on its own, using its own source's text, instead of being merged/rewritten by the
|
||||
model.
|
||||
</p>
|
||||
{#if primaryCategoryCount > 10}
|
||||
<p class="hint warn">
|
||||
@@ -120,6 +131,10 @@
|
||||
<input type="checkbox" checked={cat.isSpillover} onchange={() => toggleSpillover(cat.id)} />
|
||||
More
|
||||
</label>
|
||||
<label class="private-toggle">
|
||||
<input type="checkbox" checked={cat.disableAi} onchange={() => toggleDisableAi(cat.id)} />
|
||||
No AI
|
||||
</label>
|
||||
{/if}
|
||||
<button class="icon-btn" onclick={() => move(i, -1)} disabled={i === 0} aria-label="Move up">▲</button>
|
||||
<button
|
||||
@@ -151,6 +166,10 @@
|
||||
<input type="checkbox" bind:checked={newCategorySpillover} />
|
||||
More
|
||||
</label>
|
||||
<label class="private-toggle">
|
||||
<input type="checkbox" bind:checked={newCategoryDisableAi} />
|
||||
No AI
|
||||
</label>
|
||||
<button onclick={addCategory} disabled={addingCategory || !newCategoryName.trim()}>
|
||||
{addingCategory ? 'Adding…' : '+ Add'}
|
||||
</button>
|
||||
@@ -182,12 +201,38 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<span class="panel-title">Writing style</span>
|
||||
<p class="hint">
|
||||
Applies to AI-merged articles only — a story with just one source publishes with its
|
||||
original text untouched, no AI involved. Event recaps have their own independent writing
|
||||
style, set per tracked item under Tracked items → Edit → More.
|
||||
</p>
|
||||
<select bind:value={local.synthesisStylePreset} onchange={scheduleSave}>
|
||||
<option value="default">Default (neutral, wire-service tone)</option>
|
||||
<option value="casual">Casual</option>
|
||||
<option value="formal">Formal</option>
|
||||
</select>
|
||||
<label class="field-label" for="custom-instructions" style="margin-top: 10px;">
|
||||
Additional instructions (optional)
|
||||
</label>
|
||||
<textarea
|
||||
id="custom-instructions"
|
||||
rows="3"
|
||||
placeholder={'e.g. "Keep paragraphs under 3 sentences", "Never use the word notably"'}
|
||||
bind:value={local.synthesisCustomInstructions}
|
||||
oninput={scheduleSave}
|
||||
></textarea>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<span class="panel-title">Hold before publish</span>
|
||||
<p class="hint">Wait window to gather more sources before finalizing a story.</p>
|
||||
<select bind:value={local.holdBeforePublishMinutes} onchange={scheduleSave}>
|
||||
<option value={0}>Publish immediately</option>
|
||||
<option value={15}>Wait 15 minutes</option>
|
||||
<option value={30}>Wait 30 minutes</option>
|
||||
<option value={60}>Wait 1 hour</option>
|
||||
<option value={120}>Wait 2 hours</option>
|
||||
</select>
|
||||
</div>
|
||||
@@ -309,6 +354,12 @@
|
||||
select {
|
||||
width: 100%;
|
||||
}
|
||||
textarea {
|
||||
width: 100%;
|
||||
margin-top: 6px;
|
||||
font: inherit;
|
||||
resize: vertical;
|
||||
}
|
||||
.priority-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { AdminSettings, ModelCatalog, AiStatus } from '$lib/adminTypes';
|
||||
import { updateSettings, getAiStatus } from '$lib/adminApi';
|
||||
import { updateSettings, getAiStatus, getModelContext } from '$lib/adminApi';
|
||||
import SaveStatus from './SaveStatus.svelte';
|
||||
|
||||
let { settings, models, aiStatus: initialStatus }: { settings: AdminSettings; models: ModelCatalog; aiStatus: AiStatus } =
|
||||
@@ -30,6 +30,63 @@
|
||||
testing = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Context window / max response length — see backend/src/inference/ollama-provider.ts's
|
||||
// DEFAULT_NUM_CTX/DEFAULT_NUM_PREDICT for why these are ever explicit at all: too low a
|
||||
// num_predict silently truncates the model's output mid-sentence rather than erroring
|
||||
// (this is what a "cut off" article/recap means), and num_ctx bounds how much source
|
||||
// text can even be included in the prompt before it gets trimmed.
|
||||
let numCtx = $state(settings.synthesisNumCtx);
|
||||
let numPredict = $state(settings.synthesisNumPredict);
|
||||
let contextStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle');
|
||||
let contextSaveTimer: ReturnType<typeof setTimeout>;
|
||||
|
||||
// The selected synthesis model's own reported max context (via Ollama's /api/show) —
|
||||
// null when undetectable (older Ollama, unusual model format, unreachable), in which
|
||||
// case the slider falls back to a generous cap rather than blocking on it.
|
||||
let detectedMax = $state<number | null>(null);
|
||||
let detecting = $state(false);
|
||||
const FALLBACK_MAX_CTX = 32768;
|
||||
let maxCtx = $derived(detectedMax ?? FALLBACK_MAX_CTX);
|
||||
// num_predict counts against the same context window as the prompt — capping it well
|
||||
// under num_ctx leaves room for the prompt itself to actually fit.
|
||||
let maxPredict = $derived(Math.max(100, numCtx - 512));
|
||||
|
||||
async function detectContext(model: string) {
|
||||
if (!model) return;
|
||||
detecting = true;
|
||||
try {
|
||||
const info = await getModelContext(model);
|
||||
detectedMax = info.contextLength;
|
||||
} catch {
|
||||
detectedMax = null;
|
||||
} finally {
|
||||
detecting = false;
|
||||
}
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
detectContext(selected.synthesis);
|
||||
});
|
||||
|
||||
function scheduleContextSave() {
|
||||
contextStatus = 'saving';
|
||||
clearTimeout(contextSaveTimer);
|
||||
contextSaveTimer = setTimeout(async () => {
|
||||
try {
|
||||
await updateSettings({ synthesisNumCtx: numCtx, synthesisNumPredict: numPredict });
|
||||
contextStatus = 'saved';
|
||||
setTimeout(() => (contextStatus = 'idle'), 1500);
|
||||
} catch {
|
||||
contextStatus = 'error';
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
|
||||
function onNumCtxChange() {
|
||||
if (numPredict > numCtx - 512) numPredict = Math.max(100, numCtx - 512);
|
||||
scheduleContextSave();
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="panel">
|
||||
@@ -84,6 +141,57 @@
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="head">
|
||||
<span class="panel-title">Context window</span>
|
||||
<SaveStatus status={contextStatus} />
|
||||
</div>
|
||||
<p class="hint">
|
||||
How much text the synthesis model can take in (context window) and how long its response
|
||||
can be (max response length). Too low a response limit is why an article or event recap
|
||||
sometimes cuts off mid-sentence instead of finishing.
|
||||
{#if detecting}
|
||||
Detecting {selected.synthesis}'s limit…
|
||||
{:else if detectedMax}
|
||||
Detected max for {selected.synthesis}: {detectedMax.toLocaleString()} tokens.
|
||||
{:else}
|
||||
Couldn't detect a limit for {selected.synthesis} — defaulting the slider's ceiling to
|
||||
{FALLBACK_MAX_CTX.toLocaleString()}. Setting num_ctx above what the model actually
|
||||
supports will make Ollama reject or silently degrade requests.
|
||||
{/if}
|
||||
</p>
|
||||
|
||||
<label class="field-label" for="num-ctx">
|
||||
Context window (num_ctx) — {numCtx.toLocaleString()} tokens
|
||||
</label>
|
||||
<div class="slider-row">
|
||||
<input
|
||||
id="num-ctx"
|
||||
type="range"
|
||||
min="1024"
|
||||
max={maxCtx}
|
||||
step="512"
|
||||
bind:value={numCtx}
|
||||
oninput={onNumCtxChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<label class="field-label" for="num-predict" style="margin-top: 12px;">
|
||||
Max response length (num_predict) — {numPredict.toLocaleString()} tokens
|
||||
</label>
|
||||
<div class="slider-row">
|
||||
<input
|
||||
id="num-predict"
|
||||
type="range"
|
||||
min="100"
|
||||
max={maxPredict}
|
||||
step="50"
|
||||
bind:value={numPredict}
|
||||
oninput={scheduleContextSave}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.panel {
|
||||
background: var(--surface-1);
|
||||
@@ -121,6 +229,23 @@
|
||||
.disconnected {
|
||||
color: var(--text-danger);
|
||||
}
|
||||
.field-label {
|
||||
display: block;
|
||||
font-size: 11px;
|
||||
color: var(--text-muted);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.slider-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
.slider-row input[type='range'] {
|
||||
flex: 1;
|
||||
border: none;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
}
|
||||
select {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { AdminSettings } from '$lib/adminTypes';
|
||||
import { updateSettings, clearAllArticles, clearAllMedia } from '$lib/adminApi';
|
||||
import { updateSettings, clearAllArticles, clearAllMedia, reissueArticle } from '$lib/adminApi';
|
||||
import SaveStatus from './SaveStatus.svelte';
|
||||
|
||||
let { settings }: { settings: AdminSettings } = $props();
|
||||
@@ -11,6 +11,11 @@
|
||||
let clearing = $state<'articles' | 'media' | null>(null);
|
||||
let clearResult = $state<string | null>(null);
|
||||
|
||||
let reissueArticleId = $state('');
|
||||
let reissuing = $state(false);
|
||||
let reissueResult = $state<string | null>(null);
|
||||
let reissueError = $state<string | null>(null);
|
||||
|
||||
let telegramMediaMode = $state(settings.telegramMediaMode);
|
||||
let telegramMediaStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle');
|
||||
let telegramMediaSaveTimer: ReturnType<typeof setTimeout>;
|
||||
@@ -60,6 +65,23 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReissueArticle() {
|
||||
const id = reissueArticleId.trim();
|
||||
if (!id) return;
|
||||
reissuing = true;
|
||||
reissueResult = null;
|
||||
reissueError = null;
|
||||
try {
|
||||
const { itemsRequeued } = await reissueArticle(id);
|
||||
reissueResult = `Deleted — ${itemsRequeued} source item(s) requeued for re-publish`;
|
||||
reissueArticleId = '';
|
||||
} catch (err) {
|
||||
reissueError = /\(404\)/.test((err as Error).message) ? 'No article with that ID' : (err as Error).message;
|
||||
} finally {
|
||||
reissuing = false;
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleSave() {
|
||||
status = 'saving';
|
||||
clearTimeout(saveTimer);
|
||||
@@ -216,6 +238,39 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<span class="panel-title">Reissue an article</span>
|
||||
<p class="hint">
|
||||
Deletes one specific published article and requeues every source item it was built from, so
|
||||
they re-cluster and re-synthesize fresh on the next scheduler tick — for fixing a single bad
|
||||
publish (e.g. a garbled AI merge) without wiping anything else. Unlike a source's own "Clear
|
||||
content" action, this works regardless of how many different sources the article merged
|
||||
together. Find the article ID in its URL or via <code>GET /api/article/:id</code>.
|
||||
</p>
|
||||
<div class="clear-row">
|
||||
<input
|
||||
type="text"
|
||||
placeholder="art-…"
|
||||
bind:value={reissueArticleId}
|
||||
onkeydown={(e) => e.key === 'Enter' && handleReissueArticle()}
|
||||
style="flex: 1; min-width: 220px"
|
||||
/>
|
||||
<button
|
||||
class="danger-btn"
|
||||
onclick={handleReissueArticle}
|
||||
disabled={reissuing || !reissueArticleId.trim()}
|
||||
>
|
||||
{reissuing ? 'Reissuing…' : 'Reissue'}
|
||||
</button>
|
||||
{#if reissueResult}
|
||||
<span class="usage-label">{reissueResult}</span>
|
||||
{/if}
|
||||
{#if reissueError}
|
||||
<span class="error-label">{reissueError}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.panel {
|
||||
background: var(--surface-1);
|
||||
@@ -276,6 +331,10 @@
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.error-label {
|
||||
font-size: 12px;
|
||||
color: var(--text-danger);
|
||||
}
|
||||
.bar {
|
||||
width: 100%;
|
||||
height: 6px;
|
||||
|
||||
@@ -1,3 +1,15 @@
|
||||
/** Inverse of timeAgo — "in 3h", "in 45m" for a future timestamp; "any moment now" once it's passed. */
|
||||
export function timeUntil(iso: string): string {
|
||||
const diffMs = new Date(iso).getTime() - Date.now();
|
||||
if (diffMs <= 0) return 'any moment now';
|
||||
const mins = Math.round(diffMs / 60000);
|
||||
if (mins < 60) return `in ${mins}m`;
|
||||
const hours = Math.round(mins / 60);
|
||||
if (hours < 24) return `in ${hours}h`;
|
||||
const days = Math.round(hours / 24);
|
||||
return `in ${days}d`;
|
||||
}
|
||||
|
||||
export function timeAgo(iso: string): string {
|
||||
const diffMs = Date.now() - new Date(iso).getTime();
|
||||
const mins = Math.round(diffMs / 60000);
|
||||
|
||||
@@ -81,6 +81,8 @@ export interface TrackedEventPublic {
|
||||
name: string;
|
||||
active: boolean;
|
||||
recapIntervalHours: number | null;
|
||||
/** Timestamp of the last AI recap, or null if none has run yet — used to compute "next recap" on the /event/[id] page. */
|
||||
lastRecapAt: string | null;
|
||||
isSpillover: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,29 @@
|
||||
<script lang="ts">
|
||||
import type { PageData } from './$types';
|
||||
import InfiniteFeed from '$lib/components/InfiniteFeed.svelte';
|
||||
import { timeUntil } from '$lib/format';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
// recapIntervalHours === null means recaps are turned off for this item entirely;
|
||||
// lastRecapAt === null means the first recap hasn't happened yet (it fires once
|
||||
// there's new coverage to summarize — see backend/src/queue/eventsRecap.ts).
|
||||
let nextRecapText = $derived.by(() => {
|
||||
if (data.recapIntervalHours === null) return null;
|
||||
if (data.lastRecapAt === null) return 'first recap pending';
|
||||
const next = new Date(data.lastRecapAt).getTime() + data.recapIntervalHours * 3600_000;
|
||||
return `next recap ${timeUntil(new Date(next).toISOString())}`;
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class="head">
|
||||
<span class="title">{data.name}</span>
|
||||
<span class="sub">Tracked item — periodically recapped by AI</span>
|
||||
<span class="sub">
|
||||
Tracked item — periodically recapped by AI
|
||||
{#if nextRecapText}
|
||||
· {nextRecapText}
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<InfiniteFeed initial={data.initial} filters={data.filters} pageSize={data.pageSize} />
|
||||
|
||||
@@ -14,5 +14,12 @@ export const load: PageLoad = async ({ params, fetch, parent }) => {
|
||||
|
||||
const filters = { eventId: params.id };
|
||||
const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch);
|
||||
return { initial, filters, name: match?.name ?? 'Tracked event', pageSize: PAGE_SIZE };
|
||||
return {
|
||||
initial,
|
||||
filters,
|
||||
name: match?.name ?? 'Tracked event',
|
||||
recapIntervalHours: match?.recapIntervalHours ?? null,
|
||||
lastRecapAt: match?.lastRecapAt ?? null,
|
||||
pageSize: PAGE_SIZE
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
<script lang="ts">
|
||||
import type { PageData } from './$types';
|
||||
import InfiniteFeed from '$lib/components/InfiniteFeed.svelte';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
|
||||
<div class="head">
|
||||
<span class="title">#{data.tag.label}</span>
|
||||
</div>
|
||||
|
||||
<InfiniteFeed initial={data.initial} filters={data.filters} pageSize={data.pageSize} />
|
||||
|
||||
<style>
|
||||
.head {
|
||||
margin: 24px 0 8px;
|
||||
}
|
||||
.title {
|
||||
font-family: var(--font-voice);
|
||||
font-size: 26px;
|
||||
font-weight: 500;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,22 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { PageLoad } from './$types';
|
||||
import { getFeed, getTagBySlug } from '$lib/api';
|
||||
|
||||
const PAGE_SIZE = 15;
|
||||
|
||||
// Mirrors /category/[name] and /event/[id] — a tag chip links by slug, so the slug is
|
||||
// resolved to the real tag (id + label) via a dedicated backend lookup (GET
|
||||
// /api/tag/:slug) rather than a preloaded list, since tags aren't loaded by the root
|
||||
// layout the way categories/events are.
|
||||
export const load: PageLoad = async ({ params, fetch }) => {
|
||||
let tag;
|
||||
try {
|
||||
tag = await getTagBySlug(params.slug, fetch);
|
||||
} catch {
|
||||
throw error(404, 'Tag not found');
|
||||
}
|
||||
|
||||
const filters = { tag: tag.id };
|
||||
const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch);
|
||||
return { initial, filters, tag, pageSize: PAGE_SIZE };
|
||||
};
|
||||
Reference in New Issue
Block a user