diff --git a/backend/src/queue/scheduler.ts b/backend/src/queue/scheduler.ts index c20a905..5b69c8e 100644 --- a/backend/src/queue/scheduler.ts +++ b/backend/src/queue/scheduler.ts @@ -13,6 +13,28 @@ const POLL_TICK_MS = 60_000; // checks which sources are due every minute; each const SYNTHESIS_TICK_MS = 60_000; const RETENTION_TICK_MS = 60 * 60_000; // hourly +/** + * Runs fn on every tick, but skips a tick outright if the previous one is still in + * flight instead of overlapping it. Matters most for the synthesis tick: an item stays + * "unclustered" (cluster_id IS NULL — see contentItems.unclusteredItemsExcludingSources) + * until AFTER its cluster finishes synthesizing and publishing, so a generate() call + * that runs past the next tick (easily minutes, on CPU-only inference — see + * ollama-provider.ts) used to let the same item get picked up and republished as a + * fresh, differently-worded article by an overlapping cycle, repeatedly, until the + * first cycle's assignCluster() finally landed. Node is single-threaded, so the only + * source of "concurrent" runs here is exactly this interval overlap. + */ +function everyTickSkippingOverlap(ms: number, fn: () => Promise) { + let running = false; + setInterval(() => { + if (running) return; + running = true; + fn().finally(() => { + running = false; + }); + }, ms); +} + // Per-widget setInterval handles, keyed by widget id — lets a single widget's polling be // started/stopped independently (on live upload/delete, or an enable toggle) without // touching any other widget's interval. Exported so widgets/install.ts and @@ -53,16 +75,16 @@ export function startScheduler() { return new OllamaProvider(s.aiServiceHost, s.aiServicePort); }; - setInterval(async () => { + everyTickSkippingOverlap(POLL_TICK_MS, async () => { try { const ingested = await pollDueSources(); if (ingested > 0) logger.info('scheduler', `Poll tick: ingested ${ingested} new item(s)`); } catch (err) { logger.error('scheduler', `Poll tick failed: ${(err as Error).message}`); } - }, POLL_TICK_MS); + }); - setInterval(async () => { + everyTickSkippingOverlap(SYNTHESIS_TICK_MS, async () => { try { const settings = settingsDb.getSettings(); const p = provider(); @@ -86,16 +108,16 @@ export function startScheduler() { } catch (err) { logger.error('scheduler', `Synthesis tick failed: ${(err as Error).message}`); } - }, SYNTHESIS_TICK_MS); + }); - setInterval(() => { + everyTickSkippingOverlap(RETENTION_TICK_MS, async () => { try { runRetentionSweep(settingsDb.getSettings()); logger.info('retention', 'Retention sweep completed'); } catch (err) { logger.error('retention', `Retention tick failed: ${(err as Error).message}`); } - }, RETENTION_TICK_MS); + }); for (const plugin of loadedWidgets.values()) { startWidgetPolling(plugin);