Salastil/homefeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Act on review findings: dead code, poe2_primary_currency_name, inefficiencies, custom source type

Browse Source 
- Delete dead ArticleCard.svelte and five zero-call-site backend exports
  (getTagsByIds, unclusteredItemsForSources, itemsByCluster, deleteAllContentItems,
  latestArticleInThread)
- Drop poe2_primary_currency_name from the schema entirely (dev DB, no migration
  concerns)
- settings.ts: bind named params instead of 33 positional ?s to remove the
  reorder-and-silently-corrupt risk
- Remove the long-dead stooqToYahooSymbols unconditional startup rewrite
- Share direct-publish logic between runPassthroughCycle/runSynthesisCycle via a new
  publishItemsDirect helper, and cache sourcesDb.listSources() once per synthesis tick
  instead of a per-item getSource() call
- Remove the 'custom' source type (had no adapter of its own, aliased to api)
This commit is contained in:
Claude
2026-07-26 16:45:46 +00:00
parent c217545968
commit 24bf5afb07
12 changed files with 131 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ingestion/poller.ts
+1 -2
View File
@@ -15,8 +15,7 @@ const adapters: Record<Source['type'], SourceAdapter> = {
telegram: telegramAdapter,
api: apiAdapter,
youtube: youtubeAdapter,
nitter: nitterAdapter,
custom: apiAdapter
nitter: nitterAdapter
};
// Which source types point at a real webpage worth following for the full article,
backend/src/queue/priorityQueue.ts
+47 -37
View File
@@ -7,7 +7,7 @@ import { embedPendingItems } from '../pipeline/embedding.js';
import { clusterItems } from '../pipeline/clustering.js';
import { publishCluster, publishDirect } from '../pipeline/publish.js';
import { logger } from '../storage/db/logs.js';
import type { GlobalSettings, ContentItem, TrackedEvent } from '../storage/db/types.js';
import type { GlobalSettings, ContentItem, TrackedEvent, Source } from '../storage/db/types.js';
function partition<T>(items: T[], predicate: (item: T) => boolean): [T[], T[]] {
const matches: T[] = [];
@@ -31,8 +31,8 @@ function claimedEventId(item: ContentItem, events: TrackedEvent[]): string | nul
return match?.id ?? null;
}
function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>): number {
const source = sourcesDb.getSource(item.sourceId);
function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>, sourcesById: Map<string, Source>): number {
const source = sourcesById.get(item.sourceId);
const cats = source?.category ?? [];
let best = Infinity;
for (const cat of cats) {
@@ -45,6 +45,33 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>)
return best;
}
/**
* Shared by both the passthrough (no-AI) and synthesis direct-publish paths — same
* publish-then-tag-then-log/error shape, differing only in how the success/failure
* message describes why the item skipped merging.
*/
async function publishItemsDirect(
items: ContentItem[],
settings: GlobalSettings,
activeEvents: TrackedEvent[],
describeSuccess: (item: ContentItem) => string,
failureLabel: string
): Promise<number> {
let published = 0;
for (const item of items) {
try {
const eventId = claimedEventId(item, activeEvents) ?? undefined;
const article = await publishDirect(item, settings, { eventId });
contentItemsDb.assignCluster([item.id], article.id);
published++;
logger.info('synthesis', `Published "${article.title}" directly (${describeSuccess(item)})`);
} catch (err) {
logger.error('synthesis', `${failureLabel} for "${item.title}": ${(err as Error).message}`);
}
}
return published;
}
/**
* Fallback for when the AI service isn't reachable yet — publishes every eligible item
* immediately rather than leaving pages empty until Ollama is set up. Unlike the AI
@@ -58,28 +85,15 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
const items = contentItemsDb.unclusteredItemsExcludingSources([]);
if (items.length === 0) return 0;
const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s]));
const categories = categoriesDb.listCategories();
const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
const ranked = items
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) }))
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) }))
.sort((a, b) => a.rank - b.rank)
.map((r) => r.item);
let published = 0;
for (const item of ranked) {
try {
const eventId = claimedEventId(item, activeEvents) ?? undefined;
const article = await publishDirect(item, settings, { eventId });
contentItemsDb.assignCluster([item.id], article.id);
published++;
logger.info('synthesis', `Published "${article.title}" directly (no AI available)`);
} catch (err) {
logger.error('synthesis', `Passthrough publish failed for "${item.title}": ${(err as Error).message}`);
}
}
return published;
return publishItemsDirect(ranked, settings, activeEvents, () => 'no AI available', 'Passthrough publish failed');
}
/**
@@ -96,36 +110,32 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
const items = contentItemsDb.unclusteredItemsExcludingSources([]);
if (items.length === 0) return 0;
// One fetch of the full source list per cycle, reused below for both the
// direct-publish partition and each item's category/type lookups — avoids a
// separate sourcesDb.getSource() round-trip per item.
const sourcesById = new Map(sourcesDb.listSources().map((s) => [s.id, s]));
// YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with
// anything else — each is always its own article, same shape whether the AI service
// is up or not. Route them straight to publishDirect, same as the no-AI passthrough path.
const directPublishSourceIds = new Set(
sourcesDb
.listSources()
.filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram')
.map((s) => s.id)
[...sourcesById.values()].filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram').map((s) => s.id)
);
const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId));
let publishedDirect = 0;
for (const item of directItems) {
try {
const eventId = claimedEventId(item, activeEvents) ?? undefined;
const article = await publishDirect(item, settings, { eventId });
contentItemsDb.assignCluster([item.id], article.id);
publishedDirect++;
const source = sourcesDb.getSource(item.sourceId);
logger.info('synthesis', `Published "${article.title}" directly (${source?.type ?? 'unknown'})`);
} catch (err) {
logger.error('synthesis', `Direct publish failed for "${item.title}": ${(err as Error).message}`);
}
}
const publishedDirect = await publishItemsDirect(
directItems,
settings,
activeEvents,
(item) => sourcesById.get(item.sourceId)?.type ?? 'unknown',
'Direct publish failed'
);
const categories = categoriesDb.listCategories();
const rankByName = new Map(categories.map((c) => [c.name.toLowerCase(), c.priorityRank]));
const ranked = mergeableItems
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName) }))
.map((item) => ({ item, rank: primaryCategoryRank(item, rankByName, sourcesById) }))
.sort((a, b) => a.rank - b.rank)
.map((r) => r.item);
backend/src/storage/db/articles.ts
-7
View File
@@ -143,13 +143,6 @@ export function articlesForEventSince(eventId: string, since: string): MergedArt
return rows.map(rowToArticle);
}
export function latestArticleInThread(threadId: string): MergedArticle | null {
const row = db
.prepare('SELECT * FROM merged_articles WHERE thread_id = ? ORDER BY published_at DESC LIMIT 1')
.get(threadId);
return row ? rowToArticle(row) : null;
}
export function articlesOlderThan(days: number): MergedArticle[] {
const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
const rows = db.prepare('SELECT * FROM merged_articles WHERE published_at < ?').all(cutoff);
backend/src/storage/db/contentItems.ts
-18
View File
@@ -69,15 +69,6 @@ export function unclusteredItemsExcludingSources(excludeSourceIds: string[]): Co
return items.filter((i) => !excludeSourceIds.includes(i.sourceId));
}
export function unclusteredItemsForSources(sourceIds: string[], sinceISO: string): ContentItem[] {
if (sourceIds.length === 0) return [];
const placeholders = sourceIds.map(() => '?').join(',');
const rows = db
.prepare(`SELECT * FROM content_items WHERE cluster_id IS NULL AND source_id IN (${placeholders}) AND fetched_at > ?`)
.all(...sourceIds, sinceISO);
return rows.map(rowToItem);
}
export function setEmbedding(id: string, embedding: number[]) {
db.prepare('UPDATE content_items SET embedding = ? WHERE id = ?').run(JSON.stringify(embedding), id);
}
@@ -93,11 +84,6 @@ export function resetClusterForItems(ids: string[]) {
for (const id of ids) stmt.run(id);
}
export function itemsByCluster(clusterId: string): ContentItem[] {
const rows = db.prepare('SELECT * FROM content_items WHERE cluster_id = ?').all(clusterId);
return rows.map(rowToItem);
}
export function itemsOlderThan(days: number): ContentItem[] {
const cutoff = new Date(Date.now() - days * 86_400_000).toISOString();
const rows = db.prepare('SELECT * FROM content_items WHERE fetched_at < ?').all(cutoff);
@@ -117,7 +103,3 @@ export function itemsForSource(sourceId: string): ContentItem[] {
export function deleteContentItemsForSource(sourceId: string) {
db.prepare('DELETE FROM content_items WHERE source_id = ?').run(sourceId);
}
export function deleteAllContentItems() {
db.prepare('DELETE FROM content_items').run();
}
backend/src/storage/db/index.ts
+1 -16
View File
@@ -37,7 +37,7 @@ export function migrate() {
CREATE TABLE IF NOT EXISTS sources (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL, -- rss | api | telegram | custom
type TEXT NOT NULL, -- rss | api | telegram | youtube | nitter
category TEXT NOT NULL DEFAULT '[]', -- JSON array
url TEXT,
config TEXT NOT NULL DEFAULT '{}', -- JSON: apiKey, telegramChannelId, authHeaders
@@ -200,7 +200,6 @@ export function migrate() {
weather_updated_at TEXT, -- ISO timestamp, NULL pre-first-poll
poe2_league_id TEXT,
poe2_league_name TEXT,
poe2_primary_currency_name TEXT, -- unused since the watchlist moved to arbitrary currency pairs (no single "quoted in" currency anymore) — column kept rather than dropped, SQLite ALTER TABLE can't drop columns without a full table rebuild
poe2_updated_at TEXT
);
@@ -347,7 +346,6 @@ export function migrate() {
if (!hasColumn('global_settings', 'poe2_league_id')) {
db.exec('ALTER TABLE global_settings ADD COLUMN poe2_league_id TEXT');
db.exec('ALTER TABLE global_settings ADD COLUMN poe2_league_name TEXT');
db.exec('ALTER TABLE global_settings ADD COLUMN poe2_primary_currency_name TEXT');
db.exec('ALTER TABLE global_settings ADD COLUMN poe2_updated_at TEXT');
}
@@ -368,19 +366,6 @@ export function migrate() {
});
}
// Stocks switched data providers from Stooq (walled off behind a proof-of-work
// challenge) to Yahoo Finance, which uses different symbol syntax — rewrites only
// rows still holding exactly one of the three old Stooq-format default symbols we
// ourselves seeded, never touching a symbol the admin typed in themselves.
const stooqToYahooSymbols: [string, string][] = [
['^dji', '^DJI'],
['^spx', '^GSPC'],
['btcusd', 'BTC-USD']
];
for (const [oldSymbol, newSymbol] of stooqToYahooSymbols) {
db.prepare('UPDATE stock_tickers SET symbol = ? WHERE symbol = ?').run(newSymbol, oldSymbol);
}
// Seed default categories if none exist yet. "News" sits right under "Top stories" —
// general news sources belong here, not on "Top stories" itself, which isn't a real
// filterable tag: it's the homepage view, now scoped to only the articles whose
backend/src/storage/db/settings.ts
+50 -44
View File
@@ -60,52 +60,58 @@ export function updateSettings(patch: Partial<GlobalSettings>): GlobalSettings {
weather: { ...current.weather, ...(patch.weather ?? {}) },
poe2: { ...current.poe2, ...(patch.poe2 ?? {}) }
};
// Named params (rather than positional `?`) so this list can be reordered or
// extended without the column list and the bound-values list silently drifting
// out of sync — node:sqlite binds each by its `$name` key, not position.
db.prepare(
`UPDATE global_settings SET
merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?,
tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?,
ai_service_host=?, ai_service_port=?, selected_models=?,
nitter_media_mode=?, fxtwitter_base_url=?, telegram_media_mode=?,
published_article_max_age_days=?, raw_item_max_age_days=?,
storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=?,
weather_location_name=?, weather_latitude=?, weather_longitude=?, weather_unit=?,
weather_wind_unit=?, weather_pressure_unit=?,
weather_current=?, weather_hourly=?, weather_daily=?, weather_alerts=?, weather_updated_at=?,
poe2_league_id=?, poe2_league_name=?, poe2_updated_at=?
merge_strictness=$merge_strictness, default_poll_interval_minutes=$default_poll_interval_minutes,
hold_before_publish_minutes=$hold_before_publish_minutes,
tag_dedup_threshold=$tag_dedup_threshold, tag_expiry_days=$tag_expiry_days,
follow_up_min_hours_since_last=$follow_up_min_hours_since_last, follow_up_min_new_sources=$follow_up_min_new_sources,
ai_service_host=$ai_service_host, ai_service_port=$ai_service_port, selected_models=$selected_models,
nitter_media_mode=$nitter_media_mode, fxtwitter_base_url=$fxtwitter_base_url, telegram_media_mode=$telegram_media_mode,
published_article_max_age_days=$published_article_max_age_days, raw_item_max_age_days=$raw_item_max_age_days,
storage_cap_enabled=$storage_cap_enabled, storage_cap_value=$storage_cap_value, storage_cap_unit=$storage_cap_unit,
weather_location_name=$weather_location_name, weather_latitude=$weather_latitude, weather_longitude=$weather_longitude,
weather_unit=$weather_unit, weather_wind_unit=$weather_wind_unit, weather_pressure_unit=$weather_pressure_unit,
weather_current=$weather_current, weather_hourly=$weather_hourly, weather_daily=$weather_daily,
weather_alerts=$weather_alerts, weather_updated_at=$weather_updated_at,
poe2_league_id=$poe2_league_id, poe2_league_name=$poe2_league_name, poe2_updated_at=$poe2_updated_at
WHERE id = 1`
).run(
merged.mergeStrictness,
merged.defaultPollIntervalMinutes,
merged.holdBeforePublishMinutes,
merged.tagDedupThreshold,
merged.tagExpiryDays,
merged.followUpMinHoursSinceLast,
merged.followUpMinNewSources,
merged.aiServiceHost,
merged.aiServicePort,
JSON.stringify(merged.selectedModels),
merged.nitterMediaMode,
merged.fxtwitterBaseUrl,
merged.telegramMediaMode,
merged.retention.publishedArticleMaxAgeDays,
merged.retention.rawItemMaxAgeDays,
merged.retention.storageCapEnabled ? 1 : 0,
merged.retention.storageCapValue,
merged.retention.storageCapUnit,
merged.weather.locationName,
merged.weather.latitude,
merged.weather.longitude,
merged.weather.unit,
merged.weather.windUnit,
merged.weather.pressureUnit,
merged.weather.current ? JSON.stringify(merged.weather.current) : null,
JSON.stringify(merged.weather.hourly),
JSON.stringify(merged.weather.daily),
JSON.stringify(merged.weather.alerts),
merged.weather.updatedAt,
merged.poe2.leagueId,
merged.poe2.leagueName,
merged.poe2.updatedAt
);
).run({
$merge_strictness: merged.mergeStrictness,
$default_poll_interval_minutes: merged.defaultPollIntervalMinutes,
$hold_before_publish_minutes: merged.holdBeforePublishMinutes,
$tag_dedup_threshold: merged.tagDedupThreshold,
$tag_expiry_days: merged.tagExpiryDays,
$follow_up_min_hours_since_last: merged.followUpMinHoursSinceLast,
$follow_up_min_new_sources: merged.followUpMinNewSources,
$ai_service_host: merged.aiServiceHost,
$ai_service_port: merged.aiServicePort,
$selected_models: JSON.stringify(merged.selectedModels),
$nitter_media_mode: merged.nitterMediaMode,
$fxtwitter_base_url: merged.fxtwitterBaseUrl,
$telegram_media_mode: merged.telegramMediaMode,
$published_article_max_age_days: merged.retention.publishedArticleMaxAgeDays,
$raw_item_max_age_days: merged.retention.rawItemMaxAgeDays,
$storage_cap_enabled: merged.retention.storageCapEnabled ? 1 : 0,
$storage_cap_value: merged.retention.storageCapValue,
$storage_cap_unit: merged.retention.storageCapUnit,
$weather_location_name: merged.weather.locationName,
$weather_latitude: merged.weather.latitude,
$weather_longitude: merged.weather.longitude,
$weather_unit: merged.weather.unit,
$weather_wind_unit: merged.weather.windUnit,
$weather_pressure_unit: merged.weather.pressureUnit,
$weather_current: merged.weather.current ? JSON.stringify(merged.weather.current) : null,
$weather_hourly: JSON.stringify(merged.weather.hourly),
$weather_daily: JSON.stringify(merged.weather.daily),
$weather_alerts: JSON.stringify(merged.weather.alerts),
$weather_updated_at: merged.weather.updatedAt,
$poe2_league_id: merged.poe2.leagueId,
$poe2_league_name: merged.poe2.leagueName,
$poe2_updated_at: merged.poe2.updatedAt
});
return getSettings();
}
backend/src/storage/db/tags.ts
-7
View File
@@ -28,13 +28,6 @@ export function listActiveTags(): Tag[] {
return rows.map(rowToTag);
}
export function getTagsByIds(ids: string[]): Tag[] {
if (ids.length === 0) return [];
const placeholders = ids.map(() => '?').join(',');
const rows = db.prepare(`SELECT * FROM tags WHERE id IN (${placeholders})`).all(...ids);
return rows.map(rowToTag);
}
function cosineSimilarity(a: number[], b: number[]): number {
if (a.length === 0 || b.length === 0 || a.length !== b.length) return 0;
let dot = 0,
backend/src/storage/db/types.ts
+1 -1
View File
@@ -1,7 +1,7 @@
export interface Source {
id: string;
name: string;
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter' | 'custom';
type: 'rss' | 'api' | 'telegram' | 'youtube' | 'nitter';
category: string[];
url: string | null;
config: Record<string, unknown>;