Salastil/homefeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Many Improvements

Browse Source
This commit is contained in:
Salastil
2026-07-21 12:51:31 -04:00
parent d16cd508b5
commit b742320108
33 changed files with 1348 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/README.md
+40 -5
View File
@@ -25,6 +25,28 @@ minimum an embedding model (e.g. `nomic-embed-text`) and a generation model (e.g
the synthesis tick quietly skips each cycle until the AI service comes back — nothing
crashes, articles just don't get published.
## Full-article capture
RSS descriptions are frequently truncated teasers or ad-mangled snippets, not the
actual article. For RSS and API sources, every newly-ingested item's link gets
followed and the real page content extracted via Mozilla's Readability (the same
approach behind Firefox Reader View) — `ingestion/articleFetcher.ts`. When extraction
succeeds, the feed's own title/summary/body/image get replaced with what was actually
on the page; nav, ads, sidebars, comments, and footers are excluded, not just stripped
of tags. Also pulls the page's `og:image`/`twitter:image` meta tag as a hero image when
the feed itself didn't provide one.
Extraction fails constantly in the real world — paywalls, bot detection, JS-rendered
pages, odd markup — so this is deliberately non-fatal: on any failure it falls back to
whatever the feed itself provided (title/description), logged at `warn` level so it's
visible in the admin panel's Logs tab rather than silent. Telegram sources skip this
entirely, since a Telegram message *is* the content — there's no separate page to follow.
This adds two dependencies (`jsdom`, `@mozilla/readability`) — the one deliberate
exception to the "no heavy dependencies" preference elsewhere in this backend, since a
hand-rolled content-extraction heuristic would be meaningfully less reliable across the
range of real-world site markup this needs to handle.
## Behavior before Ollama is set up
Per the "assume Ollama arrives after the backend launches" requirement: ingestion and
@@ -32,11 +54,24 @@ publishing don't wait for it.
- **Adding a source polls it immediately**, not on the next scheduler tick — you see
results right away instead of waiting up to a minute.
- **With no AI service reachable**, the synthesis tick falls back to publishing each
item directly once it clears the hold-before-publish window — no rewriting, no
cross-source merging, no tags (there's no LLM to extract them yet), but the page
populates instead of staying empty. Media (images) still downloads normally, since
that never needed AI in the first place.
- **With no AI service reachable**, every eligible item publishes directly and
immediately — no hold-before-publish wait (that window exists to let corroborating
sources arrive before an AI merge locks in, which doesn't apply when nothing's being
merged). No rewriting, no cross-source merging, no tags (there's no LLM to extract
them yet), but the page populates right away instead of staying empty.
- **Passthrough articles show the original feed publish date** (`pubDate`/`isoDate`
from the source), not when they were ingested or published locally. Only
AI-synthesized/merged articles get a "modified" timeframe reflecting when the merge
actually happened — see `publishDirect` vs `publishCluster` in `pipeline/publish.ts`.
- **Feed content gets cleaned before storage** — RSS/API content is frequently raw (or
double-escaped) HTML; `ingestion/clean.ts` strips tags and decodes entities into
readable plain text with paragraph breaks preserved, applied centrally in
`adapters/base.ts` so every adapter benefits without duplicating the logic.
- **No image or video in the feed item?** Falls back to the site's favicon
(`{origin}/favicon.ico`, downloaded and locally hosted like any other image) rather
than leaving the article with no art at all. Doesn't parse the page's `<head>` for a
proper `<link rel="icon">` — just the conventional path, which covers most sites
without an extra HTTP round trip. See `faviconUrlFor` in `pipeline/image-selection.ts`.
- **Once Ollama becomes reachable**, the real pipeline (embed → cluster → synthesize →
tag) takes back over for anything ingested from that point on. Articles already
published via the passthrough path aren't retroactively rewritten or merged — they
backend/package-lock.json
Generated
+533
View File
@@ -11,15 +11,211 @@
"dependencies": {
"@fastify/cookie": "^11.1.1",
"@fastify/cors": "^11.3.0",
"@mozilla/readability": "^0.6.0",
"fastify": "^5.10.0",
"jsdom": "^29.1.1",
"rss-parser": "^3.13.0"
},
"devDependencies": {
"@types/jsdom": "^28.0.3",
"@types/node": "^26.1.1",
"tsx": "^4.23.0",
"typescript": "^7.0.2"
}
},
"node_modules/@asamuzakjp/css-color": {
"version": "5.1.11",
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz",
"integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==",
"license": "MIT",
"dependencies": {
"@asamuzakjp/generational-cache": "^1.0.1",
"@csstools/css-calc": "^3.2.0",
"@csstools/css-color-parser": "^4.1.0",
"@csstools/css-parser-algorithms": "^4.0.0",
"@csstools/css-tokenizer": "^4.0.0"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/dom-selector": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz",
"integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==",
"license": "MIT",
"dependencies": {
"@asamuzakjp/generational-cache": "^1.0.1",
"@asamuzakjp/nwsapi": "^2.3.9",
"bidi-js": "^1.0.3",
"css-tree": "^3.2.1",
"is-potential-custom-element-name": "^1.0.1"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/generational-cache": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz",
"integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==",
"license": "MIT",
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/nwsapi": {
"version": "2.3.9",
"resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz",
"integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==",
"license": "MIT"
},
"node_modules/@bramus/specificity": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz",
"integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==",
"license": "MIT",
"dependencies": {
"css-tree": "^3.0.0"
},
"bin": {
"specificity": "bin/cli.js"
}
},
"node_modules/@csstools/color-helpers": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz",
"integrity": "sha512-064IFJdjTfUqnjpCVpMOdbr8FLQBhinbZj6yRv2An2E41O/pLEXqfFRWqGq/SxlE5PEUYTlvWsG2r8MswAVvkg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT-0",
"engines": {
"node": ">=20.19.0"
}
},
"node_modules/@csstools/css-calc": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.1.tgz",
"integrity": "sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT",
"engines": {
"node": ">=20.19.0"
},
"peerDependencies": {
"@csstools/css-parser-algorithms": "^4.0.0",
"@csstools/css-tokenizer": "^4.0.0"
}
},
"node_modules/@csstools/css-color-parser": {
"version": "4.1.9",
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.9.tgz",
"integrity": "sha512-paQcIaOO53Rk5+YrBaBjm/SgrV4INImjo2BT1DtQRYr+XeTRbeAYlS+jxXp9drqvKmtFnWRJKIalDLhZZDu42A==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT",
"dependencies": {
"@csstools/color-helpers": "^6.1.0",
"@csstools/css-calc": "^3.2.1"
},
"engines": {
"node": ">=20.19.0"
},
"peerDependencies": {
"@csstools/css-parser-algorithms": "^4.0.0",
"@csstools/css-tokenizer": "^4.0.0"
}
},
"node_modules/@csstools/css-parser-algorithms": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz",
"integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT",
"engines": {
"node": ">=20.19.0"
},
"peerDependencies": {
"@csstools/css-tokenizer": "^4.0.0"
}
},
"node_modules/@csstools/css-syntax-patches-for-csstree": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.6.tgz",
"integrity": "sha512-TcJCWFbXLPpJYq6z7bfOyjWYJDiDg2/I4gyUC9pqPNqHFRIey0EB0q0L5cSnQDfWJg8Jd6VadakxdIez/3zkqQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT-0",
"peerDependencies": {
"css-tree": "^3.2.1"
},
"peerDependenciesMeta": {
"css-tree": {
"optional": true
}
}
},
"node_modules/@csstools/css-tokenizer": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz",
"integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/csstools"
},
{
"type": "opencollective",
"url": "https://opencollective.com/csstools"
}
],
"license": "MIT",
"engines": {
"node": ">=20.19.0"
}
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.28.1",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz",
@@ -462,6 +658,23 @@
"node": ">=18"
}
},
"node_modules/@exodus/bytes": {
"version": "1.15.1",
"resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz",
"integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==",
"license": "MIT",
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
},
"peerDependencies": {
"@noble/hashes": "^1.8.0 || ^2.0.0"
},
"peerDependenciesMeta": {
"@noble/hashes": {
"optional": true
}
}
},
"node_modules/@fastify/ajv-compiler": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.5.tgz",
@@ -613,12 +826,41 @@
"ipaddr.js": "^2.1.0"
}
},
"node_modules/@mozilla/readability": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@pinojs/redact": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
"license": "MIT"
},
"node_modules/@types/jsdom": {
"version": "28.0.3",
"resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-28.0.3.tgz",
"integrity": "sha512-/HQ2uFoetFTXuye8vzIcHw2z6Fwi7Hi/qcgC+RoS9NCyewiqxhVGqlG+ViGB6lkax481R6dmhf1I7lIGlzJStQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"@types/tough-cookie": "*",
"parse5": "^8.0.0",
"undici-types": "^7.21.0"
}
},
"node_modules/@types/jsdom/node_modules/undici-types": {
"version": "7.28.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.28.0.tgz",
"integrity": "sha512-LJAfY+2w6HGeT8d8J1wNQsUGUEGio6NWWpwdwurQe4f6oojzCFuGLizl1KSve4irsTxyLly1QhEeE6iapdaIvQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "26.1.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz",
@@ -629,6 +871,13 @@
"undici-types": "~8.3.0"
}
},
"node_modules/@types/tough-cookie": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
"integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
"dev": true,
"license": "MIT"
},
"node_modules/@typescript/typescript-aix-ppc64": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz",
@@ -1037,6 +1286,15 @@
"fastq": "^1.17.1"
}
},
"node_modules/bidi-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
"integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==",
"license": "MIT",
"dependencies": {
"require-from-string": "^2.0.2"
}
},
"node_modules/cookie": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz",
@@ -1050,6 +1308,38 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/css-tree": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
"integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==",
"license": "MIT",
"dependencies": {
"mdn-data": "2.27.1",
"source-map-js": "^1.2.1"
},
"engines": {
"node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
}
},
"node_modules/data-urls": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz",
"integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==",
"license": "MIT",
"dependencies": {
"whatwg-mimetype": "^5.0.0",
"whatwg-url": "^16.0.0"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/decimal.js": {
"version": "10.6.0",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz",
"integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==",
"license": "MIT"
},
"node_modules/dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -1274,6 +1564,18 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/html-encoding-sniffer": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz",
"integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==",
"license": "MIT",
"dependencies": {
"@exodus/bytes": "^1.6.0"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/ipaddr.js": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz",
@@ -1283,6 +1585,52 @@
"node": ">= 10"
}
},
"node_modules/is-potential-custom-element-name": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
"license": "MIT"
},
"node_modules/jsdom": {
"version": "29.1.1",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz",
"integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==",
"license": "MIT",
"dependencies": {
"@asamuzakjp/css-color": "^5.1.11",
"@asamuzakjp/dom-selector": "^7.1.1",
"@bramus/specificity": "^2.4.2",
"@csstools/css-syntax-patches-for-csstree": "^1.1.3",
"@exodus/bytes": "^1.15.0",
"css-tree": "^3.2.1",
"data-urls": "^7.0.0",
"decimal.js": "^10.6.0",
"html-encoding-sniffer": "^6.0.0",
"is-potential-custom-element-name": "^1.0.1",
"lru-cache": "^11.3.5",
"parse5": "^8.0.1",
"saxes": "^6.0.0",
"symbol-tree": "^3.2.4",
"tough-cookie": "^6.0.1",
"undici": "^7.25.0",
"w3c-xmlserializer": "^5.0.0",
"webidl-conversions": "^8.0.1",
"whatwg-mimetype": "^5.0.0",
"whatwg-url": "^16.0.1",
"xml-name-validator": "^5.0.0"
},
"engines": {
"node": "^20.19.0 || ^22.13.0 || >=24.0.0"
},
"peerDependencies": {
"canvas": "^3.0.0"
},
"peerDependenciesMeta": {
"canvas": {
"optional": true
}
}
},
"node_modules/json-schema-ref-resolver": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz",
@@ -1358,6 +1706,21 @@
],
"license": "MIT"
},
"node_modules/lru-cache": {
"version": "11.5.2",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz",
"integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==",
"license": "BlueOak-1.0.0",
"engines": {
"node": "20 || >=22"
}
},
"node_modules/mdn-data": {
"version": "2.27.1",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz",
"integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==",
"license": "CC0-1.0"
},
"node_modules/on-exit-leak-free": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
@@ -1367,6 +1730,30 @@
"node": ">=14.0.0"
}
},
"node_modules/parse5": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",
"integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==",
"license": "MIT",
"dependencies": {
"entities": "^8.0.0"
},
"funding": {
"url": "https://github.com/inikulin/parse5?sponsor=1"
}
},
"node_modules/parse5/node_modules/entities": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz",
"integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=20.19.0"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/pino": {
"version": "10.3.1",
"resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",
@@ -1420,6 +1807,15 @@
],
"license": "MIT"
},
"node_modules/punycode": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/quick-format-unescaped": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
@@ -1519,6 +1915,18 @@
"node": ">=11.0.0"
}
},
"node_modules/saxes": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
"license": "ISC",
"dependencies": {
"xmlchars": "^2.2.0"
},
"engines": {
"node": ">=v12.22.7"
}
},
"node_modules/secure-json-parse": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
@@ -1562,6 +1970,15 @@
"atomic-sleep": "^1.0.0"
}
},
"node_modules/source-map-js": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/split2": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
@@ -1571,6 +1988,12 @@
"node": ">= 10.x"
}
},
"node_modules/symbol-tree": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
"license": "MIT"
},
"node_modules/thread-stream": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz",
@@ -1589,6 +2012,24 @@
"integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==",
"license": "MIT"
},
"node_modules/tldts": {
"version": "7.4.9",
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.9.tgz",
"integrity": "sha512-3kZ8wQQ/k5DrChD4X4FVvr2D7E5uoRgAqkPyLpSCGUvqOvqu+JEdr3mwMUaVWb+vMHZaKhF5fp2PBigKsui7hA==",
"license": "MIT",
"dependencies": {
"tldts-core": "^7.4.9"
},
"bin": {
"tldts": "bin/cli.js"
}
},
"node_modules/tldts-core": {
"version": "7.4.9",
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.9.tgz",
"integrity": "sha512-DxKfPBI52p2msTEu7MPhdpdDTBhhVQg1a/8PjQckeyAvO13eMYElX545grIp6nnTGIMZlRvFZPvFhvI/WIz2Vg==",
"license": "MIT"
},
"node_modules/toad-cache": {
"version": "3.7.4",
"resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.4.tgz",
@@ -1598,6 +2039,30 @@
"node": ">=20"
}
},
"node_modules/tough-cookie": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz",
"integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==",
"license": "BSD-3-Clause",
"dependencies": {
"tldts": "^7.0.5"
},
"engines": {
"node": ">=16"
}
},
"node_modules/tr46": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz",
"integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==",
"license": "MIT",
"dependencies": {
"punycode": "^2.3.1"
},
"engines": {
"node": ">=20"
}
},
"node_modules/tsx": {
"version": "4.23.0",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.0.tgz",
@@ -1652,6 +2117,15 @@
"@typescript/typescript-win32-x64": "7.0.2"
}
},
"node_modules/undici": {
"version": "7.28.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz",
"integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==",
"license": "MIT",
"engines": {
"node": ">=20.18.1"
}
},
"node_modules/undici-types": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
@@ -1659,6 +2133,59 @@
"dev": true,
"license": "MIT"
},
"node_modules/w3c-xmlserializer": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz",
"integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==",
"license": "MIT",
"dependencies": {
"xml-name-validator": "^5.0.0"
},
"engines": {
"node": ">=18"
}
},
"node_modules/webidl-conversions": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz",
"integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=20"
}
},
"node_modules/whatwg-mimetype": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz",
"integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==",
"license": "MIT",
"engines": {
"node": ">=20"
}
},
"node_modules/whatwg-url": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz",
"integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==",
"license": "MIT",
"dependencies": {
"@exodus/bytes": "^1.11.0",
"tr46": "^6.0.0",
"webidl-conversions": "^8.0.1"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/xml-name-validator": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz",
"integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==",
"license": "Apache-2.0",
"engines": {
"node": ">=18"
}
},
"node_modules/xml2js": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz",
@@ -1680,6 +2207,12 @@
"engines": {
"node": ">=4.0"
}
},
"node_modules/xmlchars": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==",
"license": "MIT"
}
}
}
backend/package.json
+3
View File
@@ -14,10 +14,13 @@
"dependencies": {
"@fastify/cookie": "^11.1.1",
"@fastify/cors": "^11.3.0",
"@mozilla/readability": "^0.6.0",
"fastify": "^5.10.0",
"jsdom": "^29.1.1",
"rss-parser": "^3.13.0"
},
"devDependencies": {
"@types/jsdom": "^28.0.3",
"@types/node": "^26.1.1",
"tsx": "^4.23.0",
"typescript": "^7.0.2"
backend/src/api/admin.ts
+14
View File
@@ -24,6 +24,20 @@ export async function registerAdminRoutes(app: FastifyInstance) {
return { ...settings, categoryPriority: categoriesDb.listCategories() };
});
// --- Categories (add/remove — reordering is via PATCH /settings above) ---
app.post('/api/admin/categories', async (req, reply) => {
const { name } = req.body as { name?: string };
if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' });
const created = categoriesDb.createCategory(name.trim());
return reply.code(201).send(created);
});
app.delete('/api/admin/categories/:id', async (req, reply) => {
const { id } = req.params as { id: string };
categoriesDb.deleteCategory(id);
return reply.code(204).send();
});
// --- Sources ---
app.get('/api/admin/sources', async () => sourcesDb.listSources());
backend/src/api/public.ts
+16 -2
View File
@@ -2,11 +2,19 @@ import type { FastifyInstance } from 'fastify';
import * as articlesDb from '../storage/db/articles.js';
import * as tagsDb from '../storage/db/tags.js';
import * as eventsDb from '../storage/db/events.js';
import * as categoriesDb from '../storage/db/categories.js';
export async function registerPublicRoutes(app: FastifyInstance) {
app.get('/api/feed', async (req) => {
const { category, geo, eventId, tag } = req.query as Record<string, string | undefined>;
return articlesDb.queryFeed({ category, geo, eventId, tag });
const { category, geo, eventId, tag, before, limit } = req.query as Record<string, string | undefined>;
return articlesDb.queryFeed({
category,
geo,
eventId,
tag,
before,
limit: limit ? Number(limit) : undefined
});
});
app.get('/api/article/:id', async (req, reply) => {
@@ -24,4 +32,10 @@ export async function registerPublicRoutes(app: FastifyInstance) {
// Public fields only — sourceIds, cadenceTime etc. stay admin-only.
return eventsDb.listEvents().map((e) => ({ id: e.id, name: e.name, active: e.active, cadence: e.cadence }));
});
// Drives the site nav — admin-editable (add/remove/reorder) via /api/admin/categories,
// per the "user may have no interest in Business or Culture" requirement.
app.get('/api/categories', async () => {
return categoriesDb.listCategories();
});
}
backend/src/index.ts
+17 -1
View File
@@ -26,13 +26,29 @@ async function main() {
// Cross-origin is expected — see project-structure.md "Cross-origin and security
// implications". Not a wildcard: only the configured frontend origin is allowed.
await app.register(cors, { origin: FRONTEND_ORIGIN, credentials: true });
// @fastify/cors defaults to GET/HEAD/POST only — without an explicit methods list,
// every PATCH (settings saves) and DELETE (removing sources/events) gets silently
// blocked by the browser at the CORS preflight stage, before the request ever
// reaches a route handler.
await app.register(cors, {
origin: FRONTEND_ORIGIN,
credentials: true,
methods: ['GET', 'POST', 'PATCH', 'DELETE', 'PUT', 'OPTIONS']
});
await app.register(cookie);
await registerAuth(app);
await registerPublicRoutes(app);
await registerAdminRoutes(app);
// Fastify's own logger is off (see below) — without this, an unhandled exception
// in any route handler produces a bare 500 with zero trace anywhere, including the
// admin panel's own Logs tab. This is what "Save failed" with no log entry was.
app.setErrorHandler((err: Error & { statusCode?: number }, req, reply) => {
logger.error('server', `${req.method} ${req.url} failed: ${err.message}`);
reply.code(err.statusCode ?? 500).send({ error: err.message });
});
// Locally hosted media (see storage/media) — served directly rather than via a
// heavier static-file plugin, since this is a small, flat directory.
app.get('/media/:filename', async (req, reply) => {
backend/src/ingestion/adapters/base.ts
+10 -3
View File
@@ -1,4 +1,5 @@
import type { Source, ContentItem } from '../../storage/db/types.js';
import { cleanHtml, toSummary } from '../clean.js';
export interface FetchedItem {
title: string;
@@ -17,12 +18,18 @@ export interface SourceAdapter {
}
export function toContentItem(source: Source, item: FetchedItem): Omit<ContentItem, 'id'> {
// Feed content is frequently raw (or double-escaped) HTML — cleaned here once,
// centrally, so every adapter (RSS, API, Telegram once implemented) benefits
// without each needing its own cleanup logic.
const cleanBody = item.body ? cleanHtml(item.body) : null;
const cleanSummary = cleanHtml(item.summary) || (cleanBody ? toSummary(cleanBody) : '');
return {
sourceId: source.id,
type: 'article',
title: item.title,
summary: item.summary,
body: item.body,
title: cleanHtml(item.title) || item.title,
summary: cleanSummary,
body: cleanBody,
images: item.images,
videos: item.videos,
link: item.link,
backend/src/ingestion/articleFetcher.ts
+80
View File
@@ -0,0 +1,80 @@
// RSS descriptions are frequently truncated teasers, or mangled with ad markup —
// not the actual article. This follows the item's link and extracts the real page
// content using the same approach Firefox Reader View uses (Mozilla's Readability),
// which strips nav/ads/sidebars and keeps just the article itself.
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { logger } from '../storage/db/logs.js';
export interface ExtractedArticle {
title: string;
/** Raw extracted HTML — cleaned centrally by ingestion/clean.ts via toContentItem, same as feed content. */
body: string;
summary: string;
images: { url: string }[];
}
const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)';
const FETCH_TIMEOUT_MS = 12_000;
const MIN_CONTENT_LENGTH = 200; // below this, Readability probably grabbed a paywall stub or nav junk, not an article
/**
* Returns null (rather than throwing) on any failure — the caller falls back to the
* feed's own title/description, which is far better than losing the item entirely.
* Real-world scraping fails constantly (paywalls, bot detection, JS-rendered pages,
* odd markup) — that's expected, not exceptional.
*/
export async function fetchFullArticle(url: string): Promise<ExtractedArticle | null> {
try {
const res = await fetch(url, {
headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS)
});
if (!res.ok) {
logger.warn('articleFetcher', `${url} responded ${res.status} — falling back to feed summary`);
return null;
}
const contentType = res.headers.get('content-type') ?? '';
if (!contentType.includes('html')) {
logger.warn('articleFetcher', `${url} is not HTML (${contentType}) — falling back to feed summary`);
return null;
}
const html = await res.text();
const dom = new JSDOM(html, { url });
const doc = dom.window.document;
const ogImage =
doc.querySelector('meta[property="og:image"]')?.getAttribute('content') ??
doc.querySelector('meta[name="twitter:image"]')?.getAttribute('content');
const reader = new Readability(doc);
const article = reader.parse();
if (!article || !article.textContent || article.textContent.trim().length < MIN_CONTENT_LENGTH) {
logger.warn('articleFetcher', `Couldn't extract usable content from ${url} — falling back to feed summary`);
return null;
}
const images: { url: string }[] = [];
if (ogImage) {
try {
images.push({ url: new URL(ogImage, url).toString() });
} catch {
// malformed og:image URL — just skip it, not worth failing the whole extraction over
}
}
return {
title: article.title?.trim() ?? '',
body: article.content ?? article.textContent,
summary: article.excerpt?.trim() ?? '',
images
};
} catch (err) {
logger.warn('articleFetcher', `Extraction failed for ${url}: ${(err as Error).message}`);
return null;
}
}
backend/src/ingestion/clean.ts
+86
View File
@@ -0,0 +1,86 @@
// Feed content (RSS/API) frequently arrives as raw HTML — sometimes double-escaped,
// sometimes not. Rendered as plain text (which is how ContentItem.summary/body get
// displayed for passthrough articles), unstripped tags show up literally as "<p>...".
// This converts it to clean, readable plain text: block-level tags become paragraph
// breaks, everything else is stripped, and HTML entities are decoded.
const BLOCK_END_TAGS = /<\/(p|div|h[1-6]|li|ul|ol|blockquote|pre|tr)\s*>/gi;
const BREAK_TAGS = /<br\s*\/?>/gi;
const LIST_ITEM_START = /<li[^>]*>/gi;
// Whole elements to drop entirely — tag AND content, not just the tag. Script/style
// are the ad-injection case (e.g. Freestar/Google ad-slot JS showing up as plain text
// once only the tags were stripped). Figure/figcaption/time cover image captions and
// their machine-readable timestamps, which duplicate what the frontend already shows
// via its own "Image via {source}" treatment and "Published Xd ago" — not something
// worth surfacing twice, especially not as a raw ISO string in the middle of the body.
const REMOVE_ENTIRELY = /<(script|style|noscript|iframe|figure|figcaption|time)\b[^>]*>[\s\S]*?<\/\1>/gi;
// Belt-and-suspenders for malformed feeds where ad JS leaks in without proper <script>
// tags at all (happens with some feed generators). Matches on common ad-network call
// signatures rather than trying to generally detect "is this line JavaScript."
const STRAY_AD_SCRIPT_LINE = /^.*(freestar|googletag|adsbygoogle|\.push\(function|newAdSlots|fsAdCount|querySelectorAll\(["'`]\.).*$/gim;
const NAMED_ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
mdash: '—',
ndash: '–',
hellip: '…',
rsquo: '’',
lsquo: '‘',
rdquo: '”',
ldquo: '“'
};
function decodeEntities(text: string): string {
return text
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)))
.replace(/&([a-z]+);/gi, (match, name) => NAMED_ENTITIES[name.toLowerCase()] ?? match);
}
/** Strips HTML tags and decodes entities, preserving paragraph/list structure as blank-line breaks. */
export function cleanHtml(input: string | null | undefined): string {
if (!input) return '';
let text = input;
// Some feeds double-escape (e.g. "&lt;p&gt;") — decode once up front so tag
// stripping below actually sees the tags rather than their escaped form.
if (/&lt;\/?[a-z]/i.test(text)) {
text = decodeEntities(text);
}
// Drop whole elements (script/style/figure/etc.) before any other processing —
// stripping just the tags and leaving their content behind is exactly what let
// ad-network JS and duplicate image captions leak into article bodies.
text = text.replace(REMOVE_ENTIRELY, '');
text = text.replace(STRAY_AD_SCRIPT_LINE, '');
text = text.replace(LIST_ITEM_START, '\n• ');
text = text.replace(BREAK_TAGS, '\n');
text = text.replace(BLOCK_END_TAGS, '\n\n');
text = text.replace(/<[^>]+>/g, ''); // strip all remaining tags (links, spans, code, strong, etc.)
text = decodeEntities(text);
text = text
.replace(/[ \t]+/g, ' ')
.replace(/\n[ \t]+/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim();
return text;
}
/** Truncates cleaned text to a summary-appropriate length without cutting mid-word. */
export function toSummary(cleaned: string, maxLength = 400): string {
const singleLine = cleaned.replace(/\n+/g, ' ').trim();
if (singleLine.length <= maxLength) return singleLine;
const truncated = singleLine.slice(0, maxLength);
return truncated.slice(0, truncated.lastIndexOf(' ')) + '…';
}
backend/src/ingestion/poller.ts
+32 -2
View File
@@ -4,7 +4,8 @@ import { logger } from '../storage/db/logs.js';
import { rssAdapter } from './adapters/rss.js';
import { telegramAdapter } from './adapters/telegram.js';
import { apiAdapter } from './adapters/api.js';
import { toContentItem, type SourceAdapter } from './adapters/base.js';
import { toContentItem, type SourceAdapter, type FetchedItem } from './adapters/base.js';
import { fetchFullArticle } from './articleFetcher.js';
import type { Source } from '../storage/db/types.js';
const adapters: Record<Source['type'], SourceAdapter> = {
@@ -14,6 +15,10 @@ const adapters: Record<Source['type'], SourceAdapter> = {
custom: apiAdapter
};
// Which source types point at a real webpage worth following for the full article,
// as opposed to Telegram where the message itself *is* the content.
const FOLLOWS_LINK_FOR_FULL_ARTICLE: Source['type'][] = ['rss', 'api'];
export async function pollDueSources(defaultIntervalMinutes: number): Promise<number> {
const due = sourcesDb.sourcesDueForPoll(defaultIntervalMinutes);
let ingested = 0;
@@ -35,7 +40,10 @@ async function pollOne(source: Source): Promise<number> {
const fetched = await adapter.fetch(source);
for (const item of fetched) {
if (contentItemsDb.existsByLink(item.link)) continue;
contentItemsDb.insertContentItem(toContentItem(source, item));
const finalItem = FOLLOWS_LINK_FOR_FULL_ARTICLE.includes(source.type) ? await withFullArticle(item) : item;
contentItemsDb.insertContentItem(toContentItem(source, finalItem));
ingested++;
}
sourcesDb.markPolled(source.id, null);
@@ -47,3 +55,25 @@ async function pollOne(source: Source): Promise<number> {
}
return ingested;
}
/**
* Follows the item's link and replaces the feed's own title/summary/body/images with
* the actual extracted article, per the "capture the real article, not the RSS teaser"
* requirement. Falls back to the feed's own fields untouched if extraction fails.
*/
async function withFullArticle(item: FetchedItem): Promise<FetchedItem> {
const full = await fetchFullArticle(item.link);
if (!full) {
logger.warn('poller', `Using feed summary for "${item.title}" (full article capture failed)`);
return item;
}
logger.info('poller', `Captured full article for "${full.title || item.title}"`);
return {
...item,
title: full.title || item.title,
summary: full.summary || item.summary,
body: full.body || item.body,
images: full.images.length > 0 ? full.images : item.images
};
}
backend/src/pipeline/image-selection.ts
+14
View File
@@ -35,3 +35,17 @@ export function selectBestImage(items: ContentItem[]): SelectedImage | null {
return { url: best.image.url, sourceItemId: best.item.id, selectionReason: reason };
}
/**
* Best-effort favicon URL for a link's origin — used as a hero-image fallback when a
* feed article has no image or video art of its own, rather than than a placeholder.
* Doesn't parse the page's <head> for a <link rel="icon">; just tries the conventional
* /favicon.ico path, which covers most sites without needing an extra HTTP round trip.
*/
export function faviconUrlFor(link: string): string | null {
try {
return `${new URL(link).origin}/favicon.ico`;
} catch {
return null;
}
}
backend/src/pipeline/publish.ts
+48 -26
View File
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
import type { InferenceProvider } from '../inference/provider.js';
import type { Cluster } from './clustering.js';
import { synthesizeArticle } from './synthesis.js';
import { selectBestImage } from './image-selection.js';
import { selectBestImage, faviconUrlFor } from './image-selection.js';
import { downloadAndStore, promoteToPublished } from '../storage/media/index.js';
import { logger } from '../storage/db/logs.js';
import * as articles from '../storage/db/articles.js';
@@ -27,6 +27,43 @@ function deriveTitle(body: string): string {
return firstLine.length > 100 ? firstLine.slice(0, 97) + '…' : firstLine;
}
/**
* Resolves the hero image for an article: try the best candidate from the source
* items, download and locally host it; if there isn't one, fall back to the site's
* favicon rather than leaving the article with no art at all.
*/
async function resolveHeroImage(
items: ContentItem[],
primaryLink: string
): Promise<{ heroImage: MergedArticle['heroImage']; storedMediaId: string | null }> {
const selected = selectBestImage(items);
if (selected) {
const stored = await downloadAndStore(selected.url, 'published', {});
if (stored) {
return {
heroImage: { url: stored.servedPath, sourceItemId: selected.sourceItemId, selectionReason: selected.selectionReason },
storedMediaId: stored.id
};
}
// Download failed — fall back to the hotlinked URL rather than losing the image entirely.
return { heroImage: selected, storedMediaId: null };
}
const favicon = faviconUrlFor(primaryLink);
if (favicon) {
const stored = await downloadAndStore(favicon, 'published', {});
if (stored) {
return {
heroImage: { url: stored.servedPath, sourceItemId: items[0]?.id ?? '', selectionReason: 'Site favicon — no article image available' },
storedMediaId: stored.id
};
}
}
return { heroImage: null, storedMediaId: null };
}
/**
* Publishes a single item as-is, with no AI calls at all — used when the AI service
* isn't reachable (e.g. Ollama hasn't been set up yet, per the "assume it arrives
@@ -39,19 +76,10 @@ function deriveTitle(body: string): string {
*/
export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
const category = uniqueCategories([item]);
const now = new Date().toISOString();
let heroImage: MergedArticle['heroImage'] = null;
if (item.images.length > 0) {
const stored = await downloadAndStore(item.images[0].url, 'published', {});
heroImage = stored
? { url: stored.servedPath, sourceItemId: item.id, selectionReason: 'Only available image (no AI service configured yet)' }
: { url: item.images[0].url, sourceItemId: item.id, selectionReason: 'Only available image (no AI service configured yet)' };
}
const { heroImage, storedMediaId } = await resolveHeroImage([item], item.link);
const video = item.videos[0] ? { url: item.videos[0].url, provider: item.videos[0].provider, sourceItemId: item.id } : null;
return articles.insertArticle({
const article = await articles.insertArticle({
title: item.title,
body: item.body || item.summary,
heroImage,
@@ -68,14 +96,19 @@ export async function publishDirect(item: ContentItem): Promise<MergedArticle> {
publishedAt: item.publishedAt
}
],
publishedAt: now,
updatedAt: now,
// Passthrough articles show the original feed date — only AI-synthesized/merged
// articles get a "modified" publish timeframe reflecting when the merge happened.
publishedAt: item.publishedAt,
updatedAt: item.publishedAt,
mergeConfidence: 1.0,
tags: [], // no LLM available to extract tags yet — backfilling these later is a reasonable future improvement
threadId: randomUUID(),
previousArticleId: null,
nextArticleId: null
});
if (storedMediaId) promoteToPublished(storedMediaId, article.id);
return article;
}
/**
@@ -104,18 +137,7 @@ export async function publishCluster(
}
const tagIds = resolvedTags.map((t) => t.id);
const selectedImage = selectBestImage(items);
let heroImage: MergedArticle['heroImage'] = null;
let storedMediaId: string | null = null;
if (selectedImage) {
const stored = await downloadAndStore(selectedImage.url, 'published', {});
if (stored) {
storedMediaId = stored.id;
heroImage = { url: stored.servedPath, sourceItemId: selectedImage.sourceItemId, selectionReason: selectedImage.selectionReason };
} else {
heroImage = selectedImage; // fall back to the hotlinked URL if the download failed, rather than losing the image entirely
}
}
const { heroImage, storedMediaId } = await resolveHeroImage(items, items[0]?.link ?? '');
const videoItem = items.find((i) => i.videos.length > 0);
const video = videoItem
? { url: videoItem.videos[0].url, provider: videoItem.videos[0].provider, sourceItemId: videoItem.id }
backend/src/queue/priorityQueue.ts
+7 -9
View File
@@ -24,10 +24,12 @@ function primaryCategoryRank(item: ContentItem, rankByName: Map<string, number>)
}
/**
* Fallback for when the AI service isn't reachable yet — publishes eligible items
* directly (no clustering across sources, no rewriting) rather than leaving pages
* empty until Ollama is set up. Still respects category priority and the
* hold-before-publish window; just skips everything that requires an AI call.
* Fallback for when the AI service isn't reachable yet — publishes every eligible item
* immediately rather than leaving pages empty until Ollama is set up. Unlike the AI
* pipeline, this doesn't wait out the hold-before-publish window: that window exists to
* give corroborating sources time to arrive before an AI merge locks in, which doesn't
* apply here since there's no merging happening at all — each item is just itself.
* Still respects category priority.
*/
export async function runPassthroughCycle(settings: GlobalSettings): Promise<number> {
const eventSourceIds = eventsDb.listActiveEvents().flatMap((e) => e.sourceIds);
@@ -41,13 +43,9 @@ export async function runPassthroughCycle(settings: GlobalSettings): Promise<num
.sort((a, b) => a.rank - b.rank)
.map((r) => r.item);
const holdMs = settings.holdBeforePublishMinutes * 60_000;
let published = 0;
for (const item of ranked) {
const ready = Date.now() - new Date(item.fetchedAt).getTime() >= holdMs;
if (!ready) continue;
try {
const article = await publishDirect(item);
contentItemsDb.assignCluster([item.id], article.id);
@@ -122,4 +120,4 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G
}
return published;
}
}
backend/src/queue/retention.ts
+1 -1
View File
@@ -55,7 +55,7 @@ function enforceStorageCap(capValue: number, unit: 'MB' | 'GB') {
// level being absent here — media rows are cleaned up by the candidate sweep above;
// published-tier media tied to a deleted article becomes orphaned and is swept next
// cycle once its downloaded_at also ages past raw-item retention as a backstop).
const oldest = articlesDb.queryFeed({}).reverse(); // oldest first
const oldest = articlesDb.allArticlesNewestFirst().reverse(); // oldest first — retention needs everything, not the feed's page size
for (const article of oldest) {
if (used <= capBytes) break;
articlesDb.deleteArticle(article.id);
backend/src/storage/db/articles.ts
+20 -2
View File
@@ -60,7 +60,20 @@ export function getArticle(id: string): MergedArticle | null {
return row ? rowToArticle(row) : null;
}
export function queryFeed(filters: { category?: string; geo?: string; eventId?: string; tag?: string }): MergedArticle[] {
/** Unpaginated — for internal use (retention sweep), not the public feed API which caps page size. */
export function allArticlesNewestFirst(): MergedArticle[] {
const rows = db.prepare('SELECT * FROM merged_articles ORDER BY published_at DESC').all();
return rows.map(rowToArticle);
}
export function queryFeed(filters: {
category?: string;
geo?: string;
eventId?: string;
tag?: string;
before?: string;
limit?: number;
}): MergedArticle[] {
let sql = 'SELECT * FROM merged_articles WHERE 1=1';
const params: unknown[] = [];
if (filters.category) {
@@ -79,7 +92,12 @@ export function queryFeed(filters: { category?: string; geo?: string; eventId?:
sql += ' AND tags LIKE ?';
params.push(`%"${filters.tag}"%`);
}
sql += ' ORDER BY published_at DESC';
if (filters.before) {
sql += ' AND published_at < ?';
params.push(filters.before);
}
sql += ' ORDER BY published_at DESC LIMIT ?';
params.push(Math.min(filters.limit ?? 15, 50));
const rows = db.prepare(sql).all(...(params as any[]));
return rows.map(rowToArticle);
}
backend/src/storage/db/categories.ts
+12
View File
@@ -1,3 +1,4 @@
import { randomUUID } from 'node:crypto';
import { db } from './index.js';
import type { Category } from './types.js';
@@ -14,3 +15,14 @@ export function setCategoryOrder(order: { id: string; priorityRank: number }[])
const stmt = db.prepare('UPDATE categories SET priority_rank = ? WHERE id = ?');
for (const c of order) stmt.run(c.priorityRank, c.id);
}
export function createCategory(name: string): Category {
const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')}-${randomUUID().slice(0, 6)}`;
const maxRank = db.prepare('SELECT COALESCE(MAX(priority_rank), 0) as m FROM categories').get() as { m: number };
db.prepare('INSERT INTO categories (id, name, priority_rank, is_default) VALUES (?, ?, ?, 0)').run(id, name, maxRank.m + 1);
return { id, name, priorityRank: maxRank.m + 1, isDefault: false };
}
export function deleteCategory(id: string) {
db.prepare('DELETE FROM categories WHERE id = ?').run(id);
}