diff --git a/backend/README.md b/backend/README.md index 6f03ad0..17af50d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -25,6 +25,28 @@ minimum an embedding model (e.g. `nomic-embed-text`) and a generation model (e.g the synthesis tick quietly skips each cycle until the AI service comes back — nothing crashes, articles just don't get published. +## Full-article capture + +RSS descriptions are frequently truncated teasers or ad-mangled snippets, not the +actual article. For RSS and API sources, every newly-ingested item's link gets +followed and the real page content extracted via Mozilla's Readability (the same +approach behind Firefox Reader View) — `ingestion/articleFetcher.ts`. When extraction +succeeds, the feed's own title/summary/body/image get replaced with what was actually +on the page; nav, ads, sidebars, comments, and footers are excluded, not just stripped +of tags. Also pulls the page's `og:image`/`twitter:image` meta tag as a hero image when +the feed itself didn't provide one. + +Extraction fails constantly in the real world — paywalls, bot detection, JS-rendered +pages, odd markup — so this is deliberately non-fatal: on any failure it falls back to +whatever the feed itself provided (title/description), logged at `warn` level so it's +visible in the admin panel's Logs tab rather than silent. Telegram sources skip this +entirely, since a Telegram message *is* the content — there's no separate page to follow. + +This adds two dependencies (`jsdom`, `@mozilla/readability`) — the one deliberate +exception to the "no heavy dependencies" preference elsewhere in this backend, since a +hand-rolled content-extraction heuristic would be meaningfully less reliable across the +range of real-world site markup this needs to handle. + ## Behavior before Ollama is set up Per the "assume Ollama arrives after the backend launches" requirement: ingestion and @@ -32,11 +54,24 @@ publishing don't wait for it. - **Adding a source polls it immediately**, not on the next scheduler tick — you see results right away instead of waiting up to a minute. -- **With no AI service reachable**, the synthesis tick falls back to publishing each - item directly once it clears the hold-before-publish window — no rewriting, no - cross-source merging, no tags (there's no LLM to extract them yet), but the page - populates instead of staying empty. Media (images) still downloads normally, since - that never needed AI in the first place. +- **With no AI service reachable**, every eligible item publishes directly and + immediately — no hold-before-publish wait (that window exists to let corroborating + sources arrive before an AI merge locks in, which doesn't apply when nothing's being + merged). No rewriting, no cross-source merging, no tags (there's no LLM to extract + them yet), but the page populates right away instead of staying empty. +- **Passthrough articles show the original feed publish date** (`pubDate`/`isoDate` + from the source), not when they were ingested or published locally. Only + AI-synthesized/merged articles get a "modified" timeframe reflecting when the merge + actually happened — see `publishDirect` vs `publishCluster` in `pipeline/publish.ts`. +- **Feed content gets cleaned before storage** — RSS/API content is frequently raw (or + double-escaped) HTML; `ingestion/clean.ts` strips tags and decodes entities into + readable plain text with paragraph breaks preserved, applied centrally in + `adapters/base.ts` so every adapter benefits without duplicating the logic. +- **No image or video in the feed item?** Falls back to the site's favicon + (`{origin}/favicon.ico`, downloaded and locally hosted like any other image) rather + than leaving the article with no art at all. Doesn't parse the page's `` for a + proper `` — just the conventional path, which covers most sites + without an extra HTTP round trip. See `faviconUrlFor` in `pipeline/image-selection.ts`. - **Once Ollama becomes reachable**, the real pipeline (embed → cluster → synthesize → tag) takes back over for anything ingested from that point on. Articles already published via the passthrough path aren't retroactively rewritten or merged — they diff --git a/backend/package-lock.json b/backend/package-lock.json index 32174e3..99e68c6 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -11,15 +11,211 @@ "dependencies": { "@fastify/cookie": "^11.1.1", "@fastify/cors": "^11.3.0", + "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", + "jsdom": "^29.1.1", "rss-parser": "^3.13.0" }, "devDependencies": { + "@types/jsdom": "^28.0.3", "@types/node": "^26.1.1", "tsx": "^4.23.0", "typescript": "^7.0.2" } }, + "node_modules/@asamuzakjp/css-color": { + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", + "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@csstools/css-calc": "^3.2.0", + "@csstools/css-color-parser": "^4.1.0", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", + "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.2.1", + "is-potential-custom-element-name": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/generational-cache": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", + "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "license": "MIT" + }, + "node_modules/@bramus/specificity": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", + "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", + "license": "MIT", + "dependencies": { + "css-tree": "^3.0.0" + }, + "bin": { + "specificity": "bin/cli.js" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", + "integrity": "sha512-064IFJdjTfUqnjpCVpMOdbr8FLQBhinbZj6yRv2An2E41O/pLEXqfFRWqGq/SxlE5PEUYTlvWsG2r8MswAVvkg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.1.tgz", + "integrity": "sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.9.tgz", + "integrity": "sha512-paQcIaOO53Rk5+YrBaBjm/SgrV4INImjo2BT1DtQRYr+XeTRbeAYlS+jxXp9drqvKmtFnWRJKIalDLhZZDu42A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^6.1.0", + "@csstools/css-calc": "^3.2.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.6.tgz", + "integrity": "sha512-TcJCWFbXLPpJYq6z7bfOyjWYJDiDg2/I4gyUC9pqPNqHFRIey0EB0q0L5cSnQDfWJg8Jd6VadakxdIez/3zkqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", + "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -462,6 +658,23 @@ "node": ">=18" } }, + "node_modules/@exodus/bytes": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz", + "integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==", + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "@noble/hashes": "^1.8.0 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@noble/hashes": { + "optional": true + } + } + }, "node_modules/@fastify/ajv-compiler": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.5.tgz", @@ -613,12 +826,41 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@mozilla/readability": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz", + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, + "node_modules/@types/jsdom": { + "version": "28.0.3", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-28.0.3.tgz", + "integrity": "sha512-/HQ2uFoetFTXuye8vzIcHw2z6Fwi7Hi/qcgC+RoS9NCyewiqxhVGqlG+ViGB6lkax481R6dmhf1I7lIGlzJStQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^8.0.0", + "undici-types": "^7.21.0" + } + }, + "node_modules/@types/jsdom/node_modules/undici-types": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.28.0.tgz", + "integrity": "sha512-LJAfY+2w6HGeT8d8J1wNQsUGUEGio6NWWpwdwurQe4f6oojzCFuGLizl1KSve4irsTxyLly1QhEeE6iapdaIvQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "26.1.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", @@ -629,6 +871,13 @@ "undici-types": "~8.3.0" } }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@typescript/typescript-aix-ppc64": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", @@ -1037,6 +1286,15 @@ "fastq": "^1.17.1" } }, + "node_modules/bidi-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", + "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", + "license": "MIT", + "dependencies": { + "require-from-string": "^2.0.2" + } + }, "node_modules/cookie": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz", @@ -1050,6 +1308,38 @@ "url": "https://opencollective.com/express" } }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/data-urls": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", + "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -1274,6 +1564,18 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/html-encoding-sniffer": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", + "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.6.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -1283,6 +1585,52 @@ "node": ">= 10" } }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "license": "MIT" + }, + "node_modules/jsdom": { + "version": "29.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", + "integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^5.1.11", + "@asamuzakjp/dom-selector": "^7.1.1", + "@bramus/specificity": "^2.4.2", + "@csstools/css-syntax-patches-for-csstree": "^1.1.3", + "@exodus/bytes": "^1.15.0", + "css-tree": "^3.2.1", + "data-urls": "^7.0.0", + "decimal.js": "^10.6.0", + "html-encoding-sniffer": "^6.0.0", + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.3.5", + "parse5": "^8.0.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^6.0.1", + "undici": "^7.25.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^8.0.1", + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.1", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, "node_modules/json-schema-ref-resolver": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", @@ -1358,6 +1706,21 @@ ], "license": "MIT" }, + "node_modules/lru-cache": { + "version": "11.5.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", + "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "license": "CC0-1.0" + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1367,6 +1730,30 @@ "node": ">=14.0.0" } }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "license": "MIT", + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/pino": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", @@ -1420,6 +1807,15 @@ ], "license": "MIT" }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", @@ -1519,6 +1915,18 @@ "node": ">=11.0.0" } }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/secure-json-parse": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", @@ -1562,6 +1970,15 @@ "atomic-sleep": "^1.0.0" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -1571,6 +1988,12 @@ "node": ">= 10.x" } }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "license": "MIT" + }, "node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", @@ -1589,6 +2012,24 @@ "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", "license": "MIT" }, + "node_modules/tldts": { + "version": "7.4.9", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.9.tgz", + "integrity": "sha512-3kZ8wQQ/k5DrChD4X4FVvr2D7E5uoRgAqkPyLpSCGUvqOvqu+JEdr3mwMUaVWb+vMHZaKhF5fp2PBigKsui7hA==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.9" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.4.9", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.9.tgz", + "integrity": "sha512-DxKfPBI52p2msTEu7MPhdpdDTBhhVQg1a/8PjQckeyAvO13eMYElX545grIp6nnTGIMZlRvFZPvFhvI/WIz2Vg==", + "license": "MIT" + }, "node_modules/toad-cache": { "version": "3.7.4", "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.4.tgz", @@ -1598,6 +2039,30 @@ "node": ">=20" } }, + "node_modules/tough-cookie": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", + "integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/tsx": { "version": "4.23.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.0.tgz", @@ -1652,6 +2117,15 @@ "@typescript/typescript-win32-x64": "7.0.2" } }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", @@ -1659,6 +2133,59 @@ "dev": true, "license": "MIT" }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-mimetype": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", + "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/xml2js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", @@ -1680,6 +2207,12 @@ "engines": { "node": ">=4.0" } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" } } } diff --git a/backend/package.json b/backend/package.json index f108826..154f1dc 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,10 +14,13 @@ "dependencies": { "@fastify/cookie": "^11.1.1", "@fastify/cors": "^11.3.0", + "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", + "jsdom": "^29.1.1", "rss-parser": "^3.13.0" }, "devDependencies": { + "@types/jsdom": "^28.0.3", "@types/node": "^26.1.1", "tsx": "^4.23.0", "typescript": "^7.0.2" diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index f9c6966..811fcd8 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -24,6 +24,20 @@ export async function registerAdminRoutes(app: FastifyInstance) { return { ...settings, categoryPriority: categoriesDb.listCategories() }; }); + // --- Categories (add/remove — reordering is via PATCH /settings above) --- + app.post('/api/admin/categories', async (req, reply) => { + const { name } = req.body as { name?: string }; + if (!name || !name.trim()) return reply.code(400).send({ error: 'name required' }); + const created = categoriesDb.createCategory(name.trim()); + return reply.code(201).send(created); + }); + + app.delete('/api/admin/categories/:id', async (req, reply) => { + const { id } = req.params as { id: string }; + categoriesDb.deleteCategory(id); + return reply.code(204).send(); + }); + // --- Sources --- app.get('/api/admin/sources', async () => sourcesDb.listSources()); diff --git a/backend/src/api/public.ts b/backend/src/api/public.ts index 58460ea..9688653 100644 --- a/backend/src/api/public.ts +++ b/backend/src/api/public.ts @@ -2,11 +2,19 @@ import type { FastifyInstance } from 'fastify'; import * as articlesDb from '../storage/db/articles.js'; import * as tagsDb from '../storage/db/tags.js'; import * as eventsDb from '../storage/db/events.js'; +import * as categoriesDb from '../storage/db/categories.js'; export async function registerPublicRoutes(app: FastifyInstance) { app.get('/api/feed', async (req) => { - const { category, geo, eventId, tag } = req.query as Record; - return articlesDb.queryFeed({ category, geo, eventId, tag }); + const { category, geo, eventId, tag, before, limit } = req.query as Record; + return articlesDb.queryFeed({ + category, + geo, + eventId, + tag, + before, + limit: limit ? Number(limit) : undefined + }); }); app.get('/api/article/:id', async (req, reply) => { @@ -24,4 +32,10 @@ export async function registerPublicRoutes(app: FastifyInstance) { // Public fields only — sourceIds, cadenceTime etc. stay admin-only. return eventsDb.listEvents().map((e) => ({ id: e.id, name: e.name, active: e.active, cadence: e.cadence })); }); + + // Drives the site nav — admin-editable (add/remove/reorder) via /api/admin/categories, + // per the "user may have no interest in Business or Culture" requirement. + app.get('/api/categories', async () => { + return categoriesDb.listCategories(); + }); } diff --git a/backend/src/index.ts b/backend/src/index.ts index 4dfc800..3d0b1f0 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -26,13 +26,29 @@ async function main() { // Cross-origin is expected — see project-structure.md "Cross-origin and security // implications". Not a wildcard: only the configured frontend origin is allowed. - await app.register(cors, { origin: FRONTEND_ORIGIN, credentials: true }); + // @fastify/cors defaults to GET/HEAD/POST only — without an explicit methods list, + // every PATCH (settings saves) and DELETE (removing sources/events) gets silently + // blocked by the browser at the CORS preflight stage, before the request ever + // reaches a route handler. + await app.register(cors, { + origin: FRONTEND_ORIGIN, + credentials: true, + methods: ['GET', 'POST', 'PATCH', 'DELETE', 'PUT', 'OPTIONS'] + }); await app.register(cookie); await registerAuth(app); await registerPublicRoutes(app); await registerAdminRoutes(app); + // Fastify's own logger is off (see below) — without this, an unhandled exception + // in any route handler produces a bare 500 with zero trace anywhere, including the + // admin panel's own Logs tab. This is what "Save failed" with no log entry was. + app.setErrorHandler((err: Error & { statusCode?: number }, req, reply) => { + logger.error('server', `${req.method} ${req.url} failed: ${err.message}`); + reply.code(err.statusCode ?? 500).send({ error: err.message }); + }); + // Locally hosted media (see storage/media) — served directly rather than via a // heavier static-file plugin, since this is a small, flat directory. app.get('/media/:filename', async (req, reply) => { diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index 9a586f9..cb79d61 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,5 @@ import type { Source, ContentItem } from '../../storage/db/types.js'; +import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { title: string; @@ -17,12 +18,18 @@ export interface SourceAdapter { } export function toContentItem(source: Source, item: FetchedItem): Omit { + // Feed content is frequently raw (or double-escaped) HTML — cleaned here once, + // centrally, so every adapter (RSS, API, Telegram once implemented) benefits + // without each needing its own cleanup logic. + const cleanBody = item.body ? cleanHtml(item.body) : null; + const cleanSummary = cleanHtml(item.summary) || (cleanBody ? toSummary(cleanBody) : ''); + return { sourceId: source.id, type: 'article', - title: item.title, - summary: item.summary, - body: item.body, + title: cleanHtml(item.title) || item.title, + summary: cleanSummary, + body: cleanBody, images: item.images, videos: item.videos, link: item.link, diff --git a/backend/src/ingestion/articleFetcher.ts b/backend/src/ingestion/articleFetcher.ts new file mode 100644 index 0000000..dbc8e55 --- /dev/null +++ b/backend/src/ingestion/articleFetcher.ts @@ -0,0 +1,80 @@ +// RSS descriptions are frequently truncated teasers, or mangled with ad markup — +// not the actual article. This follows the item's link and extracts the real page +// content using the same approach Firefox Reader View uses (Mozilla's Readability), +// which strips nav/ads/sidebars and keeps just the article itself. + +import { JSDOM } from 'jsdom'; +import { Readability } from '@mozilla/readability'; +import { logger } from '../storage/db/logs.js'; + +export interface ExtractedArticle { + title: string; + /** Raw extracted HTML — cleaned centrally by ingestion/clean.ts via toContentItem, same as feed content. */ + body: string; + summary: string; + images: { url: string }[]; +} + +const USER_AGENT = 'Mozilla/5.0 (compatible; HomefeedBot/1.0; self-hosted RSS reader)'; +const FETCH_TIMEOUT_MS = 12_000; +const MIN_CONTENT_LENGTH = 200; // below this, Readability probably grabbed a paywall stub or nav junk, not an article + +/** + * Returns null (rather than throwing) on any failure — the caller falls back to the + * feed's own title/description, which is far better than losing the item entirely. + * Real-world scraping fails constantly (paywalls, bot detection, JS-rendered pages, + * odd markup) — that's expected, not exceptional. + */ +export async function fetchFullArticle(url: string): Promise { + try { + const res = await fetch(url, { + headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) + }); + if (!res.ok) { + logger.warn('articleFetcher', `${url} responded ${res.status} — falling back to feed summary`); + return null; + } + + const contentType = res.headers.get('content-type') ?? ''; + if (!contentType.includes('html')) { + logger.warn('articleFetcher', `${url} is not HTML (${contentType}) — falling back to feed summary`); + return null; + } + + const html = await res.text(); + const dom = new JSDOM(html, { url }); + const doc = dom.window.document; + + const ogImage = + doc.querySelector('meta[property="og:image"]')?.getAttribute('content') ?? + doc.querySelector('meta[name="twitter:image"]')?.getAttribute('content'); + + const reader = new Readability(doc); + const article = reader.parse(); + + if (!article || !article.textContent || article.textContent.trim().length < MIN_CONTENT_LENGTH) { + logger.warn('articleFetcher', `Couldn't extract usable content from ${url} — falling back to feed summary`); + return null; + } + + const images: { url: string }[] = []; + if (ogImage) { + try { + images.push({ url: new URL(ogImage, url).toString() }); + } catch { + // malformed og:image URL — just skip it, not worth failing the whole extraction over + } + } + + return { + title: article.title?.trim() ?? '', + body: article.content ?? article.textContent, + summary: article.excerpt?.trim() ?? '', + images + }; + } catch (err) { + logger.warn('articleFetcher', `Extraction failed for ${url}: ${(err as Error).message}`); + return null; + } +} diff --git a/backend/src/ingestion/clean.ts b/backend/src/ingestion/clean.ts new file mode 100644 index 0000000..1697124 --- /dev/null +++ b/backend/src/ingestion/clean.ts @@ -0,0 +1,86 @@ +// Feed content (RSS/API) frequently arrives as raw HTML — sometimes double-escaped, +// sometimes not. Rendered as plain text (which is how ContentItem.summary/body get +// displayed for passthrough articles), unstripped tags show up literally as "

...". +// This converts it to clean, readable plain text: block-level tags become paragraph +// breaks, everything else is stripped, and HTML entities are decoded. + +const BLOCK_END_TAGS = /<\/(p|div|h[1-6]|li|ul|ol|blockquote|pre|tr)\s*>/gi; +const BREAK_TAGS = //gi; +const LIST_ITEM_START = /]*>/gi; + +// Whole elements to drop entirely — tag AND content, not just the tag. Script/style +// are the ad-injection case (e.g. Freestar/Google ad-slot JS showing up as plain text +// once only the tags were stripped). Figure/figcaption/time cover image captions and +// their machine-readable timestamps, which duplicate what the frontend already shows +// via its own "Image via {source}" treatment and "Published Xd ago" — not something +// worth surfacing twice, especially not as a raw ISO string in the middle of the body. +const REMOVE_ENTIRELY = /<(script|style|noscript|iframe|figure|figcaption|time)\b[^>]*>[\s\S]*?<\/\1>/gi; + +// Belt-and-suspenders for malformed feeds where ad JS leaks in without proper + + + {#if article.heroImage} + + {:else} +

+ {/if} + +
+
+ {article.category[0] ?? ''} + · + {sourceLabel} + {#if article.video} + · + ▶ Video + {/if} +
+
{article.title}
+
{excerpt(article.body)}
+
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
+
+ + + diff --git a/frontend/src/lib/components/InfiniteFeed.svelte b/frontend/src/lib/components/InfiniteFeed.svelte new file mode 100644 index 0000000..97d938d --- /dev/null +++ b/frontend/src/lib/components/InfiniteFeed.svelte @@ -0,0 +1,90 @@ + + +
+ {#each articles as article (article.id)} + + {/each} +
+ +{#if !done} +
+{/if} + +{#if loading} +
Loading more…
+{:else if done && articles.length > 0} +
You're caught up.
+{:else if articles.length === 0} +
No stories here yet.
+{/if} + + diff --git a/frontend/src/lib/components/admin/MergeTab.svelte b/frontend/src/lib/components/admin/MergeTab.svelte index 40b2c4f..bf2a64e 100644 --- a/frontend/src/lib/components/admin/MergeTab.svelte +++ b/frontend/src/lib/components/admin/MergeTab.svelte @@ -1,6 +1,6 @@ -{#if data.hero} - -
- {#if data.hero.sourceCount > 1} - ⇄ Merged from {data.hero.sourceCount} sources - {/if} - {data.hero.category.join(', ')} -
- {#if data.hero.heroImage} - - {/if} -
{data.hero.title}
-
{timeAgo(data.hero.publishedAt)}
- -{/if} +
+ Top stories +
-
-
- Local · Philadelphia - See all -
-
- {#each data.local as article} - - {/each} -
-
- -
-
- Business - See all -
-
- {#each data.business as article} - - {/each} -
-
- -
-
- Tech - See all -
-
- {#each data.tech as article} - - {/each} -
-
+ diff --git a/frontend/src/routes/+page.ts b/frontend/src/routes/+page.ts index 5748e03..6d87179 100644 --- a/frontend/src/routes/+page.ts +++ b/frontend/src/routes/+page.ts @@ -1,19 +1,9 @@ import type { PageLoad } from './$types'; import { getFeed } from '$lib/api'; +const PAGE_SIZE = 15; + export const load: PageLoad = async ({ fetch }) => { - const [all, local] = await Promise.all([ - getFeed({}, fetch), - getFeed({ geo: 'philadelphia' }, fetch) - ]); - - const byCategory = (name: string) => - all.filter((a) => a.category.some((c) => c.toLowerCase() === name.toLowerCase())); - - return { - hero: all[0] ?? null, - local, - business: byCategory('business'), - tech: byCategory('tech') - }; + const initial = await getFeed({ limit: PAGE_SIZE }, fetch); + return { initial, pageSize: PAGE_SIZE }; }; diff --git a/frontend/src/routes/article/[id]/+page.svelte b/frontend/src/routes/article/[id]/+page.svelte index 0e38b87..c4d75ff 100644 --- a/frontend/src/routes/article/[id]/+page.svelte +++ b/frontend/src/routes/article/[id]/+page.svelte @@ -1,6 +1,7 @@
{data.name} - {data.articles.length} stories
-{#if data.articles.length === 0} -

No stories in this category yet.

-{:else} -
- {#each data.articles as article} - - {/each} -
-{/if} + diff --git a/frontend/src/routes/category/[name]/+page.ts b/frontend/src/routes/category/[name]/+page.ts index e3139a0..a33ee99 100644 --- a/frontend/src/routes/category/[name]/+page.ts +++ b/frontend/src/routes/category/[name]/+page.ts @@ -1,12 +1,12 @@ import type { PageLoad } from './$types'; import { getFeed } from '$lib/api'; +const PAGE_SIZE = 15; + export const load: PageLoad = async ({ params, fetch }) => { const name = params.name; const isLocal = name.toLowerCase() === 'local'; - const articles = await getFeed( - isLocal ? { geo: 'philadelphia' } : { category: name }, - fetch - ); - return { articles, name }; + const filters = isLocal ? { geo: 'philadelphia' } : { category: name }; + const initial = await getFeed({ ...filters, limit: PAGE_SIZE }, fetch); + return { initial, filters, name, pageSize: PAGE_SIZE }; }; diff --git a/mock-backend/admin-data.js b/mock-backend/admin-data.js index b2ea318..5c91508 100644 --- a/mock-backend/admin-data.js +++ b/mock-backend/admin-data.js @@ -25,12 +25,12 @@ let settings = { storageUsedMB: 214 }, categoryPriority: [ - { id: "cat-top", name: "Top stories", priorityRank: 1 }, - { id: "cat-local", name: "Local", priorityRank: 2 }, - { id: "cat-world", name: "World", priorityRank: 3 }, - { id: "cat-business", name: "Business", priorityRank: 4 }, - { id: "cat-tech", name: "Tech", priorityRank: 5 }, - { id: "cat-culture", name: "Culture", priorityRank: 6 } + { id: "cat-top", name: "Top stories", priorityRank: 1, isDefault: true }, + { id: "cat-local", name: "Local", priorityRank: 2, isDefault: true }, + { id: "cat-world", name: "World", priorityRank: 3, isDefault: true }, + { id: "cat-business", name: "Business", priorityRank: 4, isDefault: true }, + { id: "cat-tech", name: "Tech", priorityRank: 5, isDefault: true }, + { id: "cat-culture", name: "Culture", priorityRank: 6, isDefault: true } ] }; @@ -143,5 +143,15 @@ module.exports = { deleteEvent: (id) => { events = events.filter((e) => e.id !== id); }, - getModels: () => models + getModels: () => models, + createCategory: (name) => { + const id = `cat-${name.toLowerCase().replace(/[^a-z0-9]+/g, "-")}-${Date.now()}`; + const maxRank = Math.max(0, ...settings.categoryPriority.map((c) => c.priorityRank)); + const created = { id, name, priorityRank: maxRank + 1, isDefault: false }; + settings.categoryPriority = [...settings.categoryPriority, created]; + return created; + }, + deleteCategory: (id) => { + settings.categoryPriority = settings.categoryPriority.filter((c) => c.id !== id); + } }; diff --git a/mock-backend/server.js b/mock-backend/server.js index b6d3b3d..36270a7 100644 --- a/mock-backend/server.js +++ b/mock-backend/server.js @@ -1,9 +1,10 @@ // Mock backend — implements the public API surface from homefeed-data-schema.md // so the frontend can be built and tested before the real backend exists. -// GET /api/feed?category=&geo=&eventId=&tag= +// GET /api/feed?category=&geo=&eventId=&tag=&before=&limit= // GET /api/article/:id // GET /api/events // GET /api/tags +// GET /api/categories const express = require("express"); const cors = require("cors"); @@ -17,7 +18,7 @@ app.use(express.json()); const PORT = process.env.PORT || 4000; app.get("/api/feed", (req, res) => { - const { category, geo, eventId, tag } = req.query; + const { category, geo, eventId, tag, before, limit } = req.query; let results = articles; if (category) { @@ -34,7 +35,14 @@ app.get("/api/feed", (req, res) => { } results = [...results].sort((a, b) => new Date(b.publishedAt) - new Date(a.publishedAt)); - res.json(results); + + if (before) { + const cursor = new Date(before).getTime(); + results = results.filter((a) => new Date(a.publishedAt).getTime() < cursor); + } + + const pageSize = Math.min(Number(limit) || 15, 50); + res.json(results.slice(0, pageSize)); }); app.get("/api/article/:id", (req, res) => { @@ -51,6 +59,11 @@ app.get("/api/events", (req, res) => { res.json(events); }); +app.get("/api/categories", (req, res) => { + const { categoryPriority } = admin.getSettings(); + res.json([...categoryPriority].sort((a, b) => a.priorityRank - b.priorityRank)); +}); + // --- Admin API --- // No auth in the mock — the real backend enforces session auth on all /api/admin/* routes. @@ -104,6 +117,24 @@ app.get("/api/admin/models", (req, res) => { res.json(admin.getModels()); }); +app.post("/api/admin/categories", (req, res) => { + const { name } = req.body; + if (!name || !name.trim()) return res.status(400).json({ error: "name required" }); + res.status(201).json(admin.createCategory(name.trim())); +}); + +app.delete("/api/admin/categories/:id", (req, res) => { + admin.deleteCategory(req.params.id); + res.status(204).end(); +}); + +app.get("/api/admin/logs", (req, res) => { + // The mock backend doesn't run a scheduler/pipeline, so there's nothing to log — + // returns an empty list rather than 404ing, so the Logs tab renders its empty state + // instead of erroring when pointed at the mock backend. + res.json([]); +}); + app.get("/api/admin/ai-status", (req, res) => { // Simulates pinging the configured Ollama host const { aiServiceHost, aiServicePort } = admin.getSettings();