From f8072ab339742e92d80d31596abf08d65723fdb5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 23 Jul 2026 20:23:28 +0000 Subject: [PATCH] Build out a functional Telegram adapter, rendered like the tweet card Logs into a real Telegram account via MTProto (GramJS) rather than a bot, so it can read any public channel's history. API ID/hash and the login session are entered through the admin Connections panel and stored encrypted at rest (new storage/crypto.ts AES-256-GCM helper) rather than via .env. Messages render as their own TelegramCard (same treatment as tweets) and open the original message on Telegram instead of an internal article page; attached media/albums are downloaded and self-hosted at ingestion time since Telegram has no public hotlinkable media URL. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/package-lock.json | 526 +++++++++++++++++- backend/package.json | 3 +- backend/src/api/admin.ts | 46 ++ backend/src/index.ts | 2 + backend/src/ingestion/adapters/base.ts | 21 +- backend/src/ingestion/adapters/telegram.ts | 174 +++++- backend/src/ingestion/poller.ts | 6 +- backend/src/pipeline/publish.ts | 25 +- backend/src/queue/priorityQueue.ts | 11 +- backend/src/storage/crypto.ts | 54 ++ backend/src/storage/db/articles.ts | 10 +- backend/src/storage/db/contentItems.ts | 6 +- backend/src/storage/db/index.ts | 27 +- backend/src/storage/db/types.ts | 19 + backend/src/storage/media/index.ts | 55 +- backend/src/telegram/client.ts | 155 ++++++ backend/src/telegram/credentials.ts | 48 ++ frontend/src/lib/adminApi.ts | 38 ++ frontend/src/lib/adminTypes.ts | 6 + .../src/lib/components/ArticleListRow.svelte | 3 + .../src/lib/components/TelegramCard.svelte | 186 +++++++ .../components/admin/ConnectionsTab.svelte | 184 +++++- .../lib/components/admin/SourcesTab.svelte | 5 + frontend/src/lib/types.ts | 10 + .../src/routes/admin/settings/+page.svelte | 2 +- frontend/src/routes/admin/settings/+page.ts | 10 +- frontend/src/routes/article/[id]/+page.svelte | 13 + 27 files changed, 1598 insertions(+), 47 deletions(-) create mode 100644 backend/src/storage/crypto.ts create mode 100644 backend/src/telegram/client.ts create mode 100644 backend/src/telegram/credentials.ts create mode 100644 frontend/src/lib/components/TelegramCard.svelte diff --git a/backend/package-lock.json b/backend/package-lock.json index 5538108..3bff8b6 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -14,7 +14,8 @@ "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", "jsdom": "^29.1.1", - "rss-parser": "^3.13.0" + "rss-parser": "^3.13.0", + "telegram": "^2.26.22" }, "devDependencies": { "@types/jsdom": "^28.0.3", @@ -82,6 +83,12 @@ "specificity": "bin/cli.js" } }, + "node_modules/@cryptography/aes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@cryptography/aes/-/aes-0.1.1.tgz", + "integrity": "sha512-PcYz4FDGblO6tM2kSC+VzhhK62vml6k6/YAkiWtyPvrgJVfnDRoHGDtKn5UiaRRUrvUTTocBpvc2rRgTCqxjsg==", + "license": "GPL-3.0-or-later" + }, "node_modules/@csstools/color-helpers": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", @@ -1257,6 +1264,15 @@ } } }, + "node_modules/async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.3.1" + } + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -1286,6 +1302,26 @@ "fastq": "^1.17.1" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -1295,6 +1331,52 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/bufferutil": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.1.0.tgz", + "integrity": "sha512-ZMANVnAixE6AWWnPzlW2KpUrxhm9woycYvPOo67jWHyFowASTEd9s+QN1EIMsSDtwhIxN4sWE1jotpuDUIgyIw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/cookie": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz", @@ -1321,6 +1403,19 @@ "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" } }, + "node_modules/d": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", + "integrity": "sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==", + "license": "ISC", + "dependencies": { + "es5-ext": "^0.10.64", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -1334,6 +1429,15 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, "node_modules/decimal.js": { "version": "10.6.0", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", @@ -1349,6 +1453,61 @@ "node": ">=6" } }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/entities": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", @@ -1358,6 +1517,46 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es5-ext": { + "version": "0.10.64", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", + "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", + "hasInstallScript": true, + "license": "ISC", + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "esniff": "^2.0.1", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.4.tgz", + "integrity": "sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.2", + "ext": "^1.7.0" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -1400,6 +1599,40 @@ "@esbuild/win32-x64": "0.28.1" } }, + "node_modules/esniff": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", + "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.62", + "event-emitter": "^0.3.5", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "license": "ISC", + "dependencies": { + "type": "^2.7.2" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -1564,6 +1797,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/html-encoding-sniffer": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", @@ -1576,6 +1815,63 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -1591,6 +1887,12 @@ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", "license": "MIT" }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", + "license": "MIT" + }, "node_modules/jsdom": { "version": "29.1.1", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", @@ -1721,6 +2023,53 @@ "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", "license": "CC0-1.0" }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "license": "ISC" + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/node-localstorage": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-localstorage/-/node-localstorage-2.2.1.tgz", + "integrity": "sha512-vv8fJuOUCCvSPjDjBLlMqYMHob4aGjkmrkaE42/mZr0VT+ZAU10jRF8oTnX9+pgU9/vYJ8P7YT3Vd6ajkmzSCw==", + "license": "MIT", + "dependencies": { + "write-file-atomic": "^1.1.4" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1730,6 +2079,22 @@ "node": ">=14.0.0" } }, + "node_modules/pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "(MIT AND Zlib)" + }, "node_modules/parse5": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", @@ -1754,6 +2119,12 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/pino": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", @@ -1822,6 +2193,12 @@ "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", "license": "MIT" }, + "node_modules/real-cancellable-promise": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.2.3.tgz", + "integrity": "sha512-hBI5Gy/55VEeeMtImMgEirD7eq5UmqJf1J8dFZtbJZA/3rB0pYFZ7PayMGueb6v4UtUtpKpP+05L0VwyE1hI9Q==", + "license": "MIT" + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -1961,6 +2338,39 @@ "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", "license": "MIT" }, + "node_modules/slide": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", + "integrity": "sha512-NwrtjCg+lZoqhFU8fOwl4ay2ei8PaqCBOUV3/ektPY9trO1yQ1oXEfmHAhKArUVUr/hOHvy5f6AdP17dCM0zMw==", + "license": "ISC", + "engines": { + "node": "*" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/sonic-boom": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", @@ -1988,12 +2398,45 @@ "node": ">= 10.x" } }, + "node_modules/store2": { + "version": "2.14.4", + "resolved": "https://registry.npmjs.org/store2/-/store2-2.14.4.tgz", + "integrity": "sha512-srTItn1GOvyvOycgxjAnPA63FZNwy0PTyUBFMHRM+hVFltAeoh0LmNBz9SZqUS9mMqGk8rfyWyXn3GH5ReJ8Zw==", + "license": "MIT" + }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "license": "MIT" }, + "node_modules/telegram": { + "version": "2.26.22", + "resolved": "https://registry.npmjs.org/telegram/-/telegram-2.26.22.tgz", + "integrity": "sha512-EIj7Yrjiu0Yosa3FZ/7EyPg9s6UiTi/zDQrFmR/2Mg7pIUU+XjAit1n1u9OU9h2oRnRM5M+67/fxzQluZpaJJg==", + "deprecated": "This package is archived and no longer maintained. Development continues in teleproto (https://npmjs.com/package/teleproto), a largely compatible, actively maintained fork. See the migration guide at https://docs.teleproto.dev/migrating-from-gramjs", + "license": "MIT", + "dependencies": { + "@cryptography/aes": "^0.1.1", + "async-mutex": "^0.3.0", + "big-integer": "^1.6.48", + "buffer": "^6.0.3", + "htmlparser2": "^6.1.0", + "mime": "^3.0.0", + "node-localstorage": "^2.2.1", + "pako": "^2.0.3", + "path-browserify": "^1.0.1", + "real-cancellable-promise": "^1.1.1", + "socks": "^2.6.2", + "store2": "^2.13.0", + "ts-custom-error": "^3.2.0", + "websocket": "^1.0.34" + }, + "optionalDependencies": { + "bufferutil": "^4.0.3", + "utf-8-validate": "^5.0.5" + } + }, "node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", @@ -2063,6 +2506,21 @@ "node": ">=20" } }, + "node_modules/ts-custom-error": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.3.1.tgz", + "integrity": "sha512-5OX1tzOjxWEgsr/YEUWSuPrQ00deKLh6D7OTWcvNHm12/7QPyRh8SYpyWvA4IZv8H/+GQWQEh/kwo95Q9OVW1A==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/tsx": { "version": "4.23.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.0.tgz", @@ -2082,6 +2540,21 @@ "fsevents": "~2.3.3" } }, + "node_modules/type": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.3.tgz", + "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==", + "license": "ISC" + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "license": "MIT", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", @@ -2133,6 +2606,19 @@ "dev": true, "license": "MIT" }, + "node_modules/utf-8-validate": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", + "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", @@ -2154,6 +2640,23 @@ "node": ">=20" } }, + "node_modules/websocket": { + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.35.tgz", + "integrity": "sha512-/REy6amwPZl44DDzvRCkaI1q1bIiQB0mEFQLUrhz3z2EK91cp3n72rAjUlrTP0zV22HJIUOVHQGPxhFRjxjt+Q==", + "license": "Apache-2.0", + "dependencies": { + "bufferutil": "^4.0.1", + "debug": "^2.2.0", + "es5-ext": "^0.10.63", + "typedarray-to-buffer": "^3.1.5", + "utf-8-validate": "^5.0.2", + "yaeti": "^0.0.6" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/whatwg-mimetype": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", @@ -2177,6 +2680,17 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/write-file-atomic": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-1.3.4.tgz", + "integrity": "sha512-SdrHoC/yVBPpV0Xq/mUZQIpW2sWXAShb/V4pomcJXh92RuaO+f3UTWItiR3Px+pLnV2PvC2/bfn5cwr5X6Vfxw==", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "slide": "^1.1.5" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -2213,6 +2727,16 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" + }, + "node_modules/yaeti": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", + "integrity": "sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "engines": { + "node": ">=0.10.32" + } } } } diff --git a/backend/package.json b/backend/package.json index b5d6c27..9eca3ab 100644 --- a/backend/package.json +++ b/backend/package.json @@ -17,7 +17,8 @@ "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", "jsdom": "^29.1.1", - "rss-parser": "^3.13.0" + "rss-parser": "^3.13.0", + "telegram": "^2.26.22" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 43c6dcd..4bd71b8 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -7,6 +7,7 @@ import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMed import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; +import * as telegramClient from '../telegram/client.js'; export async function registerAdminRoutes(app: FastifyInstance) { // --- Settings --- @@ -148,6 +149,51 @@ export async function registerAdminRoutes(app: FastifyInstance) { return { connected, host: settings.aiServiceHost, port: settings.aiServicePort, ramGB: null, gpu: null }; }); + // --- Telegram account (Connections tab — see telegram/client.ts and credentials.ts. + // API ID/hash and the resulting login session are stored encrypted at rest; none of + // these routes ever echo them back to the client.) --- + app.get('/api/admin/telegram/status', async () => telegramClient.getStatus()); + + app.post('/api/admin/telegram/credentials', async (req, reply) => { + const { apiId, apiHash } = req.body as { apiId?: number; apiHash?: string }; + if (!apiId || !apiHash) return reply.code(400).send({ error: 'apiId and apiHash are required' }); + telegramClient.saveApiCredentials(apiId, apiHash); + return { credentialsConfigured: true }; + }); + + app.post('/api/admin/telegram/login/start', async (req, reply) => { + const { phoneNumber } = req.body as { phoneNumber?: string }; + if (!phoneNumber) return reply.code(400).send({ error: 'phoneNumber is required' }); + try { + await telegramClient.startLogin(phoneNumber); + return { phase: 'code-sent' }; + } catch (err) { + return reply.code(400).send({ error: (err as Error).message }); + } + }); + + app.post('/api/admin/telegram/login/verify', async (req, reply) => { + const { code, password } = req.body as { code?: string; password?: string }; + try { + if (password !== undefined) { + await telegramClient.verifyPassword(password); + return { phase: 'connected', ...telegramClient.getStatus() }; + } + if (code !== undefined) { + const phase = await telegramClient.verifyCode(code); + return phase === 'connected' ? { phase, ...telegramClient.getStatus() } : { phase }; + } + return reply.code(400).send({ error: 'code or password is required' }); + } catch (err) { + return reply.code(400).send({ error: (err as Error).message }); + } + }); + + app.post('/api/admin/telegram/logout', async (_req, reply) => { + await telegramClient.logout(); + return reply.code(200).send(telegramClient.getStatus()); + }); + // --- Logs --- app.get('/api/admin/logs', async (req) => { const { level, limit } = req.query as { level?: string; limit?: string }; diff --git a/backend/src/index.ts b/backend/src/index.ts index 0e4c485..96c73b9 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -11,6 +11,7 @@ import { registerAdminRoutes } from './api/admin.js'; import { registerMediaProxy } from './api/mediaProxy.js'; import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js'; import { startScheduler } from './queue/scheduler.js'; +import { initFromSavedSession } from './telegram/client.js'; import { logger } from './storage/db/logs.js'; const PORT = Number(process.env.PORT) || 4000; @@ -32,6 +33,7 @@ function printApiKeyBanner() { async function main() { migrate(); printApiKeyBanner(); + await initFromSavedSession(); const app = Fastify({ logger: false }); diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index e767543..bcbe135 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,4 @@ -import type { Source, ContentItem, TweetMediaItem } from '../../storage/db/types.js'; +import type { Source, ContentItem, TweetMediaItem, TelegramMediaItem } from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -11,6 +11,24 @@ export interface FetchedItem { publishedAt: string; /** Set by the Nitter adapter only — carries the tweet's author info through to ContentItem.tweet. */ tweet?: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] }; + /** Set by the Telegram adapter only — carries the channel/message info through to ContentItem.telegramMessage. */ + telegramMessage?: { + channelName: string; + channelUsername: string; + channelAvatarUrl: string | null; + messageId: string; + media: TelegramMediaItem[]; + }; + /** + * Set by the Telegram adapter only — media_assets row IDs already downloaded and + * self-hosted at fetch time (before this item's ContentItem even exists, since the + * authenticated Telegram access producing them could disappear later). Not part of + * ContentItem itself — poller.ts uses this right after insertContentItem to attach + * these rows to the real content item id (see storage/media/index.ts's + * attachToContentItem), so publishDirect can later find and promote them via + * mediaIdsForContentItem without threading ids through the JSON blob. + */ + telegramMediaAssetIds?: string[]; raw: unknown; } @@ -43,6 +61,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit { + let kind: 'photo' | 'video' | 'gif'; + let mimeType = ''; + if (message.video) { + kind = 'video'; + mimeType = (message.video as unknown as { mimeType?: string }).mimeType ?? ''; + } else if (message.gif) { + kind = 'gif'; + mimeType = (message.gif as unknown as { mimeType?: string }).mimeType ?? ''; + } else if (message.photo) { + kind = 'photo'; + } else { + return null; + } + + const buffer = await message.downloadMedia(); + if (!buffer || typeof buffer === 'string') return null; + + const ext = guessExtension(mimeType, kind); + const stored = storeMediaBuffer(buffer, ext, `telegram-message:${message.id}`, 'candidate', {}); + return { item: { type: kind, url: stored.servedPath, thumbnailUrl: null, width: null, height: null }, mediaId: stored.id }; +} + +/** Groups consecutive messages sharing a non-null groupedId (Telegram's multi-photo/video "album" concept) into one entry each. */ +function groupMessages(messages: TgMessage[]): TgMessage[][] { + const order: string[] = []; + const groups = new Map(); + for (const message of messages) { + const key = message.groupedId ? message.groupedId.toString() : `single-${message.id}`; + if (!groups.has(key)) { + groups.set(key, []); + order.push(key); + } + groups.get(key)!.push(message); + } + return order.map((key) => groups.get(key)!); +} -/** - * TODO: real implementation. Telegram channels are read via the Bot API (grammY, - * per project-structure.md) using `getUpdates` or a channel-history fetch, keyed by - * `source.config.telegramChannelId`. Left as a stub — satisfies the SourceAdapter - * interface so the poller and admin panel can already list/configure Telegram sources - * (e.g. for the "Iran war" tracked event example) before this is filled in. - */ export const telegramAdapter: SourceAdapter = { async fetch(source: Source): Promise { - logger.warn('telegram', `Adapter not yet implemented — skipping source "${source.name}"`); - return []; + const client = getClient(); + if (!client) { + logger.warn('telegram', `No connected Telegram account — skipping source "${source.name}"`); + return []; + } + if (!source.url) return []; + + const identifier = normalizeChannelIdentifier(source.url); + let entity: any; + let messages: TgMessage[]; + try { + const result = await fetchChannelMessages(identifier, FETCH_LIMIT); + entity = result.entity; + messages = result.messages as unknown as TgMessage[]; + } catch (err) { + logger.error('telegram', `Failed to resolve/fetch channel "${identifier}" for source "${source.name}": ${(err as Error).message}`); + return []; + } + + // Only channels with a public username get a permalink (https://t.me//) + // that means anything to a non-member — a private channel's t.me/c//... + // link would make "click to open on Telegram" mostly useless, so v1 restricts to + // public channels and fails soft otherwise (same style as youtube.ts's resolveFeedUrl). + const channelUsername: string | undefined = entity?.username; + if (!channelUsername) { + logger.warn('telegram', `"${source.name}" has no public username — private channels aren't supported yet, skipping`); + return []; + } + const channelName: string = entity?.title ?? channelUsername; + + let avatarBuffer: Buffer | null = null; + try { + const photo = await client.downloadProfilePhoto(entity); + if (photo && typeof photo !== 'string') avatarBuffer = photo; + } catch (err) { + logger.warn('telegram', `Failed to download avatar for "${channelName}": ${(err as Error).message}`); + } + + const items: FetchedItem[] = []; + + for (const group of groupMessages(messages)) { + const primary = group.find((m) => m.message) ?? group[0]; + const text = primary.message ?? ''; + const firstLine = text.split('\n')[0].trim(); + + const media: TelegramMediaItem[] = []; + const mediaAssetIds: string[] = []; + for (const message of group) { + if (media.length >= MAX_TELEGRAM_MEDIA) break; + const extracted = await extractMedia(message); + if (extracted) { + media.push(extracted.item); + mediaAssetIds.push(extracted.mediaId); + } + } + + let channelAvatarUrl: string | null = null; + if (avatarBuffer) { + const stored = storeMediaBuffer(avatarBuffer, '.jpg', `telegram-avatar:${channelUsername}`, 'candidate', {}); + channelAvatarUrl = stored.servedPath; + mediaAssetIds.push(stored.id); + } + + if (!text && media.length === 0) continue; // nothing worth publishing (e.g. a service message) + + items.push({ + title: firstLine || `Message from ${channelName}`, + summary: text.slice(0, 500), + body: text || null, + images: [], + videos: [], + link: `https://t.me/${channelUsername}/${group[0].id}`, + publishedAt: new Date(primary.date * 1000).toISOString(), + telegramMessage: { + channelName, + channelUsername, + channelAvatarUrl, + messageId: String(group[0].id), + media + }, + telegramMediaAssetIds: mediaAssetIds, + raw: { messageIds: group.map((m) => m.id) } + }); + } + + return items; } }; diff --git a/backend/src/ingestion/poller.ts b/backend/src/ingestion/poller.ts index 5a13c89..a8dda39 100644 --- a/backend/src/ingestion/poller.ts +++ b/backend/src/ingestion/poller.ts @@ -1,5 +1,6 @@ import * as sourcesDb from '../storage/db/sources.js'; import * as contentItemsDb from '../storage/db/contentItems.js'; +import { attachToContentItem } from '../storage/media/index.js'; import { logger } from '../storage/db/logs.js'; import { rssAdapter } from './adapters/rss.js'; import { telegramAdapter } from './adapters/telegram.js'; @@ -47,7 +48,10 @@ async function pollOne(source: Source): Promise { const finalItem = FOLLOWS_LINK_FOR_FULL_ARTICLE.includes(source.type) ? await withFullArticle(item) : item; - contentItemsDb.insertContentItem(toContentItem(source, finalItem)); + const created = contentItemsDb.insertContentItem(toContentItem(source, finalItem)); + if (finalItem.telegramMediaAssetIds?.length) { + attachToContentItem(finalItem.telegramMediaAssetIds, created.id); + } ingested++; } sourcesDb.markPolled(source.id, null); diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index 2dd2880..7deb618 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -3,7 +3,7 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { Cluster } from './clustering.js'; import { synthesizeArticle } from './synthesis.js'; import { selectBestImage, faviconUrlFor } from './image-selection.js'; -import { downloadAndStore, promoteToPublished } from '../storage/media/index.js'; +import { downloadAndStore, promoteToPublished, mediaIdsForContentItem } from '../storage/media/index.js'; import { logger } from '../storage/db/logs.js'; import * as articles from '../storage/db/articles.js'; import * as tags from '../storage/db/tags.js'; @@ -143,9 +143,10 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) const category = uniqueCategories([item]); const storedMediaIds: string[] = []; - // Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly. + // Tweets and Telegram messages never get a "hero image" — their own card components + // render tweet.media / telegramMessage.media directly. let heroImage: MergedArticle['heroImage'] = null; - if (!item.tweet) { + if (!item.tweet && !item.telegramMessage) { const resolved = await resolveHeroImage([item], item.link); heroImage = resolved.heroImage; if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); @@ -174,12 +175,29 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) }; } + // Telegram media (attached photos/videos and the channel avatar) is already + // downloaded and self-hosted by the adapter at ingestion time — there's no separate + // media-mode resolution step the way Nitter has; just find those already-stored + // assets by content item id and promote them alongside everything else below. + let telegramMessage: MergedArticle['telegramMessage'] = null; + if (item.telegramMessage) { + storedMediaIds.push(...mediaIdsForContentItem(item.id)); + telegramMessage = { + channelName: item.telegramMessage.channelName, + channelUsername: item.telegramMessage.channelUsername, + channelAvatarUrl: item.telegramMessage.channelAvatarUrl, + sourceItemId: item.id, + media: item.telegramMessage.media + }; + } + const article = await articles.insertArticle({ title: item.title, body: item.body || item.summary, heroImage, video, tweet, + telegramMessage, category, geo: item.geo, eventId: item.eventId, @@ -282,6 +300,7 @@ export async function publishCluster( heroImage, video, tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass + telegramMessage: null, // telegram messages never reach clustering either — same bypass category, geo, eventId: opts.eventId ?? items[0]?.eventId ?? null, diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 8c6a0b3..00a0d15 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -77,11 +77,14 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds); if (items.length === 0) return 0; - // YouTube videos and Nitter tweets never get LLM-merged with anything else — each - // is always its own article, same shape whether the AI service is up or not. Route - // them straight to publishDirect, same as the no-AI passthrough path. + // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with + // anything else — each is always its own article, same shape whether the AI service + // is up or not. Route them straight to publishDirect, same as the no-AI passthrough path. const directPublishSourceIds = new Set( - sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id) + sourcesDb + .listSources() + .filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram') + .map((s) => s.id) ); const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); diff --git a/backend/src/storage/crypto.ts b/backend/src/storage/crypto.ts new file mode 100644 index 0000000..37a1fd3 --- /dev/null +++ b/backend/src/storage/crypto.ts @@ -0,0 +1,54 @@ +// Generic AES-256-GCM encrypt/decrypt for at-rest secrets (currently just Telegram +// credentials — see telegram/credentials.ts — but deliberately not Telegram-specific +// here, so any future sensitive field can reuse it). The key is generated once into a +// file next to the DB but not inside it, so a naive copy of just the .db file reveals +// nothing — this is "not easily retrieved," not "safe against a fully compromised +// host": anyone with full filesystem access has both files anyway, same trust model as +// every other secret in this app. The key file must be included in backups, or +// previously-saved Telegram credentials become undecryptable (fixed by logging in +// again — no other data loss). + +import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +const DB_PATH = process.env.DB_PATH || './data/homefeed.db'; +const KEY_PATH = path.join(path.dirname(DB_PATH), '.encryption-key'); + +const ALGORITHM = 'aes-256-gcm'; +const IV_LENGTH = 12; + +let cachedKey: Buffer | null = null; + +function getKey(): Buffer { + if (cachedKey) return cachedKey; + + if (fs.existsSync(KEY_PATH)) { + cachedKey = fs.readFileSync(KEY_PATH); + return cachedKey; + } + + const key = randomBytes(32); + fs.mkdirSync(path.dirname(KEY_PATH), { recursive: true }); + fs.writeFileSync(KEY_PATH, key, { mode: 0o600 }); + cachedKey = key; + return cachedKey; +} + +export function encrypt(plaintext: string): string { + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv(ALGORITHM, getKey(), iv); + const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); + const authTag = cipher.getAuthTag(); + return Buffer.concat([iv, authTag, encrypted]).toString('base64'); +} + +export function decrypt(ciphertext: string): string { + const raw = Buffer.from(ciphertext, 'base64'); + const iv = raw.subarray(0, IV_LENGTH); + const authTag = raw.subarray(IV_LENGTH, IV_LENGTH + 16); + const encrypted = raw.subarray(IV_LENGTH + 16); + const decipher = createDecipheriv(ALGORITHM, getKey(), iv); + decipher.setAuthTag(authTag); + return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8'); +} diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 8c08ff3..62bbffe 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -23,7 +23,8 @@ function rowToArticle(row: any): MergedArticle { previousArticleId: row.previous_article_id, nextArticleId: row.next_article_id, topStories: !!row.top_stories, - tweet: row.tweet ? JSON.parse(row.tweet) : null + tweet: row.tweet ? JSON.parse(row.tweet) : null, + telegramMessage: row.telegram_message ? JSON.parse(row.telegram_message) : null }; } @@ -31,8 +32,8 @@ export function insertArticle(article: Omit): MergedArticle const id = `art-${randomUUID()}`; db.prepare( `INSERT INTO merged_articles - (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet, telegram_message) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, article.title, @@ -52,7 +53,8 @@ export function insertArticle(article: Omit): MergedArticle article.previousArticleId, article.nextArticleId, article.topStories ? 1 : 0, - article.tweet ? JSON.stringify(article.tweet) : null + article.tweet ? JSON.stringify(article.tweet) : null, + article.telegramMessage ? JSON.stringify(article.telegramMessage) : null ); if (article.previousArticleId) { db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId); diff --git a/backend/src/storage/db/contentItems.ts b/backend/src/storage/db/contentItems.ts index 471afdd..057df87 100644 --- a/backend/src/storage/db/contentItems.ts +++ b/backend/src/storage/db/contentItems.ts @@ -21,6 +21,7 @@ function rowToItem(row: any): ContentItem { eventId: row.event_id, clusterId: row.cluster_id, tweet: row.tweet ? JSON.parse(row.tweet) : null, + telegramMessage: row.telegram_message ? JSON.parse(row.telegram_message) : null, raw: row.raw ? JSON.parse(row.raw) : null }; } @@ -29,8 +30,8 @@ export function insertContentItem(item: Omit): ContentItem { const id = `ci-${randomUUID()}`; db.prepare( `INSERT INTO content_items - (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, telegram_message, raw) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, item.sourceId, @@ -49,6 +50,7 @@ export function insertContentItem(item: Omit): ContentItem { item.eventId, item.clusterId, item.tweet ? JSON.stringify(item.tweet) : null, + item.telegramMessage ? JSON.stringify(item.telegramMessage) : null, item.raw ? JSON.stringify(item.raw) : null ); return { ...item, id }; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 220dcaa..45bc9e0 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -59,6 +59,7 @@ export function migrate() { event_id TEXT, cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only + telegram_message TEXT, -- JSON {channelName, channelUsername, channelAvatarUrl, messageId, media}, telegram-sourced items only raw TEXT -- JSON, original payload ); CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id); @@ -84,7 +85,8 @@ export function migrate() { previous_article_id TEXT, next_article_id TEXT, top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?" - tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only + tweet TEXT, -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only + telegram_message TEXT -- JSON {channelName, channelUsername, channelAvatarUrl, sourceItemId, media}, telegram-sourced articles only ); CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at); CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id); @@ -172,8 +174,25 @@ export function migrate() { nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com' ); + + -- Singleton row (see storage/crypto.ts) — encrypted Telegram API credentials and + -- the resulting login session. Deliberately its own table, not part of + -- global_settings, so these encrypted blobs never ride along in the generic + -- GET /api/admin/settings payload. + CREATE TABLE IF NOT EXISTS telegram_credentials ( + id INTEGER PRIMARY KEY CHECK (id = 1), + api_id_enc TEXT, + api_hash_enc TEXT, + session_enc TEXT, + phone_enc TEXT + ); `); + const existingTelegramCreds = db.prepare('SELECT id FROM telegram_credentials WHERE id = 1').get(); + if (!existingTelegramCreds) { + db.prepare('INSERT INTO telegram_credentials (id) VALUES (1)').run(); + } + // Seed the singleton settings row if it doesn't exist yet. const existing = db.prepare('SELECT id FROM global_settings WHERE id = 1').get(); if (!existing) { @@ -207,6 +226,12 @@ export function migrate() { if (!hasColumn('categories', 'is_private')) { db.exec('ALTER TABLE categories ADD COLUMN is_private INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('content_items', 'telegram_message')) { + db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT'); + } + if (!hasColumn('merged_articles', 'telegram_message')) { + db.exec('ALTER TABLE merged_articles ADD COLUMN telegram_message TEXT'); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index a71712e..dc52707 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -24,6 +24,9 @@ export interface TweetMediaItem { height: number | null; } +/** Same shape as TweetMediaItem — distinct name for readability at Telegram call sites. */ +export type TelegramMediaItem = TweetMediaItem; + export interface ContentItem { id: string; sourceId: string; @@ -43,6 +46,14 @@ export interface ContentItem { clusterId: string | null; /** Nitter-sourced items only — null for everything else. */ tweet: { id: string; authorName: string; authorHandle: string; avatarUrl: string | null; media: TweetMediaItem[] } | null; + /** Telegram-sourced items only — null for everything else. */ + telegramMessage: { + channelName: string; + channelUsername: string; + channelAvatarUrl: string | null; + messageId: string; + media: TelegramMediaItem[]; + } | null; raw: unknown; } @@ -61,6 +72,14 @@ export interface MergedArticle { video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; /** Nitter-sourced articles only — the embed card's author info and attached media (see TweetCard.svelte). Never set alongside video. */ tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; + /** Telegram-sourced articles only — the embed card's channel info and attached media (see TelegramCard.svelte). Never set alongside video. */ + telegramMessage: { + channelName: string; + channelUsername: string; + channelAvatarUrl: string | null; + sourceItemId: string; + media: TelegramMediaItem[]; + } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/backend/src/storage/media/index.ts b/backend/src/storage/media/index.ts index a911f27..5f36cc2 100644 --- a/backend/src/storage/media/index.ts +++ b/backend/src/storage/media/index.ts @@ -29,25 +29,56 @@ export async function downloadAndStore( const res = await fetch(sourceUrl, { signal: AbortSignal.timeout(10_000) }); if (!res.ok) return null; const buffer = Buffer.from(await res.arrayBuffer()); - const ext = guessExtension(res.headers.get('content-type') ?? '', sourceUrl); - const id = randomUUID(); - const filename = `${id}${ext}`; - const localPath = path.join(MEDIA_DIR, filename); - fs.writeFileSync(localPath, buffer); - - db.prepare( - `INSERT INTO media_assets (id, source_url, local_path, size_bytes, downloaded_at, tier, content_item_id, article_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ).run(id, sourceUrl, localPath, buffer.length, new Date().toISOString(), tier, refs.contentItemId ?? null, refs.articleId ?? null); - - return { id, localPath, servedPath: `/media/${filename}`, sizeBytes: buffer.length }; + return storeMediaBuffer(buffer, ext, sourceUrl, tier, refs); } catch (err) { logger.error('media', `Failed to download ${sourceUrl}: ${(err as Error).message}`); return null; } } +/** + * Writes an already-downloaded buffer to disk and records it in media_assets — the + * common tail of downloadAndStore, extracted so the Telegram adapter (which downloads + * media itself via the authenticated GramJS session rather than a plain fetch()) can + * reuse the same storage/bookkeeping logic. + */ +export function storeMediaBuffer( + buffer: Buffer, + ext: string, + sourceUrl: string, + tier: 'candidate' | 'published', + refs: { contentItemId?: string; articleId?: string } +): StoredMedia { + const id = randomUUID(); + const filename = `${id}${ext}`; + const localPath = path.join(MEDIA_DIR, filename); + fs.writeFileSync(localPath, buffer); + + db.prepare( + `INSERT INTO media_assets (id, source_url, local_path, size_bytes, downloaded_at, tier, content_item_id, article_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run(id, sourceUrl, localPath, buffer.length, new Date().toISOString(), tier, refs.contentItemId ?? null, refs.articleId ?? null); + + return { id, localPath, servedPath: `/media/${filename}`, sizeBytes: buffer.length }; +} + +/** Finds already-downloaded media for a content item (Telegram's adapter self-hosts at ingestion time) so publishDirect can promote them without threading media-asset IDs through the JSON blob. */ +export function mediaIdsForContentItem(contentItemId: string): string[] { + const rows = db.prepare('SELECT id FROM media_assets WHERE content_item_id = ?').all(contentItemId) as { id: string }[]; + return rows.map((r) => r.id); +} + +/** + * Links media rows downloaded before their content item existed (Telegram's adapter — + * see FetchedItem.telegramMediaAssetIds) to the content item id assigned once + * insertContentItem runs. Called by poller.ts right after insertion. + */ +export function attachToContentItem(mediaIds: string[], contentItemId: string): void { + const stmt = db.prepare('UPDATE media_assets SET content_item_id = ? WHERE id = ?'); + for (const id of mediaIds) stmt.run(contentItemId, id); +} + /** Promotes a candidate-tier asset to published-tier so raw-item retention can no longer prune it. */ export function promoteToPublished(mediaId: string, articleId: string) { db.prepare("UPDATE media_assets SET tier = 'published', article_id = ? WHERE id = ?").run(articleId, mediaId); diff --git a/backend/src/telegram/client.ts b/backend/src/telegram/client.ts new file mode 100644 index 0000000..ac56a1f --- /dev/null +++ b/backend/src/telegram/client.ts @@ -0,0 +1,155 @@ +// Singleton GramJS client + login state machine. Unlike every other adapter (rss/ +// nitter/youtube — all stateless per-call HTTP fetches), Telegram needs one long-lived +// authenticated MTProto connection reused across every poll — this module owns that +// connection and the interactive phone/code/2FA login flow that produces it, consumed +// by three call sites: the adapter (getClient/fetchChannelMessages), the admin login +// routes (startLogin/verifyCode/verifyPassword/logout/getStatus), and startup +// (initFromSavedSession). + +import { TelegramClient, Api } from 'telegram'; +import { StringSession } from 'telegram/sessions/index.js'; +import { computeCheck } from 'telegram/Password.js'; +import * as credentials from './credentials.js'; +import { logger } from '../storage/db/logs.js'; + +interface PendingLogin { + client: TelegramClient; + phoneNumber: string; + phoneCodeHash: string; + phase: 'code-sent' | 'password-needed'; +} + +let client: TelegramClient | null = null; +let connectedPhone: string | null = null; +let pendingLogin: PendingLogin | null = null; + +export function telegramConfigured(): boolean { + return credentials.getApiCredentials() !== null; +} + +function newClient(sessionString = ''): TelegramClient { + const creds = credentials.getApiCredentials(); + if (!creds) throw new Error('Telegram API credentials have not been saved yet'); + return new TelegramClient(new StringSession(sessionString), creds.apiId, creds.apiHash, { + connectionRetries: 5 + }); +} + +/** Called from index.ts's main() right after migrate() — reconnects a saved session, if any. */ +export async function initFromSavedSession(): Promise { + const session = credentials.getSession(); + if (!session) return; + + try { + const c = newClient(session.sessionString); + await c.connect(); + client = c; + connectedPhone = session.phone; + logger.info('telegram', `Reconnected as ${session.phone}`); + } catch (err) { + // A network blip or transient failure shouldn't force a re-login — only an + // explicit logout() clears the saved session. Leaving client null here means the + // adapter just fails soft (same as before this feature existed) until the next + // restart or a manual reconnect. + client = null; + connectedPhone = null; + logger.error('telegram', `Failed to reconnect saved session: ${(err as Error).message}`); + } +} + +export function getClient(): TelegramClient | null { + return client; +} + +export function getStatus(): { credentialsConfigured: boolean; connected: boolean; phone: string | null } { + return { + credentialsConfigured: telegramConfigured(), + connected: client !== null, + phone: connectedPhone + }; +} + +export function saveApiCredentials(apiId: number, apiHash: string): void { + credentials.saveApiCredentials(apiId, apiHash); +} + +export async function startLogin(phoneNumber: string): Promise { + if (!telegramConfigured()) throw new Error('Save Telegram API ID/hash before logging in'); + + const creds = credentials.getApiCredentials()!; + const loginClient = newClient(); + await loginClient.connect(); + + const sent = await loginClient.sendCode({ apiId: creds.apiId, apiHash: creds.apiHash }, phoneNumber); + pendingLogin = { client: loginClient, phoneNumber, phoneCodeHash: sent.phoneCodeHash, phase: 'code-sent' }; +} + +/** Returns 'connected' or 'password-needed' depending on whether this account has 2FA enabled. */ +export async function verifyCode(code: string): Promise<'connected' | 'password-needed'> { + if (!pendingLogin || pendingLogin.phase !== 'code-sent') throw new Error('No login in progress — start again'); + + try { + await pendingLogin.client.invoke( + new Api.auth.SignIn({ + phoneNumber: pendingLogin.phoneNumber, + phoneCodeHash: pendingLogin.phoneCodeHash, + phoneCode: code + }) + ); + await completeLogin(pendingLogin.client, pendingLogin.phoneNumber); + return 'connected'; + } catch (err) { + if ((err as { errorMessage?: string }).errorMessage === 'SESSION_PASSWORD_NEEDED') { + pendingLogin.phase = 'password-needed'; + return 'password-needed'; + } + pendingLogin = null; + throw err; + } +} + +export async function verifyPassword(password: string): Promise { + if (!pendingLogin || pendingLogin.phase !== 'password-needed') throw new Error('No password step in progress — start again'); + + try { + const passwordInfo = await pendingLogin.client.invoke(new Api.account.GetPassword()); + const passwordSrpCheck = await computeCheck(passwordInfo, password); + await pendingLogin.client.invoke(new Api.auth.CheckPassword({ password: passwordSrpCheck })); + await completeLogin(pendingLogin.client, pendingLogin.phoneNumber); + } catch (err) { + pendingLogin = null; + throw err; + } +} + +async function completeLogin(loginClient: TelegramClient, phoneNumber: string): Promise { + const sessionString = loginClient.session.save() as unknown as string; + credentials.saveSession(sessionString, phoneNumber); + client = loginClient; + connectedPhone = phoneNumber; + pendingLogin = null; + logger.info('telegram', `Logged in as ${phoneNumber}`); +} + +export async function logout(): Promise { + if (client) { + try { + await client.invoke(new Api.auth.LogOut()); + } catch (err) { + logger.error('telegram', `Logout call failed (clearing local session anyway): ${(err as Error).message}`); + } + await client.disconnect(); + } + client = null; + connectedPhone = null; + pendingLogin = null; + credentials.clearSession(); +} + +/** Used by the adapter — resolves a channel and fetches its most recent messages. */ +export async function fetchChannelMessages(channelIdentifier: string, limit: number) { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelIdentifier); + const messages = await client.getMessages(entity, { limit }); + return { entity, messages }; +} diff --git a/backend/src/telegram/credentials.ts b/backend/src/telegram/credentials.ts new file mode 100644 index 0000000..577e6c2 --- /dev/null +++ b/backend/src/telegram/credentials.ts @@ -0,0 +1,48 @@ +// The only module that touches storage/crypto.ts for Telegram data — everything else +// (telegram/client.ts, the admin routes) goes through these functions and never sees +// ciphertext or handles encryption itself. + +import { db } from '../storage/db/index.js'; +import { encrypt, decrypt } from '../storage/crypto.js'; + +interface TelegramCredentialsRow { + api_id_enc: string | null; + api_hash_enc: string | null; + session_enc: string | null; + phone_enc: string | null; +} + +function getRow(): TelegramCredentialsRow { + return db.prepare('SELECT * FROM telegram_credentials WHERE id = 1').get() as unknown as TelegramCredentialsRow; +} + +export function saveApiCredentials(apiId: number, apiHash: string): void { + db.prepare('UPDATE telegram_credentials SET api_id_enc = ?, api_hash_enc = ? WHERE id = 1').run( + encrypt(String(apiId)), + encrypt(apiHash) + ); +} + +export function getApiCredentials(): { apiId: number; apiHash: string } | null { + const row = getRow(); + if (!row.api_id_enc || !row.api_hash_enc) return null; + return { apiId: Number(decrypt(row.api_id_enc)), apiHash: decrypt(row.api_hash_enc) }; +} + +export function saveSession(sessionString: string, phone: string): void { + db.prepare('UPDATE telegram_credentials SET session_enc = ?, phone_enc = ? WHERE id = 1').run( + encrypt(sessionString), + encrypt(phone) + ); +} + +export function getSession(): { sessionString: string; phone: string } | null { + const row = getRow(); + if (!row.session_enc || !row.phone_enc) return null; + return { sessionString: decrypt(row.session_enc), phone: decrypt(row.phone_enc) }; +} + +/** Logout — clears the session/phone only, keeps API ID/hash so re-login doesn't need them re-entered. */ +export function clearSession(): void { + db.prepare('UPDATE telegram_credentials SET session_enc = NULL, phone_enc = NULL WHERE id = 1').run(); +} diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 5ee90b4..5a2fd2a 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -6,6 +6,7 @@ import type { AdminTrackedEvent, ModelCatalog, AiStatus, + TelegramStatus, LogEntry } from './adminTypes'; @@ -120,6 +121,43 @@ export const getModels = (fetchFn?: typeof fetch) => export const getAiStatus = (fetchFn?: typeof fetch) => request('/api/admin/ai-status', {}, fetchFn); +// Telegram account (Connections tab) — API ID/hash and the resulting login session are +// stored encrypted at rest server-side (see backend telegram/credentials.ts); none of +// these ever come back from the server, only status flags. +export const getTelegramStatus = (fetchFn?: typeof fetch) => + request('/api/admin/telegram/status', {}, fetchFn); + +export const saveTelegramCredentials = (apiId: number, apiHash: string, fetchFn?: typeof fetch) => + request<{ credentialsConfigured: boolean }>( + '/api/admin/telegram/credentials', + { method: 'POST', body: JSON.stringify({ apiId, apiHash }) }, + fetchFn + ); + +export const startTelegramLogin = (phoneNumber: string, fetchFn?: typeof fetch) => + request<{ phase: 'code-sent' }>( + '/api/admin/telegram/login/start', + { method: 'POST', body: JSON.stringify({ phoneNumber }) }, + fetchFn + ); + +export const verifyTelegramCode = (code: string, fetchFn?: typeof fetch) => + request<{ phase: 'connected' | 'password-needed'; connected?: boolean; phone?: string | null }>( + '/api/admin/telegram/login/verify', + { method: 'POST', body: JSON.stringify({ code }) }, + fetchFn + ); + +export const verifyTelegramPassword = (password: string, fetchFn?: typeof fetch) => + request<{ phase: 'connected'; connected?: boolean; phone?: string | null }>( + '/api/admin/telegram/login/verify', + { method: 'POST', body: JSON.stringify({ password }) }, + fetchFn + ); + +export const telegramLogout = (fetchFn?: typeof fetch) => + request('/api/admin/telegram/logout', { method: 'POST' }, fetchFn); + // Logs export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: number } = {}, fetchFn?: typeof fetch) => { const qs = new URLSearchParams(filters as Record).toString(); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index f5357e2..6a2f7ef 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -71,6 +71,12 @@ export interface AiStatus { gpu: string; } +export interface TelegramStatus { + credentialsConfigured: boolean; + connected: boolean; + phone: string | null; +} + export interface LogEntry { id: number; timestamp: string; diff --git a/frontend/src/lib/components/ArticleListRow.svelte b/frontend/src/lib/components/ArticleListRow.svelte index d32248b..07b3d69 100644 --- a/frontend/src/lib/components/ArticleListRow.svelte +++ b/frontend/src/lib/components/ArticleListRow.svelte @@ -3,6 +3,7 @@ import { timeAgo, exactTime, excerpt } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; import TweetCard from './TweetCard.svelte'; + import TelegramCard from './TelegramCard.svelte'; let { article }: { article: MergedArticle } = $props(); @@ -15,6 +16,8 @@ {#if article.tweet} +{:else if article.telegramMessage} + {:else} {#if article.heroImage} diff --git a/frontend/src/lib/components/TelegramCard.svelte b/frontend/src/lib/components/TelegramCard.svelte new file mode 100644 index 0000000..4f040ec --- /dev/null +++ b/frontend/src/lib/components/TelegramCard.svelte @@ -0,0 +1,186 @@ + + + +
+ {article.category[0] ?? ''} + · + {sourceLabel} + · + ✈ Telegram +
+
+ {#if article.telegramMessage?.channelAvatarUrl} + + {:else} +
+ {/if} + {article.telegramMessage?.channelName} + @{article.telegramMessage?.channelUsername} +
+
{article.body}
+ {#if media.length > 0} +
+ {#each media as item, i (item.url)} +
+ {#if item.type === 'video' || item.type === 'gif'} + + {:else} + + {/if} +
+ {/each} +
+ {/if} +
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
+ + + diff --git a/frontend/src/lib/components/admin/ConnectionsTab.svelte b/frontend/src/lib/components/admin/ConnectionsTab.svelte index d7132e7..c6c2676 100644 --- a/frontend/src/lib/components/admin/ConnectionsTab.svelte +++ b/frontend/src/lib/components/admin/ConnectionsTab.svelte @@ -1,10 +1,22 @@
@@ -82,6 +195,63 @@
+
+
+ Telegram account + +
+

+ Reading channels requires logging into a real Telegram account (not a bot), so it can see + any public channel that account can see. Get an API ID/hash from + my.telegram.org — a + dedicated/secondary account is recommended over your primary one. Both are stored encrypted, + never sent back to this browser. +

+
+ + + +
+ {#if credentialsConfigured} +
✓ API credentials saved
+ {/if} + + +
+ diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index b6b73af..5e8c709 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -202,6 +202,8 @@ {:else if form.type === 'nitter'} + {:else if form.type === 'telegram'} + {:else} {/if} @@ -211,6 +213,9 @@ + {#if form.type === 'telegram'} + Public channels only for now — private channels without a public username aren't supported yet. + {/if}
Categories
{#each assignableCategories as cat (cat.id)} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 08ec8ab..ece038a 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -17,6 +17,9 @@ export interface TweetMediaItem { height: number | null; } +/** Same shape as TweetMediaItem — distinct name for readability at Telegram call sites. */ +export type TelegramMediaItem = TweetMediaItem; + export interface MergedArticle { id: string; title: string; @@ -24,6 +27,13 @@ export interface MergedArticle { heroImage: { url: string; sourceItemId: string; selectionReason: string } | null; video: { url: string; provider?: string; embedUrl?: string; sourceItemId: string } | null; tweet: { authorName: string; authorHandle: string; avatarUrl: string | null; sourceItemId: string; media: TweetMediaItem[] } | null; + telegramMessage: { + channelName: string; + channelUsername: string; + channelAvatarUrl: string | null; + sourceItemId: string; + media: TelegramMediaItem[]; + } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/routes/admin/settings/+page.svelte b/frontend/src/routes/admin/settings/+page.svelte index 85b35de..c6506c7 100644 --- a/frontend/src/routes/admin/settings/+page.svelte +++ b/frontend/src/routes/admin/settings/+page.svelte @@ -46,7 +46,7 @@ {:else if active === 'events'} {:else if active === 'connections'} - + {:else if active === 'logs'} {/if} diff --git a/frontend/src/routes/admin/settings/+page.ts b/frontend/src/routes/admin/settings/+page.ts index d5a9f60..4fb73f5 100644 --- a/frontend/src/routes/admin/settings/+page.ts +++ b/frontend/src/routes/admin/settings/+page.ts @@ -1,7 +1,7 @@ import { redirect } from '@sveltejs/kit'; import type { PageLoad } from './$types'; -import { getSettings, getSources, getEvents, getModels, getAiStatus, getLogs } from '$lib/adminApi'; -import type { ModelCatalog, AiStatus } from '$lib/adminTypes'; +import { getSettings, getSources, getEvents, getModels, getAiStatus, getTelegramStatus, getLogs } from '$lib/adminApi'; +import type { ModelCatalog, AiStatus, TelegramStatus } from '$lib/adminTypes'; const EMPTY_MODELS: ModelCatalog = { embedding: [], image: [], synthesis: [] }; @@ -24,7 +24,11 @@ export const load: PageLoad = async ({ fetch }) => { // swallow — surfaced instead via aiStatus.connected in the UI } - return { settings, sources, events, models, aiStatus, logs }; + const telegramStatus: TelegramStatus = await getTelegramStatus(fetch).catch( + () => ({ credentialsConfigured: false, connected: false, phone: null }) + ); + + return { settings, sources, events, models, aiStatus, telegramStatus, logs }; } catch (err) { if ((err as { status?: number }).status === 401) { throw redirect(302, '/admin/login?redirectTo=/admin/settings'); diff --git a/frontend/src/routes/article/[id]/+page.svelte b/frontend/src/routes/article/[id]/+page.svelte index 418259d..c9308a3 100644 --- a/frontend/src/routes/article/[id]/+page.svelte +++ b/frontend/src/routes/article/[id]/+page.svelte @@ -3,6 +3,7 @@ import { timeAgo, exactTime } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; import TweetCard from '$lib/components/TweetCard.svelte'; + import TelegramCard from '$lib/components/TelegramCard.svelte'; let { data }: { data: PageData } = $props(); const a = $derived(data.article); @@ -36,6 +37,18 @@ {#if a.sources[0]} View original tweet → {/if} + {:else if a.telegramMessage} + +
+ Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} +
+ + + + {#if a.sources[0]} + View original message on Telegram → + {/if} {:else if a.video?.provider === 'youtube'}

{a.title}