diff --git a/backend/package-lock.json b/backend/package-lock.json index 5538108..3bff8b6 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -14,7 +14,8 @@ "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", "jsdom": "^29.1.1", - "rss-parser": "^3.13.0" + "rss-parser": "^3.13.0", + "telegram": "^2.26.22" }, "devDependencies": { "@types/jsdom": "^28.0.3", @@ -82,6 +83,12 @@ "specificity": "bin/cli.js" } }, + "node_modules/@cryptography/aes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@cryptography/aes/-/aes-0.1.1.tgz", + "integrity": "sha512-PcYz4FDGblO6tM2kSC+VzhhK62vml6k6/YAkiWtyPvrgJVfnDRoHGDtKn5UiaRRUrvUTTocBpvc2rRgTCqxjsg==", + "license": "GPL-3.0-or-later" + }, "node_modules/@csstools/color-helpers": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", @@ -1257,6 +1264,15 @@ } } }, + "node_modules/async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.3.1" + } + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -1286,6 +1302,26 @@ "fastq": "^1.17.1" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -1295,6 +1331,52 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/bufferutil": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.1.0.tgz", + "integrity": "sha512-ZMANVnAixE6AWWnPzlW2KpUrxhm9woycYvPOo67jWHyFowASTEd9s+QN1EIMsSDtwhIxN4sWE1jotpuDUIgyIw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/cookie": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz", @@ -1321,6 +1403,19 @@ "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" } }, + "node_modules/d": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", + "integrity": "sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==", + "license": "ISC", + "dependencies": { + "es5-ext": "^0.10.64", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -1334,6 +1429,15 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, "node_modules/decimal.js": { "version": "10.6.0", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", @@ -1349,6 +1453,61 @@ "node": ">=6" } }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/entities": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", @@ -1358,6 +1517,46 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es5-ext": { + "version": "0.10.64", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", + "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", + "hasInstallScript": true, + "license": "ISC", + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "esniff": "^2.0.1", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.4.tgz", + "integrity": "sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.2", + "ext": "^1.7.0" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -1400,6 +1599,40 @@ "@esbuild/win32-x64": "0.28.1" } }, + "node_modules/esniff": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", + "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "license": "ISC", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.62", + "event-emitter": "^0.3.5", + "type": "^2.7.2" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "license": "MIT", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "license": "ISC", + "dependencies": { + "type": "^2.7.2" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -1564,6 +1797,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/html-encoding-sniffer": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", @@ -1576,6 +1815,63 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -1591,6 +1887,12 @@ "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", "license": "MIT" }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", + "license": "MIT" + }, "node_modules/jsdom": { "version": "29.1.1", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", @@ -1721,6 +2023,53 @@ "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", "license": "CC0-1.0" }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "license": "ISC" + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/node-localstorage": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-localstorage/-/node-localstorage-2.2.1.tgz", + "integrity": "sha512-vv8fJuOUCCvSPjDjBLlMqYMHob4aGjkmrkaE42/mZr0VT+ZAU10jRF8oTnX9+pgU9/vYJ8P7YT3Vd6ajkmzSCw==", + "license": "MIT", + "dependencies": { + "write-file-atomic": "^1.1.4" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1730,6 +2079,22 @@ "node": ">=14.0.0" } }, + "node_modules/pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "(MIT AND Zlib)" + }, "node_modules/parse5": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", @@ -1754,6 +2119,12 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/pino": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", @@ -1822,6 +2193,12 @@ "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", "license": "MIT" }, + "node_modules/real-cancellable-promise": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/real-cancellable-promise/-/real-cancellable-promise-1.2.3.tgz", + "integrity": "sha512-hBI5Gy/55VEeeMtImMgEirD7eq5UmqJf1J8dFZtbJZA/3rB0pYFZ7PayMGueb6v4UtUtpKpP+05L0VwyE1hI9Q==", + "license": "MIT" + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -1961,6 +2338,39 @@ "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", "license": "MIT" }, + "node_modules/slide": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", + "integrity": "sha512-NwrtjCg+lZoqhFU8fOwl4ay2ei8PaqCBOUV3/ektPY9trO1yQ1oXEfmHAhKArUVUr/hOHvy5f6AdP17dCM0zMw==", + "license": "ISC", + "engines": { + "node": "*" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/sonic-boom": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", @@ -1988,12 +2398,45 @@ "node": ">= 10.x" } }, + "node_modules/store2": { + "version": "2.14.4", + "resolved": "https://registry.npmjs.org/store2/-/store2-2.14.4.tgz", + "integrity": "sha512-srTItn1GOvyvOycgxjAnPA63FZNwy0PTyUBFMHRM+hVFltAeoh0LmNBz9SZqUS9mMqGk8rfyWyXn3GH5ReJ8Zw==", + "license": "MIT" + }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "license": "MIT" }, + "node_modules/telegram": { + "version": "2.26.22", + "resolved": "https://registry.npmjs.org/telegram/-/telegram-2.26.22.tgz", + "integrity": "sha512-EIj7Yrjiu0Yosa3FZ/7EyPg9s6UiTi/zDQrFmR/2Mg7pIUU+XjAit1n1u9OU9h2oRnRM5M+67/fxzQluZpaJJg==", + "deprecated": "This package is archived and no longer maintained. Development continues in teleproto (https://npmjs.com/package/teleproto), a largely compatible, actively maintained fork. See the migration guide at https://docs.teleproto.dev/migrating-from-gramjs", + "license": "MIT", + "dependencies": { + "@cryptography/aes": "^0.1.1", + "async-mutex": "^0.3.0", + "big-integer": "^1.6.48", + "buffer": "^6.0.3", + "htmlparser2": "^6.1.0", + "mime": "^3.0.0", + "node-localstorage": "^2.2.1", + "pako": "^2.0.3", + "path-browserify": "^1.0.1", + "real-cancellable-promise": "^1.1.1", + "socks": "^2.6.2", + "store2": "^2.13.0", + "ts-custom-error": "^3.2.0", + "websocket": "^1.0.34" + }, + "optionalDependencies": { + "bufferutil": "^4.0.3", + "utf-8-validate": "^5.0.5" + } + }, "node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", @@ -2063,6 +2506,21 @@ "node": ">=20" } }, + "node_modules/ts-custom-error": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.3.1.tgz", + "integrity": "sha512-5OX1tzOjxWEgsr/YEUWSuPrQ00deKLh6D7OTWcvNHm12/7QPyRh8SYpyWvA4IZv8H/+GQWQEh/kwo95Q9OVW1A==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/tsx": { "version": "4.23.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.0.tgz", @@ -2082,6 +2540,21 @@ "fsevents": "~2.3.3" } }, + "node_modules/type": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.3.tgz", + "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==", + "license": "ISC" + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "license": "MIT", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", @@ -2133,6 +2606,19 @@ "dev": true, "license": "MIT" }, + "node_modules/utf-8-validate": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", + "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", @@ -2154,6 +2640,23 @@ "node": ">=20" } }, + "node_modules/websocket": { + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.35.tgz", + "integrity": "sha512-/REy6amwPZl44DDzvRCkaI1q1bIiQB0mEFQLUrhz3z2EK91cp3n72rAjUlrTP0zV22HJIUOVHQGPxhFRjxjt+Q==", + "license": "Apache-2.0", + "dependencies": { + "bufferutil": "^4.0.1", + "debug": "^2.2.0", + "es5-ext": "^0.10.63", + "typedarray-to-buffer": "^3.1.5", + "utf-8-validate": "^5.0.2", + "yaeti": "^0.0.6" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/whatwg-mimetype": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", @@ -2177,6 +2680,17 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/write-file-atomic": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-1.3.4.tgz", + "integrity": "sha512-SdrHoC/yVBPpV0Xq/mUZQIpW2sWXAShb/V4pomcJXh92RuaO+f3UTWItiR3Px+pLnV2PvC2/bfn5cwr5X6Vfxw==", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "slide": "^1.1.5" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -2213,6 +2727,16 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "license": "MIT" + }, + "node_modules/yaeti": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", + "integrity": "sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "engines": { + "node": ">=0.10.32" + } } } } diff --git a/backend/package.json b/backend/package.json index b5d6c27..9eca3ab 100644 --- a/backend/package.json +++ b/backend/package.json @@ -17,7 +17,8 @@ "@mozilla/readability": "^0.6.0", "fastify": "^5.10.0", "jsdom": "^29.1.1", - "rss-parser": "^3.13.0" + "rss-parser": "^3.13.0", + "telegram": "^2.26.22" }, "devDependencies": { "@types/jsdom": "^28.0.3", diff --git a/backend/src/api/admin.ts b/backend/src/api/admin.ts index 43c6dcd..4bd71b8 100644 --- a/backend/src/api/admin.ts +++ b/backend/src/api/admin.ts @@ -7,6 +7,7 @@ import { clearSourceContent, reissueSourceContent, clearAllArticles, clearAllMed import { OllamaProvider } from '../inference/ollama-provider.js'; import { pollSourceNow } from '../ingestion/poller.js'; import { logger, listLogs } from '../storage/db/logs.js'; +import * as telegramClient from '../telegram/client.js'; export async function registerAdminRoutes(app: FastifyInstance) { // --- Settings --- @@ -148,6 +149,51 @@ export async function registerAdminRoutes(app: FastifyInstance) { return { connected, host: settings.aiServiceHost, port: settings.aiServicePort, ramGB: null, gpu: null }; }); + // --- Telegram account (Connections tab — see telegram/client.ts and credentials.ts. + // API ID/hash and the resulting login session are stored encrypted at rest; none of + // these routes ever echo them back to the client.) --- + app.get('/api/admin/telegram/status', async () => telegramClient.getStatus()); + + app.post('/api/admin/telegram/credentials', async (req, reply) => { + const { apiId, apiHash } = req.body as { apiId?: number; apiHash?: string }; + if (!apiId || !apiHash) return reply.code(400).send({ error: 'apiId and apiHash are required' }); + telegramClient.saveApiCredentials(apiId, apiHash); + return { credentialsConfigured: true }; + }); + + app.post('/api/admin/telegram/login/start', async (req, reply) => { + const { phoneNumber } = req.body as { phoneNumber?: string }; + if (!phoneNumber) return reply.code(400).send({ error: 'phoneNumber is required' }); + try { + await telegramClient.startLogin(phoneNumber); + return { phase: 'code-sent' }; + } catch (err) { + return reply.code(400).send({ error: (err as Error).message }); + } + }); + + app.post('/api/admin/telegram/login/verify', async (req, reply) => { + const { code, password } = req.body as { code?: string; password?: string }; + try { + if (password !== undefined) { + await telegramClient.verifyPassword(password); + return { phase: 'connected', ...telegramClient.getStatus() }; + } + if (code !== undefined) { + const phase = await telegramClient.verifyCode(code); + return phase === 'connected' ? { phase, ...telegramClient.getStatus() } : { phase }; + } + return reply.code(400).send({ error: 'code or password is required' }); + } catch (err) { + return reply.code(400).send({ error: (err as Error).message }); + } + }); + + app.post('/api/admin/telegram/logout', async (_req, reply) => { + await telegramClient.logout(); + return reply.code(200).send(telegramClient.getStatus()); + }); + // --- Logs --- app.get('/api/admin/logs', async (req) => { const { level, limit } = req.query as { level?: string; limit?: string }; diff --git a/backend/src/api/telegramMediaProxy.ts b/backend/src/api/telegramMediaProxy.ts new file mode 100644 index 0000000..2c55d51 --- /dev/null +++ b/backend/src/api/telegramMediaProxy.ts @@ -0,0 +1,81 @@ +// Backs the "proxy" Telegram media mode (see Retention tab / GlobalSettings.telegramMediaMode). +// Unlike media/proxy.ts (which forwards a plain HTTP request to an already-public CDN +// URL), Telegram media has no public URL at all — every request here re-authenticates +// to Telegram via the logged-in account (telegram/client.ts) and streams the result, so +// nothing is written to disk and only this server ever touches Telegram's servers. A +// small time-boxed in-memory cache absorbs repeat views of the same message/avatar +// without hitting Telegram (and its rate limits) on every single page load. + +import type { FastifyInstance } from 'fastify'; +import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js'; +import { logger } from '../storage/db/logs.js'; + +const CACHE_TTL_MS = 10 * 60_000; +const CACHE_MAX_ENTRIES = 200; + +interface CacheEntry { + buffer: Buffer; + contentType: string; + expiresAt: number; +} + +const cache = new Map(); + +function cacheGet(key: string): CacheEntry | null { + const entry = cache.get(key); + if (!entry) return null; + if (entry.expiresAt < Date.now()) { + cache.delete(key); + return null; + } + return entry; +} + +function cacheSet(key: string, entry: CacheEntry) { + if (cache.size >= CACHE_MAX_ENTRIES) { + const oldest = cache.keys().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + cache.set(key, entry); +} + +function contentTypeFor(type: string | undefined): string { + if (type === 'video' || type === 'gif') return 'video/mp4'; + return 'image/jpeg'; +} + +export async function registerTelegramMediaProxy(app: FastifyInstance) { + app.get('/media/telegram-proxy', async (req, reply) => { + const { channel, message, avatar, type } = req.query as { + channel?: string; + message?: string; + avatar?: string; + type?: string; + }; + if (!channel) return reply.code(400).send({ error: 'channel required' }); + if (!avatar && !message) return reply.code(400).send({ error: 'message or avatar required' }); + + const cacheKey = avatar ? `avatar:${channel}` : `message:${channel}:${message}`; + const cached = cacheGet(cacheKey); + if (cached) { + reply.header('content-type', cached.contentType); + reply.header('cache-control', 'private, max-age=300'); + return reply.send(cached.buffer); + } + + try { + const buffer = avatar ? await downloadChannelAvatar(channel) : await downloadMessageMedia(channel, message!); + if (!buffer) return reply.code(404).send(); + + const contentType = contentTypeFor(type); + cacheSet(cacheKey, { buffer, contentType, expiresAt: Date.now() + CACHE_TTL_MS }); + + reply.header('content-type', contentType); + reply.header('cache-control', 'private, max-age=300'); + return reply.send(buffer); + } catch (err) { + logger.error('telegram', `Proxy fetch failed for channel=${channel} message=${message ?? 'avatar'}: ${(err as Error).message}`); + return reply.code(502).send(); + } + }); +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 0e4c485..6fbe6d2 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,8 +9,10 @@ import { registerAuth } from './api/auth.js'; import { registerPublicRoutes } from './api/public.js'; import { registerAdminRoutes } from './api/admin.js'; import { registerMediaProxy } from './api/mediaProxy.js'; +import { registerTelegramMediaProxy } from './api/telegramMediaProxy.js'; import { registerPrivateAccess, privateAccessConfigured } from './api/privateAccess.js'; import { startScheduler } from './queue/scheduler.js'; +import { initFromSavedSession } from './telegram/client.js'; import { logger } from './storage/db/logs.js'; const PORT = Number(process.env.PORT) || 4000; @@ -32,6 +34,7 @@ function printApiKeyBanner() { async function main() { migrate(); printApiKeyBanner(); + await initFromSavedSession(); const app = Fastify({ logger: false }); @@ -89,10 +92,12 @@ async function main() { return reply.send(fs.createReadStream(filePath)); }); - // Static "/media/proxy" takes priority over the "/media/:filename" param route - // above regardless of registration order (find-my-way, Fastify's router, always - // prefers a static segment over a parametric one at the same depth). + // Static "/media/proxy" and "/media/telegram-proxy" take priority over the + // "/media/:filename" param route above regardless of registration order + // (find-my-way, Fastify's router, always prefers a static segment over a parametric + // one at the same depth). await registerMediaProxy(app); + await registerTelegramMediaProxy(app); app.get('/health', async () => ({ ok: true })); diff --git a/backend/src/ingestion/adapters/base.ts b/backend/src/ingestion/adapters/base.ts index 3002913..aca6602 100644 --- a/backend/src/ingestion/adapters/base.ts +++ b/backend/src/ingestion/adapters/base.ts @@ -1,4 +1,10 @@ -import type { Source, ContentItem, TweetMediaItem, QuotedTweet } from '../../storage/db/types.js'; +import type { + Source, + ContentItem, + TweetMediaItem, + QuotedTweet, + TelegramMediaRef +} from '../../storage/db/types.js'; import { cleanHtml, toSummary } from '../clean.js'; export interface FetchedItem { @@ -19,6 +25,15 @@ export interface FetchedItem { repostedByHandle: string | null; quotedTweet: QuotedTweet | null; }; + /** Set by the Telegram adapter only — carries the channel/message info through to ContentItem.telegramMessage. Media is unresolved refs; publish.ts resolves them per the admin's configured telegramMediaMode. */ + telegramMessage?: { + channelName: string; + channelUsername: string | null; + sourceChannelUsername: string; + messageId: string; + media: TelegramMediaRef[]; + repostedByHandle: string | null; + }; raw: unknown; } @@ -51,6 +66,7 @@ export function toContentItem(source: Source, item: FetchedItem): Omit a.className === 'DocumentAttributeVideo'); + return { width: attr?.w ?? null, height: attr?.h ?? null }; +} + +function photoDimensions(photo: { sizes?: { w?: number; h?: number }[] } | undefined) { + const largest = photo?.sizes?.reduce<{ w?: number; h?: number } | undefined>( + (best, size) => (!best || (size.w ?? 0) > (best.w ?? 0) ? size : best), + undefined + ); + return { width: largest?.w ?? null, height: largest?.h ?? null }; +} + +/** A reference to a single message's attached media, if any — no download, just what's needed to resolve it later. */ +function refForMessage(message: TgMessage): TelegramMediaRef | null { + if (message.video) { + const doc = message.video as unknown as { mimeType?: string; attributes?: { className?: string; w?: number; h?: number }[] }; + const { width, height } = videoDimensions(doc); + return { type: 'video', messageId: String(message.id), mimeType: doc.mimeType ?? null, width, height }; + } + if (message.gif) { + const doc = message.gif as unknown as { mimeType?: string; attributes?: { className?: string; w?: number; h?: number }[] }; + const { width, height } = videoDimensions(doc); + return { type: 'gif', messageId: String(message.id), mimeType: doc.mimeType ?? null, width, height }; + } + if (message.photo) { + const { width, height } = photoDimensions(message.photo as unknown as { sizes?: { w?: number; h?: number }[] }); + return { type: 'photo', messageId: String(message.id), mimeType: null, width, height }; + } + return null; +} + +interface ForwardOrigin { + name: string; + /** Null when the origin has no public handle (e.g. a private channel/user, or a sender who hid their identity) — the card then falls back to showing just the name, with no avatar. */ + username: string | null; +} /** - * TODO: real implementation. Telegram channels are read via the Bot API (grammY, - * per project-structure.md) using `getUpdates` or a channel-history fetch, keyed by - * `source.config.telegramChannelId`. Left as a stub — satisfies the SourceAdapter - * interface so the poller and admin panel can already list/configure Telegram sources - * (e.g. for the "Iran war" tracked event example) before this is filled in. + * Detects a forwarded message and resolves where it came from. GramJS's `message.forward` + * wraps the raw fwdFrom header using entities Telegram already sent alongside the same + * getMessages response (the whole point of that header is letting clients show forward + * attribution without a separate resolve call) — `.chat` is the origin channel/group, + * `.sender` the origin user. Falls back to fwdFrom.fromName for the rarer case where the + * origin has no resolvable identity (e.g. a user who hid their account from forwards). */ +function detectForward(message: TgMessage): ForwardOrigin | null { + const fwd = message.fwdFrom; + if (!fwd) return null; + + const chat = message.forward?.chat as { title?: string; username?: string } | undefined; + if (chat) return { name: chat.title ?? chat.username ?? 'Unknown', username: chat.username ?? null }; + + const sender = message.forward?.sender as { firstName?: string; lastName?: string; username?: string } | undefined; + if (sender) { + const name = [sender.firstName, sender.lastName].filter(Boolean).join(' ') || sender.username || 'Unknown'; + return { name, username: sender.username ?? null }; + } + + return { name: fwd.fromName ?? 'Unknown', username: null }; +} + +/** Groups consecutive messages sharing a non-null groupedId (Telegram's multi-photo/video "album" concept) into one entry each. */ +function groupMessages(messages: TgMessage[]): TgMessage[][] { + const order: string[] = []; + const groups = new Map(); + for (const message of messages) { + const key = message.groupedId ? message.groupedId.toString() : `single-${message.id}`; + if (!groups.has(key)) { + groups.set(key, []); + order.push(key); + } + groups.get(key)!.push(message); + } + return order.map((key) => groups.get(key)!); +} + export const telegramAdapter: SourceAdapter = { async fetch(source: Source): Promise { - logger.warn('telegram', `Adapter not yet implemented — skipping source "${source.name}"`); - return []; + const client = getClient(); + if (!client) { + logger.warn('telegram', `No connected Telegram account — skipping source "${source.name}"`); + return []; + } + if (!source.url) return []; + + const identifier = normalizeChannelIdentifier(source.url); + let entity: any; + let messages: TgMessage[]; + try { + const result = await fetchChannelMessages(identifier, FETCH_LIMIT); + entity = result.entity; + messages = result.messages as unknown as TgMessage[]; + } catch (err) { + logger.error('telegram', `Failed to resolve/fetch channel "${identifier}" for source "${source.name}": ${(err as Error).message}`); + return []; + } + + // Only channels with a public username get a permalink (https://t.me//) + // that means anything to a non-member — a private channel's t.me/c//... + // link would make "click to open on Telegram" mostly useless, so v1 restricts to + // public channels and fails soft otherwise (same style as youtube.ts's resolveFeedUrl). + // This is always the channel we polled — used for the permalink and, on a forward, + // as the "Forwarded by @X" attribution — never the identity actually displayed. + const pollingChannelUsername: string | undefined = entity?.username; + if (!pollingChannelUsername) { + logger.warn('telegram', `"${source.name}" has no public username — private channels aren't supported yet, skipping`); + return []; + } + const pollingChannelName: string = entity?.title ?? pollingChannelUsername; + + const items: FetchedItem[] = []; + + for (const group of groupMessages(messages)) { + const primary = group.find((m) => m.message) ?? group[0]; + const text = primary.message ?? ''; + const firstLine = text.split('\n')[0].trim(); + + const media: TelegramMediaRef[] = []; + for (const message of group) { + if (media.length >= MAX_TELEGRAM_MEDIA) break; + const ref = refForMessage(message); + if (ref) media.push(ref); + } + + if (!text && media.length === 0) continue; // nothing worth publishing (e.g. a service message) + + // A forward displays as if it were authored by the ORIGIN channel/user — same + // treatment as a retweet, where tweet.authorName is always the original tweet's + // author, never the retweeter — with a "Forwarded by @" line + // (repostedByHandle) taking the place of the retweeter's own attribution. + const origin = detectForward(primary); + const channelName = origin?.name ?? pollingChannelName; + const channelUsername = origin ? origin.username : pollingChannelUsername; + const repostedByHandle = origin ? pollingChannelUsername : null; + + items.push({ + title: firstLine || `Message from ${pollingChannelName}`, + summary: text.slice(0, 500), + body: text || null, + images: [], + videos: [], + link: `https://t.me/${pollingChannelUsername}/${group[0].id}`, + publishedAt: new Date(primary.date * 1000).toISOString(), + telegramMessage: { + channelName, + channelUsername, + sourceChannelUsername: pollingChannelUsername, + messageId: String(group[0].id), + media, + repostedByHandle + }, + raw: { messageIds: group.map((m) => m.id) } + }); + } + + return items; } }; diff --git a/backend/src/pipeline/publish.ts b/backend/src/pipeline/publish.ts index c7b1e45..a158ac7 100644 --- a/backend/src/pipeline/publish.ts +++ b/backend/src/pipeline/publish.ts @@ -3,12 +3,21 @@ import type { InferenceProvider } from '../inference/provider.js'; import type { Cluster } from './clustering.js'; import { synthesizeArticle } from './synthesis.js'; import { selectBestImage, faviconUrlFor } from './image-selection.js'; -import { downloadAndStore, promoteToPublished } from '../storage/media/index.js'; +import { downloadAndStore, promoteToPublished, storeMediaBuffer } from '../storage/media/index.js'; +import { downloadMessageMedia, downloadChannelAvatar } from '../telegram/client.js'; import { logger } from '../storage/db/logs.js'; import * as articles from '../storage/db/articles.js'; import * as tags from '../storage/db/tags.js'; import * as sources from '../storage/db/sources.js'; -import type { GlobalSettings, MergedArticle, ContentItem, TweetMediaItem, QuotedTweet } from '../storage/db/types.js'; +import type { + GlobalSettings, + MergedArticle, + ContentItem, + TweetMediaItem, + QuotedTweet, + TelegramMediaItem, + TelegramMediaRef +} from '../storage/db/types.js'; const FOLLOW_UP_LOOKBACK_DAYS = 3; @@ -129,6 +138,81 @@ async function resolveTweetMedia( return { media: resolved, storedMediaIds }; } +function guessTelegramExtension(mimeType: string | null, kind: 'photo' | 'video' | 'gif'): string { + if (mimeType?.includes('png')) return '.png'; + if (mimeType?.includes('webp')) return '.webp'; + if (mimeType?.includes('jpeg') || mimeType?.includes('jpg')) return '.jpg'; + if (mimeType?.includes('mp4')) return '.mp4'; + return kind === 'photo' ? '.jpg' : '.mp4'; +} + +/** + * Resolves a single Telegram media reference into a servable url per the admin's + * chosen telegramMediaMode (Retention tab). Unlike Nitter — which starts from an + * already-public CDN URL — Telegram media only exists behind the authenticated MTProto + * session, so there's no 'direct' hotlink option: 'self-host' downloads it now (via the + * live session) and stores it locally exactly like every other self-hosted image; + * 'proxy' doesn't touch Telegram at all here, it just builds a url the live + * telegram-proxy route resolves (via that same session) on each view. + */ +async function resolveTelegramMediaUrl( + channelUsername: string, + ref: TelegramMediaRef, + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ item: TelegramMediaItem; storedMediaId: string | null } | null> { + if (mode === 'proxy') { + const url = `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&message=${encodeURIComponent(ref.messageId)}&type=${ref.type}`; + return { item: { type: ref.type, url, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: null }; + } + + try { + const buffer = await downloadMessageMedia(channelUsername, ref.messageId); + if (!buffer) return null; + const ext = guessTelegramExtension(ref.mimeType, ref.type); + const stored = storeMediaBuffer(buffer, ext, `telegram-message:${channelUsername}:${ref.messageId}`, 'published', {}); + return { item: { type: ref.type, url: stored.servedPath, thumbnailUrl: null, width: ref.width, height: ref.height }, storedMediaId: stored.id }; + } catch (err) { + logger.error('telegram', `Failed to self-host media for message ${ref.messageId}: ${(err as Error).message}`); + return null; + } +} + +/** Resolves every attached photo/video/gif ref for one message — order preserved, failed items dropped rather than leaving a broken entry. */ +async function resolveTelegramMedia( + channelUsername: string, + refs: TelegramMediaRef[], + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ media: TelegramMediaItem[]; storedMediaIds: string[] }> { + const media: TelegramMediaItem[] = []; + const storedMediaIds: string[] = []; + for (const ref of refs) { + const resolved = await resolveTelegramMediaUrl(channelUsername, ref, mode); + if (resolved) { + media.push(resolved.item); + if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); + } + } + return { media, storedMediaIds }; +} + +async function resolveTelegramAvatarUrl( + channelUsername: string, + mode: GlobalSettings['telegramMediaMode'] +): Promise<{ url: string | null; storedMediaId: string | null }> { + if (mode === 'proxy') { + return { url: `/media/telegram-proxy?channel=${encodeURIComponent(channelUsername)}&avatar=1`, storedMediaId: null }; + } + try { + const buffer = await downloadChannelAvatar(channelUsername); + if (!buffer) return { url: null, storedMediaId: null }; + const stored = storeMediaBuffer(buffer, '.jpg', `telegram-avatar:${channelUsername}`, 'published', {}); + return { url: stored.servedPath, storedMediaId: stored.id }; + } catch (err) { + logger.error('telegram', `Failed to self-host avatar for "${channelUsername}": ${(err as Error).message}`); + return { url: null, storedMediaId: null }; + } +} + /** Resolves a quote-tweet's embedded image (if any) through the Nitter media mode, same as any other tweet media. */ async function resolveQuotedTweet( quoted: QuotedTweet, @@ -153,9 +237,10 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) const category = uniqueCategories([item]); const storedMediaIds: string[] = []; - // Tweets never get a "hero image" — TweetCard.svelte renders tweet.media directly. + // Tweets and Telegram messages never get a "hero image" — their own card components + // render tweet.media / telegramMessage.media directly. let heroImage: MergedArticle['heroImage'] = null; - if (!item.tweet) { + if (!item.tweet && !item.telegramMessage) { const resolved = await resolveHeroImage([item], item.link); heroImage = resolved.heroImage; if (resolved.storedMediaId) storedMediaIds.push(resolved.storedMediaId); @@ -194,12 +279,40 @@ export async function publishDirect(item: ContentItem, settings: GlobalSettings) }; } + let telegramMessage: MergedArticle['telegramMessage'] = null; + if (item.telegramMessage) { + const { channelUsername, sourceChannelUsername } = item.telegramMessage; + + // The avatar shown is whoever is DISPLAYED (the origin channel on a forward) — if + // that has no public handle, there's no avatar to fetch at all, regardless of mode. + const avatar = channelUsername + ? await resolveTelegramAvatarUrl(channelUsername, settings.telegramMediaMode) + : { url: null, storedMediaId: null }; + if (avatar.storedMediaId) storedMediaIds.push(avatar.storedMediaId); + + // Attached media always lives on the polled channel's own copy of the message + // (forward or not), so media resolution uses sourceChannelUsername, never the + // (possibly different, possibly null) displayed channelUsername. + const resolvedMedia = await resolveTelegramMedia(sourceChannelUsername, item.telegramMessage.media, settings.telegramMediaMode); + storedMediaIds.push(...resolvedMedia.storedMediaIds); + + telegramMessage = { + channelName: item.telegramMessage.channelName, + channelUsername, + channelAvatarUrl: avatar.url, + sourceItemId: item.id, + media: resolvedMedia.media, + repostedByHandle: item.telegramMessage.repostedByHandle + }; + } + const article = await articles.insertArticle({ title: item.title, body: item.body || item.summary, heroImage, video, tweet, + telegramMessage, category, geo: item.geo, eventId: item.eventId, @@ -302,6 +415,7 @@ export async function publishCluster( heroImage, video, tweet: null, // tweets never reach clustering — see priorityQueue.ts's direct-publish bypass + telegramMessage: null, // telegram messages never reach clustering either — same bypass category, geo, eventId: opts.eventId ?? items[0]?.eventId ?? null, diff --git a/backend/src/queue/priorityQueue.ts b/backend/src/queue/priorityQueue.ts index 8c6a0b3..00a0d15 100644 --- a/backend/src/queue/priorityQueue.ts +++ b/backend/src/queue/priorityQueue.ts @@ -77,11 +77,14 @@ export async function runSynthesisCycle(provider: InferenceProvider, settings: G const items = contentItemsDb.unclusteredItemsExcludingSources(eventSourceIds); if (items.length === 0) return 0; - // YouTube videos and Nitter tweets never get LLM-merged with anything else — each - // is always its own article, same shape whether the AI service is up or not. Route - // them straight to publishDirect, same as the no-AI passthrough path. + // YouTube videos, Nitter tweets, and Telegram messages never get LLM-merged with + // anything else — each is always its own article, same shape whether the AI service + // is up or not. Route them straight to publishDirect, same as the no-AI passthrough path. const directPublishSourceIds = new Set( - sourcesDb.listSources().filter((s) => s.type === 'youtube' || s.type === 'nitter').map((s) => s.id) + sourcesDb + .listSources() + .filter((s) => s.type === 'youtube' || s.type === 'nitter' || s.type === 'telegram') + .map((s) => s.id) ); const [directItems, mergeableItems] = partition(items, (item) => directPublishSourceIds.has(item.sourceId)); diff --git a/backend/src/storage/crypto.ts b/backend/src/storage/crypto.ts new file mode 100644 index 0000000..37a1fd3 --- /dev/null +++ b/backend/src/storage/crypto.ts @@ -0,0 +1,54 @@ +// Generic AES-256-GCM encrypt/decrypt for at-rest secrets (currently just Telegram +// credentials — see telegram/credentials.ts — but deliberately not Telegram-specific +// here, so any future sensitive field can reuse it). The key is generated once into a +// file next to the DB but not inside it, so a naive copy of just the .db file reveals +// nothing — this is "not easily retrieved," not "safe against a fully compromised +// host": anyone with full filesystem access has both files anyway, same trust model as +// every other secret in this app. The key file must be included in backups, or +// previously-saved Telegram credentials become undecryptable (fixed by logging in +// again — no other data loss). + +import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +const DB_PATH = process.env.DB_PATH || './data/homefeed.db'; +const KEY_PATH = path.join(path.dirname(DB_PATH), '.encryption-key'); + +const ALGORITHM = 'aes-256-gcm'; +const IV_LENGTH = 12; + +let cachedKey: Buffer | null = null; + +function getKey(): Buffer { + if (cachedKey) return cachedKey; + + if (fs.existsSync(KEY_PATH)) { + cachedKey = fs.readFileSync(KEY_PATH); + return cachedKey; + } + + const key = randomBytes(32); + fs.mkdirSync(path.dirname(KEY_PATH), { recursive: true }); + fs.writeFileSync(KEY_PATH, key, { mode: 0o600 }); + cachedKey = key; + return cachedKey; +} + +export function encrypt(plaintext: string): string { + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv(ALGORITHM, getKey(), iv); + const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); + const authTag = cipher.getAuthTag(); + return Buffer.concat([iv, authTag, encrypted]).toString('base64'); +} + +export function decrypt(ciphertext: string): string { + const raw = Buffer.from(ciphertext, 'base64'); + const iv = raw.subarray(0, IV_LENGTH); + const authTag = raw.subarray(IV_LENGTH, IV_LENGTH + 16); + const encrypted = raw.subarray(IV_LENGTH + 16); + const decipher = createDecipheriv(ALGORITHM, getKey(), iv); + decipher.setAuthTag(authTag); + return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8'); +} diff --git a/backend/src/storage/db/articles.ts b/backend/src/storage/db/articles.ts index 8c08ff3..62bbffe 100644 --- a/backend/src/storage/db/articles.ts +++ b/backend/src/storage/db/articles.ts @@ -23,7 +23,8 @@ function rowToArticle(row: any): MergedArticle { previousArticleId: row.previous_article_id, nextArticleId: row.next_article_id, topStories: !!row.top_stories, - tweet: row.tweet ? JSON.parse(row.tweet) : null + tweet: row.tweet ? JSON.parse(row.tweet) : null, + telegramMessage: row.telegram_message ? JSON.parse(row.telegram_message) : null }; } @@ -31,8 +32,8 @@ export function insertArticle(article: Omit): MergedArticle const id = `art-${randomUUID()}`; db.prepare( `INSERT INTO merged_articles - (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, title, body, hero_image, video, category, geo, event_id, source_count, sources, published_at, updated_at, merge_confidence, tags, thread_id, previous_article_id, next_article_id, top_stories, tweet, telegram_message) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, article.title, @@ -52,7 +53,8 @@ export function insertArticle(article: Omit): MergedArticle article.previousArticleId, article.nextArticleId, article.topStories ? 1 : 0, - article.tweet ? JSON.stringify(article.tweet) : null + article.tweet ? JSON.stringify(article.tweet) : null, + article.telegramMessage ? JSON.stringify(article.telegramMessage) : null ); if (article.previousArticleId) { db.prepare('UPDATE merged_articles SET next_article_id = ? WHERE id = ?').run(id, article.previousArticleId); diff --git a/backend/src/storage/db/contentItems.ts b/backend/src/storage/db/contentItems.ts index 471afdd..057df87 100644 --- a/backend/src/storage/db/contentItems.ts +++ b/backend/src/storage/db/contentItems.ts @@ -21,6 +21,7 @@ function rowToItem(row: any): ContentItem { eventId: row.event_id, clusterId: row.cluster_id, tweet: row.tweet ? JSON.parse(row.tweet) : null, + telegramMessage: row.telegram_message ? JSON.parse(row.telegram_message) : null, raw: row.raw ? JSON.parse(row.raw) : null }; } @@ -29,8 +30,8 @@ export function insertContentItem(item: Omit): ContentItem { const id = `ci-${randomUUID()}`; db.prepare( `INSERT INTO content_items - (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, raw) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, source_id, type, title, summary, body, images, videos, link, published_at, fetched_at, tags, geo, embedding, event_id, cluster_id, tweet, telegram_message, raw) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( id, item.sourceId, @@ -49,6 +50,7 @@ export function insertContentItem(item: Omit): ContentItem { item.eventId, item.clusterId, item.tweet ? JSON.stringify(item.tweet) : null, + item.telegramMessage ? JSON.stringify(item.telegramMessage) : null, item.raw ? JSON.stringify(item.raw) : null ); return { ...item, id }; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index 220dcaa..c6c25d6 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -59,6 +59,7 @@ export function migrate() { event_id TEXT, cluster_id TEXT, -- set once assigned to a cluster awaiting synthesis tweet TEXT, -- JSON {id, authorName, authorHandle, avatarUrl}, nitter-sourced items only + telegram_message TEXT, -- JSON {channelName, channelUsername, channelAvatarUrl, messageId, media}, telegram-sourced items only raw TEXT -- JSON, original payload ); CREATE INDEX IF NOT EXISTS idx_content_items_source ON content_items(source_id); @@ -84,7 +85,8 @@ export function migrate() { previous_article_id TEXT, next_article_id TEXT, top_stories INTEGER NOT NULL DEFAULT 0, -- true if any contributing source opted into "Push to Top Stories?" - tweet TEXT -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only + tweet TEXT, -- JSON {authorName, authorHandle, avatarUrl, sourceItemId}, nitter-sourced articles only + telegram_message TEXT -- JSON {channelName, channelUsername, channelAvatarUrl, sourceItemId, media}, telegram-sourced articles only ); CREATE INDEX IF NOT EXISTS idx_articles_published ON merged_articles(published_at); CREATE INDEX IF NOT EXISTS idx_articles_thread ON merged_articles(thread_id); @@ -170,10 +172,28 @@ export function migrate() { storage_cap_value INTEGER NOT NULL DEFAULT 500, storage_cap_unit TEXT NOT NULL DEFAULT 'GB', nitter_media_mode TEXT NOT NULL DEFAULT 'proxy', -- self-host | proxy | direct - fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com' + fxtwitter_base_url TEXT NOT NULL DEFAULT 'https://api.fxtwitter.com', + telegram_media_mode TEXT NOT NULL DEFAULT 'self-host' -- self-host | proxy (no "direct" — Telegram has no public hotlinkable media URL) + ); + + -- Singleton row (see storage/crypto.ts) — encrypted Telegram API credentials and + -- the resulting login session. Deliberately its own table, not part of + -- global_settings, so these encrypted blobs never ride along in the generic + -- GET /api/admin/settings payload. + CREATE TABLE IF NOT EXISTS telegram_credentials ( + id INTEGER PRIMARY KEY CHECK (id = 1), + api_id_enc TEXT, + api_hash_enc TEXT, + session_enc TEXT, + phone_enc TEXT ); `); + const existingTelegramCreds = db.prepare('SELECT id FROM telegram_credentials WHERE id = 1').get(); + if (!existingTelegramCreds) { + db.prepare('INSERT INTO telegram_credentials (id) VALUES (1)').run(); + } + // Seed the singleton settings row if it doesn't exist yet. const existing = db.prepare('SELECT id FROM global_settings WHERE id = 1').get(); if (!existing) { @@ -207,6 +227,15 @@ export function migrate() { if (!hasColumn('categories', 'is_private')) { db.exec('ALTER TABLE categories ADD COLUMN is_private INTEGER NOT NULL DEFAULT 0'); } + if (!hasColumn('content_items', 'telegram_message')) { + db.exec('ALTER TABLE content_items ADD COLUMN telegram_message TEXT'); + } + if (!hasColumn('merged_articles', 'telegram_message')) { + db.exec('ALTER TABLE merged_articles ADD COLUMN telegram_message TEXT'); + } + if (!hasColumn('global_settings', 'telegram_media_mode')) { + db.exec("ALTER TABLE global_settings ADD COLUMN telegram_media_mode TEXT NOT NULL DEFAULT 'self-host'"); + } // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real diff --git a/backend/src/storage/db/settings.ts b/backend/src/storage/db/settings.ts index f9fd366..13cb383 100644 --- a/backend/src/storage/db/settings.ts +++ b/backend/src/storage/db/settings.ts @@ -15,6 +15,7 @@ function rowToSettings(row: any): GlobalSettings { selectedModels: JSON.parse(row.selected_models), nitterMediaMode: row.nitter_media_mode, fxtwitterBaseUrl: row.fxtwitter_base_url, + telegramMediaMode: row.telegram_media_mode, retention: { publishedArticleMaxAgeDays: row.published_article_max_age_days, rawItemMaxAgeDays: row.raw_item_max_age_days, @@ -43,7 +44,7 @@ export function updateSettings(patch: Partial): GlobalSettings { merge_strictness=?, default_poll_interval_minutes=?, hold_before_publish_minutes=?, tag_dedup_threshold=?, tag_expiry_days=?, follow_up_min_hours_since_last=?, follow_up_min_new_sources=?, ai_service_host=?, ai_service_port=?, selected_models=?, - nitter_media_mode=?, fxtwitter_base_url=?, + nitter_media_mode=?, fxtwitter_base_url=?, telegram_media_mode=?, published_article_max_age_days=?, raw_item_max_age_days=?, storage_cap_enabled=?, storage_cap_value=?, storage_cap_unit=? WHERE id = 1` @@ -60,6 +61,7 @@ export function updateSettings(patch: Partial): GlobalSettings { JSON.stringify(merged.selectedModels), merged.nitterMediaMode, merged.fxtwitterBaseUrl, + merged.telegramMediaMode, merged.retention.publishedArticleMaxAgeDays, merged.retention.rawItemMaxAgeDays, merged.retention.storageCapEnabled ? 1 : 0, diff --git a/backend/src/storage/db/types.ts b/backend/src/storage/db/types.ts index b6e2a4f..47483dc 100644 --- a/backend/src/storage/db/types.ts +++ b/backend/src/storage/db/types.ts @@ -37,6 +37,27 @@ export interface QuotedTweet { link: string; } +/** Same shape as TweetMediaItem — distinct name for readability at Telegram call sites. */ +export type TelegramMediaItem = TweetMediaItem; + +/** + * A raw reference to a single message's attached media, captured at ingestion time — + * deliberately NOT a URL, since Telegram has no public hotlinkable media URL the way + * Twitter does; media bytes only ever come from the authenticated MTProto session. + * publish.ts resolves this into a real TelegramMediaItem (a servable url) according to + * the admin's configured telegramMediaMode, at publish time — mirroring how Nitter's + * tweet media URLs are resolved at publish time too, just starting from a message + * reference here instead of an already-public CDN URL. + */ +export interface TelegramMediaRef { + type: 'photo' | 'video' | 'gif'; + /** This media's own message id (a grouped album's items are separate messages, each individually re-fetchable). */ + messageId: string; + mimeType: string | null; + width: number | null; + height: number | null; +} + export interface ContentItem { id: string; sourceId: string; @@ -66,6 +87,25 @@ export interface ContentItem { /** Set when this item is a quote-tweet — the tweet embedded in its
. */ quotedTweet: QuotedTweet | null; } | null; + /** + * Telegram-sourced items only — null for everything else. channelName/channelUsername + * describe whoever should be *displayed* as the author — the original channel when + * this message is a forward (same as tweet.authorName always being the original + * tweet's author, not the retweeter), or the polled channel itself otherwise. + * channelUsername is null when a forward's origin has no public handle. Media is + * unresolved refs (see TelegramMediaRef); publish.ts resolves them (and the display + * avatar) per the admin's configured media mode. + */ + telegramMessage: { + channelName: string; + channelUsername: string | null; + /** The channel actually polled — always non-null, used to re-fetch this message's media (attached media lives on the polled channel's own copy of the message, forward or not). */ + sourceChannelUsername: string; + messageId: string; + media: TelegramMediaRef[]; + /** Set when this message is a forward — the polled channel's own handle, e.g. "Forwarded by @X". */ + repostedByHandle: string | null; + } | null; raw: unknown; } @@ -92,6 +132,15 @@ export interface MergedArticle { repostedByHandle: string | null; quotedTweet: QuotedTweet | null; } | null; + /** Telegram-sourced articles only — the embed card's channel info and attached media (see TelegramCard.svelte). Never set alongside video. */ + telegramMessage: { + channelName: string; + channelUsername: string | null; + channelAvatarUrl: string | null; + sourceItemId: string; + media: TelegramMediaItem[]; + repostedByHandle: string | null; + } | null; category: string[]; geo: string | null; eventId: string | null; @@ -166,6 +215,8 @@ export interface GlobalSettings { nitterMediaMode: 'self-host' | 'proxy' | 'direct'; /** Base URL of the fxtwitter-compatible enrichment API — defaults to the public instance, overridable for a self-hosted FixTweet mirror. */ fxtwitterBaseUrl: string; + /** How Telegram message media (attached photos/videos, channel avatars) is served — see pipeline/publish.ts's resolveTelegramMedia. No "direct" option: Telegram has no public hotlinkable media URL, bytes only come from the authenticated MTProto session. */ + telegramMediaMode: 'self-host' | 'proxy'; retention: { publishedArticleMaxAgeDays: number | null; rawItemMaxAgeDays: number | null; diff --git a/backend/src/storage/media/index.ts b/backend/src/storage/media/index.ts index a911f27..8d620e1 100644 --- a/backend/src/storage/media/index.ts +++ b/backend/src/storage/media/index.ts @@ -29,25 +29,41 @@ export async function downloadAndStore( const res = await fetch(sourceUrl, { signal: AbortSignal.timeout(10_000) }); if (!res.ok) return null; const buffer = Buffer.from(await res.arrayBuffer()); - const ext = guessExtension(res.headers.get('content-type') ?? '', sourceUrl); - const id = randomUUID(); - const filename = `${id}${ext}`; - const localPath = path.join(MEDIA_DIR, filename); - fs.writeFileSync(localPath, buffer); - - db.prepare( - `INSERT INTO media_assets (id, source_url, local_path, size_bytes, downloaded_at, tier, content_item_id, article_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ).run(id, sourceUrl, localPath, buffer.length, new Date().toISOString(), tier, refs.contentItemId ?? null, refs.articleId ?? null); - - return { id, localPath, servedPath: `/media/${filename}`, sizeBytes: buffer.length }; + return storeMediaBuffer(buffer, ext, sourceUrl, tier, refs); } catch (err) { logger.error('media', `Failed to download ${sourceUrl}: ${(err as Error).message}`); return null; } } +/** + * Writes an already-downloaded buffer to disk and records it in media_assets — the + * common tail of downloadAndStore, extracted so the Telegram adapter (which downloads + * media itself via the authenticated GramJS session rather than a plain fetch()) can + * reuse the same storage/bookkeeping logic. + */ +export function storeMediaBuffer( + buffer: Buffer, + ext: string, + sourceUrl: string, + tier: 'candidate' | 'published', + refs: { contentItemId?: string; articleId?: string } +): StoredMedia { + const id = randomUUID(); + const filename = `${id}${ext}`; + const localPath = path.join(MEDIA_DIR, filename); + fs.writeFileSync(localPath, buffer); + + db.prepare( + `INSERT INTO media_assets (id, source_url, local_path, size_bytes, downloaded_at, tier, content_item_id, article_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run(id, sourceUrl, localPath, buffer.length, new Date().toISOString(), tier, refs.contentItemId ?? null, refs.articleId ?? null); + + return { id, localPath, servedPath: `/media/${filename}`, sizeBytes: buffer.length }; +} + + /** Promotes a candidate-tier asset to published-tier so raw-item retention can no longer prune it. */ export function promoteToPublished(mediaId: string, articleId: string) { db.prepare("UPDATE media_assets SET tier = 'published', article_id = ? WHERE id = ?").run(articleId, mediaId); diff --git a/backend/src/telegram/client.ts b/backend/src/telegram/client.ts new file mode 100644 index 0000000..94a909c --- /dev/null +++ b/backend/src/telegram/client.ts @@ -0,0 +1,178 @@ +// Singleton GramJS client + login state machine. Unlike every other adapter (rss/ +// nitter/youtube — all stateless per-call HTTP fetches), Telegram needs one long-lived +// authenticated MTProto connection reused across every poll — this module owns that +// connection and the interactive phone/code/2FA login flow that produces it, consumed +// by three call sites: the adapter (getClient/fetchChannelMessages), the admin login +// routes (startLogin/verifyCode/verifyPassword/logout/getStatus), and startup +// (initFromSavedSession). + +import { TelegramClient, Api } from 'telegram'; +import { StringSession } from 'telegram/sessions/index.js'; +import { computeCheck } from 'telegram/Password.js'; +import * as credentials from './credentials.js'; +import { logger } from '../storage/db/logs.js'; + +interface PendingLogin { + client: TelegramClient; + phoneNumber: string; + phoneCodeHash: string; + phase: 'code-sent' | 'password-needed'; +} + +let client: TelegramClient | null = null; +let connectedPhone: string | null = null; +let pendingLogin: PendingLogin | null = null; + +export function telegramConfigured(): boolean { + return credentials.getApiCredentials() !== null; +} + +function newClient(sessionString = ''): TelegramClient { + const creds = credentials.getApiCredentials(); + if (!creds) throw new Error('Telegram API credentials have not been saved yet'); + return new TelegramClient(new StringSession(sessionString), creds.apiId, creds.apiHash, { + connectionRetries: 5 + }); +} + +/** Called from index.ts's main() right after migrate() — reconnects a saved session, if any. */ +export async function initFromSavedSession(): Promise { + const session = credentials.getSession(); + if (!session) return; + + try { + const c = newClient(session.sessionString); + await c.connect(); + client = c; + connectedPhone = session.phone; + logger.info('telegram', `Reconnected as ${session.phone}`); + } catch (err) { + // A network blip or transient failure shouldn't force a re-login — only an + // explicit logout() clears the saved session. Leaving client null here means the + // adapter just fails soft (same as before this feature existed) until the next + // restart or a manual reconnect. + client = null; + connectedPhone = null; + logger.error('telegram', `Failed to reconnect saved session: ${(err as Error).message}`); + } +} + +export function getClient(): TelegramClient | null { + return client; +} + +export function getStatus(): { credentialsConfigured: boolean; connected: boolean; phone: string | null } { + return { + credentialsConfigured: telegramConfigured(), + connected: client !== null, + phone: connectedPhone + }; +} + +export function saveApiCredentials(apiId: number, apiHash: string): void { + credentials.saveApiCredentials(apiId, apiHash); +} + +export async function startLogin(phoneNumber: string): Promise { + if (!telegramConfigured()) throw new Error('Save Telegram API ID/hash before logging in'); + + const creds = credentials.getApiCredentials()!; + const loginClient = newClient(); + await loginClient.connect(); + + const sent = await loginClient.sendCode({ apiId: creds.apiId, apiHash: creds.apiHash }, phoneNumber); + pendingLogin = { client: loginClient, phoneNumber, phoneCodeHash: sent.phoneCodeHash, phase: 'code-sent' }; +} + +/** Returns 'connected' or 'password-needed' depending on whether this account has 2FA enabled. */ +export async function verifyCode(code: string): Promise<'connected' | 'password-needed'> { + if (!pendingLogin || pendingLogin.phase !== 'code-sent') throw new Error('No login in progress — start again'); + + try { + await pendingLogin.client.invoke( + new Api.auth.SignIn({ + phoneNumber: pendingLogin.phoneNumber, + phoneCodeHash: pendingLogin.phoneCodeHash, + phoneCode: code + }) + ); + await completeLogin(pendingLogin.client, pendingLogin.phoneNumber); + return 'connected'; + } catch (err) { + if ((err as { errorMessage?: string }).errorMessage === 'SESSION_PASSWORD_NEEDED') { + pendingLogin.phase = 'password-needed'; + return 'password-needed'; + } + pendingLogin = null; + throw err; + } +} + +export async function verifyPassword(password: string): Promise { + if (!pendingLogin || pendingLogin.phase !== 'password-needed') throw new Error('No password step in progress — start again'); + + try { + const passwordInfo = await pendingLogin.client.invoke(new Api.account.GetPassword()); + const passwordSrpCheck = await computeCheck(passwordInfo, password); + await pendingLogin.client.invoke(new Api.auth.CheckPassword({ password: passwordSrpCheck })); + await completeLogin(pendingLogin.client, pendingLogin.phoneNumber); + } catch (err) { + pendingLogin = null; + throw err; + } +} + +async function completeLogin(loginClient: TelegramClient, phoneNumber: string): Promise { + const sessionString = loginClient.session.save() as unknown as string; + credentials.saveSession(sessionString, phoneNumber); + client = loginClient; + connectedPhone = phoneNumber; + pendingLogin = null; + logger.info('telegram', `Logged in as ${phoneNumber}`); +} + +export async function logout(): Promise { + if (client) { + try { + await client.invoke(new Api.auth.LogOut()); + } catch (err) { + logger.error('telegram', `Logout call failed (clearing local session anyway): ${(err as Error).message}`); + } + await client.disconnect(); + } + client = null; + connectedPhone = null; + pendingLogin = null; + credentials.clearSession(); +} + +/** Used by the adapter — resolves a channel and fetches its most recent messages. */ +export async function fetchChannelMessages(channelIdentifier: string, limit: number) { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelIdentifier); + const messages = await client.getMessages(entity, { limit }); + return { entity, messages }; +} + +/** + * Re-fetches a single message by id and downloads its attached media — used by + * pipeline/publish.ts (self-host mode, at publish time) and the live telegram-proxy + * route (proxy mode, on every view). Telegram media has no public URL; this + * authenticated call is the only way to get the bytes. + */ +export async function downloadMessageMedia(channelUsername: string, messageId: string): Promise { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelUsername); + const [message] = await client.getMessages(entity, { ids: [Number(messageId)] }); + if (!message) return null; + const buffer = await message.downloadMedia(); + return buffer && typeof buffer !== 'string' ? buffer : null; +} + +/** Downloads a channel's current avatar — same on-demand, no-public-URL reasoning as downloadMessageMedia. */ +export async function downloadChannelAvatar(channelUsername: string): Promise { + if (!client) throw new Error('Telegram client not connected'); + const entity = await client.getEntity(channelUsername); + const photo = await client.downloadProfilePhoto(entity); + return photo && typeof photo !== 'string' ? photo : null; +} diff --git a/backend/src/telegram/credentials.ts b/backend/src/telegram/credentials.ts new file mode 100644 index 0000000..577e6c2 --- /dev/null +++ b/backend/src/telegram/credentials.ts @@ -0,0 +1,48 @@ +// The only module that touches storage/crypto.ts for Telegram data — everything else +// (telegram/client.ts, the admin routes) goes through these functions and never sees +// ciphertext or handles encryption itself. + +import { db } from '../storage/db/index.js'; +import { encrypt, decrypt } from '../storage/crypto.js'; + +interface TelegramCredentialsRow { + api_id_enc: string | null; + api_hash_enc: string | null; + session_enc: string | null; + phone_enc: string | null; +} + +function getRow(): TelegramCredentialsRow { + return db.prepare('SELECT * FROM telegram_credentials WHERE id = 1').get() as unknown as TelegramCredentialsRow; +} + +export function saveApiCredentials(apiId: number, apiHash: string): void { + db.prepare('UPDATE telegram_credentials SET api_id_enc = ?, api_hash_enc = ? WHERE id = 1').run( + encrypt(String(apiId)), + encrypt(apiHash) + ); +} + +export function getApiCredentials(): { apiId: number; apiHash: string } | null { + const row = getRow(); + if (!row.api_id_enc || !row.api_hash_enc) return null; + return { apiId: Number(decrypt(row.api_id_enc)), apiHash: decrypt(row.api_hash_enc) }; +} + +export function saveSession(sessionString: string, phone: string): void { + db.prepare('UPDATE telegram_credentials SET session_enc = ?, phone_enc = ? WHERE id = 1').run( + encrypt(sessionString), + encrypt(phone) + ); +} + +export function getSession(): { sessionString: string; phone: string } | null { + const row = getRow(); + if (!row.session_enc || !row.phone_enc) return null; + return { sessionString: decrypt(row.session_enc), phone: decrypt(row.phone_enc) }; +} + +/** Logout — clears the session/phone only, keeps API ID/hash so re-login doesn't need them re-entered. */ +export function clearSession(): void { + db.prepare('UPDATE telegram_credentials SET session_enc = NULL, phone_enc = NULL WHERE id = 1').run(); +} diff --git a/frontend/src/lib/adminApi.ts b/frontend/src/lib/adminApi.ts index 5ee90b4..5a2fd2a 100644 --- a/frontend/src/lib/adminApi.ts +++ b/frontend/src/lib/adminApi.ts @@ -6,6 +6,7 @@ import type { AdminTrackedEvent, ModelCatalog, AiStatus, + TelegramStatus, LogEntry } from './adminTypes'; @@ -120,6 +121,43 @@ export const getModels = (fetchFn?: typeof fetch) => export const getAiStatus = (fetchFn?: typeof fetch) => request('/api/admin/ai-status', {}, fetchFn); +// Telegram account (Connections tab) — API ID/hash and the resulting login session are +// stored encrypted at rest server-side (see backend telegram/credentials.ts); none of +// these ever come back from the server, only status flags. +export const getTelegramStatus = (fetchFn?: typeof fetch) => + request('/api/admin/telegram/status', {}, fetchFn); + +export const saveTelegramCredentials = (apiId: number, apiHash: string, fetchFn?: typeof fetch) => + request<{ credentialsConfigured: boolean }>( + '/api/admin/telegram/credentials', + { method: 'POST', body: JSON.stringify({ apiId, apiHash }) }, + fetchFn + ); + +export const startTelegramLogin = (phoneNumber: string, fetchFn?: typeof fetch) => + request<{ phase: 'code-sent' }>( + '/api/admin/telegram/login/start', + { method: 'POST', body: JSON.stringify({ phoneNumber }) }, + fetchFn + ); + +export const verifyTelegramCode = (code: string, fetchFn?: typeof fetch) => + request<{ phase: 'connected' | 'password-needed'; connected?: boolean; phone?: string | null }>( + '/api/admin/telegram/login/verify', + { method: 'POST', body: JSON.stringify({ code }) }, + fetchFn + ); + +export const verifyTelegramPassword = (password: string, fetchFn?: typeof fetch) => + request<{ phase: 'connected'; connected?: boolean; phone?: string | null }>( + '/api/admin/telegram/login/verify', + { method: 'POST', body: JSON.stringify({ password }) }, + fetchFn + ); + +export const telegramLogout = (fetchFn?: typeof fetch) => + request('/api/admin/telegram/logout', { method: 'POST' }, fetchFn); + // Logs export const getLogs = (filters: { level?: 'info' | 'warn' | 'error'; limit?: number } = {}, fetchFn?: typeof fetch) => { const qs = new URLSearchParams(filters as Record).toString(); diff --git a/frontend/src/lib/adminTypes.ts b/frontend/src/lib/adminTypes.ts index f5357e2..ee0289d 100644 --- a/frontend/src/lib/adminTypes.ts +++ b/frontend/src/lib/adminTypes.ts @@ -28,6 +28,7 @@ export interface AdminSettings { selectedModels: { embedding: string; image: string; synthesis: string }; nitterMediaMode: 'self-host' | 'proxy' | 'direct'; fxtwitterBaseUrl: string; + telegramMediaMode: 'self-host' | 'proxy'; retention: RetentionSettings; categoryPriority: CategoryPriority[]; } @@ -71,6 +72,12 @@ export interface AiStatus { gpu: string; } +export interface TelegramStatus { + credentialsConfigured: boolean; + connected: boolean; + phone: string | null; +} + export interface LogEntry { id: number; timestamp: string; diff --git a/frontend/src/lib/components/ArticleListRow.svelte b/frontend/src/lib/components/ArticleListRow.svelte index d32248b..07b3d69 100644 --- a/frontend/src/lib/components/ArticleListRow.svelte +++ b/frontend/src/lib/components/ArticleListRow.svelte @@ -3,6 +3,7 @@ import { timeAgo, exactTime, excerpt } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; import TweetCard from './TweetCard.svelte'; + import TelegramCard from './TelegramCard.svelte'; let { article }: { article: MergedArticle } = $props(); @@ -15,6 +16,8 @@ {#if article.tweet} +{:else if article.telegramMessage} + {:else} {#if article.heroImage} diff --git a/frontend/src/lib/components/TelegramCard.svelte b/frontend/src/lib/components/TelegramCard.svelte new file mode 100644 index 0000000..aa7d494 --- /dev/null +++ b/frontend/src/lib/components/TelegramCard.svelte @@ -0,0 +1,197 @@ + + + + {#if article.telegramMessage?.repostedByHandle} +
↪️ Forwarded by @{article.telegramMessage.repostedByHandle}
+ {/if} +
+ {article.category[0] ?? ''} + · + {sourceLabel} + · + ✈ Telegram +
+
+ {#if article.telegramMessage?.channelAvatarUrl} + + {:else} +
+ {/if} + {article.telegramMessage?.channelName} + {#if article.telegramMessage?.channelUsername} + @{article.telegramMessage.channelUsername} + {/if} +
+
{article.body}
+ {#if media.length > 0} +
+ {#each media as item, i (item.url)} +
+ {#if item.type === 'video' || item.type === 'gif'} + + {:else} + + {/if} +
+ {/each} +
+ {/if} +
{timeAgo(article.publishedAt)} · {exactTime(article.publishedAt)}
+ + + diff --git a/frontend/src/lib/components/admin/ConnectionsTab.svelte b/frontend/src/lib/components/admin/ConnectionsTab.svelte index d7132e7..c6c2676 100644 --- a/frontend/src/lib/components/admin/ConnectionsTab.svelte +++ b/frontend/src/lib/components/admin/ConnectionsTab.svelte @@ -1,10 +1,22 @@
@@ -82,6 +195,63 @@
+
+
+ Telegram account + +
+

+ Reading channels requires logging into a real Telegram account (not a bot), so it can see + any public channel that account can see. Get an API ID/hash from + my.telegram.org — a + dedicated/secondary account is recommended over your primary one. Both are stored encrypted, + never sent back to this browser. +

+
+ + + +
+ {#if credentialsConfigured} +
✓ API credentials saved
+ {/if} + + +
+ diff --git a/frontend/src/lib/components/admin/RetentionTab.svelte b/frontend/src/lib/components/admin/RetentionTab.svelte index 2215d8d..a00aac9 100644 --- a/frontend/src/lib/components/admin/RetentionTab.svelte +++ b/frontend/src/lib/components/admin/RetentionTab.svelte @@ -36,6 +36,31 @@ { label: 'Direct', value: 'direct' } ]; + let telegramMediaMode = $state(settings.telegramMediaMode); + let telegramMediaStatus = $state<'idle' | 'saving' | 'saved' | 'error'>('idle'); + let telegramMediaSaveTimer: ReturnType; + + function scheduleTelegramMediaSave() { + telegramMediaStatus = 'saving'; + clearTimeout(telegramMediaSaveTimer); + telegramMediaSaveTimer = setTimeout(async () => { + try { + await updateSettings({ telegramMediaMode }); + telegramMediaStatus = 'saved'; + setTimeout(() => (telegramMediaStatus = 'idle'), 1500); + } catch { + telegramMediaStatus = 'error'; + } + }, 500); + } + + // No "Direct" option here — unlike Twitter's CDN, Telegram has no public + // hotlinkable media URL, so there's nothing to hotlink straight from. + const telegramMediaModes: { label: string; value: 'self-host' | 'proxy' }[] = [ + { label: 'Self-host', value: 'self-host' }, + { label: 'Proxy', value: 'proxy' } + ]; + async function handleClearArticles() { if (!confirm('Delete every published article and its media? Raw ingested items are kept, so sources can be re-synthesized fresh.')) return; clearing = 'articles'; @@ -200,6 +225,36 @@ +
+
+ Telegram (message media) + +
+

+ How photos/videos attached to ingested Telegram messages (and channel avatars) are served + to visitors. Telegram has no public URL for this media the way Twitter's CDN does — bytes + only ever come from the logged-in account (Connections tab), so there's no "Direct" option. + Self-hosting downloads and stores everything locally, same as regular article images. + Proxying re-fetches each request live through the logged-in account and streams it straight + through without persisting anything, so only this server ever touches Telegram's servers — + at the cost of a live round-trip to Telegram on every view (a short cache absorbs repeats). +

+
+ {#each telegramMediaModes as mode} + + {/each} +
+
+
Clear content

diff --git a/frontend/src/lib/components/admin/SourcesTab.svelte b/frontend/src/lib/components/admin/SourcesTab.svelte index b6b73af..5e8c709 100644 --- a/frontend/src/lib/components/admin/SourcesTab.svelte +++ b/frontend/src/lib/components/admin/SourcesTab.svelte @@ -202,6 +202,8 @@ {:else if form.type === 'nitter'} + {:else if form.type === 'telegram'} + {:else} {/if} @@ -211,6 +213,9 @@

+ {#if form.type === 'telegram'} + Public channels only for now — private channels without a public username aren't supported yet. + {/if}
Categories
{#each assignableCategories as cat (cat.id)} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index d0bdaa3..10d3128 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -26,6 +26,9 @@ export interface QuotedTweet { link: string; } +/** Same shape as TweetMediaItem — distinct name for readability at Telegram call sites. */ +export type TelegramMediaItem = TweetMediaItem; + export interface MergedArticle { id: string; title: string; @@ -41,6 +44,14 @@ export interface MergedArticle { repostedByHandle: string | null; quotedTweet: QuotedTweet | null; } | null; + telegramMessage: { + channelName: string; + channelUsername: string | null; + channelAvatarUrl: string | null; + sourceItemId: string; + media: TelegramMediaItem[]; + repostedByHandle: string | null; + } | null; category: string[]; geo: string | null; eventId: string | null; diff --git a/frontend/src/routes/admin/settings/+page.svelte b/frontend/src/routes/admin/settings/+page.svelte index 85b35de..c6506c7 100644 --- a/frontend/src/routes/admin/settings/+page.svelte +++ b/frontend/src/routes/admin/settings/+page.svelte @@ -46,7 +46,7 @@ {:else if active === 'events'} {:else if active === 'connections'} - + {:else if active === 'logs'} {/if} diff --git a/frontend/src/routes/admin/settings/+page.ts b/frontend/src/routes/admin/settings/+page.ts index d5a9f60..4fb73f5 100644 --- a/frontend/src/routes/admin/settings/+page.ts +++ b/frontend/src/routes/admin/settings/+page.ts @@ -1,7 +1,7 @@ import { redirect } from '@sveltejs/kit'; import type { PageLoad } from './$types'; -import { getSettings, getSources, getEvents, getModels, getAiStatus, getLogs } from '$lib/adminApi'; -import type { ModelCatalog, AiStatus } from '$lib/adminTypes'; +import { getSettings, getSources, getEvents, getModels, getAiStatus, getTelegramStatus, getLogs } from '$lib/adminApi'; +import type { ModelCatalog, AiStatus, TelegramStatus } from '$lib/adminTypes'; const EMPTY_MODELS: ModelCatalog = { embedding: [], image: [], synthesis: [] }; @@ -24,7 +24,11 @@ export const load: PageLoad = async ({ fetch }) => { // swallow — surfaced instead via aiStatus.connected in the UI } - return { settings, sources, events, models, aiStatus, logs }; + const telegramStatus: TelegramStatus = await getTelegramStatus(fetch).catch( + () => ({ credentialsConfigured: false, connected: false, phone: null }) + ); + + return { settings, sources, events, models, aiStatus, telegramStatus, logs }; } catch (err) { if ((err as { status?: number }).status === 401) { throw redirect(302, '/admin/login?redirectTo=/admin/settings'); diff --git a/frontend/src/routes/article/[id]/+page.svelte b/frontend/src/routes/article/[id]/+page.svelte index 418259d..c9308a3 100644 --- a/frontend/src/routes/article/[id]/+page.svelte +++ b/frontend/src/routes/article/[id]/+page.svelte @@ -3,6 +3,7 @@ import { timeAgo, exactTime } from '$lib/format'; import { resolveMediaUrl } from '$lib/config'; import TweetCard from '$lib/components/TweetCard.svelte'; + import TelegramCard from '$lib/components/TelegramCard.svelte'; let { data }: { data: PageData } = $props(); const a = $derived(data.article); @@ -36,6 +37,18 @@ {#if a.sources[0]} View original tweet → {/if} + {:else if a.telegramMessage} + +
+ Published {timeAgo(a.publishedAt)} · {exactTime(a.publishedAt)} +
+ + + + {#if a.sources[0]} + View original message on Telegram → + {/if} {:else if a.video?.provider === 'youtube'}

{a.title}