From 57c49f68587e2d9e3bd927eb3c79d772680887ac Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 23:47:36 +0000 Subject: [PATCH] Switch stock ticker source from Stooq to Yahoo Finance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stooq's public quote endpoint now gates every request behind a client-side proof-of-work challenge (confirmed by manual curl testing), which a plain server-side fetch can't pass and isn't worth running a headless browser to solve. Yahoo's unofficial /v8/finance/chart endpoint still works with just a browser-like User-Agent header (also confirmed manually — bare curl/fetch UAs get rate-limited immediately). One request per ticker instead of one batched request (Yahoo's batch quote endpoint needs a cookie+crumb handshake this one doesn't), and change % is now computed against the real previous close instead of the open-vs-close approximation Stooq's format forced. Existing installs get their three default tickers (Dow/S&P/Bitcoin) rewritten from Stooq to Yahoo symbol syntax automatically; any ticker an admin added themselves is left alone. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8 --- backend/src/stocks/client.ts | 83 +++++++++++-------- backend/src/stocks/poller.ts | 5 +- backend/src/storage/db/index.ts | 19 ++++- .../src/lib/components/admin/StocksTab.svelte | 8 +- 4 files changed, 70 insertions(+), 45 deletions(-) diff --git a/backend/src/stocks/client.ts b/backend/src/stocks/client.ts index 3e0eb98..637c475 100644 --- a/backend/src/stocks/client.ts +++ b/backend/src/stocks/client.ts @@ -1,11 +1,19 @@ -// Stooq (stooq.com) — free CSV quote endpoint, no account or API key required, and it -// accepts multiple symbols batched into one request. This is the only file that talks to -// it; poller.ts orchestrates when/how results get saved, same separation as -// backend/src/telegram/ keeps between the raw client and its callers. +// Yahoo Finance's unofficial chart endpoint — free, no account or API key required. +// This is the only file that talks to it; poller.ts orchestrates when/how results get +// saved, same separation as backend/src/telegram/ keeps between the raw client and its +// callers. // -// Stooq's quote line has no prior-close field, so "change %" here is computed as -// (close - open) / open * 100 — an intraday-vs-open approximation, not a true -// prior-day change. Accepted simplification for a basic ticker widget. +// Previously used Stooq's CSV quote endpoint, which started gating every request behind +// a client-side proof-of-work challenge (compute a SHA-256 hashcash puzzle in JS, POST it +// to /__verify) — not something a plain server-side fetch can pass, and not worth running +// a headless browser to poll ticker prices. Confirmed via manual curl testing that Yahoo's +// /v8/finance/chart/ endpoint still works with a plain fetch, but ONLY with a +// browser-like User-Agent header — bare `curl`/`fetch` UAs get a 429 on the very first +// request, before any real volume. This is an undocumented, unofficial API Yahoo could +// change or wall off without notice, same caveat as Stooq — if it goes the same way, +// there's no realistic simple-fetch alternative left; the fallback would be a provider +// requiring a free API key. +const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36'; export interface StockQuote { price: number; @@ -16,35 +24,38 @@ export async function fetchQuotes(symbols: string[]): Promise(); if (symbols.length === 0) return results; - const url = `https://stooq.com/q/l/?s=${symbols.map(encodeURIComponent).join(',')}&f=sd2t2ohlcv&h&e=csv`; - const res = await fetch(url); - if (!res.ok) throw new Error(`Stooq returned ${res.status}`); - const text = await res.text(); - - // Header: Symbol,Date,Time,Open,High,Low,Close,Volume — no quoted/embedded-comma - // fields in this format, so a plain split is sufficient (no CSV library needed). - const lines = text.trim().split('\n').slice(1); - const bySymbol = new Map(); - for (const line of lines) { - const cols = line.split(','); - if (cols.length < 7) continue; - bySymbol.set(cols[0].toLowerCase(), cols); - } - - for (const symbol of symbols) { - const cols = bySymbol.get(symbol.toLowerCase()); - if (!cols) { - results.set(symbol, new Error('Symbol not found in Stooq response')); - continue; - } - const open = Number(cols[3]); - const close = Number(cols[6]); - if (cols[3] === 'N/D' || cols[6] === 'N/D' || !Number.isFinite(open) || !Number.isFinite(close) || open === 0) { - results.set(symbol, new Error('Stooq has no data for this symbol')); - continue; - } - results.set(symbol, { price: close, changePercent: ((close - open) / open) * 100 }); - } + // No batch endpoint used here — Yahoo's multi-symbol /v7/finance/quote requires a + // cookie+crumb handshake first, while /v8/finance/chart/ (single symbol, no + // crumb needed) is the one confirmed to work with just a User-Agent. One request per + // ticker per poll is trivial at the scale of a sidebar widget (a handful of tickers, + // polled every 15 minutes). + await Promise.all( + symbols.map(async (symbol) => { + try { + const res = await fetch(`https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`, { + headers: { 'User-Agent': USER_AGENT } + }); + if (!res.ok) throw new Error(`Yahoo returned ${res.status}`); + const data = (await res.json()) as { + chart: { + result: { meta: { regularMarketPrice: number; previousClose?: number; chartPreviousClose?: number } }[] | null; + error: { description: string } | null; + }; + }; + if (data.chart.error) throw new Error(data.chart.error.description); + const meta = data.chart.result?.[0]?.meta; + if (!meta) throw new Error('No data returned for this symbol'); + const previousClose = meta.previousClose ?? meta.chartPreviousClose; + if (previousClose === undefined) throw new Error('No previous close available for this symbol'); + results.set(symbol, { + price: meta.regularMarketPrice, + changePercent: ((meta.regularMarketPrice - previousClose) / previousClose) * 100 + }); + } catch (err) { + results.set(symbol, err instanceof Error ? err : new Error(String(err))); + } + }) + ); return results; } diff --git a/backend/src/stocks/poller.ts b/backend/src/stocks/poller.ts index e81dbc4..84ccb26 100644 --- a/backend/src/stocks/poller.ts +++ b/backend/src/stocks/poller.ts @@ -3,8 +3,9 @@ import { logger } from '../storage/db/logs.js'; import { fetchQuotes } from './client.js'; // Called on a schedule (see queue/scheduler.ts) and immediately after the admin adds a -// ticker (see api/admin.ts) — one batched Stooq request for every configured ticker. A -// symbol Stooq can't resolve gets its own lastError, it never aborts the whole batch. +// ticker (see api/admin.ts) — one request per configured ticker (see client.ts for why +// there's no batch endpoint here). A symbol Yahoo can't resolve gets its own lastError, +// it never aborts the rest of the batch. export async function pollStocksNow(): Promise { const tickers = stocksDb.listStockTickers(); if (tickers.length === 0) return; diff --git a/backend/src/storage/db/index.ts b/backend/src/storage/db/index.ts index fecb500..3df0497 100644 --- a/backend/src/storage/db/index.ts +++ b/backend/src/storage/db/index.ts @@ -308,9 +308,9 @@ export function migrate() { const tickerCount = db.prepare('SELECT COUNT(*) as c FROM stock_tickers').get() as { c: number }; if (tickerCount.c === 0) { const defaults: [string, string][] = [ - ['Dow Jones', '^dji'], - ['S&P 500', '^spx'], - ['Bitcoin', 'btcusd'] + ['Dow Jones', '^DJI'], + ['S&P 500', '^GSPC'], + ['Bitcoin', 'BTC-USD'] ]; const stmt = db.prepare( 'INSERT INTO stock_tickers (id, label, symbol, priority_rank, created_at) VALUES (?, ?, ?, ?, ?)' @@ -320,6 +320,19 @@ export function migrate() { }); } + // Stocks switched data providers from Stooq (walled off behind a proof-of-work + // challenge) to Yahoo Finance, which uses different symbol syntax — rewrites only + // rows still holding exactly one of the three old Stooq-format default symbols we + // ourselves seeded, never touching a symbol the admin typed in themselves. + const stooqToYahooSymbols: [string, string][] = [ + ['^dji', '^DJI'], + ['^spx', '^GSPC'], + ['btcusd', 'BTC-USD'] + ]; + for (const [oldSymbol, newSymbol] of stooqToYahooSymbols) { + db.prepare('UPDATE stock_tickers SET symbol = ? WHERE symbol = ?').run(newSymbol, oldSymbol); + } + // Seed default categories if none exist yet. "News" sits right under "Top stories" — // general news sources belong here, not on "Top stories" itself, which isn't a real // filterable tag: it's the homepage view, now scoped to only the articles whose diff --git a/frontend/src/lib/components/admin/StocksTab.svelte b/frontend/src/lib/components/admin/StocksTab.svelte index 6b3dff0..dfb7c37 100644 --- a/frontend/src/lib/components/admin/StocksTab.svelte +++ b/frontend/src/lib/components/admin/StocksTab.svelte @@ -49,16 +49,16 @@
- +

- Stooq has no symbol search, so type the exact syntax: indices use a caret (^dji, ^spx), - stocks use a country suffix (aapl.us), crypto pairs have none (btcusd). Polled every 15 - minutes; a new ticker is polled immediately. + Yahoo Finance has no symbol search, so type the exact syntax: stocks are plain tickers + (AAPL), indices use a caret (^DJI, ^GSPC), crypto pairs use a dash (BTC-USD). Polled + every 15 minutes; a new ticker is polled immediately.

{/if}