Switch stock ticker source from Stooq to Yahoo Finance

Stooq's public quote endpoint now gates every request behind a
client-side proof-of-work challenge (confirmed by manual curl
testing), which a plain server-side fetch can't pass and isn't
worth running a headless browser to solve. Yahoo's unofficial
/v8/finance/chart endpoint still works with just a browser-like
User-Agent header (also confirmed manually — bare curl/fetch UAs
get rate-limited immediately).

One request per ticker instead of one batched request (Yahoo's
batch quote endpoint needs a cookie+crumb handshake this one
doesn't), and change % is now computed against the real previous
close instead of the open-vs-close approximation Stooq's format
forced. Existing installs get their three default tickers
(Dow/S&P/Bitcoin) rewritten from Stooq to Yahoo symbol syntax
automatically; any ticker an admin added themselves is left alone.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014c1L8ghNBFjfiH64UMViP8

backend/src/stocks/client.ts

@@ -1,11 +1,19 @@
-// Stooq (stooq.com) — free CSV quote endpoint, no account or API key required, and it
+// Yahoo Finance's unofficial chart endpoint — free, no account or API key required.
-// accepts multiple symbols batched into one request. This is the only file that talks to
+// This is the only file that talks to it; poller.ts orchestrates when/how results get
-// it; poller.ts orchestrates when/how results get saved, same separation as
+// saved, same separation as backend/src/telegram/ keeps between the raw client and its
-// backend/src/telegram/ keeps between the raw client and its callers.
+// callers.
 //
-// Stooq's quote line has no prior-close field, so "change %" here is computed as
+// Previously used Stooq's CSV quote endpoint, which started gating every request behind
-// (close - open) / open * 100 — an intraday-vs-open approximation, not a true
+// a client-side proof-of-work challenge (compute a SHA-256 hashcash puzzle in JS, POST it
-// prior-day change. Accepted simplification for a basic ticker widget.
+// to /__verify) — not something a plain server-side fetch can pass, and not worth running
+// a headless browser to poll ticker prices. Confirmed via manual curl testing that Yahoo's
+// /v8/finance/chart/<symbol> endpoint still works with a plain fetch, but ONLY with a
+// browser-like User-Agent header — bare `curl`/`fetch` UAs get a 429 on the very first
+// request, before any real volume. This is an undocumented, unofficial API Yahoo could
+// change or wall off without notice, same caveat as Stooq — if it goes the same way,
+// there's no realistic simple-fetch alternative left; the fallback would be a provider
+// requiring a free API key.
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
 
 export interface StockQuote {
 	price: number;
@@ -16,35 +24,38 @@ export async function fetchQuotes(symbols: string[]): Promise<Map<string, StockQ
 	const results = new Map<string, StockQuote | Error>();
 	if (symbols.length === 0) return results;
 
-	const url = `https://stooq.com/q/l/?s=${symbols.map(encodeURIComponent).join(',')}&f=sd2t2ohlcv&h&e=csv`;
+	// No batch endpoint used here — Yahoo's multi-symbol /v7/finance/quote requires a
-	const res = await fetch(url);
+	// cookie+crumb handshake first, while /v8/finance/chart/<symbol> (single symbol, no
-	if (!res.ok) throw new Error(`Stooq returned ${res.status}`);
+	// crumb needed) is the one confirmed to work with just a User-Agent. One request per
-	const text = await res.text();
+	// ticker per poll is trivial at the scale of a sidebar widget (a handful of tickers,
-
+	// polled every 15 minutes).
-	// Header: Symbol,Date,Time,Open,High,Low,Close,Volume — no quoted/embedded-comma
+	await Promise.all(
-	// fields in this format, so a plain split is sufficient (no CSV library needed).
+		symbols.map(async (symbol) => {
-	const lines = text.trim().split('\n').slice(1);
+			try {
-	const bySymbol = new Map<string, string[]>();
+				const res = await fetch(`https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`, {
-	for (const line of lines) {
+					headers: { 'User-Agent': USER_AGENT }
-		const cols = line.split(',');
+				});
-		if (cols.length < 7) continue;
+				if (!res.ok) throw new Error(`Yahoo returned ${res.status}`);
-		bySymbol.set(cols[0].toLowerCase(), cols);
+				const data = (await res.json()) as {
-	}
+					chart: {
-
+						result: { meta: { regularMarketPrice: number; previousClose?: number; chartPreviousClose?: number } }[] | null;
-	for (const symbol of symbols) {
+						error: { description: string } | null;
-		const cols = bySymbol.get(symbol.toLowerCase());
+					};
-		if (!cols) {
+				};
-			results.set(symbol, new Error('Symbol not found in Stooq response'));
+				if (data.chart.error) throw new Error(data.chart.error.description);
-			continue;
+				const meta = data.chart.result?.[0]?.meta;
-		}
+				if (!meta) throw new Error('No data returned for this symbol');
-		const open = Number(cols[3]);
+				const previousClose = meta.previousClose ?? meta.chartPreviousClose;
-		const close = Number(cols[6]);
+				if (previousClose === undefined) throw new Error('No previous close available for this symbol');
-		if (cols[3] === 'N/D' || cols[6] === 'N/D' || !Number.isFinite(open) || !Number.isFinite(close) || open === 0) {
+				results.set(symbol, {
-			results.set(symbol, new Error('Stooq has no data for this symbol'));
+					price: meta.regularMarketPrice,
-			continue;
+					changePercent: ((meta.regularMarketPrice - previousClose) / previousClose) * 100
-		}
+				});
-		results.set(symbol, { price: close, changePercent: ((close - open) / open) * 100 });
+			} catch (err) {
+				results.set(symbol, err instanceof Error ? err : new Error(String(err)));
 			}
+		})
+	);
 
 	return results;
 }

backend/src/stocks/poller.ts

@@ -3,8 +3,9 @@ import { logger } from '../storage/db/logs.js';
 import { fetchQuotes } from './client.js';
 
 // Called on a schedule (see queue/scheduler.ts) and immediately after the admin adds a
-// ticker (see api/admin.ts) — one batched Stooq request for every configured ticker. A
+// ticker (see api/admin.ts) — one request per configured ticker (see client.ts for why
-// symbol Stooq can't resolve gets its own lastError, it never aborts the whole batch.
+// there's no batch endpoint here). A symbol Yahoo can't resolve gets its own lastError,
+// it never aborts the rest of the batch.
 export async function pollStocksNow(): Promise<void> {
 	const tickers = stocksDb.listStockTickers();
 	if (tickers.length === 0) return;

backend/src/storage/db/index.ts

@@ -308,9 +308,9 @@ export function migrate() {
 	const tickerCount = db.prepare('SELECT COUNT(*) as c FROM stock_tickers').get() as { c: number };
 	if (tickerCount.c === 0) {
 		const defaults: [string, string][] = [
-			['Dow Jones', '^dji'],
+			['Dow Jones', '^DJI'],
-			['S&P 500', '^spx'],
+			['S&P 500', '^GSPC'],
-			['Bitcoin', 'btcusd']
+			['Bitcoin', 'BTC-USD']
 		];
 		const stmt = db.prepare(
 			'INSERT INTO stock_tickers (id, label, symbol, priority_rank, created_at) VALUES (?, ?, ?, ?, ?)'
@@ -320,6 +320,19 @@ export function migrate() {
 		});
 	}
 
+	// Stocks switched data providers from Stooq (walled off behind a proof-of-work
+	// challenge) to Yahoo Finance, which uses different symbol syntax — rewrites only
+	// rows still holding exactly one of the three old Stooq-format default symbols we
+	// ourselves seeded, never touching a symbol the admin typed in themselves.
+	const stooqToYahooSymbols: [string, string][] = [
+		['^dji', '^DJI'],
+		['^spx', '^GSPC'],
+		['btcusd', 'BTC-USD']
+	];
+	for (const [oldSymbol, newSymbol] of stooqToYahooSymbols) {
+		db.prepare('UPDATE stock_tickers SET symbol = ? WHERE symbol = ?').run(newSymbol, oldSymbol);
+	}
+
 	// Seed default categories if none exist yet. "News" sits right under "Top stories" —
 	// general news sources belong here, not on "Top stories" itself, which isn't a real
 	// filterable tag: it's the homepage view, now scoped to only the articles whose

frontend/src/lib/components/admin/StocksTab.svelte

@@ -49,16 +49,16 @@
 	<div class="add-panel">
 		<div class="add-grid">
 			<input placeholder="Label, e.g. Apple" bind:value={newTicker.label} />
-			<input placeholder="Stooq symbol, e.g. aapl.us" bind:value={newTicker.symbol} />
+			<input placeholder="Yahoo symbol, e.g. AAPL" bind:value={newTicker.symbol} />
 		</div>
 		<div class="add-actions">
 			<button onclick={() => (showAdd = false)}>Cancel</button>
 			<button class="primary" onclick={handleAdd}>Create</button>
 		</div>
 		<p class="hint">
-			Stooq has no symbol search, so type the exact syntax: indices use a caret (^dji, ^spx),
+			Yahoo Finance has no symbol search, so type the exact syntax: stocks are plain tickers
-			stocks use a country suffix (aapl.us), crypto pairs have none (btcusd). Polled every 15
+			(AAPL), indices use a caret (^DJI, ^GSPC), crypto pairs use a dash (BTC-USD). Polled
-			minutes; a new ticker is polled immediately.
+			every 15 minutes; a new ticker is polled immediately.
 		</p>
 	</div>
 {/if}