#!/usr/bin/env python3 """Generate sr_violentmonkey_refresh.user.js from sr_refresh_progress.json. Embeds the full SKU/tier queue and the current done_idx (as the seed default) directly into the userscript, so a fresh install already knows what's done. Re-run this any time you want to re-seed the userscript with newer DB-side progress (e.g. after ingesting a batch of downloaded results) - reinstalling just refreshes the *default*, it won't clobber progress already stored in the browser's own GM storage for an existing install. Usage: python3 gen_sr_userscript.py """ import json PROG = "sr_refresh_progress.json" OUT = "sr_violentmonkey_refresh.user.js" with open(PROG) as f: prog = json.load(f) queue = list(zip(prog["skus"], prog["tiers"])) done_idx = sorted(prog["done_idx"]) TEMPLATE = """// ==UserScript== // @name ShopRite Price Refresher (background) // @namespace shoppinglist-sr-refresh // @version 1.0 // @description Paced, backoff-aware background price refresh against storefrontgateway.shoprite.com. Runs while a shoprite.com tab is open. // @match https://www.shoprite.com/* // @grant GM_setValue // @grant GM_getValue // @grant GM_registerMenuCommand // @grant GM_download // @run-at document-idle // ==/UserScript== (function () { 'use strict'; const STORE_ID = '454'; const PACE = 15; // items/min, Tier 1-4 const PACE_TIER5 = 10; // items/min, Tier 5 (non-edible) const BASE_BACKOFF = 300000; // 5 min, ms const MAX_BACKOFF = 3600000; // 1 hr cap const PROBE_PACE = 6; // items/min while backing off const CLEAN_STREAK_TO_TRUST = 6; // [sku, tier] pairs, generated from sr_refresh_progress.json const QUEUE = __QUEUE__; // done_idx as of generation time - seeds a fresh install so it doesn't // redo work already ingested into prices.db const DEFAULT_DONE = __DEFAULT_DONE__; const KEY_DONE = 'sr_done_idx_v1'; const KEY_RESULTS = 'sr_results_v1'; const KEY_RUNNING = 'sr_running_v1'; let done = new Set(GM_getValue(KEY_DONE, DEFAULT_DONE)); let results = GM_getValue(KEY_RESULTS, []); let running = GM_getValue(KEY_RUNNING, true); let mode = 'normal'; let backoff = BASE_BACKOFF; let cleanStreak = 0; let timer = null; let loopAlive = false; // guards against a second concurrent tick chain function saveDone() { GM_setValue(KEY_DONE, Array.from(done)); } function saveResults() { GM_setValue(KEY_RESULTS, results); } function log(msg) { console.log('[sr-refresh] ' + msg); } function nextPendingIdx() { for (let i = 0; i < QUEUE.length; i++) if (!done.has(i)) return i; return -1; } async function fetchOne(sku) { const url = `https://storefrontgateway.shoprite.com/api/stores/${STORE_ID}/products/${sku}`; try { const r = await fetch(url, { headers: { 'X-Site-Host': 'www.shoprite.com' } }); if (r.status !== 200) return { sku, status: r.status }; const j = await r.json(); return { sku, status: 200, price: j.price, wasPrice: j.wasPrice, available: j.available }; } catch (e) { return { sku, status: 'err' }; } } function updateBadge(text) { let el = document.getElementById('sr-refresh-badge'); if (!el) { el = document.createElement('div'); el.id = 'sr-refresh-badge'; el.style.cssText = 'position:fixed;bottom:8px;right:8px;z-index:99999;background:#00274c;' + 'color:#fff;font:11px monospace;padding:4px 8px;border-radius:4px;opacity:0.75;pointer-events:none;'; document.body.appendChild(el); } el.textContent = 'SR refresh: ' + text; } async function tick() { if (!running || !loopAlive) return; const idx = nextPendingIdx(); if (idx === -1) { log('Queue complete.'); updateBadge(`done ${done.size}/${QUEUE.length}`); return; } const [sku, tier] = QUEUE[idx]; const result = await fetchOne(sku); if (!running || !loopAlive) return; // stopped while this fetch was in flight const blocked = result.status === 'err' || result.status === 403 || result.status === 429 || (typeof result.status === 'number' && result.status >= 500); if (blocked) { log(`BLOCKED sku=${sku} status=${result.status} -> backoff ${backoff / 1000}s`); mode = 'backoff'; cleanStreak = 0; updateBadge(`backoff ${backoff / 1000}s (${done.size}/${QUEUE.length})`); const delay = backoff; backoff = Math.min(backoff * 2, MAX_BACKOFF); timer = setTimeout(tick, delay); return; // retry same idx, do not mark done } results.push(result); done.add(idx); saveDone(); saveResults(); if (mode === 'backoff') { cleanStreak++; backoff = BASE_BACKOFF; if (cleanStreak >= CLEAN_STREAK_TO_TRUST) { mode = 'normal'; cleanStreak = 0; log('recovered -> resuming normal pace'); } } const pace = mode === 'backoff' ? PROBE_PACE : (tier === 5 ? PACE_TIER5 : PACE); const delay = 60000 / pace; if (done.size % 25 === 0) { log(`${done.size}/${QUEUE.length} done (${(done.size / QUEUE.length * 100).toFixed(1)}%), ` + `${results.length} buffered, mode=${mode}`); } updateBadge(`${done.size}/${QUEUE.length} (${mode})`); timer = setTimeout(tick, delay); } function start() { if (loopAlive) { log('already running, ignoring duplicate start'); return; } loopAlive = true; running = true; GM_setValue(KEY_RUNNING, true); log('starting/resuming'); tick(); } function stop() { running = false; loopAlive = false; GM_setValue(KEY_RUNNING, false); if (timer) clearTimeout(timer); log('paused'); updateBadge(`paused (${done.size}/${QUEUE.length})`); } function status() { log(`${done.size}/${QUEUE.length} done, ${results.length} buffered results, ` + `running=${running}, mode=${mode}`); } function downloadResults() { if (results.length === 0) { log('no buffered results to download'); return; } const blob = new Blob([JSON.stringify(results)], { type: 'application/json' }); const url = URL.createObjectURL(blob); GM_download({ url, name: `sr_results_${Date.now()}.json`, saveAs: true }); } function clearResults() { if (!confirm(`Clear ${results.length} buffered results? Only do this after downloading and ingesting them.`)) return; results = []; saveResults(); log('buffered results cleared'); } function resetProgress() { if (!confirm('Reset ALL refresh progress (done + buffered results)? This cannot be undone.')) return; done = new Set(); results = []; saveDone(); saveResults(); log('progress reset'); } GM_registerMenuCommand('SR Refresh: Start/Resume', start); GM_registerMenuCommand('SR Refresh: Pause', stop); GM_registerMenuCommand('SR Refresh: Status', status); GM_registerMenuCommand('SR Refresh: Download results JSON', downloadResults); GM_registerMenuCommand('SR Refresh: Clear buffered results (after ingest)', clearResults); GM_registerMenuCommand('SR Refresh: Reset progress', resetProgress); log(`loaded. ${done.size}/${QUEUE.length} done, ${results.length} buffered results, running=${running}.`); updateBadge(`${done.size}/${QUEUE.length}`); if (running) start(); })(); """ out = TEMPLATE.replace("__QUEUE__", json.dumps(queue, separators=(",", ":"))) out = out.replace("__DEFAULT_DONE__", json.dumps(done_idx, separators=(",", ":"))) with open(OUT, "w") as f: f.write(out) print(f"Wrote {OUT} ({len(out)/1024:.0f} KB) - {len(queue)} SKUs, {len(done_idx)} pre-marked done.")