218 lines
7.5 KiB
Python
218 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate sr_violentmonkey_refresh.user.js from sr_refresh_progress.json.
|
|
|
|
Embeds the full SKU/tier queue and the current done_idx (as the seed default)
|
|
directly into the userscript, so a fresh install already knows what's done.
|
|
Re-run this any time you want to re-seed the userscript with newer DB-side
|
|
progress (e.g. after ingesting a batch of downloaded results) - reinstalling
|
|
just refreshes the *default*, it won't clobber progress already stored in
|
|
the browser's own GM storage for an existing install.
|
|
|
|
Usage:
|
|
python3 gen_sr_userscript.py
|
|
"""
|
|
import json
|
|
|
|
PROG = "sr_refresh_progress.json"
|
|
OUT = "sr_violentmonkey_refresh.user.js"
|
|
|
|
with open(PROG) as f:
|
|
prog = json.load(f)
|
|
|
|
queue = list(zip(prog["skus"], prog["tiers"]))
|
|
done_idx = sorted(prog["done_idx"])
|
|
|
|
TEMPLATE = """// ==UserScript==
|
|
// @name ShopRite Price Refresher (background)
|
|
// @namespace shoppinglist-sr-refresh
|
|
// @version 1.0
|
|
// @description Paced, backoff-aware background price refresh against storefrontgateway.shoprite.com. Runs while a shoprite.com tab is open.
|
|
// @match https://www.shoprite.com/*
|
|
// @grant GM_setValue
|
|
// @grant GM_getValue
|
|
// @grant GM_registerMenuCommand
|
|
// @grant GM_download
|
|
// @run-at document-idle
|
|
// ==/UserScript==
|
|
|
|
(function () {
|
|
'use strict';
|
|
|
|
const STORE_ID = '454';
|
|
const PACE = 15; // items/min, Tier 1-4
|
|
const PACE_TIER5 = 10; // items/min, Tier 5 (non-edible)
|
|
const BASE_BACKOFF = 300000; // 5 min, ms
|
|
const MAX_BACKOFF = 3600000; // 1 hr cap
|
|
const PROBE_PACE = 6; // items/min while backing off
|
|
const CLEAN_STREAK_TO_TRUST = 6;
|
|
|
|
// [sku, tier] pairs, generated from sr_refresh_progress.json
|
|
const QUEUE = __QUEUE__;
|
|
|
|
// done_idx as of generation time - seeds a fresh install so it doesn't
|
|
// redo work already ingested into prices.db
|
|
const DEFAULT_DONE = __DEFAULT_DONE__;
|
|
|
|
const KEY_DONE = 'sr_done_idx_v1';
|
|
const KEY_RESULTS = 'sr_results_v1';
|
|
const KEY_RUNNING = 'sr_running_v1';
|
|
|
|
let done = new Set(GM_getValue(KEY_DONE, DEFAULT_DONE));
|
|
let results = GM_getValue(KEY_RESULTS, []);
|
|
let running = GM_getValue(KEY_RUNNING, true);
|
|
let mode = 'normal';
|
|
let backoff = BASE_BACKOFF;
|
|
let cleanStreak = 0;
|
|
let timer = null;
|
|
let loopAlive = false; // guards against a second concurrent tick chain
|
|
|
|
function saveDone() { GM_setValue(KEY_DONE, Array.from(done)); }
|
|
function saveResults() { GM_setValue(KEY_RESULTS, results); }
|
|
function log(msg) { console.log('[sr-refresh] ' + msg); }
|
|
|
|
function nextPendingIdx() {
|
|
for (let i = 0; i < QUEUE.length; i++) if (!done.has(i)) return i;
|
|
return -1;
|
|
}
|
|
|
|
async function fetchOne(sku) {
|
|
const url = `https://storefrontgateway.shoprite.com/api/stores/${STORE_ID}/products/${sku}`;
|
|
try {
|
|
const r = await fetch(url, { headers: { 'X-Site-Host': 'www.shoprite.com' } });
|
|
if (r.status !== 200) return { sku, status: r.status };
|
|
const j = await r.json();
|
|
return { sku, status: 200, price: j.price, wasPrice: j.wasPrice, available: j.available };
|
|
} catch (e) {
|
|
return { sku, status: 'err' };
|
|
}
|
|
}
|
|
|
|
function updateBadge(text) {
|
|
let el = document.getElementById('sr-refresh-badge');
|
|
if (!el) {
|
|
el = document.createElement('div');
|
|
el.id = 'sr-refresh-badge';
|
|
el.style.cssText = 'position:fixed;bottom:8px;right:8px;z-index:99999;background:#00274c;' +
|
|
'color:#fff;font:11px monospace;padding:4px 8px;border-radius:4px;opacity:0.75;pointer-events:none;';
|
|
document.body.appendChild(el);
|
|
}
|
|
el.textContent = 'SR refresh: ' + text;
|
|
}
|
|
|
|
async function tick() {
|
|
if (!running || !loopAlive) return;
|
|
const idx = nextPendingIdx();
|
|
if (idx === -1) {
|
|
log('Queue complete.');
|
|
updateBadge(`done ${done.size}/${QUEUE.length}`);
|
|
return;
|
|
}
|
|
const [sku, tier] = QUEUE[idx];
|
|
const result = await fetchOne(sku);
|
|
if (!running || !loopAlive) return; // stopped while this fetch was in flight
|
|
|
|
const blocked = result.status === 'err' || result.status === 403 || result.status === 429 ||
|
|
(typeof result.status === 'number' && result.status >= 500);
|
|
|
|
if (blocked) {
|
|
log(`BLOCKED sku=${sku} status=${result.status} -> backoff ${backoff / 1000}s`);
|
|
mode = 'backoff';
|
|
cleanStreak = 0;
|
|
updateBadge(`backoff ${backoff / 1000}s (${done.size}/${QUEUE.length})`);
|
|
const delay = backoff;
|
|
backoff = Math.min(backoff * 2, MAX_BACKOFF);
|
|
timer = setTimeout(tick, delay);
|
|
return; // retry same idx, do not mark done
|
|
}
|
|
|
|
results.push(result);
|
|
done.add(idx);
|
|
saveDone();
|
|
saveResults();
|
|
|
|
if (mode === 'backoff') {
|
|
cleanStreak++;
|
|
backoff = BASE_BACKOFF;
|
|
if (cleanStreak >= CLEAN_STREAK_TO_TRUST) {
|
|
mode = 'normal';
|
|
cleanStreak = 0;
|
|
log('recovered -> resuming normal pace');
|
|
}
|
|
}
|
|
|
|
const pace = mode === 'backoff' ? PROBE_PACE : (tier === 5 ? PACE_TIER5 : PACE);
|
|
const delay = 60000 / pace;
|
|
|
|
if (done.size % 25 === 0) {
|
|
log(`${done.size}/${QUEUE.length} done (${(done.size / QUEUE.length * 100).toFixed(1)}%), ` +
|
|
`${results.length} buffered, mode=${mode}`);
|
|
}
|
|
updateBadge(`${done.size}/${QUEUE.length} (${mode})`);
|
|
timer = setTimeout(tick, delay);
|
|
}
|
|
|
|
function start() {
|
|
if (loopAlive) {
|
|
log('already running, ignoring duplicate start');
|
|
return;
|
|
}
|
|
loopAlive = true;
|
|
running = true;
|
|
GM_setValue(KEY_RUNNING, true);
|
|
log('starting/resuming');
|
|
tick();
|
|
}
|
|
function stop() {
|
|
running = false;
|
|
loopAlive = false;
|
|
GM_setValue(KEY_RUNNING, false);
|
|
if (timer) clearTimeout(timer);
|
|
log('paused');
|
|
updateBadge(`paused (${done.size}/${QUEUE.length})`);
|
|
}
|
|
function status() {
|
|
log(`${done.size}/${QUEUE.length} done, ${results.length} buffered results, ` +
|
|
`running=${running}, mode=${mode}`);
|
|
}
|
|
function downloadResults() {
|
|
if (results.length === 0) { log('no buffered results to download'); return; }
|
|
const blob = new Blob([JSON.stringify(results)], { type: 'application/json' });
|
|
const url = URL.createObjectURL(blob);
|
|
GM_download({ url, name: `sr_results_${Date.now()}.json`, saveAs: true });
|
|
}
|
|
function clearResults() {
|
|
if (!confirm(`Clear ${results.length} buffered results? Only do this after downloading and ingesting them.`)) return;
|
|
results = [];
|
|
saveResults();
|
|
log('buffered results cleared');
|
|
}
|
|
function resetProgress() {
|
|
if (!confirm('Reset ALL refresh progress (done + buffered results)? This cannot be undone.')) return;
|
|
done = new Set();
|
|
results = [];
|
|
saveDone();
|
|
saveResults();
|
|
log('progress reset');
|
|
}
|
|
|
|
GM_registerMenuCommand('SR Refresh: Start/Resume', start);
|
|
GM_registerMenuCommand('SR Refresh: Pause', stop);
|
|
GM_registerMenuCommand('SR Refresh: Status', status);
|
|
GM_registerMenuCommand('SR Refresh: Download results JSON', downloadResults);
|
|
GM_registerMenuCommand('SR Refresh: Clear buffered results (after ingest)', clearResults);
|
|
GM_registerMenuCommand('SR Refresh: Reset progress', resetProgress);
|
|
|
|
log(`loaded. ${done.size}/${QUEUE.length} done, ${results.length} buffered results, running=${running}.`);
|
|
updateBadge(`${done.size}/${QUEUE.length}`);
|
|
if (running) start();
|
|
})();
|
|
"""
|
|
|
|
out = TEMPLATE.replace("__QUEUE__", json.dumps(queue, separators=(",", ":")))
|
|
out = out.replace("__DEFAULT_DONE__", json.dumps(done_idx, separators=(",", ":")))
|
|
|
|
with open(OUT, "w") as f:
|
|
f.write(out)
|
|
|
|
print(f"Wrote {OUT} ({len(out)/1024:.0f} KB) - {len(queue)} SKUs, {len(done_idx)} pre-marked done.")
|