shoppinglistgenerator/refresh_shoprite.py

#!/usr/bin/env python3
"""Refresh ShopRite prices directly against storefrontgateway.shoprite.com.

Run straight from your own machine - no browser/console needed. This only
works from an IP that hasn't been rate-limited by ShopRite (your machine,
not the sandboxed dev environment this project was built in).

Sequential requests only (never concurrent) - bursts trigger a block even
at a low sustained average rate. Tier-aware pacing: moderate for Tiers 1-4
(meat/dairy/bread/other-edible), conservative once Tier 5 (non-edible) is
reached. Adaptive backoff on any error/block, resuming once probes come
back clean.

Usage:
    python3 refresh_shoprite.py [--pace 15] [--pace-tier5 10]
"""
import argparse
import json
import re
import sqlite3
import sys
import time
from datetime import datetime, timezone

import requests

PROG = "sr_refresh_progress.json"
DB = "prices.db"
STORE_ID = "454"

BASE_BACKOFF = 300     # 5 min initial backoff on a block
MAX_BACKOFF = 3600     # 1 hour cap
PROBE_PACE = 6         # items/min while backing off (1 every 10s)
CLEAN_STREAK_TO_TRUST = 6

TIER_NAMES = {1: "Tier 1 (Meat/Seafood)", 2: "Tier 2 (Dairy)", 3: "Tier 3 (Bread & Bakery)",
              4: "Tier 4 (Other Edible)", 5: "Tier 5 (Non-Edible)"}

HEADERS = {"X-Site-Host": "www.shoprite.com"}


def parse_price(s):
    if not s:
        return None
    m = re.search(r"\$([\d.]+)", s)
    return float(m.group(1)) if m else None


def load_progress():
    with open(PROG) as f:
        return json.load(f)


def save_progress(prog):
    with open(PROG, "w") as f:
        json.dump(prog, f)


def dept_cat_lookup(con):
    rows = con.execute("SELECT itemno, department, category FROM products WHERE vendor='shoprite'").fetchall()
    return {itemno: (dept or "Unknown", cat or "") for itemno, dept, cat in rows}


def fetch_one(session, sku):
    url = f"https://storefrontgateway.shoprite.com/api/stores/{STORE_ID}/products/{sku}"
    try:
        r = session.get(url, headers=HEADERS, timeout=15)
        if r.status_code != 200:
            return {"sku": sku, "status": r.status_code}
        j = r.json()
        return {"sku": sku, "status": 200, "price": j.get("price"),
                 "wasPrice": j.get("wasPrice"), "available": j.get("available")}
    except requests.RequestException:
        return {"sku": sku, "status": "err"}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--pace", type=float, default=15, help="items/min for Tier 1-4 (default 15)")
    ap.add_argument("--pace-tier5", type=float, default=10, help="items/min for Tier 5 (default 10)")
    args = ap.parse_args()

    prog = load_progress()
    skus, tiers = prog["skus"], prog["tiers"]
    done = set(prog["done_idx"])
    pending_idx = [i for i in range(len(skus)) if i not in done]

    if not pending_idx:
        print("Nothing pending - queue already complete.")
        return

    con = sqlite3.connect(DB)
    labels = dept_cat_lookup(con)

    session = requests.Session()
    mode = "normal"
    backoff = BASE_BACKOFF
    clean_streak = 0
    total = len(skus)
    done_before = len(done)
    recent_ts = []

    print(f"[refresh_shoprite] {len(pending_idx)} pending of {total} total ({done_before} already done). "
          f"Pace: {args.pace}/min (Tier1-4), {args.pace_tier5}/min (Tier5).")

    i = 0
    try:
        while i < len(pending_idx):
            idx = pending_idx[i]
            sku = skus[idx]
            tier = tiers[idx]
            dept, cat = labels.get(sku, ("Unknown", ""))
            label = f"{dept} > {cat}" if cat else dept

            if mode == "backoff":
                pace = PROBE_PACE
            else:
                pace = args.pace_tier5 if tier == 5 else args.pace
            spacing = 60.0 / pace

            result = fetch_one(session, sku)
            now = time.time()
            recent_ts.append(now)
            recent_ts = [t for t in recent_ts if now - t < 60]

            blocked_codes = {403, 429}
            is_blocked = (
                result["status"] == "err"
                or (isinstance(result["status"], int) and result["status"] >= 500)
                or result["status"] in blocked_codes
            )
            if is_blocked:
                print(f"  !! BLOCKED/ERROR sku={sku} status={result['status']} -> backoff {backoff}s")
                mode = "backoff"
                clean_streak = 0
                time.sleep(backoff)
                backoff = min(backoff * 2, MAX_BACKOFF)
                continue  # retry same idx (do not advance i)

            fetched_at = datetime.now(timezone.utc).isoformat()
            if result["status"] == 200:
                price = parse_price(result.get("price"))
                was_price = parse_price(result.get("wasPrice"))
                con.execute(
                    "UPDATE products SET price=?, priceBeforeDiscount=?, is_instock=?, fetched_at=? "
                    "WHERE vendor='shoprite' AND itemno=?",
                    (price, was_price if was_price != price else None,
                     1 if result.get("available") else 0, fetched_at, sku),
                )
                con.commit()
                price_str = result.get("price") or "no price"
            elif result["status"] == 404:
                price_str = "not found"
            else:
                price_str = f"status:{result['status']}"

            done.add(idx)
            i += 1

            if mode == "backoff":
                clean_streak += 1
                backoff = BASE_BACKOFF
                if clean_streak >= CLEAN_STREAK_TO_TRUST:
                    mode = "normal"
                    clean_streak = 0
                    print("  ++ recovered -> resuming normal pace")

            done_total = done_before + i
            pct = done_total / total * 100
            print(f"[{done_total}/{total}] ({pct:.2f}%) {TIER_NAMES.get(tier, tier)} | {label} | "
                  f"{sku} | {price_str} | pace: {len(recent_ts)}/min | mode: {mode}")

            if i % 50 == 0:
                prog["done_idx"] = sorted(done)
                save_progress(prog)

            time.sleep(spacing)
    except KeyboardInterrupt:
        print("\nInterrupted - saving progress...")
    finally:
        prog["done_idx"] = sorted(done)
        save_progress(prog)
        con.close()
        print(f"Progress saved. Done: {len(done)}/{total}.")


if __name__ == "__main__":
    main()