feat: Added COOP and new SKU types

src/stores/coop.js

@@ -0,0 +1,355 @@
+"use strict";
+
+const { normalizeCspc } = require("../utils/sku");
+const { humanBytes } = require("../utils/bytes");
+const { padLeft, padRight } = require("../utils/string");
+
+const { mergeDiscoveredIntoDb } = require("../tracker/merge");
+const { buildDbObject, writeJsonAtomic } = require("../tracker/db");
+const { addCategoryResultToReport } = require("../tracker/report");
+
+/* ---------------- formatting ---------------- */
+
+function kbStr(bytes) {
+  return humanBytes(bytes).padStart(8, " ");
+}
+function secStr(ms) {
+  const s = Number.isFinite(ms) ? ms / 1000 : 0;
+  const t = Math.round(s * 10) / 10;
+  return (t < 10 ? `${t.toFixed(1)}s` : `${Math.round(s)}s`).padStart(7, " ");
+}
+function pageStr(i, total) {
+  const w = String(total).length;
+  return `${padLeft(i, w)}/${total}`;
+}
+function pctStr(done, total) {
+  const pct = total ? Math.floor((done / total) * 100) : 0;
+  return `${padLeft(pct, 3)}%`;
+}
+
+/* ---------------- co-op specifics ---------------- */
+
+const BASE = "https://shoponlinewhisky-wine.coopwinespiritsbeer.com";
+const REFERER = `${BASE}/worldofwhisky`;
+
+function coopHeaders(ctx, sourcepage) {
+  const coop = ctx.store.coop;
+  return {
+    Accept: "application/json, text/javascript, */*; q=0.01",
+    "Content-Type": "application/json",
+    Origin: BASE,
+    Referer: REFERER,
+
+    // these 4 are required on their API calls (matches browser)
+    SessionKey: coop.sessionKey,
+    chainID: coop.chainId,
+    storeID: coop.storeId,
+    appVersion: coop.appVersion,
+
+    AUTH_TOKEN: "null",
+    CONNECTION_ID: "null",
+    SESSION_ID: coop.sessionId || "null",
+    TIMESTAMP: String(Date.now()),
+    sourcepage,
+  };
+}
+
+async function fetchText(url, { headers, ua } = {}) {
+    const h = { ...(headers || {}) };
+    if (ua) h["User-Agent"] = ua;
+    const res = await fetch(url, { method: "GET", headers: h });
+    const text = await res.text();
+    return { status: res.status, text };
+  }
+  
+  function extractVar(html, re) {
+    const m = String(html || "").match(re);
+    return m ? String(m[1] || "").trim() : "";
+  }
+  
+  async function ensureCoopBootstrap(ctx) {
+    const coop = ctx.store.coop;
+    if (coop.sessionKey && coop.chainId && coop.storeId && coop.appVersion) return;
+  
+    const r = await fetchText(REFERER, {
+      ua: ctx.store.ua,
+      headers: {
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        Referer: REFERER,
+      },
+    });
+  
+    if (r.status !== 200 || !r.text) {
+      throw new Error(`coop bootstrap failed: GET ${REFERER} => ${r.status}`);
+    }
+  
+    // Values are in <script> var SESSIONKEY = "..."; etc.
+    coop.sessionKey = extractVar(r.text, /var\s+SESSIONKEY\s*=\s*"([^"]+)"/i);
+    coop.chainId = extractVar(r.text, /var\s+chainID\s*=\s*"([^"]+)"/i);
+    coop.storeId = extractVar(r.text, /var\s+store_unique_id\s*=\s*"([^"]+)"/i);
+    coop.appVersion = extractVar(r.text, /var\s+CLIENTVERSION\s*=\s*"([^"]+)"/i);
+  
+    if (!coop.sessionKey || !coop.chainId || !coop.storeId || !coop.appVersion) {
+      throw new Error(
+        `coop bootstrap missing values: sessionKey=${!!coop.sessionKey} chainId=${!!coop.chainId} storeId=${!!coop.storeId} appVersion=${!!coop.appVersion}`
+      );
+    }
+  }
+  
+
+async function ensureCoopSession(ctx) {
+  const coop = ctx.store.coop;
+  if (coop.sessionId) return;
+  await ensureCoopBootstrap(ctx);
+
+  const r = await ctx.http.fetchJsonWithRetry(
+    `${BASE}/api/account/createsession`,
+    `coop:createsession`,
+    ctx.store.ua,
+    {
+      method: "POST",
+      headers: coopHeaders(ctx, "/worldofwhisky"),
+      // browser sends Content-Length: 0; easiest equivalent:
+      body: "",
+    }
+  );
+
+  const sid =
+    r?.json?.SessionID ||
+    r?.json?.sessionID ||
+    r?.json?.sessionId ||
+    r?.json?.SessionId ||
+    "";
+
+  if (!sid) {
+    throw new Error(
+      `createSession: missing SessionID (status=${r?.status})`
+    );
+  }
+
+  coop.sessionId = sid;
+  coop.anonymousUserId = r?.json?.AnonymousUserID ?? null;
+}
+
+function normalizeAbsUrl(raw) {
+  const s = String(raw || "").trim();
+  if (!s) return "";
+  if (s.startsWith("//")) return `https:${s}`;
+  if (/^https?:\/\//i.test(s)) return s;
+  try {
+    return new URL(s, `${BASE}/`).toString();
+  } catch {
+    return s;
+  }
+}
+
+function productUrlFromId(productId) {
+  return `${REFERER}#/product/${encodeURIComponent(String(productId))}`;
+}
+
+function productFromApi(p) {
+    if (!p || p.IsActive === false) return null;
+  
+    const name = String(p.Name || "").trim();
+    if (!name) return null;
+  
+    const productId = p.ProductID;
+    if (!productId) return null;
+  
+    const url = productUrlFromId(productId);
+  
+    const price =
+      p?.CountDetails?.PriceText ||
+      (Number.isFinite(p?.Price) ? `$${Number(p.Price).toFixed(2)}` : "");
+  
+    const upc = String(p.UPC || "").trim();
+    const sku =
+      upc ||                      // use UPC if present
+      String(p.ProductStoreID);  // fallback to store-specific ID
+  
+    const img = normalizeAbsUrl(p.ImageURL);
+  
+    return {
+      name,
+      price,
+      url,
+      sku,
+      upc,
+      productId,
+      productStoreId: p.ProductStoreID || null,
+      img,
+    };
+  }
+  
+/* ---------------- scanner ---------------- */
+
+async function fetchCategoryPage(ctx, categoryId, page) {
+  await ensureCoopSession(ctx);
+
+  const doReq = () =>
+    ctx.http.fetchJsonWithRetry(
+      `${BASE}/api/v2/products/category/${categoryId}`,
+      `coop:${ctx.cat.key}:p${page}`,
+      ctx.store.ua,
+      {
+        method: "POST",
+        headers: coopHeaders(ctx, `/category/${ctx.cat.coopSlug}`),
+        body: JSON.stringify({
+          page,
+          Filters: {
+            Filters: [],
+            LastSelectedFilter: null,
+            SearchWithinTerm: null,
+          },
+          orderby: null,
+        }),
+      }
+    );
+
+  let r = await doReq();
+
+  // one fast retry on invalid_session: refresh SessionID and repeat
+  if (r?.json?.type === "invalid_session") {
+    ctx.store.coop.sessionId = "";
+    await ensureCoopSession(ctx);
+    r = await doReq();
+  }
+
+  return r;
+}
+
+function avoidMassRemoval(prevDb, discovered, ctx) {
+  const prev = prevDb?.size || 0;
+  const curr = discovered.size;
+  if (!prev || !curr) return;
+  if (curr / prev >= 0.6) return;
+
+  ctx.logger.warn(
+    `${ctx.catPrefixOut} | Partial scan (${curr}/${prev}); preserving DB`
+  );
+
+  for (const [k, v] of prevDb.entries()) {
+    if (!discovered.has(k)) discovered.set(k, v);
+  }
+}
+
+async function scanCategoryCoop(ctx, prevDb, report) {
+  const t0 = Date.now();
+  const discovered = new Map();
+
+  const maxPages =
+    ctx.config.maxPages === null ? 500 : Math.min(ctx.config.maxPages, 500);
+
+  let done = 0;
+
+  for (let page = 1; page <= maxPages; page++) {
+    let r;
+    try {
+      r = await fetchCategoryPage(ctx, ctx.cat.coopCategoryId, page);
+    } catch (e) {
+      ctx.logger.warn(
+        `${ctx.catPrefixOut} | page ${page} failed: ${e?.message || e}`
+      );
+      break;
+    }
+
+    const arr = Array.isArray(r?.json?.Products?.Result)
+      ? r.json.Products.Result
+      : [];
+
+    done++;
+
+    let kept = 0;
+    for (const p of arr) {
+      const it = productFromApi(p);
+      if (!it) continue;
+      discovered.set(it.url, it);
+      kept++;
+    }
+
+    ctx.logger.ok(
+      `${ctx.catPrefixOut} | Page ${pageStr(done, done)} | ${String(
+        r.status || ""
+      ).padEnd(3)} | items=${padLeft(kept, 3)} | bytes=${kbStr(
+        r.bytes
+      )} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
+    );
+
+    if (!arr.length) break;
+  }
+
+  if (prevDb) avoidMassRemoval(prevDb, discovered, ctx);
+
+  ctx.logger.ok(`${ctx.catPrefixOut} | Unique products: ${discovered.size}`);
+
+  const { merged, newItems, updatedItems, removedItems, restoredItems } =
+    mergeDiscoveredIntoDb(prevDb, discovered);
+
+  const dbObj = buildDbObject(ctx, merged);
+  writeJsonAtomic(ctx.dbFile, dbObj);
+
+  const elapsed = Date.now() - t0;
+
+  report.categories.push({
+    store: ctx.store.name,
+    label: ctx.cat.label,
+    key: ctx.cat.key,
+    dbFile: ctx.dbFile,
+    scannedPages: done,
+    discoveredUnique: discovered.size,
+    newCount: newItems.length,
+    updatedCount: updatedItems.length,
+    removedCount: removedItems.length,
+    restoredCount: restoredItems.length,
+    elapsedMs: elapsed,
+  });
+
+  report.totals.newCount += newItems.length;
+  report.totals.updatedCount += updatedItems.length;
+  report.totals.removedCount += removedItems.length;
+  report.totals.restoredCount += restoredItems.length;
+  
+  addCategoryResultToReport(
+    report,
+    ctx.store.name,
+    ctx.cat.label,
+    newItems,
+    updatedItems,
+    removedItems,
+    restoredItems
+  );
+}
+
+/* ---------------- store ---------------- */
+
+function createStore(defaultUa) {
+  return {
+    key: "coop",
+    name: "Co-op World of Whisky",
+    host: "shoponlinewhisky-wine.coopwinespiritsbeer.com",
+    ua: defaultUa,
+    scanCategory: scanCategoryCoop,
+
+    // put your captured values here (or pull from env)
+    coop: {
+        sessionKey: "",
+        chainId: "",
+        storeId: "",
+        appVersion: "",
+      sessionId: "", // set by ensureCoopSession()
+      anonymousUserId: null,
+    },
+
+    categories: [
+    //   { key: "canadian-whisky", label: "Canadian Whisky", coopSlug: "canadian_whisky", coopCategoryId: 4, startUrl: `${REFERER}#/category/canadian_whisky` },
+    //   { key: "bourbon-whiskey", label: "Bourbon Whiskey", coopSlug: "bourbon_whiskey", coopCategoryId: 9, startUrl: `${REFERER}#/category/bourbon_whiskey` },
+    //   { key: "scottish-single-malts", label: "Scottish Single Malts", coopSlug: "scottish_single_malts", coopCategoryId: 6, startUrl: `${REFERER}#/category/scottish_single_malts` },
+    //   { key: "scottish-blends", label: "Scottish Whisky Blends", coopSlug: "scottish_whisky_blends", coopCategoryId: 5, startUrl: `${REFERER}#/category/scottish_whisky_blends` },
+    //   { key: "american-whiskey", label: "American Whiskey", coopSlug: "american_whiskey", coopCategoryId: 8, startUrl: `${REFERER}#/category/american_whiskey` },
+    //   { key: "world-whisky", label: "World / International Whisky", coopSlug: "world_international", coopCategoryId: 10, startUrl: `${REFERER}#/category/world_international` },
+      { key: "rum", label: "Rum", coopSlug: "spirits_rum", coopCategoryId: 24, startUrl: `${REFERER}#/category/spirits_rum` },
+    ],
+  };
+}
+
+module.exports = { createStore };

src/stores/index.js

@@ -10,6 +10,7 @@ const { createStore: createBCL } = require("./bcl");
 const { createStore: createStrath } = require("./strath");
 const { createStore: createLegacy } = require("./legacyliquor");
 const { createStore: createGull } = require("./gull");
+const { createStore: createCoop } = require("./coop");
 
 function createStores({ defaultUa } = {}) {
   return [
@@ -19,6 +20,7 @@ function createStores({ defaultUa } = {}) {
     createCraftCellars(defaultUa),
     createStrath(defaultUa),
     createBSW(defaultUa),
+    createCoop(defaultUa),
     createKegNCork(defaultUa),
     createMaltsAndGrains(defaultUa),
     createBCL(defaultUa),

src/tracker/merge.js

@@ -15,6 +15,13 @@ function isRealSku(v) {
   return Boolean(normalizeCspc(v));
 }
 
+function normalizeSkuPreserve(raw) {
+  const s = String(raw || "").trim();
+  const c = normalizeCspc(s);
+  return c || s; // CSPC if present, else keep UPC/ProductStoreID/etc
+}
+  
+
 function mergeDiscoveredIntoDb(prevDb, discovered) {
   const merged = new Map(prevDb.byUrl);
 
@@ -101,7 +108,7 @@ function mergeDiscoveredIntoDb(prevDb, discovered) {
     if (!prev) {
       const now = {
         ...nowRaw,
-        sku: normalizeCspc(nowRaw.sku),
+        sku: normalizeSkuPreserve(nowRaw.sku),
         img: normImg(nowRaw.img),
         removed: false,
       };
@@ -114,7 +121,7 @@ function mergeDiscoveredIntoDb(prevDb, discovered) {
     if (prevUrlForThisItem === url && prev.removed) {
       const now = {
         ...nowRaw,
-        sku: normalizeCspc(nowRaw.sku) || normalizeCspc(prev.sku),
+        sku: normalizeSkuPreserve(nowRaw.sku) || normalizeSkuPreserve(prev.sku),
         img: normImg(nowRaw.img) || normImg(prev.img),
         removed: false,
       };
@@ -132,8 +139,8 @@ function mergeDiscoveredIntoDb(prevDb, discovered) {
     const prevPrice = normPrice(prev.price);
     const nowPrice = normPrice(nowRaw.price);
 
-    const prevSku = normalizeCspc(prev.sku);
-    const nowSku = normalizeCspc(nowRaw.sku) || prevSku;
+    const prevSku = normalizeSkuPreserve(prev.sku);
+    const nowSku = normalizeSkuPreserve(nowRaw.sku) || prevSku;
 
     const prevImg = normImg(prev.img);
     let nowImg = normImg(nowRaw.img);

src/utils/sku.js

@@ -15,6 +15,18 @@ function fnv1a32(str) {
   return (h >>> 0).toString(16).padStart(8, "0");
 }
 
+function normalizeUpc(v) {
+  const m = String(v ?? "").match(/\b(\d{12,14})\b/);
+  return m ? m[1] : "";
+}
+
+// Other stable-ish numeric IDs (e.g. ProductStoreID), keep bounded
+function normalizeNumericId(v) {
+  const m = String(v ?? "").match(/\b(\d{4,11})\b/);
+  return m ? m[1] : "";
+}
+  
+
 function makeSyntheticSkuKey({ storeLabel, url }) {
   const store = String(storeLabel || "store").trim().toLowerCase();
   let u = String(url || "").trim();
@@ -51,6 +63,8 @@ function makeSyntheticSkuKey({ storeLabel, url }) {
 /**
  * For DB + comparisons:
  * - If we can extract a real 6-digit SKU, use it.
+ * - Else if UPC-ish digits exist, store as upc:<digits> (low priority but stable)
+ * - Else if other numeric id exists, store as id:<digits>
  * - Else if v already looks like u:xxxx, keep it.
  * - Else if sku missing, generate u:hash(store|url) if possible.
  */
@@ -59,10 +73,18 @@ function normalizeSkuKey(v, { storeLabel, url } = {}) {
   const cspc = normalizeCspc(raw);
   if (cspc) return cspc;
 
+  const upc = normalizeUpc(raw);
+  if (upc) return `upc:${upc}`;
+
+  const nid = normalizeNumericId(raw);
+  if (nid) return `id:${nid}`;
+  
+  
+
   if (raw.startsWith("u:")) return raw;
 
   const syn = makeSyntheticSkuKey({ storeLabel, url });
   return syn || "";
 }
 
-module.exports = { normalizeCspc, normalizeSkuKey, makeSyntheticSkuKey };
+module.exports = { normalizeCspc, normalizeUpc, normalizeSkuKey, makeSyntheticSkuKey };

src/utils/sku_map.js

@@ -55,6 +55,12 @@ function isNumericSku(k) {
   return /^\d+$/.test(String(k || "").trim());
 }
 
+function isUpcSku(k) {
+  // UPC-A/EAN/GTIN-ish (most common: 12 or 13; sometimes 14)
+  return /^\d{12,14}$/.test(String(k || "").trim());
+}
+  
+
 function compareSku(a, b) {
   a = String(a || "").trim();
   b = String(b || "").trim();
@@ -64,6 +70,12 @@ function compareSku(a, b) {
   const bu = isUnknownSkuKey(b);
   if (au !== bu) return au ? 1 : -1; // real first
 
+
+  const aUpc = isUpcSku(a);
+  const bUpc = isUpcSku(b);
+  if (aUpc !== bUpc) return aUpc ? 1 : -1; // UPCs after other "real" keys
+  
+
   const an = isNumericSku(a);
   const bn = isNumericSku(b);
   if (an && bn) {

viz/app/linker_page.js

@@ -222,9 +222,21 @@ function skuIsBC(allRows, skuKey) {
 /* ---------------- Canonical preference (AB real > other real > BC real > u:) ---------------- */
 
 function isRealSkuKey(skuKey) {
-  return !String(skuKey || "").startsWith("u:");
+  const s = String(skuKey || "").trim();
+  return /^\d{6}$/.test(s); // only CSPC counts as "real"
 }
 
+function isSoftSkuKey(k) {
+  const s = String(k || "");
+  return s.startsWith("upc:") || s.startsWith("id:");
+}
+
+
+function isUpcSkuKey(k) {
+  return /^\d{12,14}$/.test(String(k || "").trim());
+}
+  
+
 function isABStoreLabel(label) {
   const s = String(label || "").toLowerCase();
   return (
@@ -249,7 +261,9 @@ function scoreCanonical(allRows, skuKey) {
   const real = isRealSkuKey(s) ? 1 : 0;
   const ab = skuIsAB(allRows, s) ? 1 : 0;
   const bc = skuIsBC(allRows, s) ? 1 : 0;
-  return real * 100 + ab * 25 - bc * 10 + (real ? 0 : -1000);
+  const upc = isUpcSkuKey(s) ? 1 : 0;
+  const soft = isSoftSkuKey(s) ? 1 : 0;
+  return real * 100 + ab * 25 - bc * 10 - upc * 60  - soft * 60 + (real ? 0 : -1000);
 }
 
 function pickPreferredCanonical(allRows, skuKeys) {

viz/app/sku.js

@@ -26,7 +26,8 @@ export function parsePriceToNumber(v) {
   }
   
   export function displaySku(key) {
-    return String(key || "").startsWith("u:") ? "unknown" : String(key || "");
+    const s = String(key || "");
+    return s.startsWith("u:") ? "unknown" : s;
   }
   
   export function isUnknownSkuKey(key) {