spirit-tracker/tools/build_common_listings.js

#!/usr/bin/env node
"use strict";

/*
  Build a report of canonical SKUs and how many STORES carry each one.
  - Store = storeKey (stable id derived from db filename).
  - Canonicalizes via sku_map.
  - Includes per-store numeric price (min live price per store for that SKU).
  - Writes one output file (see --out).

  Flags:
    --top N
    --min-stores N
    --require-all
    --group all|bc|ab
    --out path
*/

const fs = require("fs");
const path = require("path");

/* ---------------- helpers ---------------- */

function ensureDir(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

function readJson(p) {
  try {
    return JSON.parse(fs.readFileSync(p, "utf8"));
  } catch {
    return null;
  }
}

function listDbFiles() {
  const dir = path.join(process.cwd(), "data", "db");
  try {
    return fs
      .readdirSync(dir, { withFileTypes: true })
      .filter((e) => e.isFile() && e.name.endsWith(".json"))
      .map((e) => path.join(dir, e.name));
  } catch {
    return [];
  }
}

function priceToNumber(v) {
  const s = String(v ?? "").replace(/[^0-9.]/g, "");
  const n = Number(s);
  return Number.isFinite(n) ? n : null;
}

function hasRealSku6(s) {
  return /\b\d{6}\b/.test(String(s || ""));
}

function isSyntheticSkuKey(k) {
  return String(k || "").startsWith("u:");
}

function storeKeyFromDbPath(abs) {
  const base = path.basename(abs);
  const m = base.match(/^([^_]+)__.+\.json$/i);
  const k = m ? m[1] : base.replace(/\.json$/i, "");
  return String(k || "").toLowerCase();
}

/* ---------------- sku helpers ---------------- */

function loadSkuMapOrNull() {
  try {
    // eslint-disable-next-line node/no-missing-require
    const { loadSkuMap } = require(path.join(process.cwd(), "src/utils/sku_map"));
    return loadSkuMap({ dbDir: path.join(process.cwd(), "data/db") });
  } catch {
    return null;
  }
}

function normalizeSkuKeyOrEmpty({ skuRaw, storeLabel, url }) {
  try {
    // eslint-disable-next-line node/no-missing-require
    const { normalizeSkuKey } = require(path.join(process.cwd(), "src/utils/sku"));
    const k = normalizeSkuKey(skuRaw, { storeLabel, url });
    return k ? String(k) : "";
  } catch {
    const m = String(skuRaw ?? "").match(/\b(\d{6})\b/);
    if (m) return m[1];
    if (url) return `u:${storeLabel}:${url}`;
    return "";
  }
}

function canonicalize(k, skuMap) {
  if (!k) return "";
  if (skuMap && typeof skuMap.canonicalSku === "function") {
    return String(skuMap.canonicalSku(k) || k);
  }
  return k;
}

/* ---------------- grouping ---------------- */

const BC_STORE_KEYS = new Set([
  "gull",
  "strath",
  "bcl",
  "legacy",
  "legacyliquor",
  "tudor",
]);

function groupAllowsStore(group, storeKey) {
  const k = String(storeKey || "").toLowerCase();
  if (group === "bc") return BC_STORE_KEYS.has(k);
  if (group === "ab") return !BC_STORE_KEYS.has(k);
  return true; // all
}

/* ---------------- args ---------------- */

function parseArgs(argv) {
  const out = { top: 50, minStores: 2, requireAll: false, group: "all", out: "" };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || 50;
    else if (a === "--min-stores" && argv[i + 1]) out.minStores = Number(argv[++i]) || 2;
    else if (a === "--require-all") out.requireAll = true;
    else if (a === "--group" && argv[i + 1]) out.group = String(argv[++i] || "all").toLowerCase();
    else if (a === "--out" && argv[i + 1]) out.out = String(argv[++i] || "");
  }
  if (out.group !== "all" && out.group !== "bc" && out.group !== "ab") out.group = "all";
  return out;
}

/* ---------------- main ---------------- */

function main() {
  const args = parseArgs(process.argv.slice(2));
  const repoRoot = process.cwd();
  const reportsDir = path.join(repoRoot, "reports");
  ensureDir(reportsDir);

  const outPath = args.out ? path.join(repoRoot, args.out) : path.join(reportsDir, "common_listings.json");
  ensureDir(path.dirname(outPath));

  const dbFiles = listDbFiles();
  if (!dbFiles.length) {
    console.error("No DB files found");
    process.exitCode = 2;
    return;
  }

  const skuMap = loadSkuMapOrNull();
  console.log(`[debug] skuMap: ${skuMap ? "loaded" : "missing"}`);
  console.log(`[debug] scanning ${dbFiles.length} db files`);

  const storeToCanon = new Map(); // storeKey -> Set(canonSku)
  const canonAgg = new Map(); // canonSku -> { stores:Set, listings:[], cheapest, storeMin:Map }

  let liveRows = 0;
  let removedRows = 0;

  for (const abs of dbFiles.sort()) {
    const obj = readJson(abs);
    if (!obj) continue;

    const storeLabel = String(obj.storeLabel || obj.store || "").trim();
    if (!storeLabel) continue;

    const storeKey = storeKeyFromDbPath(abs);
    if (!groupAllowsStore(args.group, storeKey)) continue;

    if (!storeToCanon.has(storeKey)) {
      storeToCanon.set(storeKey, new Set());
    }

    const rel = path.relative(repoRoot, abs).replace(/\\/g, "/");
    const items = Array.isArray(obj.items) ? obj.items : [];

    console.log(`[debug] ${rel} storeKey="${storeKey}" storeLabel="${storeLabel}" items=${items.length}`);

    for (const it of items) {
      if (!it) continue;
      if (it.removed) {
        removedRows++;
        continue;
      }
      liveRows++;

      const skuKey = normalizeSkuKeyOrEmpty({
        skuRaw: it.sku,
        storeLabel,
        url: it.url,
      });
      if (!skuKey) continue;

      const canonSku = canonicalize(skuKey, skuMap);
      if (!canonSku) continue;

      storeToCanon.get(storeKey).add(canonSku);

      let agg = canonAgg.get(canonSku);
      if (!agg) {
        agg = { stores: new Set(), listings: [], cheapest: null, storeMin: new Map() };
        canonAgg.set(canonSku, agg);
      }

      agg.stores.add(storeKey);

      const priceNum = priceToNumber(it.price);
      if (priceNum !== null) {
        const prev = agg.storeMin.get(storeKey);
        if (prev === undefined || priceNum < prev) agg.storeMin.set(storeKey, priceNum);
      }

      const listing = {
        canonSku,
        skuKey,
        skuRaw: String(it.sku || ""),
        name: String(it.name || ""),
        price: String(it.price || ""),
        priceNum,
        url: String(it.url || ""),
        storeKey,
        storeLabel,
        categoryLabel: String(obj.categoryLabel || obj.category || ""),
        dbFile: rel,
        hasRealSku6: hasRealSku6(it.sku) && !isSyntheticSkuKey(skuKey),
      };

      agg.listings.push(listing);

      if (priceNum !== null) {
        if (!agg.cheapest || priceNum < agg.cheapest.priceNum) {
          agg.cheapest = { priceNum, item: listing };
        }
      }
    }
  }

  const stores = [...storeToCanon.keys()].sort();
  const storeCount = stores.length;

  console.log(`[debug] group="${args.group}" stores(${storeCount}): ${stores.join(", ")}`);
  console.log(`[debug] liveRows=${liveRows} removedRows=${removedRows} canonSkus=${canonAgg.size}`);

  function pickRepresentative(agg) {
    const preferred = agg.listings
      .filter((l) => l.hasRealSku6)
      .sort((a, b) => (a.priceNum ?? Infinity) - (b.priceNum ?? Infinity));

    if (preferred.length) return preferred[0];
    if (agg.cheapest) return agg.cheapest.item;
    return agg.listings[0] || null;
  }

  const rows = [];

  for (const [canonSku, agg] of canonAgg.entries()) {
    const rep = pickRepresentative(agg);
    const missingStores = stores.filter((s) => !agg.stores.has(s));

    const storePrices = {};
    for (const s of stores) {
      const p = agg.storeMin.get(s);
      if (Number.isFinite(p)) storePrices[s] = p;
    }

    rows.push({
      canonSku,
      storeCount: agg.stores.size,
      stores: [...agg.stores].sort(),
      missingStores,
      storePrices, // { [storeKey]: number } min live price per store
      representative: rep
        ? {
            name: rep.name,
            price: rep.price,
            priceNum: rep.priceNum,
            storeKey: rep.storeKey,
            storeLabel: rep.storeLabel,
            skuRaw: rep.skuRaw,
            skuKey: rep.skuKey,
            url: rep.url,
            categoryLabel: rep.categoryLabel,
            dbFile: rep.dbFile,
          }
        : null,
      cheapest: agg.cheapest
        ? {
            price: agg.cheapest.item.price,
            priceNum: agg.cheapest.priceNum,
            storeKey: agg.cheapest.item.storeKey,
            url: agg.cheapest.item.url,
          }
        : null,
    });
  }

  // Stable-ish sort: storeCount desc, then canonSku asc (stable diffs over time)
  rows.sort((a, b) => {
    if (b.storeCount !== a.storeCount) return b.storeCount - a.storeCount;
    return String(a.canonSku).localeCompare(String(b.canonSku));
  });

  const filtered = args.requireAll
    ? rows.filter((r) => r.storeCount === storeCount)
    : rows.filter((r) => r.storeCount >= args.minStores);

  const top = filtered.slice(0, args.top);

  const payload = {
    generatedAt: new Date().toISOString(),
    args: {
      top: args.top,
      minStores: args.minStores,
      requireAll: args.requireAll,
      group: args.group,
      out: path.relative(repoRoot, outPath).replace(/\\/g, "/"),
    },
    storeCount,
    stores,
    totals: {
      liveRows,
      removedRows,
      canonSkus: canonAgg.size,
      outputCount: top.length,
    },
    rows: top,
  };

  fs.writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n", "utf8");
  console.log(`Wrote ${path.relative(repoRoot, outPath)} (${top.length} rows)`);
}

main();