spirit-tracker/tools/build_viz_index.js

#!/usr/bin/env node
"use strict";

const fs = require("fs");
const path = require("path");
const { execFileSync } = require("child_process");

function ensureDir(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

function listJsonFiles(dir) {
  const out = [];
  try {
    for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
      if (!ent.isFile()) continue;
      if (!String(ent.name || "").endsWith(".json")) continue;
      out.push(path.join(dir, ent.name));
    }
  } catch {
    // ignore
  }
  return out;
}

function readJson(file) {
  try {
    return JSON.parse(fs.readFileSync(file, "utf8"));
  } catch {
    return null;
  }
}

function readDbCommitsOrNull(repoRoot) {
  const p = path.join(repoRoot, "viz", "data", "db_commits.json");
  try {
    return JSON.parse(fs.readFileSync(p, "utf8"));
  } catch {
    return null;
  }
}

function gitShowJson(sha, filePath) {
  try {
    const txt = execFileSync("git", ["show", `${sha}:${filePath}`], {
      encoding: "utf8",
      stdio: ["ignore", "pipe", "pipe"], // silence git fatal spam
    });
    return JSON.parse(txt);
  } catch {
    return null;
  }
}

function normalizeCspc(v) {
  const m = String(v ?? "").match(/\b(\d{6})\b/);
  return m ? m[1] : "";
}

function fnv1a32(str) {
  let h = 0x811c9dc5;
  for (let i = 0; i < str.length; i++) {
    h ^= str.charCodeAt(i);
    h = Math.imul(h, 0x01000193);
  }
  return (h >>> 0).toString(16).padStart(8, "0");
}

function makeSyntheticSku(storeLabel, url) {
  const store = String(storeLabel || "store");
  const u = String(url || "");
  if (!u) return "";
  return `u:${fnv1a32(`${store}|${u}`)}`;
}

function keySkuForItem(it, storeLabel) {
  const real = normalizeCspc(it?.sku);
  if (real) return real;
  return makeSyntheticSku(storeLabel, it?.url);
}

// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file).
function computeFirstSeenForDbFile({
  repoRoot,
  relDbFile,
  storeLabel,
  wantSkuKeys,
  commitsArr,
  nowIso,
}) {
  const out = new Map();
  const want = new Set(wantSkuKeys);

  // No commit history available -> treat as new today
  if (!Array.isArray(commitsArr) || !commitsArr.length) {
    for (const k of want) out.set(k, nowIso);
    return out;
  }

  // commitsArr is oldest -> newest (from db_commits.json)
  for (const c of commitsArr) {
    const sha = String(c?.sha || "");
    const ts = String(c?.ts || "");
    if (!sha || !ts) continue;

    const obj = gitShowJson(sha, relDbFile);
    const items = Array.isArray(obj?.items) ? obj.items : [];
    const sLabel = String(obj?.storeLabel || obj?.store || storeLabel || "");

    for (const it of items) {
      if (!it) continue;
      if (Boolean(it.removed)) continue; // first time it existed LIVE in this file

      const k = keySkuForItem(it, sLabel);
      if (!k) continue;
      if (!want.has(k)) continue;
      if (out.has(k)) continue;

      out.set(k, ts);
      if (out.size >= want.size) break;
    }

    if (out.size >= want.size) break;
  }

  // Anything never seen historically -> new today
  for (const k of want) if (!out.has(k)) out.set(k, nowIso);

  return out;
}

function main() {
  const repoRoot = path.resolve(__dirname, "..");
  const dbDir = path.join(repoRoot, "data", "db");
  const outDir = path.join(repoRoot, "viz", "data");
  const outFile = path.join(outDir, "index.json");

  ensureDir(outDir);

  const nowIso = new Date().toISOString();
  const commitsManifest = readDbCommitsOrNull(repoRoot);

  const items = [];
  let liveCount = 0;

  for (const file of listJsonFiles(dbDir)) {
    const obj = readJson(file);
    if (!obj) continue;

    const store = String(obj.store || "");
    const storeLabel = String(obj.storeLabel || store || "");
    const category = String(obj.category || "");
    const categoryLabel = String(obj.categoryLabel || "");
    const source = String(obj.source || "");
    const updatedAt = String(obj.updatedAt || "");

    const dbFile = path.relative(repoRoot, file).replace(/\\/g, "/"); // e.g. data/db/foo.json

    const arr = Array.isArray(obj.items) ? obj.items : [];

    // Build want keys from CURRENT file contents (includes removed rows too)
    const wantSkuKeys = [];
    for (const it of arr) {
      if (!it) continue;
      const k = keySkuForItem(it, storeLabel);
      if (k) wantSkuKeys.push(k);
    }

    const commitsArr = commitsManifest?.files?.[dbFile] || null;
    const firstSeenByKey = computeFirstSeenForDbFile({
      repoRoot,
      relDbFile: dbFile,
      storeLabel,
      wantSkuKeys,
      commitsArr,
      nowIso,
    });

    for (const it of arr) {
      if (!it) continue;

      const removed = Boolean(it.removed);
      if (!removed) liveCount++;

      const sku = String(it.sku || "").trim();
      const name = String(it.name || "").trim();
      const price = String(it.price || "").trim();
      const url = String(it.url || "").trim();
      const img = String(it.img || it.image || it.thumb || "").trim();

      const skuKey = keySkuForItem(it, storeLabel);
      const firstSeenAt = skuKey ? String(firstSeenByKey.get(skuKey) || nowIso) : nowIso;

      items.push({
        sku,
        name,
        price,
        url,
        img,
        removed, // NEW (additive): allows viz to show history / removed-only items
        store,
        storeLabel,
        category,
        categoryLabel,
        source,
        updatedAt,
        firstSeenAt, // NEW: first time this item appeared LIVE in this store/category db file (or now)
        dbFile,
      });
    }
  }

  items.sort((a, b) => {
    const ak = `${a.sku}|${a.storeLabel}|${a.removed ? 1 : 0}|${a.name}|${a.url}`;
    const bk = `${b.sku}|${b.storeLabel}|${b.removed ? 1 : 0}|${b.name}|${b.url}`;
    return ak.localeCompare(bk);
  });

  const outObj = {
    generatedAt: nowIso,
    // Additive metadata. Old readers can ignore.
    includesRemoved: true,
    count: items.length,
    countLive: liveCount,
    items,
  };

  fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8");
  process.stdout.write(
    `Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`
  );
}

module.exports = { main };

if (require.main === module) {
  main();
}