spirit-tracker/tools/build_viz_recent.js

#!/usr/bin/env node
"use strict";

const { execFileSync } = require("child_process");
const fs = require("fs");
const path = require("path");

function runGit(args) {
  return execFileSync("git", args, { encoding: "utf8" }).trimEnd();
}

function gitShowJson(sha, filePath) {
  try {
    const txt = execFileSync("git", ["show", `${sha}:${filePath}`], {
      encoding: "utf8",
      stdio: ["ignore", "pipe", "pipe"], // silence git fatal spam
    });
    return JSON.parse(txt);
  } catch {
    return null;
  }
}

function gitFileExistsAtSha(sha, filePath) {
  if (!sha) return false;
  try {
    execFileSync("git", ["cat-file", "-e", `${sha}:${filePath}`], {
      stdio: ["ignore", "ignore", "ignore"],
    });
    return true;
  } catch {
    return false;
  }
}

function gitListTreeFiles(sha, dirRel) {
  try {
    const out = runGit(["ls-tree", "-r", "--name-only", sha, dirRel]);
    return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
  } catch {
    return [];
  }
}

function readJsonFileOrNull(filePath) {
  try {
    return JSON.parse(fs.readFileSync(filePath, "utf8"));
  } catch {
    return null;
  }
}

function normalizeCspc(v) {
  const m = String(v ?? "").match(/\b(\d{6})\b/);
  return m ? m[1] : "";
}

function normPriceStr(p) {
  return String(p ?? "").trim();
}

function priceToNumber(v) {
  const s = String(v ?? "").replace(/[^0-9.]/g, "");
  const n = Number(s);
  return Number.isFinite(n) ? n : null;
}

function dateOnly(iso) {
  const m = String(iso ?? "").match(/^(\d{4}-\d{2}-\d{2})/);
  return m ? m[1] : "";
}

function fnv1a32(str) {
  let h = 0x811c9dc5;
  for (let i = 0; i < str.length; i++) {
    h ^= str.charCodeAt(i);
    h = Math.imul(h, 0x01000193);
  }
  return (h >>> 0).toString(16).padStart(8, "0");
}

function makeSyntheticSku(storeLabel, url) {
  const store = String(storeLabel || "store");
  const u = String(url || "");
  if (!u) return "";
  return `u:${fnv1a32(`${store}|${u}`)}`;
}

function keySkuForItem(it, storeLabel) {
  const real = normalizeCspc(it?.sku);
  if (real) return real;
  return makeSyntheticSku(storeLabel, it?.url);
}

function mapBySku(obj, { includeRemoved } = { includeRemoved: false }) {
  const m = new Map();
  const items = Array.isArray(obj?.items) ? obj.items : [];
  const storeLabel = String(obj?.storeLabel || obj?.store || "");

  for (const it of items) {
    if (!it) continue;

    const sku = keySkuForItem(it, storeLabel);
    if (!sku) continue;

    const removed = Boolean(it.removed);
    if (!includeRemoved && removed) continue;

    const next = {
      sku,
      name: String(it.name || ""),
      price: String(it.price || ""),
      url: String(it.url || ""),
      removed,
    };

    const prev = m.get(sku);
    if (!prev) {
      m.set(sku, next);
      continue;
    }

    // Prefer the non-removed record if both exist.
    if (prev.removed && !next.removed) {
      m.set(sku, next);
      continue;
    }
    if (!prev.removed && next.removed) {
      continue; // keep the active one
    }

    // Otherwise keep the “better” one (more complete data), deterministic.
    const prevScore = (prev.name ? 1 : 0) + (prev.price ? 1 : 0) + (prev.url ? 1 : 0);
    const nextScore = (next.name ? 1 : 0) + (next.price ? 1 : 0) + (next.url ? 1 : 0);
    if (nextScore > prevScore) m.set(sku, next);
  }

  return m;
}

function diffDb(prevObj, nextObj) {
  const prevAll = mapBySku(prevObj, { includeRemoved: true });
  const nextAll = mapBySku(nextObj, { includeRemoved: true });

  const prevLive = mapBySku(prevObj, { includeRemoved: false });
  const nextLive = mapBySku(nextObj, { includeRemoved: false });

  const newItems = [];
  const restoredItems = [];
  const removedItems = [];
  const priceChanges = [];

  for (const [sku, now] of nextLive.entries()) {
    const had = prevAll.get(sku);
    if (!had) {
      newItems.push({ ...now });
      continue;
    }
    if (had.removed) {
      restoredItems.push({ ...now });
      continue;
    }
  }

  for (const [sku, was] of prevLive.entries()) {
    const nxt = nextAll.get(sku);
    if (!nxt || nxt.removed) {
      removedItems.push({ ...was });
    }
  }

  for (const [sku, now] of nextLive.entries()) {
    const was = prevLive.get(sku);
    if (!was) continue;

    const a = normPriceStr(was.price);
    const b = normPriceStr(now.price);
    if (a === b) continue;

    const aN = priceToNumber(a);
    const bN = priceToNumber(b);

    let kind = "price_change";
    if (aN !== null && bN !== null) {
      if (bN < aN) kind = "price_down";
      else if (bN > aN) kind = "price_up";
      else kind = "price_change";
    }

    priceChanges.push({
      kind,
      sku,
      name: now.name || was.name || "",
      oldPrice: a,
      newPrice: b,
      url: now.url || was.url || "",
    });
  }

  return { newItems, restoredItems, removedItems, priceChanges };
}

function getHeadShaOrEmpty() {
  try {
    return runGit(["rev-parse", "--verify", "HEAD"]);
  } catch {
    return "";
  }
}

function firstParentSha(sha) {
  try {
    const out = runGit(["rev-list", "--parents", "-n", "1", sha]);
    const parts = out.split(/\s+/).filter(Boolean);
    return parts.length >= 2 ? parts[1] : "";
  } catch {
    return "";
  }
}

function listChangedDbFiles(fromSha, toSha) {
  if (!fromSha && toSha && toSha !== "WORKTREE") {
    return gitListTreeFiles(toSha, "data/db");
  }

  if (!fromSha && toSha === "WORKTREE") {
    try {
      return fs
        .readdirSync(path.join(process.cwd(), "data", "db"), { withFileTypes: true })
        .filter((e) => e.isFile() && e.name.endsWith(".json"))
        .map((e) => path.posix.join("data/db", e.name));
    } catch {
      return [];
    }
  }

  try {
    if (toSha === "WORKTREE") {
      const out = runGit(["diff", "--name-only", fromSha, "--", "data/db"]);
      return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
    }
    const out = runGit(["diff", "--name-only", fromSha, toSha, "--", "data/db"]);
    return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
  } catch {
    return [];
  }
}

function logDbCommitsSince(sinceIso) {
  try {
    const out = runGit(["log", `--since=${sinceIso}`, "--format=%H %cI", "--", "data/db"]);
    const lines = out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
    const arr = [];
    for (const line of lines) {
      const m = line.match(/^([0-9a-f]{7,40})\s+(.+)$/i);
      if (!m) continue;
      const sha = m[1];
      const ts = m[2];
      const d = dateOnly(ts);
      arr.push({ sha, ts, date: d });
    }
    arr.reverse();
    return arr;
  } catch {
    return [];
  }
}

function main() {
  const repoRoot = process.cwd();
  const outDir = path.join(repoRoot, "viz", "data");
  const outFile = path.join(outDir, "recent.json");
  fs.mkdirSync(outDir, { recursive: true });

  const windowDays = Math.max(1, Number(process.env.RECENT_DAYS || 3));
  const maxItems = Math.max(1, Number(process.env.RECENT_MAX_ITEMS || 500));

  const now = new Date();
  const since = new Date(now.getTime() - windowDays * 24 * 3600 * 1000);
  const sinceIso = since.toISOString();

  const headSha = getHeadShaOrEmpty();
  const items = [];

  const commits = headSha ? logDbCommitsSince(sinceIso) : [];

  const pairs = [];

  if (commits.length) {
    const first = commits[0];
    const parent = firstParentSha(first.sha);
    pairs.push({
      fromSha: parent || "",
      toSha: first.sha,
      ts: first.ts,
      date: first.date,
    });

    for (let i = 1; i < commits.length; i++) {
      pairs.push({
        fromSha: commits[i - 1].sha,
        toSha: commits[i].sha,
        ts: commits[i].ts,
        date: commits[i].date,
      });
    }
  }

  if (headSha) {
    pairs.push({
      fromSha: headSha,
      toSha: "WORKTREE",
      ts: now.toISOString(),
      date: dateOnly(now.toISOString()),
    });
  }

  for (const p of pairs) {
    const fromSha = p.fromSha;
    const toSha = p.toSha;
    const ts = p.ts;
    const d = p.date;

    const files = listChangedDbFiles(fromSha, toSha);
    if (!files.length) continue;

    for (const file of files) {
      let prevObj = null;
      let nextObj = null;

      if (toSha === "WORKTREE") {
        prevObj = fromSha ? gitShowJson(fromSha, file) : null;
        nextObj = readJsonFileOrNull(path.join(repoRoot, file));
      } else {
        prevObj = fromSha ? gitShowJson(fromSha, file) : null;
        nextObj = gitShowJson(toSha, file);
      }

      // NEW: if the DB file itself doesn't exist at "to", skip (prevents mass "removed")
      const nextExists =
        toSha === "WORKTREE"
          ? fs.existsSync(path.join(repoRoot, file))
          : gitFileExistsAtSha(toSha, file);
      if (!nextExists) continue;

      if (!prevObj && !nextObj) continue;

      const storeLabel = String(nextObj?.storeLabel || nextObj?.store || prevObj?.storeLabel || prevObj?.store || "");
      const categoryLabel = String(nextObj?.categoryLabel || nextObj?.category || prevObj?.categoryLabel || prevObj?.category || "");

      // NEW FEATURE:
      // If this DB file did not exist at fromSha, then treat it as a "new store/category file"
      // and DO NOT emit its "new"/"restored" items into recent.json (frontpage).
      // (Report text is unaffected elsewhere.)
      const isNewStoreFile =
        Boolean(fromSha) &&
        !gitFileExistsAtSha(fromSha, file) &&
        (toSha === "WORKTREE" ? fs.existsSync(path.join(repoRoot, file)) : gitFileExistsAtSha(toSha, file));

      let { newItems, restoredItems, removedItems, priceChanges } = diffDb(prevObj, nextObj);

      if (isNewStoreFile) {
        newItems = [];
        restoredItems = [];
      }

      for (const it of newItems) {
        items.push({
          ts,
          date: d,
          fromSha: fromSha || "",
          toSha,
          kind: "new",
          sku: it.sku,
          name: it.name,
          storeLabel,
          categoryLabel,
          price: normPriceStr(it.price),
          url: it.url,
          dbFile: file,
        });
      }

      for (const it of restoredItems) {
        items.push({
          ts,
          date: d,
          fromSha: fromSha || "",
          toSha,
          kind: "restored",
          sku: it.sku,
          name: it.name,
          storeLabel,
          categoryLabel,
          price: normPriceStr(it.price),
          url: it.url,
          dbFile: file,
        });
      }

      for (const it of removedItems) {
        items.push({
          ts,
          date: d,
          fromSha: fromSha || "",
          toSha,
          kind: "removed",
          sku: it.sku,
          name: it.name,
          storeLabel,
          categoryLabel,
          price: normPriceStr(it.price),
          url: it.url,
          dbFile: file,
        });
      }

      for (const u of priceChanges) {
        items.push({
          ts,
          date: d,
          fromSha: fromSha || "",
          toSha,
          kind: u.kind,
          sku: u.sku,
          name: u.name,
          storeLabel,
          categoryLabel,
          oldPrice: normPriceStr(u.oldPrice),
          newPrice: normPriceStr(u.newPrice),
          url: u.url,
          dbFile: file,
        });
      }
    }

  }

  items.sort((a, b) => String(b.ts).localeCompare(String(a.ts)));

  const trimmed = items.slice(0, maxItems);

  const payload = {
    generatedAt: now.toISOString(),
    windowDays,
    since: sinceIso,
    headSha,
    count: trimmed.length,
    items: trimmed,
  };

  fs.writeFileSync(outFile, JSON.stringify(payload, null, 2) + "\n", "utf8");
  process.stdout.write(`Wrote ${outFile} (${trimmed.length} items)\n`);
}

main();