diff --git a/tools/build_viz_index.js b/tools/build_viz_index.js index 5c01454..bb72f53 100755 --- a/tools/build_viz_index.js +++ b/tools/build_viz_index.js @@ -3,6 +3,7 @@ const fs = require("fs"); const path = require("path"); +const { execFileSync } = require("child_process"); function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); @@ -30,6 +31,104 @@ function readJson(file) { } } +function readDbCommitsOrNull(repoRoot) { + const p = path.join(repoRoot, "viz", "data", "db_commits.json"); + try { + return JSON.parse(fs.readFileSync(p, "utf8")); + } catch { + return null; + } +} + +function gitShowJson(sha, filePath) { + try { + const txt = execFileSync("git", ["show", `${sha}:${filePath}`], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], // silence git fatal spam + }); + return JSON.parse(txt); + } catch { + return null; + } +} + +function normalizeCspc(v) { + const m = String(v ?? "").match(/\b(\d{6})\b/); + return m ? m[1] : ""; +} + +function fnv1a32(str) { + let h = 0x811c9dc5; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return (h >>> 0).toString(16).padStart(8, "0"); +} + +function makeSyntheticSku(storeLabel, url) { + const store = String(storeLabel || "store"); + const u = String(url || ""); + if (!u) return ""; + return `u:${fnv1a32(`${store}|${u}`)}`; +} + +function keySkuForItem(it, storeLabel) { + const real = normalizeCspc(it?.sku); + if (real) return real; + return makeSyntheticSku(storeLabel, it?.url); +} + +// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file). +function computeFirstSeenForDbFile({ + repoRoot, + relDbFile, + storeLabel, + wantSkuKeys, + commitsArr, + nowIso, +}) { + const out = new Map(); + const want = new Set(wantSkuKeys); + + // No commit history available -> treat as new today + if (!Array.isArray(commitsArr) || !commitsArr.length) { + for (const k of want) out.set(k, nowIso); + return out; + } + + // commitsArr is oldest -> newest (from db_commits.json) + for (const c of commitsArr) { + const sha = String(c?.sha || ""); + const ts = String(c?.ts || ""); + if (!sha || !ts) continue; + + const obj = gitShowJson(sha, relDbFile); + const items = Array.isArray(obj?.items) ? obj.items : []; + const sLabel = String(obj?.storeLabel || obj?.store || storeLabel || ""); + + for (const it of items) { + if (!it) continue; + if (Boolean(it.removed)) continue; // first time it existed LIVE in this file + + const k = keySkuForItem(it, sLabel); + if (!k) continue; + if (!want.has(k)) continue; + if (out.has(k)) continue; + + out.set(k, ts); + if (out.size >= want.size) break; + } + + if (out.size >= want.size) break; + } + + // Anything never seen historically -> new today + for (const k of want) if (!out.has(k)) out.set(k, nowIso); + + return out; +} + function main() { const repoRoot = path.resolve(__dirname, ".."); const dbDir = path.join(repoRoot, "data", "db"); @@ -38,6 +137,9 @@ function main() { ensureDir(outDir); + const nowIso = new Date().toISOString(); + const commitsManifest = readDbCommitsOrNull(repoRoot); + const items = []; let liveCount = 0; @@ -52,11 +154,28 @@ function main() { const source = String(obj.source || ""); const updatedAt = String(obj.updatedAt || ""); - const dbFile = path - .relative(repoRoot, file) - .replace(/\\/g, "/"); + const dbFile = path.relative(repoRoot, file).replace(/\\/g, "/"); // e.g. data/db/foo.json const arr = Array.isArray(obj.items) ? obj.items : []; + + // Build want keys from CURRENT file contents (includes removed rows too) + const wantSkuKeys = []; + for (const it of arr) { + if (!it) continue; + const k = keySkuForItem(it, storeLabel); + if (k) wantSkuKeys.push(k); + } + + const commitsArr = commitsManifest?.files?.[dbFile] || null; + const firstSeenByKey = computeFirstSeenForDbFile({ + repoRoot, + relDbFile: dbFile, + storeLabel, + wantSkuKeys, + commitsArr, + nowIso, + }); + for (const it of arr) { if (!it) continue; @@ -69,6 +188,9 @@ function main() { const url = String(it.url || "").trim(); const img = String(it.img || it.image || it.thumb || "").trim(); + const skuKey = keySkuForItem(it, storeLabel); + const firstSeenAt = skuKey ? String(firstSeenByKey.get(skuKey) || nowIso) : nowIso; + items.push({ sku, name, @@ -82,6 +204,7 @@ function main() { categoryLabel, source, updatedAt, + firstSeenAt, // NEW: first time this item appeared LIVE in this store/category db file (or now) dbFile, }); } @@ -94,7 +217,7 @@ function main() { }); const outObj = { - generatedAt: new Date().toISOString(), + generatedAt: nowIso, // Additive metadata. Old readers can ignore. includesRemoved: true, count: items.length, @@ -103,7 +226,9 @@ function main() { }; fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8"); - process.stdout.write(`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`); + process.stdout.write( + `Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n` + ); } module.exports = { main }; diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 4be11e5..aa4a3bb 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -60,7 +60,8 @@ let rulesCache = null; export async function renderStore($app, storeLabelRaw) { const storeLabel = String(storeLabelRaw || "").trim(); - const storeLabelShort = abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store"); + const storeLabelShort = + abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store"); $app.innerHTML = `
@@ -181,7 +182,8 @@ export async function renderStore($app, storeLabelRaw) { // Persist max price per store (clamped later once bounds known) const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`; const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE); - let savedMaxPrice = savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null; + let savedMaxPrice = + savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null; if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null; // Persist exclusives sort per store @@ -205,32 +207,26 @@ export async function renderStore($app, storeLabelRaw) { const liveAll = listingsAll.filter((r) => r && !r.removed); function dateMsFromRow(r) { - if (!r) return null; - - // Match renderItem() semantics: - // 1) prefer precise ts - const t = String(r?.ts || ""); + const t = String(r?.firstSeenAt || ""); const ms = t ? Date.parse(t) : NaN; - if (Number.isFinite(ms)) return ms; - - // 2) fall back to date-only (treat as end of day UTC so ordering within day is stable) - const d = String(r?.date || ""); - const ms2 = d ? Date.parse(d + "T23:59:59Z") : NaN; - return Number.isFinite(ms2) ? ms2 : null; + return Number.isFinite(ms) ? ms : null; } - - // Build earliest "first in DB" timestamp per canonical SKU (includes removed rows) - const firstSeenBySku = new Map(); // sku -> ms + + // Build earliest "first in DB (for this store)" timestamp per canonical SKU (includes removed rows) + const firstSeenBySkuInStore = new Map(); // sku -> ms for (const r of listingsAll) { if (!r) continue; + const store = normStoreLabel(r.storeLabel || r.store || ""); + if (store !== storeNorm) continue; + const skuKey = keySkuForRow(r); const sku = String(rules.canonicalSku(skuKey) || skuKey); const ms = dateMsFromRow(r); if (ms === null) continue; - const prev = firstSeenBySku.get(sku); - if (prev === undefined || ms < prev) firstSeenBySku.set(sku, ms); + const prev = firstSeenBySkuInStore.get(sku); + if (prev === undefined || ms < prev) firstSeenBySkuInStore.set(sku, ms); } // Build "ever seen" store presence per canonical SKU (includes removed rows) @@ -307,29 +303,37 @@ export async function renderStore($app, storeLabelRaw) { const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]); const everStoreSet = everStoresBySku.get(sku) || liveStoreSet; - const soloLiveHere = liveStoreSet.size === 1 && liveStoreSet.has(storeNorm); + const soloLiveHere = + liveStoreSet.size === 1 && liveStoreSet.has(storeNorm); const lastStock = soloLiveHere && everStoreSet.size > 1; const exclusive = soloLiveHere && !lastStock; - const storePrice = Number.isFinite(it.cheapestPriceNum) ? it.cheapestPriceNum : null; + const storePrice = Number.isFinite(it.cheapestPriceNum) + ? it.cheapestPriceNum + : null; const bestAll = bestAllPrice(sku); const other = bestOtherPrice(sku, storeNorm); - const isBest = storePrice !== null && bestAll !== null ? storePrice <= bestAll + EPS : false; + const isBest = + storePrice !== null && bestAll !== null + ? storePrice <= bestAll + EPS + : false; - const diffVsOtherDollar = storePrice !== null && other !== null ? storePrice - other : null; + const diffVsOtherDollar = + storePrice !== null && other !== null ? storePrice - other : null; const diffVsOtherPct = storePrice !== null && other !== null && other > 0 ? ((storePrice - other) / other) * 100 : null; - const diffVsBestDollar = storePrice !== null && bestAll !== null ? storePrice - bestAll : null; + const diffVsBestDollar = + storePrice !== null && bestAll !== null ? storePrice - bestAll : null; const diffVsBestPct = storePrice !== null && bestAll !== null && bestAll > 0 ? ((storePrice - bestAll) / bestAll) * 100 : null; - const firstSeenMs = firstSeenBySku.get(sku); + const firstSeenMs = firstSeenBySkuInStore.get(sku); const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null; return { @@ -447,7 +451,8 @@ export async function renderStore($app, storeLabelRaw) { // ---- Listing display price: keep cents (no rounding) ---- function listingPriceStr(it) { const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; - if (p === null) return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)"; + if (p === null) + return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)"; return `$${p.toFixed(2)}`; } @@ -494,8 +499,8 @@ export async function renderStore($app, storeLabelRaw) { const specialBadge = it._lastStock ? `Last Stock` : it._exclusive - ? `Exclusive` - : ""; + ? `Exclusive` + : ""; const bestBadge = !it._exclusive && !it._lastStock && it._isBest @@ -513,7 +518,9 @@ export async function renderStore($app, storeLabelRaw) {
${esc(it.name || "(no name)")}
${esc(displaySku(it.sku))} + href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc( + displaySku(it.sku) + )}
${specialBadge} @@ -522,7 +529,11 @@ export async function renderStore($app, storeLabelRaw) { ${esc(price)} ${ href - ? `${esc(storeLabelShort)}` + ? `${esc( + storeLabelShort + )}` : `` }
@@ -556,7 +567,9 @@ export async function renderStore($app, storeLabelRaw) { } if (pageMax !== null) { - $status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(selectedMaxPrice)}).`; + $status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars( + selectedMaxPrice + )}).`; return; } @@ -571,8 +584,14 @@ export async function renderStore($app, storeLabelRaw) { shownCompare = 0; } - const sliceEx = filteredExclusive.slice(shownExclusive, shownExclusive + PAGE_EACH); - const sliceCo = filteredCompare.slice(shownCompare, shownCompare + PAGE_EACH); + const sliceEx = filteredExclusive.slice( + shownExclusive, + shownExclusive + PAGE_EACH + ); + const sliceCo = filteredCompare.slice( + shownCompare, + shownCompare + PAGE_EACH + ); shownExclusive += sliceEx.length; shownCompare += sliceCo.length; @@ -615,8 +634,10 @@ export async function renderStore($app, storeLabelRaw) { arr.sort((a, b) => { const ap = Number.isFinite(a._storePrice) ? a._storePrice : null; const bp = Number.isFinite(b._storePrice) ? b._storePrice : null; - const aKey = ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; - const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; + const aKey = + ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; + const bKey = + bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); @@ -627,8 +648,10 @@ export async function renderStore($app, storeLabelRaw) { arr.sort((a, b) => { const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null; const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null; - const aKey = ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; - const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; + const aKey = + ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; + const bKey = + bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); @@ -638,10 +661,8 @@ export async function renderStore($app, storeLabelRaw) { function sortCompareInPlace(arr) { const mode = compareMode(); arr.sort((a, b) => { - const da = - mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar; - const db = - mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar; + const da = mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar; + const db = mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar; const sa = da === null || !Number.isFinite(da) ? 999999 : da; const sb = db === null || !Number.isFinite(db) ? 999999 : db; @@ -702,13 +723,13 @@ export async function renderStore($app, storeLabelRaw) { $clearSearch.addEventListener("click", () => { let changed = false; - + if ($q.value) { $q.value = ""; localStorage.setItem(LS_KEY, ""); changed = true; } - + // reset max price too (only if slider is active) if (pageMax !== null) { selectedMaxPrice = clampAndRound(boundMax); @@ -717,11 +738,11 @@ export async function renderStore($app, storeLabelRaw) { updateMaxPriceLabel(); changed = true; } - + if (changed) applyFilter(); $q.focus(); }); - + $exSort.addEventListener("change", () => { localStorage.setItem(LS_EX_SORT, String($exSort.value || "")); applyFilter();