mirror of
https://github.com/samsonjs/spirit-tracker.git
synced 2026-03-25 09:25:51 +00:00
feat: V1 of tracking when items first appeared
This commit is contained in:
Brennan Wilkes (Text Groove)
parent
f88cd290c3
commit
6f919ff7fe
2 changed files with 195 additions and 49 deletions
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { execFileSync } = require("child_process");
|
||||
|
||||
function ensureDir(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
|
@ -30,6 +31,104 @@ function readJson(file) {
|
|||
}
|
||||
}
|
||||
|
||||
function readDbCommitsOrNull(repoRoot) {
|
||||
const p = path.join(repoRoot, "viz", "data", "db_commits.json");
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(p, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function gitShowJson(sha, filePath) {
|
||||
try {
|
||||
const txt = execFileSync("git", ["show", `${sha}:${filePath}`], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"], // silence git fatal spam
|
||||
});
|
||||
return JSON.parse(txt);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCspc(v) {
|
||||
const m = String(v ?? "").match(/\b(\d{6})\b/);
|
||||
return m ? m[1] : "";
|
||||
}
|
||||
|
||||
function fnv1a32(str) {
|
||||
let h = 0x811c9dc5;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
h ^= str.charCodeAt(i);
|
||||
h = Math.imul(h, 0x01000193);
|
||||
}
|
||||
return (h >>> 0).toString(16).padStart(8, "0");
|
||||
}
|
||||
|
||||
function makeSyntheticSku(storeLabel, url) {
|
||||
const store = String(storeLabel || "store");
|
||||
const u = String(url || "");
|
||||
if (!u) return "";
|
||||
return `u:${fnv1a32(`${store}|${u}`)}`;
|
||||
}
|
||||
|
||||
function keySkuForItem(it, storeLabel) {
|
||||
const real = normalizeCspc(it?.sku);
|
||||
if (real) return real;
|
||||
return makeSyntheticSku(storeLabel, it?.url);
|
||||
}
|
||||
|
||||
// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file).
|
||||
function computeFirstSeenForDbFile({
|
||||
repoRoot,
|
||||
relDbFile,
|
||||
storeLabel,
|
||||
wantSkuKeys,
|
||||
commitsArr,
|
||||
nowIso,
|
||||
}) {
|
||||
const out = new Map();
|
||||
const want = new Set(wantSkuKeys);
|
||||
|
||||
// No commit history available -> treat as new today
|
||||
if (!Array.isArray(commitsArr) || !commitsArr.length) {
|
||||
for (const k of want) out.set(k, nowIso);
|
||||
return out;
|
||||
}
|
||||
|
||||
// commitsArr is oldest -> newest (from db_commits.json)
|
||||
for (const c of commitsArr) {
|
||||
const sha = String(c?.sha || "");
|
||||
const ts = String(c?.ts || "");
|
||||
if (!sha || !ts) continue;
|
||||
|
||||
const obj = gitShowJson(sha, relDbFile);
|
||||
const items = Array.isArray(obj?.items) ? obj.items : [];
|
||||
const sLabel = String(obj?.storeLabel || obj?.store || storeLabel || "");
|
||||
|
||||
for (const it of items) {
|
||||
if (!it) continue;
|
||||
if (Boolean(it.removed)) continue; // first time it existed LIVE in this file
|
||||
|
||||
const k = keySkuForItem(it, sLabel);
|
||||
if (!k) continue;
|
||||
if (!want.has(k)) continue;
|
||||
if (out.has(k)) continue;
|
||||
|
||||
out.set(k, ts);
|
||||
if (out.size >= want.size) break;
|
||||
}
|
||||
|
||||
if (out.size >= want.size) break;
|
||||
}
|
||||
|
||||
// Anything never seen historically -> new today
|
||||
for (const k of want) if (!out.has(k)) out.set(k, nowIso);
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const repoRoot = path.resolve(__dirname, "..");
|
||||
const dbDir = path.join(repoRoot, "data", "db");
|
||||
|
|
@ -38,6 +137,9 @@ function main() {
|
|||
|
||||
ensureDir(outDir);
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const commitsManifest = readDbCommitsOrNull(repoRoot);
|
||||
|
||||
const items = [];
|
||||
let liveCount = 0;
|
||||
|
||||
|
|
@ -52,11 +154,28 @@ function main() {
|
|||
const source = String(obj.source || "");
|
||||
const updatedAt = String(obj.updatedAt || "");
|
||||
|
||||
const dbFile = path
|
||||
.relative(repoRoot, file)
|
||||
.replace(/\\/g, "/");
|
||||
const dbFile = path.relative(repoRoot, file).replace(/\\/g, "/"); // e.g. data/db/foo.json
|
||||
|
||||
const arr = Array.isArray(obj.items) ? obj.items : [];
|
||||
|
||||
// Build want keys from CURRENT file contents (includes removed rows too)
|
||||
const wantSkuKeys = [];
|
||||
for (const it of arr) {
|
||||
if (!it) continue;
|
||||
const k = keySkuForItem(it, storeLabel);
|
||||
if (k) wantSkuKeys.push(k);
|
||||
}
|
||||
|
||||
const commitsArr = commitsManifest?.files?.[dbFile] || null;
|
||||
const firstSeenByKey = computeFirstSeenForDbFile({
|
||||
repoRoot,
|
||||
relDbFile: dbFile,
|
||||
storeLabel,
|
||||
wantSkuKeys,
|
||||
commitsArr,
|
||||
nowIso,
|
||||
});
|
||||
|
||||
for (const it of arr) {
|
||||
if (!it) continue;
|
||||
|
||||
|
|
@ -69,6 +188,9 @@ function main() {
|
|||
const url = String(it.url || "").trim();
|
||||
const img = String(it.img || it.image || it.thumb || "").trim();
|
||||
|
||||
const skuKey = keySkuForItem(it, storeLabel);
|
||||
const firstSeenAt = skuKey ? String(firstSeenByKey.get(skuKey) || nowIso) : nowIso;
|
||||
|
||||
items.push({
|
||||
sku,
|
||||
name,
|
||||
|
|
@ -82,6 +204,7 @@ function main() {
|
|||
categoryLabel,
|
||||
source,
|
||||
updatedAt,
|
||||
firstSeenAt, // NEW: first time this item appeared LIVE in this store/category db file (or now)
|
||||
dbFile,
|
||||
});
|
||||
}
|
||||
|
|
@ -94,7 +217,7 @@ function main() {
|
|||
});
|
||||
|
||||
const outObj = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
generatedAt: nowIso,
|
||||
// Additive metadata. Old readers can ignore.
|
||||
includesRemoved: true,
|
||||
count: items.length,
|
||||
|
|
@ -103,7 +226,9 @@ function main() {
|
|||
};
|
||||
|
||||
fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8");
|
||||
process.stdout.write(`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`);
|
||||
process.stdout.write(
|
||||
`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = { main };
|
||||
|
|
|
|||
|
|
@ -60,7 +60,8 @@ let rulesCache = null;
|
|||
|
||||
export async function renderStore($app, storeLabelRaw) {
|
||||
const storeLabel = String(storeLabelRaw || "").trim();
|
||||
const storeLabelShort = abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
|
||||
const storeLabelShort =
|
||||
abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
|
||||
|
||||
$app.innerHTML = `
|
||||
<div class="container">
|
||||
|
|
@ -181,7 +182,8 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
// Persist max price per store (clamped later once bounds known)
|
||||
const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`;
|
||||
const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE);
|
||||
let savedMaxPrice = savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
|
||||
let savedMaxPrice =
|
||||
savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
|
||||
if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null;
|
||||
|
||||
// Persist exclusives sort per store
|
||||
|
|
@ -205,32 +207,26 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
const liveAll = listingsAll.filter((r) => r && !r.removed);
|
||||
|
||||
function dateMsFromRow(r) {
|
||||
if (!r) return null;
|
||||
|
||||
// Match renderItem() semantics:
|
||||
// 1) prefer precise ts
|
||||
const t = String(r?.ts || "");
|
||||
const t = String(r?.firstSeenAt || "");
|
||||
const ms = t ? Date.parse(t) : NaN;
|
||||
if (Number.isFinite(ms)) return ms;
|
||||
|
||||
// 2) fall back to date-only (treat as end of day UTC so ordering within day is stable)
|
||||
const d = String(r?.date || "");
|
||||
const ms2 = d ? Date.parse(d + "T23:59:59Z") : NaN;
|
||||
return Number.isFinite(ms2) ? ms2 : null;
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
// Build earliest "first in DB" timestamp per canonical SKU (includes removed rows)
|
||||
const firstSeenBySku = new Map(); // sku -> ms
|
||||
// Build earliest "first in DB (for this store)" timestamp per canonical SKU (includes removed rows)
|
||||
const firstSeenBySkuInStore = new Map(); // sku -> ms
|
||||
for (const r of listingsAll) {
|
||||
if (!r) continue;
|
||||
const store = normStoreLabel(r.storeLabel || r.store || "");
|
||||
if (store !== storeNorm) continue;
|
||||
|
||||
const skuKey = keySkuForRow(r);
|
||||
const sku = String(rules.canonicalSku(skuKey) || skuKey);
|
||||
|
||||
const ms = dateMsFromRow(r);
|
||||
if (ms === null) continue;
|
||||
|
||||
const prev = firstSeenBySku.get(sku);
|
||||
if (prev === undefined || ms < prev) firstSeenBySku.set(sku, ms);
|
||||
const prev = firstSeenBySkuInStore.get(sku);
|
||||
if (prev === undefined || ms < prev) firstSeenBySkuInStore.set(sku, ms);
|
||||
}
|
||||
|
||||
// Build "ever seen" store presence per canonical SKU (includes removed rows)
|
||||
|
|
@ -307,29 +303,37 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]);
|
||||
const everStoreSet = everStoresBySku.get(sku) || liveStoreSet;
|
||||
|
||||
const soloLiveHere = liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
|
||||
const soloLiveHere =
|
||||
liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
|
||||
const lastStock = soloLiveHere && everStoreSet.size > 1;
|
||||
const exclusive = soloLiveHere && !lastStock;
|
||||
|
||||
const storePrice = Number.isFinite(it.cheapestPriceNum) ? it.cheapestPriceNum : null;
|
||||
const storePrice = Number.isFinite(it.cheapestPriceNum)
|
||||
? it.cheapestPriceNum
|
||||
: null;
|
||||
const bestAll = bestAllPrice(sku);
|
||||
const other = bestOtherPrice(sku, storeNorm);
|
||||
|
||||
const isBest = storePrice !== null && bestAll !== null ? storePrice <= bestAll + EPS : false;
|
||||
const isBest =
|
||||
storePrice !== null && bestAll !== null
|
||||
? storePrice <= bestAll + EPS
|
||||
: false;
|
||||
|
||||
const diffVsOtherDollar = storePrice !== null && other !== null ? storePrice - other : null;
|
||||
const diffVsOtherDollar =
|
||||
storePrice !== null && other !== null ? storePrice - other : null;
|
||||
const diffVsOtherPct =
|
||||
storePrice !== null && other !== null && other > 0
|
||||
? ((storePrice - other) / other) * 100
|
||||
: null;
|
||||
|
||||
const diffVsBestDollar = storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
|
||||
const diffVsBestDollar =
|
||||
storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
|
||||
const diffVsBestPct =
|
||||
storePrice !== null && bestAll !== null && bestAll > 0
|
||||
? ((storePrice - bestAll) / bestAll) * 100
|
||||
: null;
|
||||
|
||||
const firstSeenMs = firstSeenBySku.get(sku);
|
||||
const firstSeenMs = firstSeenBySkuInStore.get(sku);
|
||||
const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null;
|
||||
|
||||
return {
|
||||
|
|
@ -447,7 +451,8 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
// ---- Listing display price: keep cents (no rounding) ----
|
||||
function listingPriceStr(it) {
|
||||
const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null;
|
||||
if (p === null) return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
|
||||
if (p === null)
|
||||
return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
|
||||
return `$${p.toFixed(2)}`;
|
||||
}
|
||||
|
||||
|
|
@ -494,8 +499,8 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
const specialBadge = it._lastStock
|
||||
? `<span class="badge badgeLastStock">Last Stock</span>`
|
||||
: it._exclusive
|
||||
? `<span class="badge badgeExclusive">Exclusive</span>`
|
||||
: "";
|
||||
? `<span class="badge badgeExclusive">Exclusive</span>`
|
||||
: "";
|
||||
|
||||
const bestBadge =
|
||||
!it._exclusive && !it._lastStock && it._isBest
|
||||
|
|
@ -513,7 +518,9 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
<div class="itemTop">
|
||||
<div class="itemName">${esc(it.name || "(no name)")}</div>
|
||||
<a class="badge mono skuLink" target="_blank" rel="noopener noreferrer"
|
||||
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(displaySku(it.sku))}</a>
|
||||
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(
|
||||
displaySku(it.sku)
|
||||
)}</a>
|
||||
</div>
|
||||
<div class="metaRow">
|
||||
${specialBadge}
|
||||
|
|
@ -522,7 +529,11 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
<span class="mono price">${esc(price)}</span>
|
||||
${
|
||||
href
|
||||
? `<a class="badge" href="${esc(href)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(storeLabelShort)}</a>`
|
||||
? `<a class="badge" href="${esc(
|
||||
href
|
||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||
storeLabelShort
|
||||
)}</a>`
|
||||
: ``
|
||||
}
|
||||
</div>
|
||||
|
|
@ -556,7 +567,9 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
}
|
||||
|
||||
if (pageMax !== null) {
|
||||
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(selectedMaxPrice)}).`;
|
||||
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(
|
||||
selectedMaxPrice
|
||||
)}).`;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -571,8 +584,14 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
shownCompare = 0;
|
||||
}
|
||||
|
||||
const sliceEx = filteredExclusive.slice(shownExclusive, shownExclusive + PAGE_EACH);
|
||||
const sliceCo = filteredCompare.slice(shownCompare, shownCompare + PAGE_EACH);
|
||||
const sliceEx = filteredExclusive.slice(
|
||||
shownExclusive,
|
||||
shownExclusive + PAGE_EACH
|
||||
);
|
||||
const sliceCo = filteredCompare.slice(
|
||||
shownCompare,
|
||||
shownCompare + PAGE_EACH
|
||||
);
|
||||
|
||||
shownExclusive += sliceEx.length;
|
||||
shownCompare += sliceCo.length;
|
||||
|
|
@ -615,8 +634,10 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
arr.sort((a, b) => {
|
||||
const ap = Number.isFinite(a._storePrice) ? a._storePrice : null;
|
||||
const bp = Number.isFinite(b._storePrice) ? b._storePrice : null;
|
||||
const aKey = ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
|
||||
const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
|
||||
const aKey =
|
||||
ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
|
||||
const bKey =
|
||||
bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
|
||||
if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey;
|
||||
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
||||
});
|
||||
|
|
@ -627,8 +648,10 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
arr.sort((a, b) => {
|
||||
const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null;
|
||||
const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null;
|
||||
const aKey = ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
|
||||
const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
|
||||
const aKey =
|
||||
ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
|
||||
const bKey =
|
||||
bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
|
||||
if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey;
|
||||
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
||||
});
|
||||
|
|
@ -638,10 +661,8 @@ export async function renderStore($app, storeLabelRaw) {
|
|||
function sortCompareInPlace(arr) {
|
||||
const mode = compareMode();
|
||||
arr.sort((a, b) => {
|
||||
const da =
|
||||
mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar;
|
||||
const db =
|
||||
mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar;
|
||||
const da = mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar;
|
||||
const db = mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar;
|
||||
|
||||
const sa = da === null || !Number.isFinite(da) ? 999999 : da;
|
||||
const sb = db === null || !Number.isFinite(db) ? 999999 : db;
|
||||
|
|
|
|||
Loading…
Reference in a new issue