diff --git a/tools/build_email_alert.js b/tools/build_email_alert.js index 8a15cd6..ecde6fc 100755 --- a/tools/build_email_alert.js +++ b/tools/build_email_alert.js @@ -299,7 +299,7 @@ function renderHtml({ title, subtitle, uniqueNews, bigSales, commitUrl, pagesUrl const name = htmlEscape(it.name || ""); const store = htmlEscape(it.storeLabel || ""); const cat = htmlEscape(it.categoryLabel || ""); - const price = htmlEscape(it.price || it.newPrice || ""); + const price = htmlEscape(it.price || ""); const url = htmlEscape(it.url || ""); return ` diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index cd925cb..d5c476a 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -1,23 +1,11 @@ #!/usr/bin/env node "use strict"; -/* - Print local link URLs for SKUs with largest rank discrepancy between AB and BC lists. - - Usage: - node scripts/rank_discrepency_links.js \ - --ab reports/common_listings_ab_top1000.json \ - --bc reports/common_listings_bc_top1000.json \ - --top 50 \ - --base "http://127.0.0.1:8080/#/link/?left=" - - Output: - http://127.0.0.1:8080/#/link/?left= -*/ - const fs = require("fs"); const path = require("path"); +/* ---------------- IO ---------------- */ + function readJson(p) { return JSON.parse(fs.readFileSync(p, "utf8")); } @@ -26,47 +14,511 @@ function parseArgs(argv) { const out = { ab: "reports/common_listings_ab_top1000.json", bc: "reports/common_listings_bc_top1000.json", + meta: "data/sku_links.json", + top: 50, minDiscrep: 1, includeMissing: false, + + // similarityScore is NOT 0..1. + minScore: 9.0, + minContain: 0.75, + + // only consider suggestions from the opposite list (AB->BC or BC->AB) + requireCrossGroup: true, + base: "http://127.0.0.1:8080/#/link/?left=", + + debug: false, + debugN: 25, + debugPayload: false, + debugBest: false, + dumpScores: false, }; + for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--ab" && argv[i + 1]) out.ab = argv[++i]; else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i]; + else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i]; + else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top; else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep; + else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore; + else if (a === "--min-contain" && argv[i + 1]) out.minContain = Number(argv[++i]) || out.minContain; + else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); + + else if (a === "--no-cross-group") out.requireCrossGroup = false; + + else if (a === "--debug") out.debug = true; + else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN; + else if (a === "--debug-payload") out.debugPayload = true; + else if (a === "--debug-best") out.debugBest = true; + else if (a === "--dump-scores") out.dumpScores = true; } + return out; } +/* ---------------- row extraction ---------------- */ + +function extractRows(payload) { + if (Array.isArray(payload)) return payload; + const candidates = [payload?.rows, payload?.data?.rows, payload?.data, payload?.items, payload?.list, payload?.results]; + for (const x of candidates) if (Array.isArray(x)) return x; + return []; +} + +function rowKey(r) { + const k = r?.canonSku ?? r?.sku ?? r?.canon ?? r?.id ?? r?.key; + return k ? String(k) : ""; +} + function buildRankMap(payload) { - const rows = Array.isArray(payload?.rows) ? payload.rows : []; + const rows = extractRows(payload); const map = new Map(); for (let i = 0; i < rows.length; i++) { const r = rows[i]; - const k = r?.canonSku; + const k = rowKey(r); if (!k) continue; map.set(String(k), { rank: i + 1, row: r }); } - return map; + return { map, rowsLen: rows.length, rows }; } +function pickName(row) { + if (!row) return ""; + const repName = row?.representative?.name; + if (typeof repName === "string" && repName.trim()) return repName.trim(); + const cheapName = row?.cheapest?.name; + if (typeof cheapName === "string" && cheapName.trim()) return cheapName.trim(); + + const direct = ["name","title","productName","displayName","itemName","label","desc","description"]; + for (const k of direct) { + const v = row[k]; + if (typeof v === "string" && v.trim()) return v.trim(); + } + return ""; +} + +/* ---------------- sku_links union-find grouping + ignores ---------------- */ + +function normalizeImplicitSkuKey(k) { + const s = String(k || "").trim(); + const m = s.match(/^id:(\d{1,6})$/i); + if (m) return String(m[1]).padStart(6, "0"); + return s; +} + +function canonicalPairKey(a, b) { + const x = normalizeImplicitSkuKey(a); + const y = normalizeImplicitSkuKey(b); + if (!x || !y) return ""; + return x < y ? `${x}|${y}` : `${y}|${x}`; +} + +function buildIgnoreSet(meta) { + const ignores = Array.isArray(meta?.ignores) ? meta.ignores : []; + const s = new Set(); + for (const x of ignores) { + const a = String(x?.skuA || x?.a || x?.left || "").trim(); + const b = String(x?.skuB || x?.b || x?.right || "").trim(); + const k = canonicalPairKey(a, b); + if (k) s.add(k); + } + return s; +} + +class DSU { + constructor() { + this.parent = new Map(); + this.rank = new Map(); + } + _add(x) { + if (!this.parent.has(x)) { + this.parent.set(x, x); + this.rank.set(x, 0); + } + } + find(x) { + x = String(x || "").trim(); + if (!x) return ""; + this._add(x); + let p = this.parent.get(x); + if (p !== x) { + p = this.find(p); + this.parent.set(x, p); + } + return p; + } + union(a, b) { + a = String(a || "").trim(); + b = String(b || "").trim(); + if (!a || !b || a === b) return; + const ra = this.find(a); + const rb = this.find(b); + if (!ra || !rb || ra === rb) return; + + const rka = this.rank.get(ra) || 0; + const rkb = this.rank.get(rb) || 0; + + if (rka < rkb) this.parent.set(ra, rb); + else if (rkb < rka) this.parent.set(rb, ra); + else { + this.parent.set(rb, ra); + this.rank.set(ra, rka + 1); + } + } +} + +function compareSku(a, b) { + a = String(a || "").trim(); + b = String(b || "").trim(); + if (a === b) return 0; + + const aUnknown = a.startsWith("u:"); + const bUnknown = b.startsWith("u:"); + if (aUnknown !== bUnknown) return aUnknown ? 1 : -1; + + const aNum = /^\d+$/.test(a); + const bNum = /^\d+$/.test(b); + if (aNum && bNum) { + const na = Number(a), nb = Number(b); + if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1; + } + return a < b ? -1 : 1; +} + +function buildCanonicalSkuFnFromMeta(meta) { + const links = Array.isArray(meta?.links) ? meta.links : []; + if (!links.length) return (sku) => normalizeImplicitSkuKey(sku); + + const dsu = new DSU(); + const all = new Set(); + + for (const x of links) { + const a = normalizeImplicitSkuKey(x?.fromSku); + const b = normalizeImplicitSkuKey(x?.toSku); + if (!a || !b || a === b) continue; + all.add(a); + all.add(b); + dsu.union(a, b); + } + + const groupsByRoot = new Map(); + for (const s of all) { + const r = dsu.find(s); + if (!r) continue; + let set = groupsByRoot.get(r); + if (!set) groupsByRoot.set(r, (set = new Set())); + set.add(s); + } + + const repByRoot = new Map(); + for (const [root, members] of groupsByRoot.entries()) { + const arr = Array.from(members); + arr.sort(compareSku); + repByRoot.set(root, arr[0] || root); + } + + const canonBySku = new Map(); + for (const [root, members] of groupsByRoot.entries()) { + const rep = repByRoot.get(root) || root; + for (const s of members) canonBySku.set(s, rep); + canonBySku.set(rep, rep); + } + + return (sku) => { + const s = normalizeImplicitSkuKey(sku); + return canonBySku.get(s) || s; + }; +} + +/* ---------------- similarity (same math as viz/app/linker/similarity.js) ---------------- */ + +function normSearchText(s) { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function tokenizeQuery(q) { + const n = normSearchText(q); + return n ? n.split(" ").filter(Boolean) : []; +} + +const SIM_STOP_TOKENS = new Set([ + "the","a","an","and","of","to","in","for","with", + "year","years","yr","yrs","old", + "whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof", + "anniversary", +]); + +const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i; + +function numKey(t) { + const s = String(t || "").trim().toLowerCase(); + if (!s) return ""; + if (/^\d+$/.test(s)) return s; + const m = s.match(ORDINAL_RE); + return m ? m[1] : ""; +} + +function extractAgeFromText(normName) { + const s = String(normName || ""); + if (!s) return ""; + + const m = s.match(/\b(?:aged\s*)?(\d{1,2})\s*(?:yr|yrs|year|years)\b/i); + if (m && m[1]) return String(parseInt(m[1], 10)); + + const m2 = s.match(/\b(\d{1,2})\s*yo\b/i); + if (m2 && m2[1]) return String(parseInt(m2[1], 10)); + + return ""; +} + +function filterSimTokens(tokens) { + const out = []; + const seen = new Set(); + + const SIM_EQUIV = new Map([ + ["years", "yr"], + ["year", "yr"], + ["yrs", "yr"], + ["yr", "yr"], + ["whiskey", "whisky"], + ["whisky", "whisky"], + ["bourbon", "bourbon"], + ]); + + const VOL_UNIT = new Set(["ml","l","cl","oz","liter","liters","litre","litres"]); + const VOL_INLINE_RE = /^\d+(?:\.\d+)?(?:ml|l|cl|oz)$/i; + const PCT_INLINE_RE = /^\d+(?:\.\d+)?%$/; + + const arr = Array.isArray(tokens) ? tokens : []; + + for (let i = 0; i < arr.length; i++) { + let t = String(arr[i] || "").trim().toLowerCase(); + if (!t) continue; + if (!/[a-z0-9]/i.test(t)) continue; + if (VOL_INLINE_RE.test(t)) continue; + if (PCT_INLINE_RE.test(t)) continue; + + t = SIM_EQUIV.get(t) || t; + + const nk = numKey(t); + if (nk) t = nk; + + if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue; + + if (/^\d+(?:\.\d+)?$/.test(t)) { + const next = String(arr[i + 1] || "").trim().toLowerCase(); + const nextNorm = SIM_EQUIV.get(next) || next; + if (VOL_UNIT.has(nextNorm)) { + i++; + continue; + } + } + + if (!numKey(t) && SIM_STOP_TOKENS.has(t)) continue; + + if (seen.has(t)) continue; + seen.add(t); + out.push(t); + } + + return out; +} + +function tokenContainmentScore(aTokens, bTokens) { + const A = filterSimTokens(aTokens || []); + const B = filterSimTokens(bTokens || []); + if (!A.length || !B.length) return 0; + + const aSet = new Set(A); + const bSet = new Set(B); + + const small = aSet.size <= bSet.size ? aSet : bSet; + const big = aSet.size <= bSet.size ? bSet : aSet; + + let hit = 0; + for (const t of small) if (big.has(t)) hit++; + + const recall = hit / Math.max(1, small.size); + const precision = hit / Math.max(1, big.size); + const f1 = (2 * precision * recall) / Math.max(1e-9, precision + recall); + + return f1; +} + +function levenshtein(a, b) { + a = String(a || ""); + b = String(b || ""); + const n = a.length, m = b.length; + if (!n) return m; + if (!m) return n; + + const dp = new Array(m + 1); + for (let j = 0; j <= m; j++) dp[j] = j; + + for (let i = 1; i <= n; i++) { + let prev = dp[0]; + dp[0] = i; + const ca = a.charCodeAt(i - 1); + for (let j = 1; j <= m; j++) { + const tmp = dp[j]; + const cost = ca === b.charCodeAt(j - 1) ? 0 : 1; + dp[j] = Math.min(dp[j] + 1, dp[j - 1] + 1, prev + cost); + prev = tmp; + } + } + return dp[m]; +} + +function numberMismatchPenalty(aTokens, bTokens) { + const aNums = new Set((aTokens || []).map(numKey).filter(Boolean)); + const bNums = new Set((bTokens || []).map(numKey).filter(Boolean)); + if (!aNums.size || !bNums.size) return 1.0; + for (const n of aNums) if (bNums.has(n)) return 1.0; + return 0.28; +} + +function similarityScore(aName, bName) { + const a = normSearchText(aName); + const b = normSearchText(bName); + if (!a || !b) return 0; + + const aAge = extractAgeFromText(a); + const bAge = extractAgeFromText(b); + const ageBoth = !!(aAge && bAge); + const ageMatch = ageBoth && aAge === bAge; + const ageMismatch = ageBoth && aAge !== bAge; + + const aToksRaw = tokenizeQuery(a); + const bToksRaw = tokenizeQuery(b); + + const aToks = filterSimTokens(aToksRaw); + const bToks = filterSimTokens(bToksRaw); + if (!aToks.length || !bToks.length) return 0; + + const contain = tokenContainmentScore(aToksRaw, bToksRaw); + + const aFirst = aToks[0] || ""; + const bFirst = bToks[0] || ""; + const firstMatch = aFirst && bFirst && aFirst === bFirst ? 1 : 0; + + const A = new Set(aToks.slice(1)); + const B = new Set(bToks.slice(1)); + let inter = 0; + for (const w of A) if (B.has(w)) inter++; + const denom = Math.max(1, Math.max(A.size, B.size)); + const overlapTail = inter / denom; + + const d = levenshtein(a, b); + const maxLen = Math.max(1, Math.max(a.length, b.length)); + const levSim = 1 - d / maxLen; + + let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain); + const smallN = Math.min(aToks.length, bToks.length); + if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18; + + const numGate = numberMismatchPenalty(aToks, bToks); + + let s = + numGate * + (firstMatch * 3.0 + + overlapTail * 2.2 * gate + + levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain))); + + if (ageMatch) s *= 2.2; + else if (ageMismatch) s *= 0.18; + + s *= 1 + 0.9 * contain; + + return s; +} + +/* ---------------- debug helpers ---------------- */ + +function eprintln(...args) { console.error(...args); } +function truncate(s, n) { s = String(s || ""); return s.length <= n ? s : s.slice(0, n - 1) + "…"; } + +/* ---------------- main ---------------- */ + function main() { const args = parseArgs(process.argv.slice(2)); const repoRoot = process.cwd(); const abPath = path.isAbsolute(args.ab) ? args.ab : path.join(repoRoot, args.ab); const bcPath = path.isAbsolute(args.bc) ? args.bc : path.join(repoRoot, args.bc); + const metaPath = args.meta ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) : ""; const ab = readJson(abPath); const bc = readJson(bcPath); - const abMap = buildRankMap(ab); - const bcMap = buildRankMap(bc); + const meta = metaPath ? readJson(metaPath) : null; + const canonicalSku = meta ? buildCanonicalSkuFnFromMeta(meta) : (sku) => normalizeImplicitSkuKey(sku); + + const ignoreSet = meta ? buildIgnoreSet(meta) : new Set(); + function isIgnoredPair(a, b) { + const k = canonicalPairKey(a, b); + return k ? ignoreSet.has(k) : false; + } + + const abBuilt = buildRankMap(ab); + const bcBuilt = buildRankMap(bc); + + const abMap = abBuilt.map; + const bcMap = bcBuilt.map; + + // SKU pools for “cross group” matching + const abSkus = new Set(abMap.keys()); + const bcSkus = new Set(bcMap.keys()); + + // union SKU -> row (for name lookup) + const rowBySku = new Map(); + for (const m of [abMap, bcMap]) { + for (const [canonSku, v] of m.entries()) { + if (!rowBySku.has(canonSku)) rowBySku.set(canonSku, v.row); + } + } + + const allSkus = Array.from(rowBySku.keys()); + const allNames = new Map(); + let namedCount = 0; + for (const sku of allSkus) { + const n = pickName(rowBySku.get(sku)); + allNames.set(sku, n); + if (n) namedCount++; + } + + if (args.debug) { + eprintln("[rank_discrepency] inputs:", { + abPath, bcPath, metaPath: metaPath || "(none)", + linkCount: Array.isArray(meta?.links) ? meta.links.length : 0, + ignoreCount: Array.isArray(meta?.ignores) ? meta.ignores.length : 0, + ignoreSetSize: ignoreSet.size, + minDiscrep: args.minDiscrep, + minScore: args.minScore, + minContain: args.minContain, + requireCrossGroup: args.requireCrossGroup, + top: args.top, + includeMissing: args.includeMissing, + }); + eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, abKeys: abMap.size, bcKeys: bcMap.size }); + eprintln("[rank_discrepency] name coverage:", { totalSkus: allSkus.length, named: namedCount, unnamed: allSkus.length - namedCount }); + } + + if (args.debugPayload) { + const ab0 = abBuilt.rows[0]; + const bc0 = bcBuilt.rows[0]; + eprintln("[rank_discrepency] sample AB rep.name:", truncate(ab0?.representative?.name || "", 120)); + eprintln("[rank_discrepency] sample BC rep.name:", truncate(bc0?.representative?.name || "", 120)); + } const keys = new Set([...abMap.keys(), ...bcMap.keys()]); const diffs = []; @@ -74,23 +526,14 @@ function main() { for (const canonSku of keys) { const a = abMap.get(canonSku); const b = bcMap.get(canonSku); - if (!args.includeMissing && (!a || !b)) continue; const rankAB = a ? a.rank : null; const rankBC = b ? b.rank : null; - - const discrep = - rankAB !== null && rankBC !== null ? Math.abs(rankAB - rankBC) : Infinity; - + const discrep = rankAB !== null && rankBC !== null ? Math.abs(rankAB - rankBC) : Infinity; if (discrep !== Infinity && discrep < args.minDiscrep) continue; - diffs.push({ - canonSku, - discrep, - // tie-breakers - sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9), - }); + diffs.push({ canonSku, discrep, rankAB, rankBC, sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9) }); } diffs.sort((x, y) => { @@ -99,14 +542,145 @@ function main() { return String(x.canonSku).localeCompare(String(y.canonSku)); }); - const top = diffs.slice(0, args.top); + if (args.debug) { + eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length }); + eprintln("[rank_discrepency] top discrep sample:", + diffs.slice(0, 5).map((d) => ({ + sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, + name: truncate(allNames.get(String(d.canonSku)) || "", 80), + })) + ); + } - for (const d of top) { - // examples: - // 884096 -> left=884096 - // id:1049355 -> left=id%3A1049355 - // u:bb504a62 -> left=u%3Abb504a62 - console.log(args.base + encodeURIComponent(d.canonSku)); + if (args.debugBest && diffs.length) { + const skuA = String(diffs[0].canonSku); + const nameA = allNames.get(skuA) || ""; + const groupA = canonicalSku(skuA); + const aInAB = abSkus.has(skuA); + const pool = args.requireCrossGroup ? (aInAB ? bcSkus : abSkus) : new Set(allSkus); + + const aRaw = tokenizeQuery(nameA); + const scored = []; + + for (const skuB of pool) { + if (skuB === skuA) continue; + if (canonicalSku(skuB) === groupA) continue; + if (isIgnoredPair(skuA, skuB)) continue; + + const nameB = allNames.get(skuB) || ""; + if (!nameB) continue; + + const contain = tokenContainmentScore(aRaw, tokenizeQuery(nameB)); + if (contain < args.minContain) continue; + + const s = similarityScore(nameA, nameB); + scored.push({ skuB, s, contain, nameB }); + } + + scored.sort((a, b) => b.s - a.s); + eprintln("[rank_discrepency] debug-best for first discrep:", { + skuA, + side: aInAB ? "AB" : "BC", + nameA: truncate(nameA, 120), + minContain: args.minContain, + top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })), + }); + } + + const filtered = []; + const debugLines = []; + + for (const d of diffs) { + const skuA = String(d.canonSku); + const nameA = allNames.get(skuA) || ""; + if (!nameA) continue; + + const aInAB = abSkus.has(skuA); + const pool = args.requireCrossGroup ? (aInAB ? bcSkus : abSkus) : new Set(allSkus); + + const groupA = canonicalSku(skuA); + const aRaw = tokenizeQuery(nameA); + + let best = 0, bestSku = "", bestName = "", bestContain = 0; + let bestWasIgnored = false; + + for (const skuB of pool) { + if (skuB === skuA) continue; + if (canonicalSku(skuB) === groupA) continue; + + if (isIgnoredPair(skuA, skuB)) { + // critical: ignored pairs must NOT satisfy the requirement + bestWasIgnored = true; + continue; + } + + const nameB = allNames.get(skuB) || ""; + if (!nameB) continue; + + const contain = tokenContainmentScore(aRaw, tokenizeQuery(nameB)); + if (contain < args.minContain) continue; + + const s = similarityScore(nameA, nameB); + if (s > best) { + best = s; + bestSku = skuB; + bestName = nameB; + bestContain = contain; + } + } + + const pass = bestSku && best >= args.minScore; + + if (args.debug && debugLines.length < args.debugN) { + debugLines.push({ + sku: skuA, + side: aInAB ? "AB" : "BC", + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + nameA: truncate(nameA, 52), + best, + bestContain, + bestSku, + bestSide: abSkus.has(bestSku) ? "AB" : "BC", + bestName: truncate(bestName, 52), + sawIgnoredPairs: bestWasIgnored, + pass, + }); + } + + if (!pass) continue; + + filtered.push({ ...d, best, bestSku, bestName, bestContain }); + if (filtered.length >= args.top) break; + } + + if (args.debug) { + eprintln("[rank_discrepency] filter results:", { + filtered: filtered.length, + minScore: args.minScore, + minContain: args.minContain, + requireCrossGroup: args.requireCrossGroup, + minDiscrep: args.minDiscrep, + }); + eprintln("[rank_discrepency] debug sample (first N checked):"); + for (const x of debugLines) eprintln(" ", x); + } + + for (const d of filtered) { + if (args.dumpScores) { + eprintln("[rank_discrepency] emit", JSON.stringify({ + sku: d.canonSku, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + best: d.best, + bestContain: d.bestContain, + bestSku: d.bestSku, + bestName: truncate(d.bestName, 120), + })); + } + console.log(args.base + encodeURIComponent(String(d.canonSku))); } } diff --git a/viz/app/search_page.js b/viz/app/search_page.js index 82f6845..a8a16a1 100644 --- a/viz/app/search_page.js +++ b/viz/app/search_page.js @@ -347,7 +347,7 @@ export function renderSearch($app) { typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x; const nowMs = Date.now(); - const cutoffMs = nowMs - 24 * 60 * 60 * 1000; + const cutoffMs = nowMs - 3 * 24 * 60 * 60 * 1000; function eventMs(r) { const t = String(r?.ts || ""); @@ -420,7 +420,7 @@ export function renderSearch($app) { const limited = ranked.slice(0, 140); $results.innerHTML = - `
Recently changed (last 24 hours):
` + + `
Recently changed (last 3 days):
` + limited .map(({ r, meta }) => { const kindLabel = diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index 3920d2d..0c0092f 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -438,9 +438,8 @@ function computeSeriesFromRaw(raw, filter) { /* ---------------- y-axis bounds ---------------- */ -function computeYBounds(seriesByStore, defaultAbs) { - let mn = Infinity; - let mx = -Infinity; +function computeYBounds(seriesByStore, minSpan = 6, pad = 1) { + let mn = Infinity, mx = -Infinity; for (const arr of Object.values(seriesByStore || {})) { if (!Array.isArray(arr)) continue; @@ -451,13 +450,27 @@ function computeYBounds(seriesByStore, defaultAbs) { } } - if (mn === Infinity) return { min: -defaultAbs, max: defaultAbs }; + if (mn === Infinity) return { min: -minSpan / 2, max: minSpan / 2 }; - const min = Math.min(-defaultAbs, Math.floor(mn)); - const max = Math.max(defaultAbs, Math.ceil(mx)); - return { min, max }; + mn = Math.min(mn, 0); + mx = Math.max(mx, 0); + + // pad a bit so lines aren't glued to edges + mn = Math.floor(mn - pad); + mx = Math.ceil(mx + pad); + + // enforce a minimum visible range so it doesn't get *too* tight + const span = mx - mn; + if (span < minSpan) { + const mid = (mn + mx) / 2; + mn = Math.floor(mid - minSpan / 2); + mx = Math.ceil(mid + minSpan / 2); + } + + return { min: mn, max: mx }; } + /* ---------------- prefs ---------------- */ const LS_GROUP = "stviz:v1:stats:group"; @@ -498,7 +511,7 @@ export async function renderStats($app) { const pref = loadPrefs(); $app.innerHTML = ` -
+
@@ -557,8 +570,8 @@ export async function renderStats($app) {
-
-
+
+
@@ -772,9 +785,21 @@ export async function renderStats($app) { min: yBounds?.min, max: yBounds?.max, title: { display: true, text: "Avg % vs per-SKU median" }, + ticks: { + stepSize: 1, + precision: 0, + autoSkip: false, // <- don't skip integer ticks callback: (v) => `${Number(v).toFixed(0)}%`, - maxTicksLimit: 12, + }, + + grid: { + drawBorder: false, + color: (ctx) => + ctx.tick.value === 0 + ? "rgba(154,166,178,0.35)" + : "rgba(154,166,178,0.18)", + lineWidth: 1, }, }, }, @@ -841,10 +866,10 @@ export async function renderStats($app) { maxPrice: selectedMaxPrice, }); - const abs = group === "all" ? 12 : 8; - const yBounds = computeYBounds(series.seriesByStore, abs); + const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); + _chart?.resize(); const short = `Loaded ${series.labels.length} day(s). Filtered SKUs: ${series.newestUsed}/${series.newestTotal}.`; onStatus(short); @@ -877,10 +902,10 @@ export async function renderStats($app) { maxPrice: selectedMaxPrice, }); - const abs = group === "all" ? 12 : 8; - const yBounds = computeYBounds(series.seriesByStore, abs); + const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); + _chart?.resize(); const short = `Loaded ${series.labels.length} day(s). Filtered SKUs: ${series.newestUsed}/${series.newestTotal}.`; onStatus(short); diff --git a/viz/app/storeColors.js b/viz/app/storeColors.js index 49dd4c0..a336b13 100644 --- a/viz/app/storeColors.js +++ b/viz/app/storeColors.js @@ -1,166 +1,183 @@ function normalizeId(s) { - return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, ""); - } - - // Your pinned colors (exact / “roughly right”) - const OVERRIDES = { - strath: "#76B7FF", - bsw: "#E9DF7A", - kensingtonwinemarket: "#F2C200", - vessel: "#FFFFFF", - gullliquor: "#6B0F1A", - kegncork: "#111111", - legacyliquor: "#7B4A12", - vintagespirits: "#E34A2C", - - craftcellars: "#E31B23", // bright red - maltsandgrains: "#A67C52", // faded brown - - // aliases - gull: "#6B0F1A", - legacy: "#7B4A12", - vintage: "#E34A2C", - kwm: "#F2C200", - }; - - // High-contrast qualitative palette (distinct hues). - // (Avoids whites/blacks/yellows that clash w/ your overrides by filtering below.) - const PALETTE = [ - "#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD", - "#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22", - "#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5", - "#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D", - // extras to reduce wrap risk - "#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173", - "#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D", - ]; - - function uniq(arr) { - return [...new Set(arr)]; - } - - function buildUniverse(base, extra) { - const a = Array.isArray(base) ? base : []; - const b = Array.isArray(extra) ? extra : []; - return uniq([...a, ...b].map(normalizeId).filter(Boolean)); - } - - // Default known ids (keeps mapping stable even if a page only sees a subset) - const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [ - "bcl", - "bsw", - "coop", - "craftcellars", - "gullliquor", - "gull", - "kegncork", - "kwm", - "kensingtonwinemarket", - "legacy", - "legacyliquor", - "maltsandgrains", - "sierrasprings", - "strath", - "tudor", - "vessel", - "vintage", - "vintagespirits", - "willowpark", - ]); - - function isWhiteHex(c) { - return String(c || "").trim().toUpperCase() === "#FFFFFF"; - } - - export function buildStoreColorMap(extraUniverse = []) { - const universe = buildUniverse(DEFAULT_UNIVERSE, extraUniverse).sort(); - - const used = new Set(); - const map = new Map(); - - // pin overrides first - for (const id of universe) { - const c = OVERRIDES[id]; - if (c) { - map.set(id, c); - used.add(String(c).toUpperCase()); - } - } - - // filter palette to avoid exact collisions with overrides (and keep white reserved for Vessel) - const palette = PALETTE - .map((c) => String(c).toUpperCase()) - .filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111"); - - let pi = 0; - for (const id of universe) { - if (map.has(id)) continue; - if (pi >= palette.length) { - // If you ever exceed palette size, just reuse (rare). Still deterministic. - pi = 0; - } - const c = palette[pi++]; + return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, ""); +} + +// Map normalized store *labels* to canonical ids used by OVERRIDES +const ALIASES = { + strathliquor: "strath", + vesselliquor: "vessel", + tudorhouse: "tudor", + coopworldofwhisky: "coop", + + kensingtonwinemarket: "kensingtonwinemarket", + gullliquor: "gullliquor", + legacyliquor: "legacyliquor", + vintagespirits: "vintagespirits", + kegncork: "kegncork", + + // short forms + gull: "gullliquor", + legacy: "legacyliquor", + vintage: "vintagespirits", + kwm: "kensingtonwinemarket", +}; + +// Your pinned colors +const OVERRIDES = { + strath: "#76B7FF", + bsw: "#E9DF7A", + kensingtonwinemarket: "#F2C200", + vessel: "#FFFFFF", + gullliquor: "#6B0F1A", + kegncork: "#111111", + legacyliquor: "#7B4A12", + vintagespirits: "#E34A2C", + + craftcellars: "#E31B23", + maltsandgrains: "#A67C52", + + // aliases + gull: "#6B0F1A", + legacy: "#7B4A12", + vintage: "#E34A2C", + kwm: "#F2C200", +}; + +// High-contrast qualitative palette +const PALETTE = [ + "#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD", + "#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22", + "#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5", + "#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D", + "#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173", + "#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D", +]; + +function uniq(arr) { + return [...new Set(arr)]; +} + +function canonicalId(s) { + const id = normalizeId(s); + return ALIASES[id] || id; +} + +function buildUniverse(base, extra) { + const a = Array.isArray(base) ? base : []; + const b = Array.isArray(extra) ? extra : []; + return uniq([...a, ...b].map(canonicalId).filter(Boolean)); +} + +// Keep mapping stable even if page sees a subset +const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [ + "bcl", + "bsw", + "coop", + "craftcellars", + "gullliquor", + "gull", + "kegncork", + "kwm", + "kensingtonwinemarket", + "legacy", + "legacyliquor", + "maltsandgrains", + "sierrasprings", + "strath", + "tudor", + "vessel", + "vintage", + "vintagespirits", + "willowpark", +]); + +function isWhiteHex(c) { + return String(c || "").trim().toUpperCase() === "#FFFFFF"; +} + +export function buildStoreColorMap(extraUniverse = []) { + const universe = buildUniverse(DEFAULT_UNIVERSE, extraUniverse).sort(); + + const used = new Set(); + const map = new Map(); + + // Pin overrides first + for (const id of universe) { + const c = OVERRIDES[id]; + if (c) { map.set(id, c); - used.add(c); + used.add(String(c).toUpperCase()); } - - return map; } - - export function storeColor(storeKeyOrLabel, colorMap) { - const id = normalizeId(storeKeyOrLabel); - if (!id) return "#7F7F7F"; - - const forced = OVERRIDES[id]; - if (forced) return forced; - - if (colorMap && typeof colorMap.get === "function") { - const c = colorMap.get(id); - if (c) return c; - } - - // fallback: deterministic but not “no conflicts” - return PALETTE[(id.length + id.charCodeAt(0)) % PALETTE.length]; + + // Filter palette to avoid collisions and keep white/black reserved + const palette = PALETTE + .map((c) => String(c).toUpperCase()) + .filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111"); + + let pi = 0; + for (const id of universe) { + if (map.has(id)) continue; + if (pi >= palette.length) pi = 0; + const c = palette[pi++]; + map.set(id, c); + used.add(c); } - - export function datasetStrokeWidth(color) { - return isWhiteHex(color) ? 2.5 : 1.5; + + return map; +} + +export function storeColor(storeKeyOrLabel, colorMap) { + const id = canonicalId(storeKeyOrLabel); + if (!id) return "#7F7F7F"; + + const forced = OVERRIDES[id]; + if (forced) return forced; + + if (colorMap && typeof colorMap.get === "function") { + const c = colorMap.get(id); + if (c) return c; } - - export function datasetPointRadius(color) { - return isWhiteHex(color) ? 2.8 : 2.2; - } - - function clamp(v, lo, hi) { - return Math.max(lo, Math.min(hi, v)); - } - - function hexToRgb(hex) { - const m = String(hex).replace("#", ""); - if (m.length !== 6) return null; - const n = parseInt(m, 16); - return { - r: (n >> 16) & 255, - g: (n >> 8) & 255, - b: n & 255, - }; - } - - function rgbToHex({ r, g, b }) { - const h = (x) => clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0"); - return `#${h(r)}${h(g)}${h(b)}`; - } - - // lighten by mixing with white (amount 0–1) - export function lighten(hex, amount = 0.25) { - const rgb = hexToRgb(hex); - if (!rgb) return hex; - return rgbToHex({ - r: rgb.r + (255 - rgb.r) * amount, - g: rgb.g + (255 - rgb.g) * amount, - b: rgb.b + (255 - rgb.b) * amount, - }); - } - - - \ No newline at end of file + + return PALETTE[(id.length + id.charCodeAt(0)) % PALETTE.length]; +} + +export function datasetStrokeWidth(color) { + return isWhiteHex(color) ? 2.5 : 1.5; +} + +export function datasetPointRadius(color) { + return isWhiteHex(color) ? 2.8 : 2.2; +} + +function clamp(v, lo, hi) { + return Math.max(lo, Math.min(hi, v)); +} + +function hexToRgb(hex) { + const m = String(hex).replace("#", ""); + if (m.length !== 6) return null; + const n = parseInt(m, 16); + return { + r: (n >> 16) & 255, + g: (n >> 8) & 255, + b: n & 255, + }; +} + +function rgbToHex({ r, g, b }) { + const h = (x) => + clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0"); + return `#${h(r)}${h(g)}${h(b)}`; +} + +// Lighten by mixing with white (0–1) +export function lighten(hex, amount = 0.25) { + const rgb = hexToRgb(hex); + if (!rgb) return hex; + return rgbToHex({ + r: rgb.r + (255 - rgb.r) * amount, + g: rgb.g + (255 - rgb.g) * amount, + b: rgb.b + (255 - rgb.b) * amount, + }); +} diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 481217c..6cadaaa 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -6,7 +6,7 @@ import { keySkuForRow, parsePriceToNumber, } from "./sku.js"; -import { loadIndex } from "./state.js"; +import { loadIndex, loadRecent } from "./state.js"; import { aggregateBySku } from "./catalog.js"; import { loadSkuRules } from "./mapping.js"; @@ -127,6 +127,8 @@ export async function renderStore($app, storeLabelRaw) { + +
@@ -203,6 +205,84 @@ export async function renderStore($app, storeLabelRaw) { rulesCache = await loadSkuRules(); const rules = rulesCache; + // --- Recent (7d), most-recent per canonicalSku + store --- + const recent = await loadRecent().catch(() => null); + const recentItems = Array.isArray(recent?.items) ? recent.items : []; + + function eventMs(r) { + const t = String(r?.ts || ""); + const ms = t ? Date.parse(t) : NaN; + if (Number.isFinite(ms)) return ms; + + const d = String(r?.date || ""); + const ms2 = d ? Date.parse(d + "T00:00:00Z") : NaN; + return Number.isFinite(ms2) ? ms2 : 0; + } + + const RECENT_DAYS = 7; + const nowMs = Date.now(); + const cutoffMs = nowMs - RECENT_DAYS * 24 * 60 * 60 * 1000; + + // canonicalSku -> storeNorm -> recentRow (latest) + const recentBySkuStore = new Map(); + + for (const r of recentItems) { + const ms = eventMs(r); + if (!(ms >= cutoffMs && ms <= nowMs)) continue; + + const rawSku = String(r?.sku || "").trim(); + if (!rawSku) continue; + const sku = String(rules.canonicalSku(rawSku) || rawSku); + + const stNorm = normStoreLabel(r?.storeLabel || r?.store || ""); + if (!stNorm) continue; + + let sm = recentBySkuStore.get(sku); + if (!sm) recentBySkuStore.set(sku, (sm = new Map())); + + const prev = sm.get(stNorm); + if (!prev || eventMs(prev) < ms) sm.set(stNorm, r); + } + + function normalizeKindForPrice(r) { + let kind = String(r?.kind || ""); + if (kind === "price_change") { + const o = parsePriceToNumber(r?.oldPrice || ""); + const n = parsePriceToNumber(r?.newPrice || ""); + if (Number.isFinite(o) && Number.isFinite(n)) { + if (n < o) kind = "price_down"; + else if (n > o) kind = "price_up"; + else kind = "price_change"; + } + } + return kind; + } + + function saleMetaFor(it) { + const sku = String(it?.sku || ""); + const r = recentBySkuStore.get(sku)?.get(storeNorm) || null; + if (!r) return null; + + const kind = normalizeKindForPrice(r); + if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") + return null; + + const oldStr = String(r?.oldPrice || "").trim(); + const newStr = String(r?.newPrice || "").trim(); + const oldN = parsePriceToNumber(oldStr); + const newN = parsePriceToNumber(newStr); + if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) + return null; + + const delta = newN - oldN; // negative = down + const pct = Math.round(((newN - oldN) / oldN) * 100); // negative = down + + return { + _saleDelta: Number.isFinite(delta) ? delta : 0, + _salePct: Number.isFinite(pct) ? pct : 0, + }; + } + const listingsAll = Array.isArray(idx.items) ? idx.items : []; const liveAll = listingsAll.filter((r) => r && !r.removed); @@ -336,6 +416,8 @@ export async function renderStore($app, storeLabelRaw) { const firstSeenMs = firstSeenBySkuInStore.get(sku); const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null; + const sm = saleMetaFor(it); // { _saleDelta, _salePct } or null + return { ...it, _exclusive: exclusive, @@ -349,6 +431,9 @@ export async function renderStore($app, storeLabelRaw) { _diffVsBestDollar: diffVsBestDollar, _diffVsBestPct: diffVsBestPct, _firstSeenMs: firstSeen, + _saleDelta: sm ? sm._saleDelta : 0, + _salePct: sm ? sm._salePct : 0, + _hasSaleMeta: !!sm, }; }); @@ -492,9 +577,41 @@ export async function renderStore($app, storeLabelRaw) { return `$${esc(dollars)} higher`; } + function exclusiveAnnotHtml(it) { + const mode = String($exSort.value || "priceDesc"); + + // Sale sorts: show price change for THIS store (7d recent), unchanged => nothing. + if (mode === "salePct") { + const p = Number.isFinite(it._salePct) ? it._salePct : 0; + if (!p) return ""; + const abs = Math.abs(p); + if (p < 0) return `${esc(abs)}% off`; + return `+${esc(abs)}%`; + } + + if (mode === "saleAbs") { + const d = Number.isFinite(it._saleDelta) ? it._saleDelta : 0; + if (!d) return ""; + const abs = Math.round(Math.abs(d)); + if (!abs) return ""; + if (d < 0) return `$${esc(abs)} off`; + return `+$${esc(abs)}`; + } + + // Any NON-sale sort: still show the % badge (same as Sale %) when there was a change. + const p = Number.isFinite(it._salePct) ? it._salePct : 0; + if (!p) return ""; + const abs = Math.abs(p); + if (p < 0) return `${esc(abs)}% off`; + return `+${esc(abs)}%`; + } + function renderCard(it) { const price = listingPriceStr(it); - const href = String(it.sampleUrl || "").trim(); + + // Link the store badge consistently (respects SKU linking / canonical SKU) + const storeHref = readLinkHrefForSkuInStore(liveAll, String(it.sku || ""), storeNorm); + const href = storeHref || String(it.sampleUrl || "").trim(); const specialBadge = it._lastStock ? `Last Stock` @@ -508,6 +625,7 @@ export async function renderStore($app, storeLabelRaw) { : ""; const diffBadge = priceBadgeHtml(it); + const exAnnot = it._exclusive || it._lastStock ? exclusiveAnnotHtml(it) : ""; const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`; return ` @@ -526,6 +644,7 @@ export async function renderStore($app, storeLabelRaw) { ${specialBadge} ${bestBadge} ${diffBadge} + ${exAnnot} ${esc(price)} ${ href @@ -630,6 +749,31 @@ export async function renderStore($app, storeLabelRaw) { function sortExclusiveInPlace(arr) { const mode = String($exSort.value || "priceDesc"); + + if (mode === "salePct") { + arr.sort((a, b) => { + const ap = Number.isFinite(a._salePct) ? a._salePct : 0; // negative = better + const bp = Number.isFinite(b._salePct) ? b._salePct : 0; + if (ap !== bp) return ap - bp; // best deal first + const an = (String(a.name) + a.sku).toLowerCase(); + const bn = (String(b.name) + b.sku).toLowerCase(); + return an.localeCompare(bn); + }); + return; + } + + if (mode === "saleAbs") { + arr.sort((a, b) => { + const ad = Number.isFinite(a._saleDelta) ? a._saleDelta : 0; // negative = better + const bd = Number.isFinite(b._saleDelta) ? b._saleDelta : 0; + if (ad !== bd) return ad - bd; // best deal first + const an = (String(a.name) + a.sku).toLowerCase(); + const bn = (String(b.name) + b.sku).toLowerCase(); + return an.localeCompare(bn); + }); + return; + } + if (mode === "priceAsc" || mode === "priceDesc") { arr.sort((a, b) => { const ap = Number.isFinite(a._storePrice) ? a._storePrice : null; @@ -638,7 +782,8 @@ export async function renderStore($app, storeLabelRaw) { ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; - if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey; + if (aKey !== bKey) + return mode === "priceAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); return; @@ -652,7 +797,8 @@ export async function renderStore($app, storeLabelRaw) { ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; - if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey; + if (aKey !== bKey) + return mode === "dateAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); } diff --git a/viz/style.css b/viz/style.css index c8864e6..af6b48b 100644 --- a/viz/style.css +++ b/viz/style.css @@ -549,3 +549,27 @@ html { overflow-y: scroll; } .rangeDual input[type="range"]::-moz-range-track { background: transparent; } .rangeDual input[type="range"]::-moz-range-progress { background: transparent; } .rangeDual input[type="range"]::-moz-range-thumb { pointer-events: all; } + + +/* Stats page: make chart fill remaining viewport height */ +.containerFull{ + min-height: 100vh; + display: flex; + flex-direction: column; +} + +.cardFill{ + flex: 1 1 auto; + min-height: 320px; /* safety */ + display: flex; +} + +.chartFill{ + flex: 1 1 auto; + min-height: 0; /* IMPORTANT so flex children can actually shrink/grow */ +} + +.chartFill canvas{ + width: 100% !important; + height: 100% !important; +}