From b2a4afa8904bd2b897cd4dbfb97c1b2df894ad6f Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 09:52:27 -0800 Subject: [PATCH 01/19] UX Improvements --- tools/rank_discrepency.js | 402 +++++++++++++++++++++++++++++++++++++- 1 file changed, 394 insertions(+), 8 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index cd925cb..239b87d 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -2,12 +2,16 @@ "use strict"; /* - Print local link URLs for SKUs with largest rank discrepancy between AB and BC lists. + Print local link URLs for SKUs with largest rank discrepancy between AB and BC lists, + but ONLY when there exists another *different* listing (not in same linked group) + with a reasonably high similarity score by name. Usage: node scripts/rank_discrepency_links.js \ --ab reports/common_listings_ab_top1000.json \ --bc reports/common_listings_bc_top1000.json \ + --meta viz/data/sku_meta.json \ + --min-score 0.75 \ --top 50 \ --base "http://127.0.0.1:8080/#/link/?left=" @@ -18,6 +22,8 @@ const fs = require("fs"); const path = require("path"); +/* ---------------- IO ---------------- */ + function readJson(p) { return JSON.parse(fs.readFileSync(p, "utf8")); } @@ -26,17 +32,21 @@ function parseArgs(argv) { const out = { ab: "reports/common_listings_ab_top1000.json", bc: "reports/common_listings_bc_top1000.json", + meta: "", // optional sku_meta containing {links:[{fromSku,toSku}], ignores:...} top: 50, minDiscrep: 1, includeMissing: false, base: "http://127.0.0.1:8080/#/link/?left=", + minScore: 0.75, // similarity threshold for "reasonably high" }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--ab" && argv[i + 1]) out.ab = argv[++i]; else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i]; + else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i]; else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top; else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep; + else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore; else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); } @@ -55,19 +65,375 @@ function buildRankMap(payload) { return map; } +function pickName(row) { + if (!row) return ""; + return String(row.name || row.title || row.productName || row.displayName || ""); +} + +/* ---------------- sku_meta grouping (optional) ---------------- */ + +function normalizeImplicitSkuKey(k) { + const s = String(k || "").trim(); + const m = s.match(/^id:(\d{1,6})$/i); + if (m) return String(m[1]).padStart(6, "0"); + return s; +} + +class DSU { + constructor() { + this.parent = new Map(); + this.rank = new Map(); + } + _add(x) { + if (!this.parent.has(x)) { + this.parent.set(x, x); + this.rank.set(x, 0); + } + } + find(x) { + x = String(x || "").trim(); + if (!x) return ""; + this._add(x); + let p = this.parent.get(x); + if (p !== x) { + p = this.find(p); + this.parent.set(x, p); + } + return p; + } + union(a, b) { + a = String(a || "").trim(); + b = String(b || "").trim(); + if (!a || !b || a === b) return; + const ra = this.find(a); + const rb = this.find(b); + if (!ra || !rb || ra === rb) return; + + const rka = this.rank.get(ra) || 0; + const rkb = this.rank.get(rb) || 0; + + if (rka < rkb) this.parent.set(ra, rb); + else if (rkb < rka) this.parent.set(rb, ra); + else { + this.parent.set(rb, ra); + this.rank.set(ra, rka + 1); + } + } +} + +// Choose a stable representative (good enough for filtering “same-linked”) +function compareSku(a, b) { + a = String(a || "").trim(); + b = String(b || "").trim(); + if (a === b) return 0; + + const aUnknown = a.startsWith("u:"); + const bUnknown = b.startsWith("u:"); + if (aUnknown !== bUnknown) return aUnknown ? 1 : -1; + + const aNum = /^\d+$/.test(a); + const bNum = /^\d+$/.test(b); + if (aNum && bNum) { + const na = Number(a), nb = Number(b); + if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1; + } + return a < b ? -1 : 1; +} + +function buildCanonicalSkuFnFromMeta(meta) { + const links = Array.isArray(meta?.links) ? meta.links : []; + if (!links.length) return (sku) => normalizeImplicitSkuKey(sku); + + const dsu = new DSU(); + const all = new Set(); + + for (const x of links) { + const a = normalizeImplicitSkuKey(x?.fromSku); + const b = normalizeImplicitSkuKey(x?.toSku); + if (!a || !b || a === b) continue; + all.add(a); + all.add(b); + dsu.union(a, b); + } + + // root -> members + const groupsByRoot = new Map(); + for (const s of all) { + const r = dsu.find(s); + if (!r) continue; + let set = groupsByRoot.get(r); + if (!set) groupsByRoot.set(r, (set = new Set())); + set.add(s); + } + + // root -> representative + const repByRoot = new Map(); + for (const [root, members] of groupsByRoot.entries()) { + const arr = Array.from(members); + arr.sort(compareSku); + repByRoot.set(root, arr[0] || root); + } + + // sku -> rep + const canonBySku = new Map(); + for (const [root, members] of groupsByRoot.entries()) { + const rep = repByRoot.get(root) || root; + for (const s of members) canonBySku.set(s, rep); + canonBySku.set(rep, rep); + } + + return (sku) => { + const s = normalizeImplicitSkuKey(sku); + return canonBySku.get(s) || s; + }; +} + +/* ---------------- similarity (copied from viz/app) ---------------- */ + +// Normalize for search: lowercase, punctuation -> space, collapse spaces +function normSearchText(s) { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function tokenizeQuery(q) { + const n = normSearchText(q); + return n ? n.split(" ").filter(Boolean) : []; +} + +const SIM_STOP_TOKENS = new Set([ + "the","a","an","and","of","to","in","for","with", + "year","years","yr","yrs","old", + "whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof", + "anniversary", +]); + +const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i; + +function numKey(t) { + const s = String(t || "").trim().toLowerCase(); + if (!s) return ""; + if (/^\d+$/.test(s)) return s; + const m = s.match(ORDINAL_RE); + return m ? m[1] : ""; +} + +function extractAgeFromText(normName) { + const s = String(normName || ""); + if (!s) return ""; + + const m = s.match(/\b(?:aged\s*)?(\d{1,2})\s*(?:yr|yrs|year|years)\b/i); + if (m && m[1]) return String(parseInt(m[1], 10)); + + const m2 = s.match(/\b(\d{1,2})\s*yo\b/i); + if (m2 && m2[1]) return String(parseInt(m2[1], 10)); + + return ""; +} + +function filterSimTokens(tokens) { + const out = []; + const seen = new Set(); + + const SIM_EQUIV = new Map([ + ["years", "yr"], + ["year", "yr"], + ["yrs", "yr"], + ["yr", "yr"], + ["whiskey", "whisky"], + ["whisky", "whisky"], + ["bourbon", "bourbon"], + ]); + + const VOL_UNIT = new Set(["ml","l","cl","oz","liter","liters","litre","litres"]); + const VOL_INLINE_RE = /^\d+(?:\.\d+)?(?:ml|l|cl|oz)$/i; + const PCT_INLINE_RE = /^\d+(?:\.\d+)?%$/; + + const arr = Array.isArray(tokens) ? tokens : []; + + for (let i = 0; i < arr.length; i++) { + const raw = arr[i]; + let t = String(raw || "").trim().toLowerCase(); + if (!t) continue; + + if (!/[a-z0-9]/i.test(t)) continue; + + if (VOL_INLINE_RE.test(t)) continue; + if (PCT_INLINE_RE.test(t)) continue; + + t = SIM_EQUIV.get(t) || t; + + const nk = numKey(t); + if (nk) t = nk; + + if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue; + + if (/^\d+(?:\.\d+)?$/.test(t)) { + const next = String(arr[i + 1] || "").trim().toLowerCase(); + const nextNorm = SIM_EQUIV.get(next) || next; + if (VOL_UNIT.has(nextNorm)) { + i++; + continue; + } + } + + if (!numKey(t) && SIM_STOP_TOKENS.has(t)) continue; + + if (seen.has(t)) continue; + seen.add(t); + out.push(t); + } + + return out; +} + +function tokenContainmentScore(aTokens, bTokens) { + const A = filterSimTokens(aTokens || []); + const B = filterSimTokens(bTokens || []); + if (!A.length || !B.length) return 0; + + const aSet = new Set(A); + const bSet = new Set(B); + + const small = aSet.size <= bSet.size ? aSet : bSet; + const big = aSet.size <= bSet.size ? bSet : aSet; + + let hit = 0; + for (const t of small) if (big.has(t)) hit++; + + const recall = hit / Math.max(1, small.size); + const precision = hit / Math.max(1, big.size); + const f1 = (2 * precision * recall) / Math.max(1e-9, precision + recall); + + return f1; +} + +function levenshtein(a, b) { + a = String(a || ""); + b = String(b || ""); + const n = a.length, m = b.length; + if (!n) return m; + if (!m) return n; + + const dp = new Array(m + 1); + for (let j = 0; j <= m; j++) dp[j] = j; + + for (let i = 1; i <= n; i++) { + let prev = dp[0]; + dp[0] = i; + const ca = a.charCodeAt(i - 1); + for (let j = 1; j <= m; j++) { + const tmp = dp[j]; + const cost = ca === b.charCodeAt(j - 1) ? 0 : 1; + dp[j] = Math.min(dp[j] + 1, dp[j - 1] + 1, prev + cost); + prev = tmp; + } + } + return dp[m]; +} + +function numberMismatchPenalty(aTokens, bTokens) { + const aNums = new Set((aTokens || []).map(numKey).filter(Boolean)); + const bNums = new Set((bTokens || []).map(numKey).filter(Boolean)); + if (!aNums.size || !bNums.size) return 1.0; + for (const n of aNums) if (bNums.has(n)) return 1.0; + return 0.28; +} + +// Same structure/weights as viz/app/linker/similarity.js +function similarityScore(aName, bName) { + const a = normSearchText(aName); + const b = normSearchText(bName); + if (!a || !b) return 0; + + const aAge = extractAgeFromText(a); + const bAge = extractAgeFromText(b); + const ageBoth = !!(aAge && bAge); + const ageMatch = ageBoth && aAge === bAge; + const ageMismatch = ageBoth && aAge !== bAge; + + const aToksRaw = tokenizeQuery(a); + const bToksRaw = tokenizeQuery(b); + + const aToks = filterSimTokens(aToksRaw); + const bToks = filterSimTokens(bToksRaw); + if (!aToks.length || !bToks.length) return 0; + + const contain = tokenContainmentScore(aToksRaw, bToksRaw); + + const aFirst = aToks[0] || ""; + const bFirst = bToks[0] || ""; + const firstMatch = aFirst && bFirst && aFirst === bFirst ? 1 : 0; + + const A = new Set(aToks.slice(1)); + const B = new Set(bToks.slice(1)); + let inter = 0; + for (const w of A) if (B.has(w)) inter++; + const denom = Math.max(1, Math.max(A.size, B.size)); + const overlapTail = inter / denom; + + const d = levenshtein(a, b); + const maxLen = Math.max(1, Math.max(a.length, b.length)); + const levSim = 1 - d / maxLen; + + let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain); + + const smallN = Math.min(aToks.length, bToks.length); + if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18; + + const numGate = numberMismatchPenalty(aToks, bToks); + + let s = + numGate * + (firstMatch * 3.0 + + overlapTail * 2.2 * gate + + levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain))); + + if (ageMatch) s *= 2.2; + else if (ageMismatch) s *= 0.18; + + s *= 1 + 0.9 * contain; + + return s; +} + +/* ---------------- main logic ---------------- */ + function main() { const args = parseArgs(process.argv.slice(2)); const repoRoot = process.cwd(); const abPath = path.isAbsolute(args.ab) ? args.ab : path.join(repoRoot, args.ab); const bcPath = path.isAbsolute(args.bc) ? args.bc : path.join(repoRoot, args.bc); + const metaPath = args.meta + ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) + : ""; const ab = readJson(abPath); const bc = readJson(bcPath); + const canonicalSku = metaPath + ? buildCanonicalSkuFnFromMeta(readJson(metaPath)) + : (sku) => normalizeImplicitSkuKey(sku); + const abMap = buildRankMap(ab); const bcMap = buildRankMap(bc); + // Build a flat pool of candidates from AB+BC (unique by canonSku) + const rowBySku = new Map(); + for (const m of [abMap, bcMap]) { + for (const [canonSku, v] of m.entries()) { + if (!rowBySku.has(canonSku)) rowBySku.set(canonSku, v.row); + } + } + + const allSkus = Array.from(rowBySku.keys()); + const allNames = new Map(); + for (const sku of allSkus) allNames.set(sku, pickName(rowBySku.get(sku))); + const keys = new Set([...abMap.keys(), ...bcMap.keys()]); const diffs = []; @@ -88,7 +454,6 @@ function main() { diffs.push({ canonSku, discrep, - // tie-breakers sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9), }); } @@ -99,13 +464,34 @@ function main() { return String(x.canonSku).localeCompare(String(y.canonSku)); }); - const top = diffs.slice(0, args.top); + // Keep only discrepancies that have a high-scoring "other" candidate not in same linked group + const filtered = []; + for (const d of diffs) { + const skuA = String(d.canonSku); + const nameA = allNames.get(skuA) || pickName(abMap.get(skuA)?.row) || pickName(bcMap.get(skuA)?.row); + if (!nameA) continue; - for (const d of top) { - // examples: - // 884096 -> left=884096 - // id:1049355 -> left=id%3A1049355 - // u:bb504a62 -> left=u%3Abb504a62 + const groupA = canonicalSku(skuA); + + let best = 0; + for (const skuB of allSkus) { + if (skuB === skuA) continue; + + // not same-linked group + if (canonicalSku(skuB) === groupA) continue; + + const nameB = allNames.get(skuB) || ""; + if (!nameB) continue; + + const s = similarityScore(nameA, nameB); + if (s > best) best = s; + } + + if (best >= args.minScore) filtered.push(d); + if (filtered.length >= args.top) break; + } + + for (const d of filtered) { console.log(args.base + encodeURIComponent(d.canonSku)); } } From c4673a05b096bc9597c53db1c8db48727c3515dd Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 09:58:35 -0800 Subject: [PATCH 02/19] UX Improvements --- tools/rank_discrepency.js | 309 ++++++++++++++++++++++++++++++++------ 1 file changed, 261 insertions(+), 48 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index 239b87d..cf1af9c 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -3,20 +3,23 @@ /* Print local link URLs for SKUs with largest rank discrepancy between AB and BC lists, - but ONLY when there exists another *different* listing (not in same linked group) + BUT only when there exists another *different* listing (not in same linked group) with a reasonably high similarity score by name. Usage: - node scripts/rank_discrepency_links.js \ + node ./tools/rank_discrepency.js \ --ab reports/common_listings_ab_top1000.json \ --bc reports/common_listings_bc_top1000.json \ --meta viz/data/sku_meta.json \ - --min-score 0.75 \ + --min 10 \ + --min-score 0.7 \ --top 50 \ - --base "http://127.0.0.1:8080/#/link/?left=" + --base "http://127.0.0.1:8080/#/link/?left=" \ + --debug - Output: - http://127.0.0.1:8080/#/link/?left= + Notes: + - If --meta is not provided, "same-linked" filtering is disabled (each SKU is its own group). + - Debug output goes to STDERR so your STDOUT stays as just links. */ const fs = require("fs"); @@ -32,13 +35,21 @@ function parseArgs(argv) { const out = { ab: "reports/common_listings_ab_top1000.json", bc: "reports/common_listings_bc_top1000.json", - meta: "", // optional sku_meta containing {links:[{fromSku,toSku}], ignores:...} + meta: "", + top: 50, minDiscrep: 1, includeMissing: false, + + minScore: 0.75, base: "http://127.0.0.1:8080/#/link/?left=", - minScore: 0.75, // similarity threshold for "reasonably high" + + debug: false, + debugN: 20, // how many discrepancy candidates to dump debug lines for + debugPayload: false, // show payload structure details + dumpScores: false, // dump best match info per emitted link }; + for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--ab" && argv[i + 1]) out.ab = argv[++i]; @@ -49,25 +60,69 @@ function parseArgs(argv) { else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore; else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); + + else if (a === "--debug") out.debug = true; + else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN; + else if (a === "--debug-payload") out.debugPayload = true; + else if (a === "--dump-scores") out.dumpScores = true; } return out; } -function buildRankMap(payload) { - const rows = Array.isArray(payload?.rows) ? payload.rows : []; - const map = new Map(); - for (let i = 0; i < rows.length; i++) { - const r = rows[i]; - const k = r?.canonSku; - if (!k) continue; - map.set(String(k), { rank: i + 1, row: r }); - } - return map; +/* ---------------- row extraction ---------------- */ + +function extractRows(payload) { + // Most likely shapes: + // - [ ... ] + // - { rows: [...] } + // - { data: { rows: [...] } } + // - { data: [...] } (sometimes) + // - { items: [...] } / { results: [...] } etc. + if (Array.isArray(payload)) return payload; + + const candidates = [ + payload?.rows, + payload?.data?.rows, + payload?.data, + payload?.items, + payload?.list, + payload?.results, + ]; + for (const x of candidates) if (Array.isArray(x)) return x; + + return []; +} + +function rowKey(r) { + // Prefer canonSku if present (this script works in canonSku space). + // Fall back to sku/id-like fields. + const k = r?.canonSku ?? r?.sku ?? r?.canon ?? r?.id ?? r?.key; + return k ? String(k) : ""; } function pickName(row) { if (!row) return ""; - return String(row.name || row.title || row.productName || row.displayName || ""); + return String( + row.name ?? + row.title ?? + row.productName ?? + row.displayName ?? + row.itemName ?? + row.text ?? + "" + ); +} + +function buildRankMap(payload) { + const rows = extractRows(payload); + const map = new Map(); + for (let i = 0; i < rows.length; i++) { + const r = rows[i]; + const k = rowKey(r); + if (!k) continue; + map.set(String(k), { rank: i + 1, row: r }); + } + return { map, rowsLen: rows.length }; } /* ---------------- sku_meta grouping (optional) ---------------- */ @@ -121,7 +176,6 @@ class DSU { } } -// Choose a stable representative (good enough for filtering “same-linked”) function compareSku(a, b) { a = String(a || "").trim(); b = String(b || "").trim(); @@ -134,7 +188,8 @@ function compareSku(a, b) { const aNum = /^\d+$/.test(a); const bNum = /^\d+$/.test(b); if (aNum && bNum) { - const na = Number(a), nb = Number(b); + const na = Number(a), + nb = Number(b); if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1; } return a < b ? -1 : 1; @@ -156,7 +211,6 @@ function buildCanonicalSkuFnFromMeta(meta) { dsu.union(a, b); } - // root -> members const groupsByRoot = new Map(); for (const s of all) { const r = dsu.find(s); @@ -166,7 +220,6 @@ function buildCanonicalSkuFnFromMeta(meta) { set.add(s); } - // root -> representative const repByRoot = new Map(); for (const [root, members] of groupsByRoot.entries()) { const arr = Array.from(members); @@ -174,7 +227,6 @@ function buildCanonicalSkuFnFromMeta(meta) { repByRoot.set(root, arr[0] || root); } - // sku -> rep const canonBySku = new Map(); for (const [root, members] of groupsByRoot.entries()) { const rep = repByRoot.get(root) || root; @@ -190,7 +242,6 @@ function buildCanonicalSkuFnFromMeta(meta) { /* ---------------- similarity (copied from viz/app) ---------------- */ -// Normalize for search: lowercase, punctuation -> space, collapse spaces function normSearchText(s) { return String(s ?? "") .toLowerCase() @@ -205,9 +256,33 @@ function tokenizeQuery(q) { } const SIM_STOP_TOKENS = new Set([ - "the","a","an","and","of","to","in","for","with", - "year","years","yr","yrs","old", - "whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof", + "the", + "a", + "an", + "and", + "of", + "to", + "in", + "for", + "with", + "year", + "years", + "yr", + "yrs", + "old", + "whisky", + "whiskey", + "scotch", + "single", + "malt", + "cask", + "finish", + "edition", + "release", + "batch", + "strength", + "abv", + "proof", "anniversary", ]); @@ -248,7 +323,7 @@ function filterSimTokens(tokens) { ["bourbon", "bourbon"], ]); - const VOL_UNIT = new Set(["ml","l","cl","oz","liter","liters","litre","litres"]); + const VOL_UNIT = new Set(["ml", "l", "cl", "oz", "liter", "liters", "litre", "litres"]); const VOL_INLINE_RE = /^\d+(?:\.\d+)?(?:ml|l|cl|oz)$/i; const PCT_INLINE_RE = /^\d+(?:\.\d+)?%$/; @@ -314,7 +389,8 @@ function tokenContainmentScore(aTokens, bTokens) { function levenshtein(a, b) { a = String(a || ""); b = String(b || ""); - const n = a.length, m = b.length; + const n = a.length, + m = b.length; if (!n) return m; if (!m) return n; @@ -343,7 +419,6 @@ function numberMismatchPenalty(aTokens, bTokens) { return 0.28; } -// Same structure/weights as viz/app/linker/similarity.js function similarityScore(aName, bName) { const a = normSearchText(aName); const b = normSearchText(bName); @@ -379,7 +454,7 @@ function similarityScore(aName, bName) { const maxLen = Math.max(1, Math.max(a.length, b.length)); const levSim = 1 - d / maxLen; - let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain); + let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain); const smallN = Math.min(aToks.length, bToks.length); if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18; @@ -390,7 +465,7 @@ function similarityScore(aName, bName) { numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + - levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain))); + levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain)); if (ageMatch) s *= 2.2; else if (ageMismatch) s *= 0.18; @@ -400,7 +475,24 @@ function similarityScore(aName, bName) { return s; } -/* ---------------- main logic ---------------- */ +/* ---------------- debug helpers ---------------- */ + +function briefObjShape(x) { + if (Array.isArray(x)) return { type: "array", len: x.length }; + if (x && typeof x === "object") return { type: "object", keys: Object.keys(x).slice(0, 30) }; + return { type: typeof x }; +} + +function eprintln(...args) { + console.error(...args); +} + +function truncate(s, n) { + s = String(s || ""); + return s.length <= n ? s : s.slice(0, n - 1) + "…"; +} + +/* ---------------- main ---------------- */ function main() { const args = parseArgs(process.argv.slice(2)); @@ -409,7 +501,9 @@ function main() { const abPath = path.isAbsolute(args.ab) ? args.ab : path.join(repoRoot, args.ab); const bcPath = path.isAbsolute(args.bc) ? args.bc : path.join(repoRoot, args.bc); const metaPath = args.meta - ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) + ? path.isAbsolute(args.meta) + ? args.meta + : path.join(repoRoot, args.meta) : ""; const ab = readJson(abPath); @@ -419,8 +513,49 @@ function main() { ? buildCanonicalSkuFnFromMeta(readJson(metaPath)) : (sku) => normalizeImplicitSkuKey(sku); - const abMap = buildRankMap(ab); - const bcMap = buildRankMap(bc); + const abBuilt = buildRankMap(ab); + const bcBuilt = buildRankMap(bc); + const abMap = abBuilt.map; + const bcMap = bcBuilt.map; + + if (args.debug || args.debugPayload) { + eprintln("[rank_discrepency] inputs:", { + abPath, + bcPath, + metaPath: metaPath || "(none)", + minDiscrep: args.minDiscrep, + minScore: args.minScore, + top: args.top, + includeMissing: args.includeMissing, + }); + eprintln("[rank_discrepency] payload shapes:", { + ab: briefObjShape(ab), + bc: briefObjShape(bc), + }); + eprintln("[rank_discrepency] extracted rows:", { + abRows: abBuilt.rowsLen, + bcRows: bcBuilt.rowsLen, + abKeys: abMap.size, + bcKeys: bcMap.size, + }); + + if (args.debugPayload) { + // show a tiny sample row keys + fields + const abRows = extractRows(ab); + const bcRows = extractRows(bc); + eprintln("[rank_discrepency] sample AB row[0] keys:", abRows[0] && typeof abRows[0] === "object" ? Object.keys(abRows[0]).slice(0, 40) : abRows[0]); + eprintln("[rank_discrepency] sample BC row[0] keys:", bcRows[0] && typeof bcRows[0] === "object" ? Object.keys(bcRows[0]).slice(0, 40) : bcRows[0]); + eprintln("[rank_discrepency] sample AB rowKey:", rowKey(abRows[0])); + eprintln("[rank_discrepency] sample BC rowKey:", rowKey(bcRows[0])); + eprintln("[rank_discrepency] sample AB name:", truncate(pickName(abRows[0]), 120)); + eprintln("[rank_discrepency] sample BC name:", truncate(pickName(bcRows[0]), 120)); + } + } + + if (!abMap.size || !bcMap.size) { + eprintln("[rank_discrepency] ERROR: empty rank maps. Your JSON shape probably isn't {rows:[...]}. Try --debug-payload."); + process.exit(2); + } // Build a flat pool of candidates from AB+BC (unique by canonSku) const rowBySku = new Map(); @@ -446,14 +581,14 @@ function main() { const rankAB = a ? a.rank : null; const rankBC = b ? b.rank : null; - const discrep = - rankAB !== null && rankBC !== null ? Math.abs(rankAB - rankBC) : Infinity; - + const discrep = rankAB !== null && rankBC !== null ? Math.abs(rankAB - rankBC) : Infinity; if (discrep !== Infinity && discrep < args.minDiscrep) continue; diffs.push({ canonSku, discrep, + rankAB, + rankBC, sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9), }); } @@ -464,35 +599,113 @@ function main() { return String(x.canonSku).localeCompare(String(y.canonSku)); }); - // Keep only discrepancies that have a high-scoring "other" candidate not in same linked group + if (args.debug) { + eprintln("[rank_discrepency] discrepancy candidates:", { + unionKeys: keys.size, + diffsAfterMin: diffs.length, + topDiscrepSample: diffs.slice(0, 5).map((d) => ({ + sku: d.canonSku, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + name: truncate(allNames.get(String(d.canonSku)) || "", 90), + })), + }); + } + const filtered = []; + const debugLines = []; + for (const d of diffs) { const skuA = String(d.canonSku); - const nameA = allNames.get(skuA) || pickName(abMap.get(skuA)?.row) || pickName(bcMap.get(skuA)?.row); - if (!nameA) continue; + const nameA = + allNames.get(skuA) || + pickName(abMap.get(skuA)?.row) || + pickName(bcMap.get(skuA)?.row) || + ""; + if (!nameA) { + if (args.debug && debugLines.length < args.debugN) { + debugLines.push({ sku: skuA, reason: "no-name" }); + } + continue; + } const groupA = canonicalSku(skuA); let best = 0; + let bestSku = ""; + let bestName = ""; + for (const skuB of allSkus) { if (skuB === skuA) continue; - - // not same-linked group if (canonicalSku(skuB) === groupA) continue; const nameB = allNames.get(skuB) || ""; if (!nameB) continue; const s = similarityScore(nameA, nameB); - if (s > best) best = s; + if (s > best) { + best = s; + bestSku = skuB; + bestName = nameB; + } } - if (best >= args.minScore) filtered.push(d); + const pass = best >= args.minScore; + if (args.debug && debugLines.length < args.debugN) { + debugLines.push({ + sku: skuA, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + nameA: truncate(nameA, 80), + groupA, + best, + bestSku, + bestGroup: bestSku ? canonicalSku(bestSku) : "", + bestName: truncate(bestName, 80), + pass, + }); + } + + if (!pass) continue; + + filtered.push({ ...d, best, bestSku, bestName }); if (filtered.length >= args.top) break; } + if (args.debug) { + eprintln("[rank_discrepency] filter results:", { + filtered: filtered.length, + minScore: args.minScore, + minDiscrep: args.minDiscrep, + }); + eprintln("[rank_discrepency] debug sample (first N checked):"); + for (const x of debugLines) eprintln(" ", x); + } + + // STDOUT: links (and optionally score dumps) for (const d of filtered) { - console.log(args.base + encodeURIComponent(d.canonSku)); + if (args.dumpScores) { + // keep link first so it's easy to pipe + eprintln( + "[rank_discrepency] emit", + JSON.stringify({ + sku: d.canonSku, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + best: d.best, + bestSku: d.bestSku, + bestName: truncate(d.bestName, 120), + }) + ); + } + console.log(args.base + encodeURIComponent(String(d.canonSku))); + } + + if (args.debug) { + eprintln("[rank_discrepency] done."); } } From d6dd47d0773b142bcc9a23e2bded31b00caf161c Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 10:02:43 -0800 Subject: [PATCH 03/19] UX Improvements --- tools/rank_discrepency.js | 315 +++++++++++++++++++++++--------------- 1 file changed, 193 insertions(+), 122 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index cf1af9c..e9af760 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -2,24 +2,14 @@ "use strict"; /* - Print local link URLs for SKUs with largest rank discrepancy between AB and BC lists, - BUT only when there exists another *different* listing (not in same linked group) - with a reasonably high similarity score by name. + Rank discrepancy links, filtered by existence of a high-similarity "other" listing. - Usage: - node ./tools/rank_discrepency.js \ - --ab reports/common_listings_ab_top1000.json \ - --bc reports/common_listings_bc_top1000.json \ - --meta viz/data/sku_meta.json \ - --min 10 \ - --min-score 0.7 \ - --top 50 \ - --base "http://127.0.0.1:8080/#/link/?left=" \ - --debug + Debug is verbose and goes to STDERR so STDOUT stays as emitted links. - Notes: - - If --meta is not provided, "same-linked" filtering is disabled (each SKU is its own group). - - Debug output goes to STDERR so your STDOUT stays as just links. + Examples: + node ./tools/rank_discrepency.js --debug --debug-payload + node ./tools/rank_discrepency.js --min-score 0.2 --debug + node ./tools/rank_discrepency.js --name-field "product.title" --debug */ const fs = require("fs"); @@ -44,10 +34,14 @@ function parseArgs(argv) { minScore: 0.75, base: "http://127.0.0.1:8080/#/link/?left=", + // name picking + nameField: "", // optional dotted path override, e.g. "product.title" + + // debug debug: false, - debugN: 20, // how many discrepancy candidates to dump debug lines for - debugPayload: false, // show payload structure details - dumpScores: false, // dump best match info per emitted link + debugN: 25, + debugPayload: false, + dumpScores: false, }; for (let i = 0; i < argv.length; i++) { @@ -55,29 +49,26 @@ function parseArgs(argv) { if (a === "--ab" && argv[i + 1]) out.ab = argv[++i]; else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i]; else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i]; + else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top; else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep; else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore; else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); + else if (a === "--name-field" && argv[i + 1]) out.nameField = String(argv[++i] || ""); else if (a === "--debug") out.debug = true; else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN; else if (a === "--debug-payload") out.debugPayload = true; else if (a === "--dump-scores") out.dumpScores = true; } + return out; } /* ---------------- row extraction ---------------- */ function extractRows(payload) { - // Most likely shapes: - // - [ ... ] - // - { rows: [...] } - // - { data: { rows: [...] } } - // - { data: [...] } (sometimes) - // - { items: [...] } / { results: [...] } etc. if (Array.isArray(payload)) return payload; const candidates = [ @@ -94,25 +85,10 @@ function extractRows(payload) { } function rowKey(r) { - // Prefer canonSku if present (this script works in canonSku space). - // Fall back to sku/id-like fields. const k = r?.canonSku ?? r?.sku ?? r?.canon ?? r?.id ?? r?.key; return k ? String(k) : ""; } -function pickName(row) { - if (!row) return ""; - return String( - row.name ?? - row.title ?? - row.productName ?? - row.displayName ?? - row.itemName ?? - row.text ?? - "" - ); -} - function buildRankMap(payload) { const rows = extractRows(payload); const map = new Map(); @@ -122,7 +98,103 @@ function buildRankMap(payload) { if (!k) continue; map.set(String(k), { rank: i + 1, row: r }); } - return { map, rowsLen: rows.length }; + return { map, rowsLen: rows.length, rows }; +} + +/* ---------------- name picking ---------------- */ + +function getByPath(obj, dotted) { + if (!obj || !dotted) return undefined; + const parts = String(dotted).split(".").filter(Boolean); + let cur = obj; + for (const p of parts) { + if (!cur || typeof cur !== "object") return undefined; + cur = cur[p]; + } + return cur; +} + +function pickFirstString(obj, paths) { + for (const p of paths) { + const v = getByPath(obj, p); + if (typeof v === "string" && v.trim()) return v.trim(); + } + return ""; +} + +// Tries hard to find a display name in common listing rows. +// Your debug showed `name: ''` for top discrepancies, so the field is elsewhere. +function pickName(row, nameFieldOverride) { + if (!row) return ""; + + if (nameFieldOverride) { + const forced = getByPath(row, nameFieldOverride); + if (typeof forced === "string" && forced.trim()) return forced.trim(); + } + + // Common direct fields + const direct = [ + "name", + "title", + "productName", + "displayName", + "itemName", + "label", + "desc", + "description", + "query", + ]; + for (const k of direct) { + const v = row[k]; + if (typeof v === "string" && v.trim()) return v.trim(); + } + + // Common nested patterns used in listing aggregations + const nested = [ + "product.name", + "product.title", + "product.displayName", + "item.name", + "item.title", + "listing.name", + "listing.title", + "canon.name", + "canon.title", + "best.name", + "best.title", + "top.name", + "top.title", + "meta.name", + "meta.title", + "agg.name", + "agg.title", + ]; + const got = pickFirstString(row, nested); + if (got) return got; + + // If rows have a "bestRow" or "example" child object, probe that too + const children = ["bestRow", "example", "sample", "row", "source", "picked", "winner"]; + for (const c of children) { + const child = row[c]; + if (child && typeof child === "object") { + const g2 = pickName(child, ""); + if (g2) return g2; + } + } + + // Last resort: sometimes there is an array like `listings` or `rows` with objects containing name/title + const arrays = ["listings", "sources", "items", "matches"]; + for (const a of arrays) { + const arr = row[a]; + if (Array.isArray(arr) && arr.length) { + for (let i = 0; i < Math.min(arr.length, 5); i++) { + const g3 = pickName(arr[i], ""); + if (g3) return g3; + } + } + } + + return ""; } /* ---------------- sku_meta grouping (optional) ---------------- */ @@ -188,8 +260,7 @@ function compareSku(a, b) { const aNum = /^\d+$/.test(a); const bNum = /^\d+$/.test(b); if (aNum && bNum) { - const na = Number(a), - nb = Number(b); + const na = Number(a), nb = Number(b); if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1; } return a < b ? -1 : 1; @@ -256,33 +327,9 @@ function tokenizeQuery(q) { } const SIM_STOP_TOKENS = new Set([ - "the", - "a", - "an", - "and", - "of", - "to", - "in", - "for", - "with", - "year", - "years", - "yr", - "yrs", - "old", - "whisky", - "whiskey", - "scotch", - "single", - "malt", - "cask", - "finish", - "edition", - "release", - "batch", - "strength", - "abv", - "proof", + "the","a","an","and","of","to","in","for","with", + "year","years","yr","yrs","old", + "whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof", "anniversary", ]); @@ -323,19 +370,17 @@ function filterSimTokens(tokens) { ["bourbon", "bourbon"], ]); - const VOL_UNIT = new Set(["ml", "l", "cl", "oz", "liter", "liters", "litre", "litres"]); + const VOL_UNIT = new Set(["ml","l","cl","oz","liter","liters","litre","litres"]); const VOL_INLINE_RE = /^\d+(?:\.\d+)?(?:ml|l|cl|oz)$/i; const PCT_INLINE_RE = /^\d+(?:\.\d+)?%$/; const arr = Array.isArray(tokens) ? tokens : []; for (let i = 0; i < arr.length; i++) { - const raw = arr[i]; - let t = String(raw || "").trim().toLowerCase(); + let t = String(arr[i] || "").trim().toLowerCase(); if (!t) continue; if (!/[a-z0-9]/i.test(t)) continue; - if (VOL_INLINE_RE.test(t)) continue; if (PCT_INLINE_RE.test(t)) continue; @@ -389,8 +434,7 @@ function tokenContainmentScore(aTokens, bTokens) { function levenshtein(a, b) { a = String(a || ""); b = String(b || ""); - const n = a.length, - m = b.length; + const n = a.length, m = b.length; if (!n) return m; if (!m) return n; @@ -454,7 +498,7 @@ function similarityScore(aName, bName) { const maxLen = Math.max(1, Math.max(a.length, b.length)); const levSim = 1 - d / maxLen; - let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain); + let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain); const smallN = Math.min(aToks.length, bToks.length); if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18; @@ -465,7 +509,7 @@ function similarityScore(aName, bName) { numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + - levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain)); + levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain))); if (ageMatch) s *= 2.2; else if (ageMismatch) s *= 0.18; @@ -477,12 +521,6 @@ function similarityScore(aName, bName) { /* ---------------- debug helpers ---------------- */ -function briefObjShape(x) { - if (Array.isArray(x)) return { type: "array", len: x.length }; - if (x && typeof x === "object") return { type: "object", keys: Object.keys(x).slice(0, 30) }; - return { type: typeof x }; -} - function eprintln(...args) { console.error(...args); } @@ -492,6 +530,26 @@ function truncate(s, n) { return s.length <= n ? s : s.slice(0, n - 1) + "…"; } +function briefObjShape(x) { + if (Array.isArray(x)) return { type: "array", len: x.length }; + if (x && typeof x === "object") return { type: "object", keys: Object.keys(x).slice(0, 30) }; + return { type: typeof x }; +} + +function trimForPrint(obj, maxKeys = 40, maxStr = 180) { + if (!obj || typeof obj !== "object") return obj; + const out = {}; + const keys = Object.keys(obj).slice(0, maxKeys); + for (const k of keys) { + const v = obj[k]; + if (typeof v === "string") out[k] = truncate(v, maxStr); + else if (Array.isArray(v)) out[k] = `[array len=${v.length}]`; + else if (v && typeof v === "object") out[k] = `{object keys=${Object.keys(v).slice(0, 12).join(",")}}`; + else out[k] = v; + } + return out; +} + /* ---------------- main ---------------- */ function main() { @@ -527,37 +585,35 @@ function main() { minScore: args.minScore, top: args.top, includeMissing: args.includeMissing, + nameField: args.nameField || "(auto)", }); - eprintln("[rank_discrepency] payload shapes:", { - ab: briefObjShape(ab), - bc: briefObjShape(bc), - }); + eprintln("[rank_discrepency] payload shapes:", { ab: briefObjShape(ab), bc: briefObjShape(bc) }); eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, abKeys: abMap.size, bcKeys: bcMap.size, }); - - if (args.debugPayload) { - // show a tiny sample row keys + fields - const abRows = extractRows(ab); - const bcRows = extractRows(bc); - eprintln("[rank_discrepency] sample AB row[0] keys:", abRows[0] && typeof abRows[0] === "object" ? Object.keys(abRows[0]).slice(0, 40) : abRows[0]); - eprintln("[rank_discrepency] sample BC row[0] keys:", bcRows[0] && typeof bcRows[0] === "object" ? Object.keys(bcRows[0]).slice(0, 40) : bcRows[0]); - eprintln("[rank_discrepency] sample AB rowKey:", rowKey(abRows[0])); - eprintln("[rank_discrepency] sample BC rowKey:", rowKey(bcRows[0])); - eprintln("[rank_discrepency] sample AB name:", truncate(pickName(abRows[0]), 120)); - eprintln("[rank_discrepency] sample BC name:", truncate(pickName(bcRows[0]), 120)); - } } if (!abMap.size || !bcMap.size) { - eprintln("[rank_discrepency] ERROR: empty rank maps. Your JSON shape probably isn't {rows:[...]}. Try --debug-payload."); + eprintln("[rank_discrepency] ERROR: empty rank maps. JSON shape issue."); process.exit(2); } - // Build a flat pool of candidates from AB+BC (unique by canonSku) + // If asked, print sample row structure for AB/BC so you can see where the name is. + if (args.debugPayload) { + const ab0 = abBuilt.rows[0]; + const bc0 = bcBuilt.rows[0]; + eprintln("[rank_discrepency] sample AB row[0] keys:", ab0 && typeof ab0 === "object" ? Object.keys(ab0).slice(0, 80) : ab0); + eprintln("[rank_discrepency] sample BC row[0] keys:", bc0 && typeof bc0 === "object" ? Object.keys(bc0).slice(0, 80) : bc0); + eprintln("[rank_discrepency] sample AB row[0] trimmed:", trimForPrint(ab0)); + eprintln("[rank_discrepency] sample BC row[0] trimmed:", trimForPrint(bc0)); + eprintln("[rank_discrepency] sample AB name(auto):", truncate(pickName(ab0, args.nameField), 160)); + eprintln("[rank_discrepency] sample BC name(auto):", truncate(pickName(bc0, args.nameField), 160)); + } + + // Build pool of unique rows by sku key const rowBySku = new Map(); for (const m of [abMap, bcMap]) { for (const [canonSku, v] of m.entries()) { @@ -567,7 +623,10 @@ function main() { const allSkus = Array.from(rowBySku.keys()); const allNames = new Map(); - for (const sku of allSkus) allNames.set(sku, pickName(rowBySku.get(sku))); + for (const sku of allSkus) { + const n = pickName(rowBySku.get(sku), args.nameField); + allNames.set(sku, n); + } const keys = new Set([...abMap.keys(), ...bcMap.keys()]); const diffs = []; @@ -603,7 +662,7 @@ function main() { eprintln("[rank_discrepency] discrepancy candidates:", { unionKeys: keys.size, diffsAfterMin: diffs.length, - topDiscrepSample: diffs.slice(0, 5).map((d) => ({ + topDiscrepSample: diffs.slice(0, 8).map((d) => ({ sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, @@ -613,20 +672,35 @@ function main() { }); } + // BIG DEBUG: if we keep seeing empty names, dump the actual row objects for top discrepancies + if (args.debugPayload) { + for (const d of diffs.slice(0, Math.min(args.debugN, diffs.length))) { + const sku = String(d.canonSku); + const row = rowBySku.get(sku) || abMap.get(sku)?.row || bcMap.get(sku)?.row; + const nm = pickName(row, args.nameField); + if (!nm) { + eprintln("[rank_discrepency] no-name row example:", { + sku, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + rowKeys: row && typeof row === "object" ? Object.keys(row).slice(0, 80) : typeof row, + rowTrim: trimForPrint(row), + }); + break; // one is enough to reveal the name field + } + } + } + + // Filter by having a good "other" match not in same linked group const filtered = []; const debugLines = []; for (const d of diffs) { const skuA = String(d.canonSku); - const nameA = - allNames.get(skuA) || - pickName(abMap.get(skuA)?.row) || - pickName(bcMap.get(skuA)?.row) || - ""; + const nameA = allNames.get(skuA) || ""; if (!nameA) { - if (args.debug && debugLines.length < args.debugN) { - debugLines.push({ sku: skuA, reason: "no-name" }); - } + if (args.debug && debugLines.length < args.debugN) debugLines.push({ sku: skuA, reason: "no-name" }); continue; } @@ -658,12 +732,10 @@ function main() { discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, - nameA: truncate(nameA, 80), - groupA, + nameA: truncate(nameA, 90), best, bestSku, - bestGroup: bestSku ? canonicalSku(bestSku) : "", - bestName: truncate(bestName, 80), + bestName: truncate(bestName, 90), pass, }); } @@ -679,15 +751,16 @@ function main() { filtered: filtered.length, minScore: args.minScore, minDiscrep: args.minDiscrep, + totalDiffs: diffs.length, + totalNamed: Array.from(allNames.values()).filter(Boolean).length, }); eprintln("[rank_discrepency] debug sample (first N checked):"); for (const x of debugLines) eprintln(" ", x); } - // STDOUT: links (and optionally score dumps) + // Emit links on STDOUT for (const d of filtered) { if (args.dumpScores) { - // keep link first so it's easy to pipe eprintln( "[rank_discrepency] emit", JSON.stringify({ @@ -697,16 +770,14 @@ function main() { rankBC: d.rankBC, best: d.best, bestSku: d.bestSku, - bestName: truncate(d.bestName, 120), + bestName: truncate(d.bestName, 160), }) ); } console.log(args.base + encodeURIComponent(String(d.canonSku))); } - if (args.debug) { - eprintln("[rank_discrepency] done."); - } + if (args.debug) eprintln("[rank_discrepency] done."); } main(); From f2174d86167c436247d001baf313786c8c67a69a Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 10:12:49 -0800 Subject: [PATCH 04/19] UX Improvements --- tools/rank_discrepency.js | 245 ++++++++++++-------------------------- 1 file changed, 78 insertions(+), 167 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index e9af760..408812e 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -2,14 +2,13 @@ "use strict"; /* - Rank discrepancy links, filtered by existence of a high-similarity "other" listing. - - Debug is verbose and goes to STDERR so STDOUT stays as emitted links. + Rank discrepancy links, filtered by existence of a high-similarity "other" listing + that is NOT in the same linked group (using sku_links.json union-find). Examples: - node ./tools/rank_discrepency.js --debug --debug-payload - node ./tools/rank_discrepency.js --min-score 0.2 --debug - node ./tools/rank_discrepency.js --name-field "product.title" --debug + node ./tools/rank_discrepency.js --debug + node ./tools/rank_discrepency.js --min-score 0.35 --top 100 --debug + node ./tools/rank_discrepency.js --meta data/sku_links.json --debug-best --debug */ const fs = require("fs"); @@ -25,7 +24,9 @@ function parseArgs(argv) { const out = { ab: "reports/common_listings_ab_top1000.json", bc: "reports/common_listings_bc_top1000.json", - meta: "", + + // default to your real links file + meta: "data/sku_links.json", top: 50, minDiscrep: 1, @@ -34,14 +35,11 @@ function parseArgs(argv) { minScore: 0.75, base: "http://127.0.0.1:8080/#/link/?left=", - // name picking - nameField: "", // optional dotted path override, e.g. "product.title" - - // debug debug: false, debugN: 25, debugPayload: false, - dumpScores: false, + debugBest: false, // dump top 5 candidate matches for first discrepancy item + dumpScores: false, // emit per-link score info to STDERR }; for (let i = 0; i < argv.length; i++) { @@ -56,10 +54,10 @@ function parseArgs(argv) { else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); - else if (a === "--name-field" && argv[i + 1]) out.nameField = String(argv[++i] || ""); else if (a === "--debug") out.debug = true; else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN; else if (a === "--debug-payload") out.debugPayload = true; + else if (a === "--debug-best") out.debugBest = true; else if (a === "--dump-scores") out.dumpScores = true; } @@ -101,38 +99,20 @@ function buildRankMap(payload) { return { map, rowsLen: rows.length, rows }; } -/* ---------------- name picking ---------------- */ +/* ---------------- name picking (FIXED) ---------------- */ -function getByPath(obj, dotted) { - if (!obj || !dotted) return undefined; - const parts = String(dotted).split(".").filter(Boolean); - let cur = obj; - for (const p of parts) { - if (!cur || typeof cur !== "object") return undefined; - cur = cur[p]; - } - return cur; -} - -function pickFirstString(obj, paths) { - for (const p of paths) { - const v = getByPath(obj, p); - if (typeof v === "string" && v.trim()) return v.trim(); - } - return ""; -} - -// Tries hard to find a display name in common listing rows. -// Your debug showed `name: ''` for top discrepancies, so the field is elsewhere. -function pickName(row, nameFieldOverride) { +function pickName(row) { if (!row) return ""; - if (nameFieldOverride) { - const forced = getByPath(row, nameFieldOverride); - if (typeof forced === "string" && forced.trim()) return forced.trim(); - } + // ✅ common_listings_* puts display name here + const repName = row?.representative?.name; + if (typeof repName === "string" && repName.trim()) return repName.trim(); - // Common direct fields + // fallback: sometimes cheapest has a name (rare) + const cheapName = row?.cheapest?.name; + if (typeof cheapName === "string" && cheapName.trim()) return cheapName.trim(); + + // old fallbacks (keep) const direct = [ "name", "title", @@ -142,62 +122,16 @@ function pickName(row, nameFieldOverride) { "label", "desc", "description", - "query", ]; for (const k of direct) { const v = row[k]; if (typeof v === "string" && v.trim()) return v.trim(); } - // Common nested patterns used in listing aggregations - const nested = [ - "product.name", - "product.title", - "product.displayName", - "item.name", - "item.title", - "listing.name", - "listing.title", - "canon.name", - "canon.title", - "best.name", - "best.title", - "top.name", - "top.title", - "meta.name", - "meta.title", - "agg.name", - "agg.title", - ]; - const got = pickFirstString(row, nested); - if (got) return got; - - // If rows have a "bestRow" or "example" child object, probe that too - const children = ["bestRow", "example", "sample", "row", "source", "picked", "winner"]; - for (const c of children) { - const child = row[c]; - if (child && typeof child === "object") { - const g2 = pickName(child, ""); - if (g2) return g2; - } - } - - // Last resort: sometimes there is an array like `listings` or `rows` with objects containing name/title - const arrays = ["listings", "sources", "items", "matches"]; - for (const a of arrays) { - const arr = row[a]; - if (Array.isArray(arr) && arr.length) { - for (let i = 0; i < Math.min(arr.length, 5); i++) { - const g3 = pickName(arr[i], ""); - if (g3) return g3; - } - } - } - return ""; } -/* ---------------- sku_meta grouping (optional) ---------------- */ +/* ---------------- sku_links union-find grouping ---------------- */ function normalizeImplicitSkuKey(k) { const s = String(k || "").trim(); @@ -311,7 +245,7 @@ function buildCanonicalSkuFnFromMeta(meta) { }; } -/* ---------------- similarity (copied from viz/app) ---------------- */ +/* ---------------- similarity (from viz/app/linker/similarity.js) ---------------- */ function normSearchText(s) { return String(s ?? "") @@ -530,26 +464,6 @@ function truncate(s, n) { return s.length <= n ? s : s.slice(0, n - 1) + "…"; } -function briefObjShape(x) { - if (Array.isArray(x)) return { type: "array", len: x.length }; - if (x && typeof x === "object") return { type: "object", keys: Object.keys(x).slice(0, 30) }; - return { type: typeof x }; -} - -function trimForPrint(obj, maxKeys = 40, maxStr = 180) { - if (!obj || typeof obj !== "object") return obj; - const out = {}; - const keys = Object.keys(obj).slice(0, maxKeys); - for (const k of keys) { - const v = obj[k]; - if (typeof v === "string") out[k] = truncate(v, maxStr); - else if (Array.isArray(v)) out[k] = `[array len=${v.length}]`; - else if (v && typeof v === "object") out[k] = `{object keys=${Object.keys(v).slice(0, 12).join(",")}}`; - else out[k] = v; - } - return out; -} - /* ---------------- main ---------------- */ function main() { @@ -559,35 +473,31 @@ function main() { const abPath = path.isAbsolute(args.ab) ? args.ab : path.join(repoRoot, args.ab); const bcPath = path.isAbsolute(args.bc) ? args.bc : path.join(repoRoot, args.bc); const metaPath = args.meta - ? path.isAbsolute(args.meta) - ? args.meta - : path.join(repoRoot, args.meta) + ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) : ""; const ab = readJson(abPath); const bc = readJson(bcPath); - const canonicalSku = metaPath - ? buildCanonicalSkuFnFromMeta(readJson(metaPath)) - : (sku) => normalizeImplicitSkuKey(sku); + const meta = metaPath ? readJson(metaPath) : null; + const canonicalSku = meta ? buildCanonicalSkuFnFromMeta(meta) : (sku) => normalizeImplicitSkuKey(sku); const abBuilt = buildRankMap(ab); const bcBuilt = buildRankMap(bc); const abMap = abBuilt.map; const bcMap = bcBuilt.map; - if (args.debug || args.debugPayload) { + if (args.debug) { eprintln("[rank_discrepency] inputs:", { abPath, bcPath, metaPath: metaPath || "(none)", + linkCount: Array.isArray(meta?.links) ? meta.links.length : 0, minDiscrep: args.minDiscrep, minScore: args.minScore, top: args.top, includeMissing: args.includeMissing, - nameField: args.nameField || "(auto)", }); - eprintln("[rank_discrepency] payload shapes:", { ab: briefObjShape(ab), bc: briefObjShape(bc) }); eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, @@ -597,23 +507,18 @@ function main() { } if (!abMap.size || !bcMap.size) { - eprintln("[rank_discrepency] ERROR: empty rank maps. JSON shape issue."); + eprintln("[rank_discrepency] ERROR: empty rank maps."); process.exit(2); } - // If asked, print sample row structure for AB/BC so you can see where the name is. if (args.debugPayload) { const ab0 = abBuilt.rows[0]; const bc0 = bcBuilt.rows[0]; - eprintln("[rank_discrepency] sample AB row[0] keys:", ab0 && typeof ab0 === "object" ? Object.keys(ab0).slice(0, 80) : ab0); - eprintln("[rank_discrepency] sample BC row[0] keys:", bc0 && typeof bc0 === "object" ? Object.keys(bc0).slice(0, 80) : bc0); - eprintln("[rank_discrepency] sample AB row[0] trimmed:", trimForPrint(ab0)); - eprintln("[rank_discrepency] sample BC row[0] trimmed:", trimForPrint(bc0)); - eprintln("[rank_discrepency] sample AB name(auto):", truncate(pickName(ab0, args.nameField), 160)); - eprintln("[rank_discrepency] sample BC name(auto):", truncate(pickName(bc0, args.nameField), 160)); + eprintln("[rank_discrepency] sample AB rep.name:", truncate(ab0?.representative?.name || "", 120)); + eprintln("[rank_discrepency] sample BC rep.name:", truncate(bc0?.representative?.name || "", 120)); } - // Build pool of unique rows by sku key + // Build unique sku pool from AB+BC const rowBySku = new Map(); for (const m of [abMap, bcMap]) { for (const [canonSku, v] of m.entries()) { @@ -623,9 +528,19 @@ function main() { const allSkus = Array.from(rowBySku.keys()); const allNames = new Map(); + let namedCount = 0; for (const sku of allSkus) { - const n = pickName(rowBySku.get(sku), args.nameField); + const n = pickName(rowBySku.get(sku)); allNames.set(sku, n); + if (n) namedCount++; + } + + if (args.debug) { + eprintln("[rank_discrepency] name coverage:", { + totalSkus: allSkus.length, + named: namedCount, + unnamed: allSkus.length - namedCount, + }); } const keys = new Set([...abMap.keys(), ...bcMap.keys()]); @@ -659,50 +574,50 @@ function main() { }); if (args.debug) { - eprintln("[rank_discrepency] discrepancy candidates:", { - unionKeys: keys.size, - diffsAfterMin: diffs.length, - topDiscrepSample: diffs.slice(0, 8).map((d) => ({ + eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length }); + eprintln( + "[rank_discrepency] top discrep sample:", + diffs.slice(0, 5).map((d) => ({ sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, - name: truncate(allNames.get(String(d.canonSku)) || "", 90), - })), + name: truncate(allNames.get(String(d.canonSku)) || "", 80), + })) + ); + } + + // Optional: show top 5 matches for the first discrep SKU (helps tune min-score) + if (args.debugBest && diffs.length) { + const skuA = String(diffs[0].canonSku); + const nameA = allNames.get(skuA) || ""; + const groupA = canonicalSku(skuA); + + const scored = []; + for (const skuB of allSkus) { + if (skuB === skuA) continue; + if (canonicalSku(skuB) === groupA) continue; + const nameB = allNames.get(skuB) || ""; + if (!nameB) continue; + const s = similarityScore(nameA, nameB); + scored.push({ skuB, s, nameB }); + } + scored.sort((a, b) => b.s - a.s); + eprintln("[rank_discrepency] debug-best for first discrep:", { + skuA, + nameA: truncate(nameA, 120), + top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, name: truncate(x.nameB, 120) })), }); } - // BIG DEBUG: if we keep seeing empty names, dump the actual row objects for top discrepancies - if (args.debugPayload) { - for (const d of diffs.slice(0, Math.min(args.debugN, diffs.length))) { - const sku = String(d.canonSku); - const row = rowBySku.get(sku) || abMap.get(sku)?.row || bcMap.get(sku)?.row; - const nm = pickName(row, args.nameField); - if (!nm) { - eprintln("[rank_discrepency] no-name row example:", { - sku, - discrep: d.discrep, - rankAB: d.rankAB, - rankBC: d.rankBC, - rowKeys: row && typeof row === "object" ? Object.keys(row).slice(0, 80) : typeof row, - rowTrim: trimForPrint(row), - }); - break; // one is enough to reveal the name field - } - } - } - - // Filter by having a good "other" match not in same linked group + // Filter by “has a high scoring other candidate not in same linked group” const filtered = []; const debugLines = []; for (const d of diffs) { const skuA = String(d.canonSku); const nameA = allNames.get(skuA) || ""; - if (!nameA) { - if (args.debug && debugLines.length < args.debugN) debugLines.push({ sku: skuA, reason: "no-name" }); - continue; - } + if (!nameA) continue; const groupA = canonicalSku(skuA); @@ -726,16 +641,17 @@ function main() { } const pass = best >= args.minScore; + if (args.debug && debugLines.length < args.debugN) { debugLines.push({ sku: skuA, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, - nameA: truncate(nameA, 90), + nameA: truncate(nameA, 70), best, bestSku, - bestName: truncate(bestName, 90), + bestName: truncate(bestName, 70), pass, }); } @@ -751,14 +667,11 @@ function main() { filtered: filtered.length, minScore: args.minScore, minDiscrep: args.minDiscrep, - totalDiffs: diffs.length, - totalNamed: Array.from(allNames.values()).filter(Boolean).length, }); eprintln("[rank_discrepency] debug sample (first N checked):"); for (const x of debugLines) eprintln(" ", x); } - // Emit links on STDOUT for (const d of filtered) { if (args.dumpScores) { eprintln( @@ -770,14 +683,12 @@ function main() { rankBC: d.rankBC, best: d.best, bestSku: d.bestSku, - bestName: truncate(d.bestName, 160), + bestName: truncate(d.bestName, 120), }) ); } console.log(args.base + encodeURIComponent(String(d.canonSku))); } - - if (args.debug) eprintln("[rank_discrepency] done."); } main(); From a4dba47295a2c95aa7e9a84e426f75cb1204261f Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 10:21:43 -0800 Subject: [PATCH 05/19] UX Improvements --- tools/rank_discrepency.js | 222 ++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 132 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index 408812e..e316f90 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -1,16 +1,6 @@ #!/usr/bin/env node "use strict"; -/* - Rank discrepancy links, filtered by existence of a high-similarity "other" listing - that is NOT in the same linked group (using sku_links.json union-find). - - Examples: - node ./tools/rank_discrepency.js --debug - node ./tools/rank_discrepency.js --min-score 0.35 --top 100 --debug - node ./tools/rank_discrepency.js --meta data/sku_links.json --debug-best --debug -*/ - const fs = require("fs"); const path = require("path"); @@ -24,22 +14,26 @@ function parseArgs(argv) { const out = { ab: "reports/common_listings_ab_top1000.json", bc: "reports/common_listings_bc_top1000.json", - - // default to your real links file meta: "data/sku_links.json", top: 50, minDiscrep: 1, includeMissing: false, - minScore: 0.75, + // IMPORTANT: similarityScore is NOT 0..1. defaults should be high. + minScore: 9.0, + minContain: 0.75, + + // only consider suggestions from the opposite list (AB->BC or BC->AB) + requireCrossGroup: true, + base: "http://127.0.0.1:8080/#/link/?left=", debug: false, debugN: 25, debugPayload: false, - debugBest: false, // dump top 5 candidate matches for first discrepancy item - dumpScores: false, // emit per-link score info to STDERR + debugBest: false, + dumpScores: false, }; for (let i = 0; i < argv.length; i++) { @@ -51,9 +45,13 @@ function parseArgs(argv) { else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top; else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep; else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore; + else if (a === "--min-contain" && argv[i + 1]) out.minContain = Number(argv[++i]) || out.minContain; + else if (a === "--include-missing") out.includeMissing = true; else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base); + else if (a === "--no-cross-group") out.requireCrossGroup = false; + else if (a === "--debug") out.debug = true; else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN; else if (a === "--debug-payload") out.debugPayload = true; @@ -68,17 +66,8 @@ function parseArgs(argv) { function extractRows(payload) { if (Array.isArray(payload)) return payload; - - const candidates = [ - payload?.rows, - payload?.data?.rows, - payload?.data, - payload?.items, - payload?.list, - payload?.results, - ]; + const candidates = [payload?.rows, payload?.data?.rows, payload?.data, payload?.items, payload?.list, payload?.results]; for (const x of candidates) if (Array.isArray(x)) return x; - return []; } @@ -99,35 +88,18 @@ function buildRankMap(payload) { return { map, rowsLen: rows.length, rows }; } -/* ---------------- name picking (FIXED) ---------------- */ - function pickName(row) { if (!row) return ""; - - // ✅ common_listings_* puts display name here const repName = row?.representative?.name; if (typeof repName === "string" && repName.trim()) return repName.trim(); - - // fallback: sometimes cheapest has a name (rare) const cheapName = row?.cheapest?.name; if (typeof cheapName === "string" && cheapName.trim()) return cheapName.trim(); - // old fallbacks (keep) - const direct = [ - "name", - "title", - "productName", - "displayName", - "itemName", - "label", - "desc", - "description", - ]; + const direct = ["name","title","productName","displayName","itemName","label","desc","description"]; for (const k of direct) { const v = row[k]; if (typeof v === "string" && v.trim()) return v.trim(); } - return ""; } @@ -245,7 +217,7 @@ function buildCanonicalSkuFnFromMeta(meta) { }; } -/* ---------------- similarity (from viz/app/linker/similarity.js) ---------------- */ +/* ---------------- similarity (same math as viz/app/linker/similarity.js) ---------------- */ function normSearchText(s) { return String(s ?? "") @@ -313,7 +285,6 @@ function filterSimTokens(tokens) { for (let i = 0; i < arr.length; i++) { let t = String(arr[i] || "").trim().toLowerCase(); if (!t) continue; - if (!/[a-z0-9]/i.test(t)) continue; if (VOL_INLINE_RE.test(t)) continue; if (PCT_INLINE_RE.test(t)) continue; @@ -433,7 +404,6 @@ function similarityScore(aName, bName) { const levSim = 1 - d / maxLen; let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain); - const smallN = Math.min(aToks.length, bToks.length); if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18; @@ -455,14 +425,8 @@ function similarityScore(aName, bName) { /* ---------------- debug helpers ---------------- */ -function eprintln(...args) { - console.error(...args); -} - -function truncate(s, n) { - s = String(s || ""); - return s.length <= n ? s : s.slice(0, n - 1) + "…"; -} +function eprintln(...args) { console.error(...args); } +function truncate(s, n) { s = String(s || ""); return s.length <= n ? s : s.slice(0, n - 1) + "…"; } /* ---------------- main ---------------- */ @@ -472,9 +436,7 @@ function main() { const abPath = path.isAbsolute(args.ab) ? args.ab : path.join(repoRoot, args.ab); const bcPath = path.isAbsolute(args.bc) ? args.bc : path.join(repoRoot, args.bc); - const metaPath = args.meta - ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) - : ""; + const metaPath = args.meta ? (path.isAbsolute(args.meta) ? args.meta : path.join(repoRoot, args.meta)) : ""; const ab = readJson(abPath); const bc = readJson(bcPath); @@ -484,41 +446,15 @@ function main() { const abBuilt = buildRankMap(ab); const bcBuilt = buildRankMap(bc); + const abMap = abBuilt.map; const bcMap = bcBuilt.map; - if (args.debug) { - eprintln("[rank_discrepency] inputs:", { - abPath, - bcPath, - metaPath: metaPath || "(none)", - linkCount: Array.isArray(meta?.links) ? meta.links.length : 0, - minDiscrep: args.minDiscrep, - minScore: args.minScore, - top: args.top, - includeMissing: args.includeMissing, - }); - eprintln("[rank_discrepency] extracted rows:", { - abRows: abBuilt.rowsLen, - bcRows: bcBuilt.rowsLen, - abKeys: abMap.size, - bcKeys: bcMap.size, - }); - } + // SKU pools for “cross group” matching + const abSkus = new Set(abMap.keys()); + const bcSkus = new Set(bcMap.keys()); - if (!abMap.size || !bcMap.size) { - eprintln("[rank_discrepency] ERROR: empty rank maps."); - process.exit(2); - } - - if (args.debugPayload) { - const ab0 = abBuilt.rows[0]; - const bc0 = bcBuilt.rows[0]; - eprintln("[rank_discrepency] sample AB rep.name:", truncate(ab0?.representative?.name || "", 120)); - eprintln("[rank_discrepency] sample BC rep.name:", truncate(bc0?.representative?.name || "", 120)); - } - - // Build unique sku pool from AB+BC + // union SKU -> row (for name lookup) const rowBySku = new Map(); for (const m of [abMap, bcMap]) { for (const [canonSku, v] of m.entries()) { @@ -536,11 +472,25 @@ function main() { } if (args.debug) { - eprintln("[rank_discrepency] name coverage:", { - totalSkus: allSkus.length, - named: namedCount, - unnamed: allSkus.length - namedCount, + eprintln("[rank_discrepency] inputs:", { + abPath, bcPath, metaPath: metaPath || "(none)", + linkCount: Array.isArray(meta?.links) ? meta.links.length : 0, + minDiscrep: args.minDiscrep, + minScore: args.minScore, + minContain: args.minContain, + requireCrossGroup: args.requireCrossGroup, + top: args.top, + includeMissing: args.includeMissing, }); + eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, abKeys: abMap.size, bcKeys: bcMap.size }); + eprintln("[rank_discrepency] name coverage:", { totalSkus: allSkus.length, named: namedCount, unnamed: allSkus.length - namedCount }); + } + + if (args.debugPayload) { + const ab0 = abBuilt.rows[0]; + const bc0 = bcBuilt.rows[0]; + eprintln("[rank_discrepency] sample AB rep.name:", truncate(ab0?.representative?.name || "", 120)); + eprintln("[rank_discrepency] sample BC rep.name:", truncate(bc0?.representative?.name || "", 120)); } const keys = new Set([...abMap.keys(), ...bcMap.keys()]); @@ -549,22 +499,14 @@ function main() { for (const canonSku of keys) { const a = abMap.get(canonSku); const b = bcMap.get(canonSku); - if (!args.includeMissing && (!a || !b)) continue; const rankAB = a ? a.rank : null; const rankBC = b ? b.rank : null; - const discrep = rankAB !== null && rankBC !== null ? Math.abs(rankAB - rankBC) : Infinity; if (discrep !== Infinity && discrep < args.minDiscrep) continue; - diffs.push({ - canonSku, - discrep, - rankAB, - rankBC, - sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9), - }); + diffs.push({ canonSku, discrep, rankAB, rankBC, sumRank: (rankAB ?? 1e9) + (rankBC ?? 1e9) }); } diffs.sort((x, y) => { @@ -575,42 +517,48 @@ function main() { if (args.debug) { eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length }); - eprintln( - "[rank_discrepency] top discrep sample:", + eprintln("[rank_discrepency] top discrep sample:", diffs.slice(0, 5).map((d) => ({ - sku: d.canonSku, - discrep: d.discrep, - rankAB: d.rankAB, - rankBC: d.rankBC, + sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, name: truncate(allNames.get(String(d.canonSku)) || "", 80), })) ); } - // Optional: show top 5 matches for the first discrep SKU (helps tune min-score) + // debug-best (top 5) for first discrep SKU, but restricted to cross-group + contain threshold if (args.debugBest && diffs.length) { const skuA = String(diffs[0].canonSku); const nameA = allNames.get(skuA) || ""; const groupA = canonicalSku(skuA); + const aInAB = abSkus.has(skuA); + const pool = args.requireCrossGroup ? (aInAB ? bcSkus : abSkus) : new Set(allSkus); + const aRaw = tokenizeQuery(nameA); const scored = []; - for (const skuB of allSkus) { + + for (const skuB of pool) { if (skuB === skuA) continue; if (canonicalSku(skuB) === groupA) continue; const nameB = allNames.get(skuB) || ""; if (!nameB) continue; + + const contain = tokenContainmentScore(aRaw, tokenizeQuery(nameB)); + if (contain < args.minContain) continue; + const s = similarityScore(nameA, nameB); - scored.push({ skuB, s, nameB }); + scored.push({ skuB, s, contain, nameB }); } + scored.sort((a, b) => b.s - a.s); eprintln("[rank_discrepency] debug-best for first discrep:", { skuA, + side: aInAB ? "AB" : "BC", nameA: truncate(nameA, 120), - top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, name: truncate(x.nameB, 120) })), + minContain: args.minContain, + top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })), }); } - // Filter by “has a high scoring other candidate not in same linked group” const filtered = []; const debugLines = []; @@ -619,46 +567,56 @@ function main() { const nameA = allNames.get(skuA) || ""; if (!nameA) continue; + const aInAB = abSkus.has(skuA); + const pool = args.requireCrossGroup ? (aInAB ? bcSkus : abSkus) : new Set(allSkus); + const groupA = canonicalSku(skuA); + const aRaw = tokenizeQuery(nameA); - let best = 0; - let bestSku = ""; - let bestName = ""; + let best = 0, bestSku = "", bestName = "", bestContain = 0; - for (const skuB of allSkus) { + for (const skuB of pool) { if (skuB === skuA) continue; if (canonicalSku(skuB) === groupA) continue; const nameB = allNames.get(skuB) || ""; if (!nameB) continue; + const contain = tokenContainmentScore(aRaw, tokenizeQuery(nameB)); + if (contain < args.minContain) continue; + const s = similarityScore(nameA, nameB); if (s > best) { best = s; bestSku = skuB; bestName = nameB; + bestContain = contain; } } - const pass = best >= args.minScore; + const pass = bestSku && best >= args.minScore; if (args.debug && debugLines.length < args.debugN) { debugLines.push({ sku: skuA, + side: aInAB ? "AB" : "BC", discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC, - nameA: truncate(nameA, 70), + nameA: truncate(nameA, 52), best, + bestContain, bestSku, - bestName: truncate(bestName, 70), + bestSide: abSkus.has(bestSku) ? "AB" : "BC", + bestName: truncate(bestName, 52), + sameGroupBlocked: bestSku ? (canonicalSku(bestSku) === groupA) : false, pass, }); } if (!pass) continue; - filtered.push({ ...d, best, bestSku, bestName }); + filtered.push({ ...d, best, bestSku, bestName, bestContain }); if (filtered.length >= args.top) break; } @@ -666,6 +624,8 @@ function main() { eprintln("[rank_discrepency] filter results:", { filtered: filtered.length, minScore: args.minScore, + minContain: args.minContain, + requireCrossGroup: args.requireCrossGroup, minDiscrep: args.minDiscrep, }); eprintln("[rank_discrepency] debug sample (first N checked):"); @@ -674,18 +634,16 @@ function main() { for (const d of filtered) { if (args.dumpScores) { - eprintln( - "[rank_discrepency] emit", - JSON.stringify({ - sku: d.canonSku, - discrep: d.discrep, - rankAB: d.rankAB, - rankBC: d.rankBC, - best: d.best, - bestSku: d.bestSku, - bestName: truncate(d.bestName, 120), - }) - ); + eprintln("[rank_discrepency] emit", JSON.stringify({ + sku: d.canonSku, + discrep: d.discrep, + rankAB: d.rankAB, + rankBC: d.rankBC, + best: d.best, + bestContain: d.bestContain, + bestSku: d.bestSku, + bestName: truncate(d.bestName, 120), + })); } console.log(args.base + encodeURIComponent(String(d.canonSku))); } From a553cb4d4e0b57bbf7fe98fa28681f3856b20322 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Tue, 3 Feb 2026 11:00:37 -0800 Subject: [PATCH 06/19] UX Improvements --- tools/rank_discrepency.js | 43 +++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/tools/rank_discrepency.js b/tools/rank_discrepency.js index e316f90..d5c476a 100644 --- a/tools/rank_discrepency.js +++ b/tools/rank_discrepency.js @@ -20,7 +20,7 @@ function parseArgs(argv) { minDiscrep: 1, includeMissing: false, - // IMPORTANT: similarityScore is NOT 0..1. defaults should be high. + // similarityScore is NOT 0..1. minScore: 9.0, minContain: 0.75, @@ -103,7 +103,7 @@ function pickName(row) { return ""; } -/* ---------------- sku_links union-find grouping ---------------- */ +/* ---------------- sku_links union-find grouping + ignores ---------------- */ function normalizeImplicitSkuKey(k) { const s = String(k || "").trim(); @@ -112,6 +112,25 @@ function normalizeImplicitSkuKey(k) { return s; } +function canonicalPairKey(a, b) { + const x = normalizeImplicitSkuKey(a); + const y = normalizeImplicitSkuKey(b); + if (!x || !y) return ""; + return x < y ? `${x}|${y}` : `${y}|${x}`; +} + +function buildIgnoreSet(meta) { + const ignores = Array.isArray(meta?.ignores) ? meta.ignores : []; + const s = new Set(); + for (const x of ignores) { + const a = String(x?.skuA || x?.a || x?.left || "").trim(); + const b = String(x?.skuB || x?.b || x?.right || "").trim(); + const k = canonicalPairKey(a, b); + if (k) s.add(k); + } + return s; +} + class DSU { constructor() { this.parent = new Map(); @@ -444,6 +463,12 @@ function main() { const meta = metaPath ? readJson(metaPath) : null; const canonicalSku = meta ? buildCanonicalSkuFnFromMeta(meta) : (sku) => normalizeImplicitSkuKey(sku); + const ignoreSet = meta ? buildIgnoreSet(meta) : new Set(); + function isIgnoredPair(a, b) { + const k = canonicalPairKey(a, b); + return k ? ignoreSet.has(k) : false; + } + const abBuilt = buildRankMap(ab); const bcBuilt = buildRankMap(bc); @@ -475,6 +500,8 @@ function main() { eprintln("[rank_discrepency] inputs:", { abPath, bcPath, metaPath: metaPath || "(none)", linkCount: Array.isArray(meta?.links) ? meta.links.length : 0, + ignoreCount: Array.isArray(meta?.ignores) ? meta.ignores.length : 0, + ignoreSetSize: ignoreSet.size, minDiscrep: args.minDiscrep, minScore: args.minScore, minContain: args.minContain, @@ -525,7 +552,6 @@ function main() { ); } - // debug-best (top 5) for first discrep SKU, but restricted to cross-group + contain threshold if (args.debugBest && diffs.length) { const skuA = String(diffs[0].canonSku); const nameA = allNames.get(skuA) || ""; @@ -539,6 +565,8 @@ function main() { for (const skuB of pool) { if (skuB === skuA) continue; if (canonicalSku(skuB) === groupA) continue; + if (isIgnoredPair(skuA, skuB)) continue; + const nameB = allNames.get(skuB) || ""; if (!nameB) continue; @@ -574,11 +602,18 @@ function main() { const aRaw = tokenizeQuery(nameA); let best = 0, bestSku = "", bestName = "", bestContain = 0; + let bestWasIgnored = false; for (const skuB of pool) { if (skuB === skuA) continue; if (canonicalSku(skuB) === groupA) continue; + if (isIgnoredPair(skuA, skuB)) { + // critical: ignored pairs must NOT satisfy the requirement + bestWasIgnored = true; + continue; + } + const nameB = allNames.get(skuB) || ""; if (!nameB) continue; @@ -609,7 +644,7 @@ function main() { bestSku, bestSide: abSkus.has(bestSku) ? "AB" : "BC", bestName: truncate(bestName, 52), - sameGroupBlocked: bestSku ? (canonicalSku(bestSku) === groupA) : false, + sawIgnoredPairs: bestWasIgnored, pass, }); } From bb99881a7ccd3e6a3c7ad1e935c58880cd8e73d3 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:24:25 -0800 Subject: [PATCH 07/19] UX Improvements --- viz/app/search_page.js | 4 +- viz/app/store_page.js | 1296 ++++++++++++++++++++-------------------- 2 files changed, 643 insertions(+), 657 deletions(-) diff --git a/viz/app/search_page.js b/viz/app/search_page.js index 82f6845..a8a16a1 100644 --- a/viz/app/search_page.js +++ b/viz/app/search_page.js @@ -347,7 +347,7 @@ export function renderSearch($app) { typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x; const nowMs = Date.now(); - const cutoffMs = nowMs - 24 * 60 * 60 * 1000; + const cutoffMs = nowMs - 3 * 24 * 60 * 60 * 1000; function eventMs(r) { const t = String(r?.ts || ""); @@ -420,7 +420,7 @@ export function renderSearch($app) { const limited = ranked.slice(0, 140); $results.innerHTML = - `
Recently changed (last 24 hours):
` + + `
Recently changed (last 3 days):
` + limited .map(({ r, meta }) => { const kindLabel = diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 481217c..4cb6456 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -1,4 +1,4 @@ -import { esc, renderThumbHtml } from "./dom.js"; +import { esc, renderThumbHtml, prettyTs } from "./dom.js"; import { tokenizeQuery, matchesAllTokens, @@ -6,536 +6,464 @@ import { keySkuForRow, parsePriceToNumber, } from "./sku.js"; -import { loadIndex } from "./state.js"; +import { loadIndex, loadRecent, saveQuery, loadSavedQuery } from "./state.js"; import { aggregateBySku } from "./catalog.js"; import { loadSkuRules } from "./mapping.js"; -function normStoreLabel(s) { - return String(s || "").trim().toLowerCase(); -} - -function abbrevStoreLabel(s) { - const t = String(s || "").trim(); - if (!t) return ""; - return t.split(/\s+/)[0] || t; -} - -function readLinkHrefForSkuInStore(listingsLive, canonSku, storeLabelNorm) { - // Prefer the most recent-ish url if multiple exist; stable enough for viz. - let bestUrl = ""; - let bestScore = -1; - - function scoreUrl(u) { - if (!u) return -1; - let s = u.length; - if (/\bproduct\/\d+\//.test(u)) s += 50; - if (/[a-z0-9-]{8,}/i.test(u)) s += 10; - return s; - } - - for (const r of listingsLive) { - if (!r || r.removed) continue; - const store = normStoreLabel(r.storeLabel || r.store || ""); - if (store !== storeLabelNorm) continue; - - const skuKey = String( - rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r) - ); - if (skuKey !== canonSku) continue; - - const u = String(r.url || "").trim(); - const sc = scoreUrl(u); - if (sc > bestScore) { - bestScore = sc; - bestUrl = u; - } else if (sc === bestScore && u && bestUrl && u < bestUrl) { - bestUrl = u; - } - } - return bestUrl; -} - -// small module-level cache so we can reuse in readLinkHrefForSkuInStore -let rulesCache = null; - -export async function renderStore($app, storeLabelRaw) { - const storeLabel = String(storeLabelRaw || "").trim(); - const storeLabelShort = - abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store"); - +export function renderStore($app, storeLabelParam) { $app.innerHTML = `
-
- - ${esc(storeLabelShort || "Store")} +
+
+
+

Store

+
Loading…
+
+ + +
-
-
-
-
Max price
- - - -
-
- -
- +
+ +
+
+
+
-
- -
-
-
-
- Exclusive - and - Last Stock -
-
- Sort - -
-
-
+ +
+
+
Exclusives / Last stock
+
-
-
- Price compare -
- Comparison - -
-
-
-
+
Exclusives
+
+ +
Last stock
+
- -
`; - document.getElementById("back").addEventListener("click", () => { - sessionStorage.setItem("viz:lastRoute", location.hash); - location.hash = "#/"; - }); - const $q = document.getElementById("q"); - const $status = document.getElementById("status"); - const $resultsExclusive = document.getElementById("resultsExclusive"); - const $resultsCompare = document.getElementById("resultsCompare"); - const $sentinel = document.getElementById("sentinel"); - const $resultsWrap = document.getElementById("results"); - - const $maxPrice = document.getElementById("maxPrice"); - const $maxPriceLabel = document.getElementById("maxPriceLabel"); - const $priceWrap = document.getElementById("priceWrap"); - + const $storeSub = document.getElementById("storeSub"); + const $storeResults = document.getElementById("storeResults"); + const $exclusiveResults = document.getElementById("exclusiveResults"); + const $lastStockResults = document.getElementById("lastStockResults"); const $clearSearch = document.getElementById("clearSearch"); - const $exSort = document.getElementById("exSort"); - const $cmpMode = document.getElementById("cmpMode"); + const $rightSort = document.getElementById("rightSort"); - // Persist query per store - const storeNorm = normStoreLabel(storeLabel); - const LS_KEY = `viz:storeQuery:${storeNorm}`; - const savedQ = String(localStorage.getItem(LS_KEY) || ""); - if (savedQ) $q.value = savedQ; + // Keep store-page filter consistent with the app's saved query (optional). + $q.value = loadSavedQuery() || ""; - // Persist max price per store (clamped later once bounds known) - const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`; - const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE); - let savedMaxPrice = - savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null; - if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null; + const SORT_KEY = "viz:storeRightSort"; + $rightSort.value = sessionStorage.getItem(SORT_KEY) || "time"; - // Persist exclusives sort per store - const LS_EX_SORT = `viz:storeExclusiveSort:${storeNorm}`; - const savedExSort = String(localStorage.getItem(LS_EX_SORT) || ""); - if (savedExSort) $exSort.value = savedExSort; + let indexReady = false; - // Persist comparison technique per store - const LS_CMP_MODE = `viz:storeCompareMode:${storeNorm}`; - const savedCmpMode = String(localStorage.getItem(LS_CMP_MODE) || ""); - if (savedCmpMode) $cmpMode.value = savedCmpMode; + let STORE_LABEL = String(storeLabelParam || "").trim(); + let CANON = (x) => x; - $resultsExclusive.innerHTML = `
Loading…
`; - $resultsCompare.innerHTML = ``; + // canonicalSku -> agg + let aggBySku = new Map(); + // canonicalSku -> storeLabel -> { priceNum, priceStr, url } + let PRICE_BY_SKU_STORE = new Map(); + // canonicalSku -> storeLabel -> url (for badge href) + let URL_BY_SKU_STORE = new Map(); - const idx = await loadIndex(); - rulesCache = await loadSkuRules(); - const rules = rulesCache; + // canonicalSku -> storeLabel -> most recent recent-row (within 7d) + let RECENT_BY_SKU_STORE = new Map(); - const listingsAll = Array.isArray(idx.items) ? idx.items : []; - const liveAll = listingsAll.filter((r) => r && !r.removed); + // For left list: canonicalSku -> best row for this store (cheapest priceNum; tie -> newest) + let STORE_ROW_BY_SKU = new Map(); - function dateMsFromRow(r) { - const t = String(r?.firstSeenAt || ""); + // Derived right-side lists + let exclusives = []; + let lastStock = []; + let storeItems = []; // left list items + + function normStoreLabel(s) { + return String(s || "").trim().toLowerCase(); + } + + function resolveStoreLabel(listings, wanted) { + const w = normStoreLabel(wanted); + if (!w) return ""; + for (const r of Array.isArray(listings) ? listings : []) { + const lab = String(r?.storeLabel || r?.store || "").trim(); + if (lab && normStoreLabel(lab) === w) return lab; + } + return wanted; + } + + function tsValue(r) { + const t = String(r?.ts || ""); const ms = t ? Date.parse(t) : NaN; - return Number.isFinite(ms) ? ms : null; + if (Number.isFinite(ms)) return ms; + const d = String(r?.date || ""); + const ms2 = d ? Date.parse(d) : NaN; + return Number.isFinite(ms2) ? ms2 : 0; } - // Build earliest "first in DB (for this store)" timestamp per canonical SKU (includes removed rows) - const firstSeenBySkuInStore = new Map(); // sku -> ms - for (const r of listingsAll) { - if (!r) continue; - const store = normStoreLabel(r.storeLabel || r.store || ""); - if (store !== storeNorm) continue; + function eventMs(r) { + const t = String(r?.ts || ""); + const ms = t ? Date.parse(t) : NaN; + if (Number.isFinite(ms)) return ms; - const skuKey = keySkuForRow(r); - const sku = String(rules.canonicalSku(skuKey) || skuKey); - - const ms = dateMsFromRow(r); - if (ms === null) continue; - - const prev = firstSeenBySkuInStore.get(sku); - if (prev === undefined || ms < prev) firstSeenBySkuInStore.set(sku, ms); + const d = String(r?.date || ""); + const ms2 = d ? Date.parse(d + "T00:00:00Z") : NaN; + return Number.isFinite(ms2) ? ms2 : 0; } - // Build "ever seen" store presence per canonical SKU (includes removed rows) - const everStoresBySku = new Map(); // sku -> Set(storeLabelNorm) - for (const r of listingsAll) { - if (!r) continue; - const store = normStoreLabel(r.storeLabel || r.store || ""); - if (!store) continue; + function buildUrlMap(listings, canonicalSkuFn) { + const out = new Map(); + for (const r of Array.isArray(listings) ? listings : []) { + if (!r || r.removed) continue; - const skuKey = keySkuForRow(r); - const sku = String(rules.canonicalSku(skuKey) || skuKey); + const skuKey = String(keySkuForRow(r) || "").trim(); + if (!skuKey) continue; - let ss = everStoresBySku.get(sku); - if (!ss) everStoresBySku.set(sku, (ss = new Set())); - ss.add(store); - } + const sku = String(canonicalSkuFn ? canonicalSkuFn(skuKey) : skuKey).trim(); + if (!sku) continue; - // Build global per-canonical-SKU live store presence + min prices - const storesBySku = new Map(); // sku -> Set(storeLabelNorm) - const minPriceBySkuStore = new Map(); // sku -> Map(storeLabelNorm -> minPrice) + const storeLabel = String(r.storeLabel || r.store || "").trim(); + const url = String(r.url || "").trim(); + if (!storeLabel || !url) continue; - for (const r of liveAll) { - const store = normStoreLabel(r.storeLabel || r.store || ""); - if (!store) continue; - - const skuKey = keySkuForRow(r); - const sku = String(rules.canonicalSku(skuKey) || skuKey); - - let ss = storesBySku.get(sku); - if (!ss) storesBySku.set(sku, (ss = new Set())); - ss.add(store); - - const p = parsePriceToNumber(r.price); - if (p !== null) { - let m = minPriceBySkuStore.get(sku); - if (!m) minPriceBySkuStore.set(sku, (m = new Map())); - const prev = m.get(store); - if (prev === undefined || p < prev) m.set(store, p); + let m = out.get(sku); + if (!m) out.set(sku, (m = new Map())); + if (!m.has(storeLabel)) m.set(storeLabel, url); } + return out; } - function bestAllPrice(sku) { - const m = minPriceBySkuStore.get(sku); - if (!m) return null; - let best = null; - for (const v of m.values()) best = best === null ? v : Math.min(best, v); - return best; + function urlForSkuStore(sku, storeLabel) { + const s = String(sku || ""); + const lab = String(storeLabel || ""); + return URL_BY_SKU_STORE.get(s)?.get(lab) || ""; } - function bestOtherPrice(sku, store) { - const m = minPriceBySkuStore.get(sku); - if (!m) return null; - let best = null; - for (const [k, v] of m.entries()) { - if (k === store) continue; - best = best === null ? v : Math.min(best, v); + function buildPriceMap(listings, canonicalSkuFn) { + const out = new Map(); + for (const r of Array.isArray(listings) ? listings : []) { + if (!r || r.removed) continue; + + const rawSku = String(keySkuForRow(r) || "").trim(); + if (!rawSku) continue; + + const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); + if (!sku) continue; + + const storeLabel = String(r.storeLabel || r.store || "").trim(); + if (!storeLabel) continue; + + const priceStr = String(r.price || "").trim(); + const priceNum = parsePriceToNumber(priceStr); + if (!Number.isFinite(priceNum)) continue; + + const url = String(r.url || "").trim(); + + let m = out.get(sku); + if (!m) out.set(sku, (m = new Map())); + + const prev = m.get(storeLabel); + if (!prev || priceNum < prev.priceNum) { + m.set(storeLabel, { priceNum, priceStr, url }); + } } - return best; + return out; } - // Store-specific live rows only (in-stock for that store) - const rowsStoreLive = liveAll.filter( - (r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm - ); + function pickBestStoreRowForSku(listings, canonicalSkuFn, storeLabel) { + const out = new Map(); + const want = normStoreLabel(storeLabel); - // Aggregate in this store, grouped by canonical SKU (so mappings count as same bottle) - let items = aggregateBySku(rowsStoreLive, rules.canonicalSku); + for (const r of Array.isArray(listings) ? listings : []) { + if (!r || r.removed) continue; - // Decorate each item with pricing comparisons + exclusivity - const EPS = 0.01; + const lab = String(r.storeLabel || r.store || "").trim(); + if (!lab || normStoreLabel(lab) !== want) continue; - items = items.map((it) => { - const sku = String(it.sku || ""); - const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]); - const everStoreSet = everStoresBySku.get(sku) || liveStoreSet; + const rawSku = String(keySkuForRow(r) || "").trim(); + if (!rawSku) continue; - const soloLiveHere = - liveStoreSet.size === 1 && liveStoreSet.has(storeNorm); - const lastStock = soloLiveHere && everStoreSet.size > 1; - const exclusive = soloLiveHere && !lastStock; + const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); + if (!sku) continue; - const storePrice = Number.isFinite(it.cheapestPriceNum) - ? it.cheapestPriceNum - : null; - const bestAll = bestAllPrice(sku); - const other = bestOtherPrice(sku, storeNorm); + const priceStr = String(r.price || "").trim(); + const priceNum = parsePriceToNumber(priceStr); + const ms = tsValue(r); - const isBest = - storePrice !== null && bestAll !== null - ? storePrice <= bestAll + EPS - : false; + const prev = out.get(sku); + if (!prev) { + out.set(sku, { r, priceNum, ms }); + continue; + } - const diffVsOtherDollar = - storePrice !== null && other !== null ? storePrice - other : null; - const diffVsOtherPct = - storePrice !== null && other !== null && other > 0 - ? ((storePrice - other) / other) * 100 - : null; + const prevPrice = prev.priceNum; + const prevMs = prev.ms; - const diffVsBestDollar = - storePrice !== null && bestAll !== null ? storePrice - bestAll : null; - const diffVsBestPct = - storePrice !== null && bestAll !== null && bestAll > 0 - ? ((storePrice - bestAll) / bestAll) * 100 - : null; + const priceOk = Number.isFinite(priceNum); + const prevOk = Number.isFinite(prevPrice); - const firstSeenMs = firstSeenBySkuInStore.get(sku); - const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null; + if (priceOk && !prevOk) out.set(sku, { r, priceNum, ms }); + else if (priceOk && prevOk && priceNum < prevPrice) out.set(sku, { r, priceNum, ms }); + else if ( + (priceOk && prevOk && Math.abs(priceNum - prevPrice) <= 0.01 && ms > prevMs) || + (!priceOk && !prevOk && ms > prevMs) + ) { + out.set(sku, { r, priceNum, ms }); + } + } + + return out; + } + + function buildRecentBySkuStore(recentItems, canonicalSkuFn, days) { + const nowMs = Date.now(); + const cutoffMs = nowMs - days * 24 * 60 * 60 * 1000; + + // sku -> storeLabel -> row + const out = new Map(); + + for (const r of Array.isArray(recentItems) ? recentItems : []) { + const ms = eventMs(r); + if (!(ms >= cutoffMs && ms <= nowMs)) continue; + + const rawSku = String(r?.sku || "").trim(); + if (!rawSku) continue; + + const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); + if (!sku) continue; + + const storeLabel = String(r?.storeLabel || r?.store || "").trim(); + if (!storeLabel) continue; + + let storeMap = out.get(sku); + if (!storeMap) out.set(sku, (storeMap = new Map())); + + const prev = storeMap.get(storeLabel); + if (!prev || eventMs(prev) < ms) storeMap.set(storeLabel, r); + } + + return out; + } + + function normalizeKindForPrice(r) { + let kind = String(r?.kind || ""); + if (kind === "price_change") { + const o = parsePriceToNumber(r?.oldPrice || ""); + const n = parsePriceToNumber(r?.newPrice || ""); + if (Number.isFinite(o) && Number.isFinite(n)) { + if (n < o) kind = "price_down"; + else if (n > o) kind = "price_up"; + else kind = "price_change"; + } + } + return kind; + } + + function salePctOff(oldRaw, newRaw) { + const oldN = parsePriceToNumber(oldRaw); + const newN = parsePriceToNumber(newRaw); + if (!Number.isFinite(oldN) || !Number.isFinite(newN)) return null; + if (!(oldN > 0)) return null; + if (!(newN < oldN)) return null; + const pct = Math.round(((oldN - newN) / oldN) * 100); + return Number.isFinite(pct) && pct > 0 ? pct : null; + } + + function pctChange(oldRaw, newRaw) { + const oldN = parsePriceToNumber(oldRaw); + const newN = parsePriceToNumber(newRaw); + if (!Number.isFinite(oldN) || !Number.isFinite(newN)) return null; + if (!(oldN > 0)) return null; + const pct = Math.round(((newN - oldN) / oldN) * 100); + return Number.isFinite(pct) ? pct : null; + } + + function fmtUsd(n) { + const abs = Math.abs(Number(n) || 0); + if (!Number.isFinite(abs)) return ""; + const s = abs.toFixed(2); + return s.endsWith(".00") ? s.slice(0, -3) : s; + } + + function recentSaleMetaForSkuStore(sku, storeLabel) { + const r = RECENT_BY_SKU_STORE.get(String(sku || ""))?.get(String(storeLabel || "")); + if (!r) return null; + + const kind = normalizeKindForPrice(r); + if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") return null; + + const oldStr = String(r?.oldPrice || "").trim(); + const newStr = String(r?.newPrice || "").trim(); + const oldN = parsePriceToNumber(oldStr); + const newN = parsePriceToNumber(newStr); + + if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) return null; + + // signedPct: down => negative; up => positive; unchanged => 0 + let signedPct = 0; + let signedDelta = newN - oldN; // down => negative + + if (newN < oldN) { + const off = salePctOff(oldStr, newStr); + signedPct = off !== null ? -off : Math.round(((newN - oldN) / oldN) * 100); + } else if (newN > oldN) { + const up = pctChange(oldStr, newStr); + signedPct = up !== null ? up : Math.round(((newN - oldN) / oldN) * 100); + } else { + signedPct = 0; + signedDelta = 0; + } + + const when = r.ts ? prettyTs(r.ts) : r.date || ""; return { - ...it, - _exclusive: exclusive, - _lastStock: lastStock, - _storePrice: storePrice, - _bestAll: bestAll, - _bestOther: other, - _isBest: isBest, - _diffVsOtherDollar: diffVsOtherDollar, - _diffVsOtherPct: diffVsOtherPct, - _diffVsBestDollar: diffVsBestDollar, - _diffVsBestPct: diffVsBestPct, - _firstSeenMs: firstSeen, + r, + kind, + oldStr, + newStr, + oldN, + newN, + signedPct, + signedDelta, + when, }; - }); + } - // ---- Max price slider (exponential mapping + clicky rounding) ---- - const MIN_PRICE = 25; + function pctOffVsNextBest(sku, storeLabel) { + const m = PRICE_BY_SKU_STORE.get(String(sku || "")); + if (!m) return null; - function maxStorePriceOnPage() { - let mx = null; - for (const it of items) { - const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; - if (p === null) continue; - mx = mx === null ? p : Math.max(mx, p); + const here = m.get(String(storeLabel || "")); + if (!here || !Number.isFinite(here.priceNum)) return null; + + const prices = []; + for (const v of m.values()) { + if (Number.isFinite(v?.priceNum)) prices.push(v.priceNum); } - return mx; - } + prices.sort((a, b) => a - b); - const pageMax = maxStorePriceOnPage(); - const boundMax = pageMax !== null ? Math.max(MIN_PRICE, pageMax) : MIN_PRICE; + if (!prices.length) return null; - function stepForPrice(p) { - const x = Number.isFinite(p) ? p : boundMax; - if (x < 120) return 5; - if (x < 250) return 10; - if (x < 600) return 25; - return 100; - } - function roundToStep(p) { - const step = stepForPrice(p); - return Math.round(p / step) * step; - } + const EPS = 0.01; + const min = prices[0]; + if (Math.abs(here.priceNum - min) > EPS) return null; - function priceFromT(t) { - t = Math.max(0, Math.min(1, t)); - if (boundMax <= MIN_PRICE) return MIN_PRICE; - const ratio = boundMax / MIN_PRICE; - return MIN_PRICE * Math.exp(Math.log(ratio) * t); - } - function tFromPrice(price) { - if (!Number.isFinite(price)) return 1; - if (boundMax <= MIN_PRICE) return 1; - const p = Math.max(MIN_PRICE, Math.min(boundMax, price)); - const ratio = boundMax / MIN_PRICE; - return Math.log(p / MIN_PRICE) / Math.log(ratio); - } - - function clampPrice(p) { - if (!Number.isFinite(p)) return boundMax; - return Math.max(MIN_PRICE, Math.min(boundMax, p)); - } - - function clampAndRound(p) { - const c = clampPrice(p); - const r = roundToStep(c); - return clampPrice(r); - } - - function formatDollars(p) { - if (!Number.isFinite(p)) return ""; - return `$${Math.round(p)}`; - } - - let selectedMaxPrice = clampAndRound( - savedMaxPrice !== null ? savedMaxPrice : boundMax - ); - - function setSliderFromPrice(p) { - const t = tFromPrice(p); - const v = Math.round(t * 1000); - $maxPrice.value = String(v); - } - - function getRawPriceFromSlider() { - const v = Number($maxPrice.value); - const t = Number.isFinite(v) ? v / 1000 : 1; - return clampPrice(priceFromT(t)); - } - - function updateMaxPriceLabel() { - if (pageMax === null) { - $maxPriceLabel.textContent = "No prices"; - return; - } - $maxPriceLabel.textContent = `${formatDollars(selectedMaxPrice)}`; - } - - if (pageMax === null) { - $maxPrice.disabled = true; - $priceWrap.title = "No priced items in this store."; - selectedMaxPrice = boundMax; - setSliderFromPrice(boundMax); - localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); - updateMaxPriceLabel(); - } else { - selectedMaxPrice = clampAndRound(selectedMaxPrice); - localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); - setSliderFromPrice(selectedMaxPrice); - updateMaxPriceLabel(); - } - - // ---- Listing display price: keep cents (no rounding) ---- - function listingPriceStr(it) { - const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; - if (p === null) - return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)"; - return `$${p.toFixed(2)}`; - } - - function compareMode() { - return $cmpMode && $cmpMode.value === "percent" ? "percent" : "dollar"; - } - - function priceBadgeHtml(it) { - if (it._exclusive || it._lastStock) return ""; - - const mode = compareMode(); - - if (mode === "percent") { - const d = it._diffVsOtherPct; - if (d === null || !Number.isFinite(d)) return ""; - const abs = Math.abs(d); - if (abs <= 5) { - return `within 5%`; + // find second distinct + let second = null; + for (let i = 1; i < prices.length; i++) { + if (Math.abs(prices[i] - min) > EPS) { + second = prices[i]; + break; } - const pct = Math.round(abs); - if (d < 0) return `${esc(pct)}% lower`; - return `${esc(pct)}% higher`; } + if (!Number.isFinite(second) || !(second > 0) || !(min < second)) return null; - const d = it._diffVsOtherDollar; - if (d === null || !Number.isFinite(d)) return ""; - - const abs = Math.abs(d); - if (abs <= 5) { - return `within $5`; - } - - const dollars = Math.round(abs); - if (d < 0) { - return `$${esc(dollars)} lower`; - } - return `$${esc(dollars)} higher`; + const pct = Math.round(((second - min) / second) * 100); + return Number.isFinite(pct) && pct > 0 ? pct : null; } - function renderCard(it) { - const price = listingPriceStr(it); - const href = String(it.sampleUrl || "").trim(); + function badgeHtmlForSale(sortMode, saleMeta) { + if (!saleMeta) return ""; + if (sortMode === "sale_pct") { + const v = saleMeta.signedPct; + if (!Number.isFinite(v) || v === 0) return ""; + const isDown = v < 0; + const abs = Math.abs(v); + const style = isDown + ? ` style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);"` + : ` style="margin-left:6px; color:rgba(170,40,40,0.95); background:rgba(170,40,40,0.10); border:1px solid rgba(170,40,40,0.20);"`; + const txt = isDown ? `[${abs}% Off]` : `[+${abs}%]`; + return `${esc(txt)}`; + } - const specialBadge = it._lastStock - ? `Last Stock` - : it._exclusive - ? `Exclusive` - : ""; + if (sortMode === "sale_abs") { + const d = saleMeta.signedDelta; + if (!Number.isFinite(d) || d === 0) return ""; + const isDown = d < 0; + const abs = fmtUsd(d); + if (!abs) return ""; + const style = isDown + ? ` style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);"` + : ` style="margin-left:6px; color:rgba(170,40,40,0.95); background:rgba(170,40,40,0.10); border:1px solid rgba(170,40,40,0.20);"`; + const txt = isDown ? `[$${abs} Off]` : `[+$${abs}]`; + return `${esc(txt)}`; + } - const bestBadge = - !it._exclusive && !it._lastStock && it._isBest - ? `Best Price` - : ""; + return ""; + } - const diffBadge = priceBadgeHtml(it); + function badgeHtmlForExclusivePctOff(sku) { + const pct = pctOffVsNextBest(sku, STORE_LABEL); + if (!Number.isFinite(pct) || pct <= 0) return ""; + return `[${esc( + pct + )}% Off]`; + } + + function itemCardHtml(it, { annotateMode }) { + // annotateMode: "sale_pct" | "sale_abs" | "default" + const sku = String(it?.sku || ""); + const name = String(it?.name || "(no name)"); + const img = String(it?.img || ""); + const priceStr = it.priceStr ? it.priceStr : "(no price)"; + + const href = urlForSkuStore(sku, STORE_LABEL) || String(it?.url || "").trim(); + const storeBadge = href + ? `${esc( + STORE_LABEL + )}` + : `${esc(STORE_LABEL)}`; + + const skuLink = `#/link/?left=${encodeURIComponent(sku)}`; + + let annot = ""; + if (annotateMode === "sale_pct" || annotateMode === "sale_abs") { + annot = badgeHtmlForSale(annotateMode, it.saleMeta); + } else { + // default annotation is % off for exclusives only (and only if >0) + if (it.isExclusive) annot = badgeHtmlForExclusivePctOff(sku); + } - const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`; return ` -
+
-
${renderThumbHtml(it.img)}
+
+ ${renderThumbHtml(img)} +
- ${specialBadge} - ${bestBadge} - ${diffBadge} - ${esc(price)} - ${ - href - ? `${esc( - storeLabelShort - )}` - : `` - } + ${esc(priceStr)} + ${annot} + ${storeBadge}
@@ -543,242 +471,300 @@ export async function renderStore($app, storeLabelRaw) { `; } - // ---- Infinite scroll paging (shared across both columns) ---- - const PAGE_SIZE = 140; - const PAGE_EACH = Math.max(1, Math.floor(PAGE_SIZE / 2)); - - let filteredExclusive = []; - let filteredCompare = []; - let shownExclusive = 0; - let shownCompare = 0; - - function totalFiltered() { - return filteredExclusive.length + filteredCompare.length; - } - function totalShown() { - return shownExclusive + shownCompare; - } - - function setStatus() { - const total = totalFiltered(); - if (!total) { - $status.textContent = "No in-stock items for this store."; + function renderList($el, items, annotateMode) { + if (!items.length) { + $el.innerHTML = `
No matches.
`; return; } - if (pageMax !== null) { - $status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars( - selectedMaxPrice - )}).`; - return; - } + const limited = items.slice(0, 80); + $el.innerHTML = limited.map((it) => itemCardHtml(it, { annotateMode })).join(""); - $status.textContent = `In stock: ${total} item(s).`; - } - - function renderNext(reset) { - if (reset) { - $resultsExclusive.innerHTML = ""; - $resultsCompare.innerHTML = ""; - shownExclusive = 0; - shownCompare = 0; - } - - const sliceEx = filteredExclusive.slice( - shownExclusive, - shownExclusive + PAGE_EACH - ); - const sliceCo = filteredCompare.slice( - shownCompare, - shownCompare + PAGE_EACH - ); - - shownExclusive += sliceEx.length; - shownCompare += sliceCo.length; - - if (sliceEx.length) - $resultsExclusive.insertAdjacentHTML( - "beforeend", - sliceEx.map(renderCard).join("") - ); - if (sliceCo.length) - $resultsCompare.insertAdjacentHTML( - "beforeend", - sliceCo.map(renderCard).join("") - ); - - const total = totalFiltered(); - const shown = totalShown(); - - if (!total) { - $sentinel.textContent = ""; - } else if (shown >= total) { - $sentinel.textContent = `Showing ${shown} / ${total}`; - } else { - $sentinel.textContent = `Showing ${shown} / ${total}…`; - } - } - - $resultsWrap.addEventListener("click", (e) => { - const el = e.target.closest(".item"); - if (!el) return; - const sku = el.getAttribute("data-sku") || ""; - if (!sku) return; - sessionStorage.setItem("viz:lastRoute", location.hash); - location.hash = `#/item/${encodeURIComponent(sku)}`; - }); - - function sortExclusiveInPlace(arr) { - const mode = String($exSort.value || "priceDesc"); - if (mode === "priceAsc" || mode === "priceDesc") { - arr.sort((a, b) => { - const ap = Number.isFinite(a._storePrice) ? a._storePrice : null; - const bp = Number.isFinite(b._storePrice) ? b._storePrice : null; - const aKey = - ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; - const bKey = - bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; - if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey; - return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); - }); - return; - } - - if (mode === "dateAsc" || mode === "dateDesc") { - arr.sort((a, b) => { - const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null; - const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null; - const aKey = - ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; - const bKey = - bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; - if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey; - return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); + for (const el of Array.from($el.querySelectorAll(".item"))) { + el.addEventListener("click", () => { + const sku = el.getAttribute("data-sku") || ""; + if (!sku) return; + saveQuery($q.value); + sessionStorage.setItem("viz:lastRoute", location.hash); + location.hash = `#/item/${encodeURIComponent(sku)}`; }); } } - function sortCompareInPlace(arr) { - const mode = compareMode(); - arr.sort((a, b) => { - const da = mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar; - const db = mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar; + function renderStoreCatalog(items) { + if (!items.length) { + $storeResults.innerHTML = `
No matches.
`; + return; + } - const sa = da === null || !Number.isFinite(da) ? 999999 : da; - const sb = db === null || !Number.isFinite(db) ? 999999 : db; - if (sa !== sb) return sa - sb; + const limited = items.slice(0, 160); + $storeResults.innerHTML = limited + .map((it) => { + const sku = String(it?.sku || ""); + const href = urlForSkuStore(sku, STORE_LABEL) || String(it?.url || "").trim(); + const storeBadge = href + ? `${esc( + STORE_LABEL + )}` + : `${esc(STORE_LABEL)}`; - return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); + const skuLink = `#/link/?left=${encodeURIComponent(sku)}`; + + return ` +
+
+
+ ${renderThumbHtml(it.img)} +
+
+
+
${esc(it.name || "(no name)")}
+ ${esc( + displaySku(sku) + )} +
+
+ ${esc(it.priceStr || "(no price)")} + ${storeBadge} +
+
+
+
+ `; + }) + .join(""); + + for (const el of Array.from($storeResults.querySelectorAll(".item"))) { + el.addEventListener("click", () => { + const sku = el.getAttribute("data-sku") || ""; + if (!sku) return; + saveQuery($q.value); + sessionStorage.setItem("viz:lastRoute", location.hash); + location.hash = `#/item/${encodeURIComponent(sku)}`; + }); + } + } + + function sortRightList(items, mode) { + const m = String(mode || "time"); + + const getTime = (it) => Number(it?.timeMs || 0); + const getPrice = (it) => (Number.isFinite(it?.priceNum) ? it.priceNum : Number.POSITIVE_INFINITY); + + const getSignedPct = (it) => { + const v = it?.saleMeta?.signedPct; + return Number.isFinite(v) ? v : 0; // unchanged/no-event => 0 (middle) + }; + + const getSignedDelta = (it) => { + const v = it?.saleMeta?.signedDelta; + return Number.isFinite(v) ? v : 0; // unchanged/no-event => 0 (middle) + }; + + const out = items.slice(); + + if (m === "sale_pct") { + out.sort((a, b) => { + // deals (negative) first; unchanged (0) middle; increases (+) last + const av = getSignedPct(a); + const bv = getSignedPct(b); + if (av !== bv) return av - bv; + const ap = getPrice(a); + const bp = getPrice(b); + if (ap !== bp) return ap - bp; + return String(a.sku || "").localeCompare(String(b.sku || "")); + }); + return out; + } + + if (m === "sale_abs") { + out.sort((a, b) => { + const av = getSignedDelta(a); + const bv = getSignedDelta(b); + if (av !== bv) return av - bv; // negative (down) first, positive (up) last + const ap = getPrice(a); + const bp = getPrice(b); + if (ap !== bp) return ap - bp; + return String(a.sku || "").localeCompare(String(b.sku || "")); + }); + return out; + } + + if (m === "price") { + out.sort((a, b) => { + const ap = getPrice(a); + const bp = getPrice(b); + if (ap !== bp) return ap - bp; + return String(a.sku || "").localeCompare(String(b.sku || "")); + }); + return out; + } + + // time (default): newest first + out.sort((a, b) => { + const at = getTime(a); + const bt = getTime(b); + if (bt !== at) return bt - at; + return String(a.sku || "").localeCompare(String(b.sku || "")); }); + return out; } - function applyFilter() { - const raw = String($q.value || ""); - localStorage.setItem(LS_KEY, raw); + function rebuildDerivedLists() { + const tokens = tokenizeQuery($q.value); - const tokens = tokenizeQuery(raw); + const filteredStoreItems = !tokens.length + ? storeItems + : storeItems.filter((it) => matchesAllTokens(it.searchText, tokens)); - let base = items; + renderStoreCatalog(filteredStoreItems); - if (tokens.length) { - base = base.filter((it) => matchesAllTokens(it.searchText, tokens)); - } + const rightSortMode = String($rightSort.value || "time"); + const annotMode = + rightSortMode === "sale_pct" + ? "sale_pct" + : rightSortMode === "sale_abs" + ? "sale_abs" + : "default"; - if (pageMax !== null && Number.isFinite(selectedMaxPrice)) { - const cap = selectedMaxPrice + 0.0001; - base = base.filter((it) => { - const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; - return p === null ? true : p <= cap; - }); - } + const ex = !tokens.length + ? exclusives + : exclusives.filter((it) => matchesAllTokens(it.searchText, tokens)); - filteredExclusive = base.filter((it) => it._exclusive || it._lastStock); - filteredCompare = base.filter((it) => !it._exclusive && !it._lastStock); + const ls = !tokens.length + ? lastStock + : lastStock.filter((it) => matchesAllTokens(it.searchText, tokens)); - sortExclusiveInPlace(filteredExclusive); - sortCompareInPlace(filteredCompare); - - setStatus(); - renderNext(true); + renderList($exclusiveResults, sortRightList(ex, rightSortMode), annotMode); + renderList($lastStockResults, sortRightList(ls, rightSortMode), annotMode); } - applyFilter(); + $storeResults.innerHTML = `
Loading…
`; + $exclusiveResults.innerHTML = `
Loading…
`; + $lastStockResults.innerHTML = `
Loading…
`; - const io = new IntersectionObserver( - (entries) => { - const hit = entries.some((x) => x.isIntersecting); - if (!hit) return; - if (totalShown() >= totalFiltered()) return; - renderNext(false); - }, - { root: null, rootMargin: "600px 0px", threshold: 0.01 } - ); - io.observe($sentinel); + Promise.all([loadIndex(), loadSkuRules(), loadRecent()]) + .then(([idx, rules, recent]) => { + const listings = Array.isArray(idx?.items) ? idx.items : []; - let t = null; - $q.addEventListener("input", () => { - if (t) clearTimeout(t); - t = setTimeout(applyFilter, 60); - }); + CANON = typeof rules?.canonicalSku === "function" ? rules.canonicalSku : (x) => x; + + STORE_LABEL = resolveStoreLabel(listings, STORE_LABEL); + $storeSub.textContent = STORE_LABEL ? `Browsing: ${STORE_LABEL}` : `Browsing store`; + + // Global aggregates (for "exclusive" / "last stock" determination) + const allAgg = aggregateBySku(listings, CANON); + aggBySku = new Map(allAgg.map((x) => [String(x.sku || ""), x])); + + URL_BY_SKU_STORE = buildUrlMap(listings, CANON); + PRICE_BY_SKU_STORE = buildPriceMap(listings, CANON); + + // Store rows + STORE_ROW_BY_SKU = pickBestStoreRowForSku(listings, CANON, STORE_LABEL); + + // Recent (7 days) + const recentItems = Array.isArray(recent?.items) ? recent.items : []; + RECENT_BY_SKU_STORE = buildRecentBySkuStore(recentItems, CANON, 7); + + // Build store item objects + storeItems = []; + exclusives = []; + lastStock = []; + + for (const [sku, best] of STORE_ROW_BY_SKU.entries()) { + const r = best?.r || null; + if (!r) continue; + + const global = aggBySku.get(String(sku || "")) || null; + const globalStoreCount = global?.stores?.size || 0; + + const storePriceStr = String(r.price || "").trim(); + const storePriceNum = parsePriceToNumber(storePriceStr); + + const saleMeta = recentSaleMetaForSkuStore(sku, STORE_LABEL); + + const searchText = String( + [ + r.name || global?.name || "", + r.url || "", + sku, + STORE_LABEL, + ].join(" ") + ).toLowerCase(); + + const it = { + sku: String(sku || ""), + name: r.name || global?.name || "", + img: global?.img || r.img || "", + url: r.url || "", + priceStr: storePriceStr, + priceNum: storePriceNum, + timeMs: tsValue(r), + searchText, + saleMeta, + isExclusive: false, + isLastStock: false, + globalStoreCount, + }; + + // Determine exclusives / last stock + const EPS = 0.01; + const bestGlobalNum = Number.isFinite(global?.cheapestPriceNum) ? global.cheapestPriceNum : null; + const storeIsCheapest = + Number.isFinite(storePriceNum) && Number.isFinite(bestGlobalNum) + ? Math.abs(storePriceNum - bestGlobalNum) <= EPS + : false; + + if (globalStoreCount <= 1) { + it.isLastStock = true; + lastStock.push(it); + } else if (storeIsCheapest) { + it.isExclusive = true; + exclusives.push(it); + } + + storeItems.push(it); + } + + // Default sort for store catalog: by name + storeItems.sort((a, b) => String(a.name || "").localeCompare(String(b.name || ""))); + indexReady = true; + $q.focus(); + rebuildDerivedLists(); + }) + .catch((e) => { + const msg = `Failed to load: ${esc(e?.message || String(e))}`; + $storeResults.innerHTML = `
${msg}
`; + $exclusiveResults.innerHTML = `
${msg}
`; + $lastStockResults.innerHTML = `
${msg}
`; + }); $clearSearch.addEventListener("click", () => { - let changed = false; - if ($q.value) { $q.value = ""; - localStorage.setItem(LS_KEY, ""); - changed = true; + saveQuery(""); + rebuildDerivedLists(); } - - // reset max price too (only if slider is active) - if (pageMax !== null) { - selectedMaxPrice = clampAndRound(boundMax); - localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); - setSliderFromPrice(selectedMaxPrice); - updateMaxPriceLabel(); - changed = true; - } - - if (changed) applyFilter(); $q.focus(); }); - $exSort.addEventListener("change", () => { - localStorage.setItem(LS_EX_SORT, String($exSort.value || "")); - applyFilter(); + let t = null; + $q.addEventListener("input", () => { + saveQuery($q.value); + if (t) clearTimeout(t); + t = setTimeout(() => { + if (!indexReady) return; + rebuildDerivedLists(); + }, 50); }); - $cmpMode.addEventListener("change", () => { - localStorage.setItem(LS_CMP_MODE, String($cmpMode.value || "")); - applyFilter(); - }); - - let tp = null; - function setSelectedMaxPriceFromSlider() { - const raw = getRawPriceFromSlider(); - const rounded = clampAndRound(raw); - if (Math.abs(rounded - selectedMaxPrice) > 0.001) { - selectedMaxPrice = rounded; - localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); - updateMaxPriceLabel(); - } else { - updateMaxPriceLabel(); - } - } - - $maxPrice.addEventListener("input", () => { - if (pageMax === null) return; - setSelectedMaxPriceFromSlider(); - - if (tp) clearTimeout(tp); - tp = setTimeout(applyFilter, 40); - }); - - $maxPrice.addEventListener("change", () => { - if (pageMax === null) return; - setSelectedMaxPriceFromSlider(); - setSliderFromPrice(selectedMaxPrice); - updateMaxPriceLabel(); - applyFilter(); + $rightSort.addEventListener("change", () => { + sessionStorage.setItem(SORT_KEY, String($rightSort.value || "time")); + if (!indexReady) return; + rebuildDerivedLists(); }); } From 09a3a07c8062a4a741b54b723d1e35e109542ff4 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:24:52 -0800 Subject: [PATCH 08/19] UX Improvements --- viz/app/store_page.js | 1320 +++++++++++++++++++++-------------------- 1 file changed, 667 insertions(+), 653 deletions(-) diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 4cb6456..481217c 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -1,4 +1,4 @@ -import { esc, renderThumbHtml, prettyTs } from "./dom.js"; +import { esc, renderThumbHtml } from "./dom.js"; import { tokenizeQuery, matchesAllTokens, @@ -6,464 +6,536 @@ import { keySkuForRow, parsePriceToNumber, } from "./sku.js"; -import { loadIndex, loadRecent, saveQuery, loadSavedQuery } from "./state.js"; +import { loadIndex } from "./state.js"; import { aggregateBySku } from "./catalog.js"; import { loadSkuRules } from "./mapping.js"; -export function renderStore($app, storeLabelParam) { +function normStoreLabel(s) { + return String(s || "").trim().toLowerCase(); +} + +function abbrevStoreLabel(s) { + const t = String(s || "").trim(); + if (!t) return ""; + return t.split(/\s+/)[0] || t; +} + +function readLinkHrefForSkuInStore(listingsLive, canonSku, storeLabelNorm) { + // Prefer the most recent-ish url if multiple exist; stable enough for viz. + let bestUrl = ""; + let bestScore = -1; + + function scoreUrl(u) { + if (!u) return -1; + let s = u.length; + if (/\bproduct\/\d+\//.test(u)) s += 50; + if (/[a-z0-9-]{8,}/i.test(u)) s += 10; + return s; + } + + for (const r of listingsLive) { + if (!r || r.removed) continue; + const store = normStoreLabel(r.storeLabel || r.store || ""); + if (store !== storeLabelNorm) continue; + + const skuKey = String( + rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r) + ); + if (skuKey !== canonSku) continue; + + const u = String(r.url || "").trim(); + const sc = scoreUrl(u); + if (sc > bestScore) { + bestScore = sc; + bestUrl = u; + } else if (sc === bestScore && u && bestUrl && u < bestUrl) { + bestUrl = u; + } + } + return bestUrl; +} + +// small module-level cache so we can reuse in readLinkHrefForSkuInStore +let rulesCache = null; + +export async function renderStore($app, storeLabelRaw) { + const storeLabel = String(storeLabelRaw || "").trim(); + const storeLabelShort = + abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store"); + $app.innerHTML = `
-
-
-
-

Store

-
Loading…
-
- - -
+
+ + ${esc(storeLabelShort || "Store")}
-
- -
-
- +
+
+
+
Max price
+ + + +
+
+ +
+
-
- -
-
-
Exclusives / Last stock
- +
+ +
+
+
+
+ Exclusive + and + Last Stock +
+
+ Sort + +
+
+
-
Exclusives
-
- -
Last stock
-
+
+
+ Price compare +
+ Comparison + +
+
+
+
+ +
`; + document.getElementById("back").addEventListener("click", () => { + sessionStorage.setItem("viz:lastRoute", location.hash); + location.hash = "#/"; + }); + const $q = document.getElementById("q"); - const $storeSub = document.getElementById("storeSub"); - const $storeResults = document.getElementById("storeResults"); - const $exclusiveResults = document.getElementById("exclusiveResults"); - const $lastStockResults = document.getElementById("lastStockResults"); + const $status = document.getElementById("status"); + const $resultsExclusive = document.getElementById("resultsExclusive"); + const $resultsCompare = document.getElementById("resultsCompare"); + const $sentinel = document.getElementById("sentinel"); + const $resultsWrap = document.getElementById("results"); + + const $maxPrice = document.getElementById("maxPrice"); + const $maxPriceLabel = document.getElementById("maxPriceLabel"); + const $priceWrap = document.getElementById("priceWrap"); + const $clearSearch = document.getElementById("clearSearch"); - const $rightSort = document.getElementById("rightSort"); + const $exSort = document.getElementById("exSort"); + const $cmpMode = document.getElementById("cmpMode"); - // Keep store-page filter consistent with the app's saved query (optional). - $q.value = loadSavedQuery() || ""; + // Persist query per store + const storeNorm = normStoreLabel(storeLabel); + const LS_KEY = `viz:storeQuery:${storeNorm}`; + const savedQ = String(localStorage.getItem(LS_KEY) || ""); + if (savedQ) $q.value = savedQ; - const SORT_KEY = "viz:storeRightSort"; - $rightSort.value = sessionStorage.getItem(SORT_KEY) || "time"; + // Persist max price per store (clamped later once bounds known) + const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`; + const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE); + let savedMaxPrice = + savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null; + if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null; - let indexReady = false; + // Persist exclusives sort per store + const LS_EX_SORT = `viz:storeExclusiveSort:${storeNorm}`; + const savedExSort = String(localStorage.getItem(LS_EX_SORT) || ""); + if (savedExSort) $exSort.value = savedExSort; - let STORE_LABEL = String(storeLabelParam || "").trim(); - let CANON = (x) => x; + // Persist comparison technique per store + const LS_CMP_MODE = `viz:storeCompareMode:${storeNorm}`; + const savedCmpMode = String(localStorage.getItem(LS_CMP_MODE) || ""); + if (savedCmpMode) $cmpMode.value = savedCmpMode; - // canonicalSku -> agg - let aggBySku = new Map(); - // canonicalSku -> storeLabel -> { priceNum, priceStr, url } - let PRICE_BY_SKU_STORE = new Map(); - // canonicalSku -> storeLabel -> url (for badge href) - let URL_BY_SKU_STORE = new Map(); + $resultsExclusive.innerHTML = `
Loading…
`; + $resultsCompare.innerHTML = ``; - // canonicalSku -> storeLabel -> most recent recent-row (within 7d) - let RECENT_BY_SKU_STORE = new Map(); + const idx = await loadIndex(); + rulesCache = await loadSkuRules(); + const rules = rulesCache; - // For left list: canonicalSku -> best row for this store (cheapest priceNum; tie -> newest) - let STORE_ROW_BY_SKU = new Map(); + const listingsAll = Array.isArray(idx.items) ? idx.items : []; + const liveAll = listingsAll.filter((r) => r && !r.removed); - // Derived right-side lists - let exclusives = []; - let lastStock = []; - let storeItems = []; // left list items - - function normStoreLabel(s) { - return String(s || "").trim().toLowerCase(); - } - - function resolveStoreLabel(listings, wanted) { - const w = normStoreLabel(wanted); - if (!w) return ""; - for (const r of Array.isArray(listings) ? listings : []) { - const lab = String(r?.storeLabel || r?.store || "").trim(); - if (lab && normStoreLabel(lab) === w) return lab; - } - return wanted; - } - - function tsValue(r) { - const t = String(r?.ts || ""); + function dateMsFromRow(r) { + const t = String(r?.firstSeenAt || ""); const ms = t ? Date.parse(t) : NaN; - if (Number.isFinite(ms)) return ms; - const d = String(r?.date || ""); - const ms2 = d ? Date.parse(d) : NaN; - return Number.isFinite(ms2) ? ms2 : 0; + return Number.isFinite(ms) ? ms : null; } - function eventMs(r) { - const t = String(r?.ts || ""); - const ms = t ? Date.parse(t) : NaN; - if (Number.isFinite(ms)) return ms; + // Build earliest "first in DB (for this store)" timestamp per canonical SKU (includes removed rows) + const firstSeenBySkuInStore = new Map(); // sku -> ms + for (const r of listingsAll) { + if (!r) continue; + const store = normStoreLabel(r.storeLabel || r.store || ""); + if (store !== storeNorm) continue; - const d = String(r?.date || ""); - const ms2 = d ? Date.parse(d + "T00:00:00Z") : NaN; - return Number.isFinite(ms2) ? ms2 : 0; + const skuKey = keySkuForRow(r); + const sku = String(rules.canonicalSku(skuKey) || skuKey); + + const ms = dateMsFromRow(r); + if (ms === null) continue; + + const prev = firstSeenBySkuInStore.get(sku); + if (prev === undefined || ms < prev) firstSeenBySkuInStore.set(sku, ms); } - function buildUrlMap(listings, canonicalSkuFn) { - const out = new Map(); - for (const r of Array.isArray(listings) ? listings : []) { - if (!r || r.removed) continue; + // Build "ever seen" store presence per canonical SKU (includes removed rows) + const everStoresBySku = new Map(); // sku -> Set(storeLabelNorm) + for (const r of listingsAll) { + if (!r) continue; + const store = normStoreLabel(r.storeLabel || r.store || ""); + if (!store) continue; - const skuKey = String(keySkuForRow(r) || "").trim(); - if (!skuKey) continue; + const skuKey = keySkuForRow(r); + const sku = String(rules.canonicalSku(skuKey) || skuKey); - const sku = String(canonicalSkuFn ? canonicalSkuFn(skuKey) : skuKey).trim(); - if (!sku) continue; + let ss = everStoresBySku.get(sku); + if (!ss) everStoresBySku.set(sku, (ss = new Set())); + ss.add(store); + } - const storeLabel = String(r.storeLabel || r.store || "").trim(); - const url = String(r.url || "").trim(); - if (!storeLabel || !url) continue; + // Build global per-canonical-SKU live store presence + min prices + const storesBySku = new Map(); // sku -> Set(storeLabelNorm) + const minPriceBySkuStore = new Map(); // sku -> Map(storeLabelNorm -> minPrice) - let m = out.get(sku); - if (!m) out.set(sku, (m = new Map())); - if (!m.has(storeLabel)) m.set(storeLabel, url); + for (const r of liveAll) { + const store = normStoreLabel(r.storeLabel || r.store || ""); + if (!store) continue; + + const skuKey = keySkuForRow(r); + const sku = String(rules.canonicalSku(skuKey) || skuKey); + + let ss = storesBySku.get(sku); + if (!ss) storesBySku.set(sku, (ss = new Set())); + ss.add(store); + + const p = parsePriceToNumber(r.price); + if (p !== null) { + let m = minPriceBySkuStore.get(sku); + if (!m) minPriceBySkuStore.set(sku, (m = new Map())); + const prev = m.get(store); + if (prev === undefined || p < prev) m.set(store, p); } - return out; } - function urlForSkuStore(sku, storeLabel) { - const s = String(sku || ""); - const lab = String(storeLabel || ""); - return URL_BY_SKU_STORE.get(s)?.get(lab) || ""; + function bestAllPrice(sku) { + const m = minPriceBySkuStore.get(sku); + if (!m) return null; + let best = null; + for (const v of m.values()) best = best === null ? v : Math.min(best, v); + return best; } - function buildPriceMap(listings, canonicalSkuFn) { - const out = new Map(); - for (const r of Array.isArray(listings) ? listings : []) { - if (!r || r.removed) continue; - - const rawSku = String(keySkuForRow(r) || "").trim(); - if (!rawSku) continue; - - const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); - if (!sku) continue; - - const storeLabel = String(r.storeLabel || r.store || "").trim(); - if (!storeLabel) continue; - - const priceStr = String(r.price || "").trim(); - const priceNum = parsePriceToNumber(priceStr); - if (!Number.isFinite(priceNum)) continue; - - const url = String(r.url || "").trim(); - - let m = out.get(sku); - if (!m) out.set(sku, (m = new Map())); - - const prev = m.get(storeLabel); - if (!prev || priceNum < prev.priceNum) { - m.set(storeLabel, { priceNum, priceStr, url }); - } + function bestOtherPrice(sku, store) { + const m = minPriceBySkuStore.get(sku); + if (!m) return null; + let best = null; + for (const [k, v] of m.entries()) { + if (k === store) continue; + best = best === null ? v : Math.min(best, v); } - return out; + return best; } - function pickBestStoreRowForSku(listings, canonicalSkuFn, storeLabel) { - const out = new Map(); - const want = normStoreLabel(storeLabel); + // Store-specific live rows only (in-stock for that store) + const rowsStoreLive = liveAll.filter( + (r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm + ); - for (const r of Array.isArray(listings) ? listings : []) { - if (!r || r.removed) continue; + // Aggregate in this store, grouped by canonical SKU (so mappings count as same bottle) + let items = aggregateBySku(rowsStoreLive, rules.canonicalSku); - const lab = String(r.storeLabel || r.store || "").trim(); - if (!lab || normStoreLabel(lab) !== want) continue; + // Decorate each item with pricing comparisons + exclusivity + const EPS = 0.01; - const rawSku = String(keySkuForRow(r) || "").trim(); - if (!rawSku) continue; + items = items.map((it) => { + const sku = String(it.sku || ""); + const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]); + const everStoreSet = everStoresBySku.get(sku) || liveStoreSet; - const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); - if (!sku) continue; + const soloLiveHere = + liveStoreSet.size === 1 && liveStoreSet.has(storeNorm); + const lastStock = soloLiveHere && everStoreSet.size > 1; + const exclusive = soloLiveHere && !lastStock; - const priceStr = String(r.price || "").trim(); - const priceNum = parsePriceToNumber(priceStr); - const ms = tsValue(r); + const storePrice = Number.isFinite(it.cheapestPriceNum) + ? it.cheapestPriceNum + : null; + const bestAll = bestAllPrice(sku); + const other = bestOtherPrice(sku, storeNorm); - const prev = out.get(sku); - if (!prev) { - out.set(sku, { r, priceNum, ms }); - continue; - } + const isBest = + storePrice !== null && bestAll !== null + ? storePrice <= bestAll + EPS + : false; - const prevPrice = prev.priceNum; - const prevMs = prev.ms; + const diffVsOtherDollar = + storePrice !== null && other !== null ? storePrice - other : null; + const diffVsOtherPct = + storePrice !== null && other !== null && other > 0 + ? ((storePrice - other) / other) * 100 + : null; - const priceOk = Number.isFinite(priceNum); - const prevOk = Number.isFinite(prevPrice); + const diffVsBestDollar = + storePrice !== null && bestAll !== null ? storePrice - bestAll : null; + const diffVsBestPct = + storePrice !== null && bestAll !== null && bestAll > 0 + ? ((storePrice - bestAll) / bestAll) * 100 + : null; - if (priceOk && !prevOk) out.set(sku, { r, priceNum, ms }); - else if (priceOk && prevOk && priceNum < prevPrice) out.set(sku, { r, priceNum, ms }); - else if ( - (priceOk && prevOk && Math.abs(priceNum - prevPrice) <= 0.01 && ms > prevMs) || - (!priceOk && !prevOk && ms > prevMs) - ) { - out.set(sku, { r, priceNum, ms }); - } - } - - return out; - } - - function buildRecentBySkuStore(recentItems, canonicalSkuFn, days) { - const nowMs = Date.now(); - const cutoffMs = nowMs - days * 24 * 60 * 60 * 1000; - - // sku -> storeLabel -> row - const out = new Map(); - - for (const r of Array.isArray(recentItems) ? recentItems : []) { - const ms = eventMs(r); - if (!(ms >= cutoffMs && ms <= nowMs)) continue; - - const rawSku = String(r?.sku || "").trim(); - if (!rawSku) continue; - - const sku = String(canonicalSkuFn ? canonicalSkuFn(rawSku) : rawSku).trim(); - if (!sku) continue; - - const storeLabel = String(r?.storeLabel || r?.store || "").trim(); - if (!storeLabel) continue; - - let storeMap = out.get(sku); - if (!storeMap) out.set(sku, (storeMap = new Map())); - - const prev = storeMap.get(storeLabel); - if (!prev || eventMs(prev) < ms) storeMap.set(storeLabel, r); - } - - return out; - } - - function normalizeKindForPrice(r) { - let kind = String(r?.kind || ""); - if (kind === "price_change") { - const o = parsePriceToNumber(r?.oldPrice || ""); - const n = parsePriceToNumber(r?.newPrice || ""); - if (Number.isFinite(o) && Number.isFinite(n)) { - if (n < o) kind = "price_down"; - else if (n > o) kind = "price_up"; - else kind = "price_change"; - } - } - return kind; - } - - function salePctOff(oldRaw, newRaw) { - const oldN = parsePriceToNumber(oldRaw); - const newN = parsePriceToNumber(newRaw); - if (!Number.isFinite(oldN) || !Number.isFinite(newN)) return null; - if (!(oldN > 0)) return null; - if (!(newN < oldN)) return null; - const pct = Math.round(((oldN - newN) / oldN) * 100); - return Number.isFinite(pct) && pct > 0 ? pct : null; - } - - function pctChange(oldRaw, newRaw) { - const oldN = parsePriceToNumber(oldRaw); - const newN = parsePriceToNumber(newRaw); - if (!Number.isFinite(oldN) || !Number.isFinite(newN)) return null; - if (!(oldN > 0)) return null; - const pct = Math.round(((newN - oldN) / oldN) * 100); - return Number.isFinite(pct) ? pct : null; - } - - function fmtUsd(n) { - const abs = Math.abs(Number(n) || 0); - if (!Number.isFinite(abs)) return ""; - const s = abs.toFixed(2); - return s.endsWith(".00") ? s.slice(0, -3) : s; - } - - function recentSaleMetaForSkuStore(sku, storeLabel) { - const r = RECENT_BY_SKU_STORE.get(String(sku || ""))?.get(String(storeLabel || "")); - if (!r) return null; - - const kind = normalizeKindForPrice(r); - if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") return null; - - const oldStr = String(r?.oldPrice || "").trim(); - const newStr = String(r?.newPrice || "").trim(); - const oldN = parsePriceToNumber(oldStr); - const newN = parsePriceToNumber(newStr); - - if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) return null; - - // signedPct: down => negative; up => positive; unchanged => 0 - let signedPct = 0; - let signedDelta = newN - oldN; // down => negative - - if (newN < oldN) { - const off = salePctOff(oldStr, newStr); - signedPct = off !== null ? -off : Math.round(((newN - oldN) / oldN) * 100); - } else if (newN > oldN) { - const up = pctChange(oldStr, newStr); - signedPct = up !== null ? up : Math.round(((newN - oldN) / oldN) * 100); - } else { - signedPct = 0; - signedDelta = 0; - } - - const when = r.ts ? prettyTs(r.ts) : r.date || ""; + const firstSeenMs = firstSeenBySkuInStore.get(sku); + const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null; return { - r, - kind, - oldStr, - newStr, - oldN, - newN, - signedPct, - signedDelta, - when, + ...it, + _exclusive: exclusive, + _lastStock: lastStock, + _storePrice: storePrice, + _bestAll: bestAll, + _bestOther: other, + _isBest: isBest, + _diffVsOtherDollar: diffVsOtherDollar, + _diffVsOtherPct: diffVsOtherPct, + _diffVsBestDollar: diffVsBestDollar, + _diffVsBestPct: diffVsBestPct, + _firstSeenMs: firstSeen, }; + }); + + // ---- Max price slider (exponential mapping + clicky rounding) ---- + const MIN_PRICE = 25; + + function maxStorePriceOnPage() { + let mx = null; + for (const it of items) { + const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; + if (p === null) continue; + mx = mx === null ? p : Math.max(mx, p); + } + return mx; } - function pctOffVsNextBest(sku, storeLabel) { - const m = PRICE_BY_SKU_STORE.get(String(sku || "")); - if (!m) return null; + const pageMax = maxStorePriceOnPage(); + const boundMax = pageMax !== null ? Math.max(MIN_PRICE, pageMax) : MIN_PRICE; - const here = m.get(String(storeLabel || "")); - if (!here || !Number.isFinite(here.priceNum)) return null; + function stepForPrice(p) { + const x = Number.isFinite(p) ? p : boundMax; + if (x < 120) return 5; + if (x < 250) return 10; + if (x < 600) return 25; + return 100; + } + function roundToStep(p) { + const step = stepForPrice(p); + return Math.round(p / step) * step; + } - const prices = []; - for (const v of m.values()) { - if (Number.isFinite(v?.priceNum)) prices.push(v.priceNum); + function priceFromT(t) { + t = Math.max(0, Math.min(1, t)); + if (boundMax <= MIN_PRICE) return MIN_PRICE; + const ratio = boundMax / MIN_PRICE; + return MIN_PRICE * Math.exp(Math.log(ratio) * t); + } + function tFromPrice(price) { + if (!Number.isFinite(price)) return 1; + if (boundMax <= MIN_PRICE) return 1; + const p = Math.max(MIN_PRICE, Math.min(boundMax, price)); + const ratio = boundMax / MIN_PRICE; + return Math.log(p / MIN_PRICE) / Math.log(ratio); + } + + function clampPrice(p) { + if (!Number.isFinite(p)) return boundMax; + return Math.max(MIN_PRICE, Math.min(boundMax, p)); + } + + function clampAndRound(p) { + const c = clampPrice(p); + const r = roundToStep(c); + return clampPrice(r); + } + + function formatDollars(p) { + if (!Number.isFinite(p)) return ""; + return `$${Math.round(p)}`; + } + + let selectedMaxPrice = clampAndRound( + savedMaxPrice !== null ? savedMaxPrice : boundMax + ); + + function setSliderFromPrice(p) { + const t = tFromPrice(p); + const v = Math.round(t * 1000); + $maxPrice.value = String(v); + } + + function getRawPriceFromSlider() { + const v = Number($maxPrice.value); + const t = Number.isFinite(v) ? v / 1000 : 1; + return clampPrice(priceFromT(t)); + } + + function updateMaxPriceLabel() { + if (pageMax === null) { + $maxPriceLabel.textContent = "No prices"; + return; } - prices.sort((a, b) => a - b); + $maxPriceLabel.textContent = `${formatDollars(selectedMaxPrice)}`; + } - if (!prices.length) return null; + if (pageMax === null) { + $maxPrice.disabled = true; + $priceWrap.title = "No priced items in this store."; + selectedMaxPrice = boundMax; + setSliderFromPrice(boundMax); + localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); + updateMaxPriceLabel(); + } else { + selectedMaxPrice = clampAndRound(selectedMaxPrice); + localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); + setSliderFromPrice(selectedMaxPrice); + updateMaxPriceLabel(); + } - const EPS = 0.01; - const min = prices[0]; - if (Math.abs(here.priceNum - min) > EPS) return null; + // ---- Listing display price: keep cents (no rounding) ---- + function listingPriceStr(it) { + const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; + if (p === null) + return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)"; + return `$${p.toFixed(2)}`; + } - // find second distinct - let second = null; - for (let i = 1; i < prices.length; i++) { - if (Math.abs(prices[i] - min) > EPS) { - second = prices[i]; - break; + function compareMode() { + return $cmpMode && $cmpMode.value === "percent" ? "percent" : "dollar"; + } + + function priceBadgeHtml(it) { + if (it._exclusive || it._lastStock) return ""; + + const mode = compareMode(); + + if (mode === "percent") { + const d = it._diffVsOtherPct; + if (d === null || !Number.isFinite(d)) return ""; + const abs = Math.abs(d); + if (abs <= 5) { + return `within 5%`; } + const pct = Math.round(abs); + if (d < 0) return `${esc(pct)}% lower`; + return `${esc(pct)}% higher`; } - if (!Number.isFinite(second) || !(second > 0) || !(min < second)) return null; - const pct = Math.round(((second - min) / second) * 100); - return Number.isFinite(pct) && pct > 0 ? pct : null; + const d = it._diffVsOtherDollar; + if (d === null || !Number.isFinite(d)) return ""; + + const abs = Math.abs(d); + if (abs <= 5) { + return `within $5`; + } + + const dollars = Math.round(abs); + if (d < 0) { + return `$${esc(dollars)} lower`; + } + return `$${esc(dollars)} higher`; } - function badgeHtmlForSale(sortMode, saleMeta) { - if (!saleMeta) return ""; - if (sortMode === "sale_pct") { - const v = saleMeta.signedPct; - if (!Number.isFinite(v) || v === 0) return ""; - const isDown = v < 0; - const abs = Math.abs(v); - const style = isDown - ? ` style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);"` - : ` style="margin-left:6px; color:rgba(170,40,40,0.95); background:rgba(170,40,40,0.10); border:1px solid rgba(170,40,40,0.20);"`; - const txt = isDown ? `[${abs}% Off]` : `[+${abs}%]`; - return `${esc(txt)}`; - } + function renderCard(it) { + const price = listingPriceStr(it); + const href = String(it.sampleUrl || "").trim(); - if (sortMode === "sale_abs") { - const d = saleMeta.signedDelta; - if (!Number.isFinite(d) || d === 0) return ""; - const isDown = d < 0; - const abs = fmtUsd(d); - if (!abs) return ""; - const style = isDown - ? ` style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);"` - : ` style="margin-left:6px; color:rgba(170,40,40,0.95); background:rgba(170,40,40,0.10); border:1px solid rgba(170,40,40,0.20);"`; - const txt = isDown ? `[$${abs} Off]` : `[+$${abs}]`; - return `${esc(txt)}`; - } + const specialBadge = it._lastStock + ? `Last Stock` + : it._exclusive + ? `Exclusive` + : ""; - return ""; - } + const bestBadge = + !it._exclusive && !it._lastStock && it._isBest + ? `Best Price` + : ""; - function badgeHtmlForExclusivePctOff(sku) { - const pct = pctOffVsNextBest(sku, STORE_LABEL); - if (!Number.isFinite(pct) || pct <= 0) return ""; - return `[${esc( - pct - )}% Off]`; - } - - function itemCardHtml(it, { annotateMode }) { - // annotateMode: "sale_pct" | "sale_abs" | "default" - const sku = String(it?.sku || ""); - const name = String(it?.name || "(no name)"); - const img = String(it?.img || ""); - const priceStr = it.priceStr ? it.priceStr : "(no price)"; - - const href = urlForSkuStore(sku, STORE_LABEL) || String(it?.url || "").trim(); - const storeBadge = href - ? `${esc( - STORE_LABEL - )}` - : `${esc(STORE_LABEL)}`; - - const skuLink = `#/link/?left=${encodeURIComponent(sku)}`; - - let annot = ""; - if (annotateMode === "sale_pct" || annotateMode === "sale_abs") { - annot = badgeHtmlForSale(annotateMode, it.saleMeta); - } else { - // default annotation is % off for exclusives only (and only if >0) - if (it.isExclusive) annot = badgeHtmlForExclusivePctOff(sku); - } + const diffBadge = priceBadgeHtml(it); + const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`; return ` -
+
-
- ${renderThumbHtml(img)} -
+
${renderThumbHtml(it.img)}
- ${esc(priceStr)} - ${annot} - ${storeBadge} + ${specialBadge} + ${bestBadge} + ${diffBadge} + ${esc(price)} + ${ + href + ? `${esc( + storeLabelShort + )}` + : `` + }
@@ -471,300 +543,242 @@ export function renderStore($app, storeLabelParam) { `; } - function renderList($el, items, annotateMode) { - if (!items.length) { - $el.innerHTML = `
No matches.
`; + // ---- Infinite scroll paging (shared across both columns) ---- + const PAGE_SIZE = 140; + const PAGE_EACH = Math.max(1, Math.floor(PAGE_SIZE / 2)); + + let filteredExclusive = []; + let filteredCompare = []; + let shownExclusive = 0; + let shownCompare = 0; + + function totalFiltered() { + return filteredExclusive.length + filteredCompare.length; + } + function totalShown() { + return shownExclusive + shownCompare; + } + + function setStatus() { + const total = totalFiltered(); + if (!total) { + $status.textContent = "No in-stock items for this store."; return; } - const limited = items.slice(0, 80); - $el.innerHTML = limited.map((it) => itemCardHtml(it, { annotateMode })).join(""); - - for (const el of Array.from($el.querySelectorAll(".item"))) { - el.addEventListener("click", () => { - const sku = el.getAttribute("data-sku") || ""; - if (!sku) return; - saveQuery($q.value); - sessionStorage.setItem("viz:lastRoute", location.hash); - location.hash = `#/item/${encodeURIComponent(sku)}`; - }); - } - } - - function renderStoreCatalog(items) { - if (!items.length) { - $storeResults.innerHTML = `
No matches.
`; + if (pageMax !== null) { + $status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars( + selectedMaxPrice + )}).`; return; } - const limited = items.slice(0, 160); - $storeResults.innerHTML = limited - .map((it) => { - const sku = String(it?.sku || ""); - const href = urlForSkuStore(sku, STORE_LABEL) || String(it?.url || "").trim(); - const storeBadge = href - ? `${esc( - STORE_LABEL - )}` - : `${esc(STORE_LABEL)}`; + $status.textContent = `In stock: ${total} item(s).`; + } - const skuLink = `#/link/?left=${encodeURIComponent(sku)}`; + function renderNext(reset) { + if (reset) { + $resultsExclusive.innerHTML = ""; + $resultsCompare.innerHTML = ""; + shownExclusive = 0; + shownCompare = 0; + } - return ` -
-
-
- ${renderThumbHtml(it.img)} -
-
-
-
${esc(it.name || "(no name)")}
- ${esc( - displaySku(sku) - )} -
-
- ${esc(it.priceStr || "(no price)")} - ${storeBadge} -
-
-
-
- `; - }) - .join(""); + const sliceEx = filteredExclusive.slice( + shownExclusive, + shownExclusive + PAGE_EACH + ); + const sliceCo = filteredCompare.slice( + shownCompare, + shownCompare + PAGE_EACH + ); - for (const el of Array.from($storeResults.querySelectorAll(".item"))) { - el.addEventListener("click", () => { - const sku = el.getAttribute("data-sku") || ""; - if (!sku) return; - saveQuery($q.value); - sessionStorage.setItem("viz:lastRoute", location.hash); - location.hash = `#/item/${encodeURIComponent(sku)}`; - }); + shownExclusive += sliceEx.length; + shownCompare += sliceCo.length; + + if (sliceEx.length) + $resultsExclusive.insertAdjacentHTML( + "beforeend", + sliceEx.map(renderCard).join("") + ); + if (sliceCo.length) + $resultsCompare.insertAdjacentHTML( + "beforeend", + sliceCo.map(renderCard).join("") + ); + + const total = totalFiltered(); + const shown = totalShown(); + + if (!total) { + $sentinel.textContent = ""; + } else if (shown >= total) { + $sentinel.textContent = `Showing ${shown} / ${total}`; + } else { + $sentinel.textContent = `Showing ${shown} / ${total}…`; } } - function sortRightList(items, mode) { - const m = String(mode || "time"); - - const getTime = (it) => Number(it?.timeMs || 0); - const getPrice = (it) => (Number.isFinite(it?.priceNum) ? it.priceNum : Number.POSITIVE_INFINITY); - - const getSignedPct = (it) => { - const v = it?.saleMeta?.signedPct; - return Number.isFinite(v) ? v : 0; // unchanged/no-event => 0 (middle) - }; - - const getSignedDelta = (it) => { - const v = it?.saleMeta?.signedDelta; - return Number.isFinite(v) ? v : 0; // unchanged/no-event => 0 (middle) - }; - - const out = items.slice(); - - if (m === "sale_pct") { - out.sort((a, b) => { - // deals (negative) first; unchanged (0) middle; increases (+) last - const av = getSignedPct(a); - const bv = getSignedPct(b); - if (av !== bv) return av - bv; - const ap = getPrice(a); - const bp = getPrice(b); - if (ap !== bp) return ap - bp; - return String(a.sku || "").localeCompare(String(b.sku || "")); - }); - return out; - } - - if (m === "sale_abs") { - out.sort((a, b) => { - const av = getSignedDelta(a); - const bv = getSignedDelta(b); - if (av !== bv) return av - bv; // negative (down) first, positive (up) last - const ap = getPrice(a); - const bp = getPrice(b); - if (ap !== bp) return ap - bp; - return String(a.sku || "").localeCompare(String(b.sku || "")); - }); - return out; - } - - if (m === "price") { - out.sort((a, b) => { - const ap = getPrice(a); - const bp = getPrice(b); - if (ap !== bp) return ap - bp; - return String(a.sku || "").localeCompare(String(b.sku || "")); - }); - return out; - } - - // time (default): newest first - out.sort((a, b) => { - const at = getTime(a); - const bt = getTime(b); - if (bt !== at) return bt - at; - return String(a.sku || "").localeCompare(String(b.sku || "")); - }); - return out; - } - - function rebuildDerivedLists() { - const tokens = tokenizeQuery($q.value); - - const filteredStoreItems = !tokens.length - ? storeItems - : storeItems.filter((it) => matchesAllTokens(it.searchText, tokens)); - - renderStoreCatalog(filteredStoreItems); - - const rightSortMode = String($rightSort.value || "time"); - const annotMode = - rightSortMode === "sale_pct" - ? "sale_pct" - : rightSortMode === "sale_abs" - ? "sale_abs" - : "default"; - - const ex = !tokens.length - ? exclusives - : exclusives.filter((it) => matchesAllTokens(it.searchText, tokens)); - - const ls = !tokens.length - ? lastStock - : lastStock.filter((it) => matchesAllTokens(it.searchText, tokens)); - - renderList($exclusiveResults, sortRightList(ex, rightSortMode), annotMode); - renderList($lastStockResults, sortRightList(ls, rightSortMode), annotMode); - } - - $storeResults.innerHTML = `
Loading…
`; - $exclusiveResults.innerHTML = `
Loading…
`; - $lastStockResults.innerHTML = `
Loading…
`; - - Promise.all([loadIndex(), loadSkuRules(), loadRecent()]) - .then(([idx, rules, recent]) => { - const listings = Array.isArray(idx?.items) ? idx.items : []; - - CANON = typeof rules?.canonicalSku === "function" ? rules.canonicalSku : (x) => x; - - STORE_LABEL = resolveStoreLabel(listings, STORE_LABEL); - $storeSub.textContent = STORE_LABEL ? `Browsing: ${STORE_LABEL}` : `Browsing store`; - - // Global aggregates (for "exclusive" / "last stock" determination) - const allAgg = aggregateBySku(listings, CANON); - aggBySku = new Map(allAgg.map((x) => [String(x.sku || ""), x])); - - URL_BY_SKU_STORE = buildUrlMap(listings, CANON); - PRICE_BY_SKU_STORE = buildPriceMap(listings, CANON); - - // Store rows - STORE_ROW_BY_SKU = pickBestStoreRowForSku(listings, CANON, STORE_LABEL); - - // Recent (7 days) - const recentItems = Array.isArray(recent?.items) ? recent.items : []; - RECENT_BY_SKU_STORE = buildRecentBySkuStore(recentItems, CANON, 7); - - // Build store item objects - storeItems = []; - exclusives = []; - lastStock = []; - - for (const [sku, best] of STORE_ROW_BY_SKU.entries()) { - const r = best?.r || null; - if (!r) continue; - - const global = aggBySku.get(String(sku || "")) || null; - const globalStoreCount = global?.stores?.size || 0; - - const storePriceStr = String(r.price || "").trim(); - const storePriceNum = parsePriceToNumber(storePriceStr); - - const saleMeta = recentSaleMetaForSkuStore(sku, STORE_LABEL); - - const searchText = String( - [ - r.name || global?.name || "", - r.url || "", - sku, - STORE_LABEL, - ].join(" ") - ).toLowerCase(); - - const it = { - sku: String(sku || ""), - name: r.name || global?.name || "", - img: global?.img || r.img || "", - url: r.url || "", - priceStr: storePriceStr, - priceNum: storePriceNum, - timeMs: tsValue(r), - searchText, - saleMeta, - isExclusive: false, - isLastStock: false, - globalStoreCount, - }; - - // Determine exclusives / last stock - const EPS = 0.01; - const bestGlobalNum = Number.isFinite(global?.cheapestPriceNum) ? global.cheapestPriceNum : null; - const storeIsCheapest = - Number.isFinite(storePriceNum) && Number.isFinite(bestGlobalNum) - ? Math.abs(storePriceNum - bestGlobalNum) <= EPS - : false; - - if (globalStoreCount <= 1) { - it.isLastStock = true; - lastStock.push(it); - } else if (storeIsCheapest) { - it.isExclusive = true; - exclusives.push(it); - } - - storeItems.push(it); - } - - // Default sort for store catalog: by name - storeItems.sort((a, b) => String(a.name || "").localeCompare(String(b.name || ""))); - indexReady = true; - $q.focus(); - rebuildDerivedLists(); - }) - .catch((e) => { - const msg = `Failed to load: ${esc(e?.message || String(e))}`; - $storeResults.innerHTML = `
${msg}
`; - $exclusiveResults.innerHTML = `
${msg}
`; - $lastStockResults.innerHTML = `
${msg}
`; - }); - - $clearSearch.addEventListener("click", () => { - if ($q.value) { - $q.value = ""; - saveQuery(""); - rebuildDerivedLists(); - } - $q.focus(); + $resultsWrap.addEventListener("click", (e) => { + const el = e.target.closest(".item"); + if (!el) return; + const sku = el.getAttribute("data-sku") || ""; + if (!sku) return; + sessionStorage.setItem("viz:lastRoute", location.hash); + location.hash = `#/item/${encodeURIComponent(sku)}`; }); + function sortExclusiveInPlace(arr) { + const mode = String($exSort.value || "priceDesc"); + if (mode === "priceAsc" || mode === "priceDesc") { + arr.sort((a, b) => { + const ap = Number.isFinite(a._storePrice) ? a._storePrice : null; + const bp = Number.isFinite(b._storePrice) ? b._storePrice : null; + const aKey = + ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; + const bKey = + bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; + if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey; + return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); + }); + return; + } + + if (mode === "dateAsc" || mode === "dateDesc") { + arr.sort((a, b) => { + const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null; + const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null; + const aKey = + ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; + const bKey = + bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; + if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey; + return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); + }); + } + } + + function sortCompareInPlace(arr) { + const mode = compareMode(); + arr.sort((a, b) => { + const da = mode === "percent" ? a._diffVsOtherPct : a._diffVsOtherDollar; + const db = mode === "percent" ? b._diffVsOtherPct : b._diffVsOtherDollar; + + const sa = da === null || !Number.isFinite(da) ? 999999 : da; + const sb = db === null || !Number.isFinite(db) ? 999999 : db; + if (sa !== sb) return sa - sb; + + return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); + }); + } + + function applyFilter() { + const raw = String($q.value || ""); + localStorage.setItem(LS_KEY, raw); + + const tokens = tokenizeQuery(raw); + + let base = items; + + if (tokens.length) { + base = base.filter((it) => matchesAllTokens(it.searchText, tokens)); + } + + if (pageMax !== null && Number.isFinite(selectedMaxPrice)) { + const cap = selectedMaxPrice + 0.0001; + base = base.filter((it) => { + const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null; + return p === null ? true : p <= cap; + }); + } + + filteredExclusive = base.filter((it) => it._exclusive || it._lastStock); + filteredCompare = base.filter((it) => !it._exclusive && !it._lastStock); + + sortExclusiveInPlace(filteredExclusive); + sortCompareInPlace(filteredCompare); + + setStatus(); + renderNext(true); + } + + applyFilter(); + + const io = new IntersectionObserver( + (entries) => { + const hit = entries.some((x) => x.isIntersecting); + if (!hit) return; + if (totalShown() >= totalFiltered()) return; + renderNext(false); + }, + { root: null, rootMargin: "600px 0px", threshold: 0.01 } + ); + io.observe($sentinel); + let t = null; $q.addEventListener("input", () => { - saveQuery($q.value); if (t) clearTimeout(t); - t = setTimeout(() => { - if (!indexReady) return; - rebuildDerivedLists(); - }, 50); + t = setTimeout(applyFilter, 60); }); - $rightSort.addEventListener("change", () => { - sessionStorage.setItem(SORT_KEY, String($rightSort.value || "time")); - if (!indexReady) return; - rebuildDerivedLists(); + $clearSearch.addEventListener("click", () => { + let changed = false; + + if ($q.value) { + $q.value = ""; + localStorage.setItem(LS_KEY, ""); + changed = true; + } + + // reset max price too (only if slider is active) + if (pageMax !== null) { + selectedMaxPrice = clampAndRound(boundMax); + localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); + setSliderFromPrice(selectedMaxPrice); + updateMaxPriceLabel(); + changed = true; + } + + if (changed) applyFilter(); + $q.focus(); + }); + + $exSort.addEventListener("change", () => { + localStorage.setItem(LS_EX_SORT, String($exSort.value || "")); + applyFilter(); + }); + + $cmpMode.addEventListener("change", () => { + localStorage.setItem(LS_CMP_MODE, String($cmpMode.value || "")); + applyFilter(); + }); + + let tp = null; + function setSelectedMaxPriceFromSlider() { + const raw = getRawPriceFromSlider(); + const rounded = clampAndRound(raw); + if (Math.abs(rounded - selectedMaxPrice) > 0.001) { + selectedMaxPrice = rounded; + localStorage.setItem(LS_MAX_PRICE, String(selectedMaxPrice)); + updateMaxPriceLabel(); + } else { + updateMaxPriceLabel(); + } + } + + $maxPrice.addEventListener("input", () => { + if (pageMax === null) return; + setSelectedMaxPriceFromSlider(); + + if (tp) clearTimeout(tp); + tp = setTimeout(applyFilter, 40); + }); + + $maxPrice.addEventListener("change", () => { + if (pageMax === null) return; + setSelectedMaxPriceFromSlider(); + setSliderFromPrice(selectedMaxPrice); + updateMaxPriceLabel(); + applyFilter(); }); } From 947491b21c4a3f73f5fce64bf4126a65fae3fc73 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:30:29 -0800 Subject: [PATCH 09/19] UX Improvements --- viz/app/store_page.js | 157 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 4 deletions(-) diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 481217c..bf23190 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -6,7 +6,7 @@ import { keySkuForRow, parsePriceToNumber, } from "./sku.js"; -import { loadIndex } from "./state.js"; +import { loadIndex, loadRecent } from "./state.js"; import { aggregateBySku } from "./catalog.js"; import { loadSkuRules } from "./mapping.js"; @@ -127,6 +127,8 @@ export async function renderStore($app, storeLabelRaw) { + +
@@ -203,6 +205,84 @@ export async function renderStore($app, storeLabelRaw) { rulesCache = await loadSkuRules(); const rules = rulesCache; + // --- Recent (7d), most-recent per canonicalSku + store --- + const recent = await loadRecent().catch(() => null); + const recentItems = Array.isArray(recent?.items) ? recent.items : []; + + function eventMs(r) { + const t = String(r?.ts || ""); + const ms = t ? Date.parse(t) : NaN; + if (Number.isFinite(ms)) return ms; + + const d = String(r?.date || ""); + const ms2 = d ? Date.parse(d + "T00:00:00Z") : NaN; + return Number.isFinite(ms2) ? ms2 : 0; + } + + const RECENT_DAYS = 7; + const nowMs = Date.now(); + const cutoffMs = nowMs - RECENT_DAYS * 24 * 60 * 60 * 1000; + + // canonicalSku -> storeNorm -> recentRow (latest) + const recentBySkuStore = new Map(); + + for (const r of recentItems) { + const ms = eventMs(r); + if (!(ms >= cutoffMs && ms <= nowMs)) continue; + + const rawSku = String(r?.sku || "").trim(); + if (!rawSku) continue; + const sku = String(rules.canonicalSku(rawSku) || rawSku); + + const stNorm = normStoreLabel(r?.storeLabel || r?.store || ""); + if (!stNorm) continue; + + let sm = recentBySkuStore.get(sku); + if (!sm) recentBySkuStore.set(sku, (sm = new Map())); + + const prev = sm.get(stNorm); + if (!prev || eventMs(prev) < ms) sm.set(stNorm, r); + } + + function normalizeKindForPrice(r) { + let kind = String(r?.kind || ""); + if (kind === "price_change") { + const o = parsePriceToNumber(r?.oldPrice || ""); + const n = parsePriceToNumber(r?.newPrice || ""); + if (Number.isFinite(o) && Number.isFinite(n)) { + if (n < o) kind = "price_down"; + else if (n > o) kind = "price_up"; + else kind = "price_change"; + } + } + return kind; + } + + function saleMetaFor(it) { + const sku = String(it?.sku || ""); + const r = recentBySkuStore.get(sku)?.get(storeNorm) || null; + if (!r) return null; + + const kind = normalizeKindForPrice(r); + if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") + return null; + + const oldStr = String(r?.oldPrice || "").trim(); + const newStr = String(r?.newPrice || "").trim(); + const oldN = parsePriceToNumber(oldStr); + const newN = parsePriceToNumber(newStr); + if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) + return null; + + const delta = newN - oldN; // negative = down + const pct = Math.round(((newN - oldN) / oldN) * 100); // negative = down + + return { + _saleDelta: Number.isFinite(delta) ? delta : 0, + _salePct: Number.isFinite(pct) ? pct : 0, + }; + } + const listingsAll = Array.isArray(idx.items) ? idx.items : []; const liveAll = listingsAll.filter((r) => r && !r.removed); @@ -336,6 +416,8 @@ export async function renderStore($app, storeLabelRaw) { const firstSeenMs = firstSeenBySkuInStore.get(sku); const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null; + const sm = saleMetaFor(it); // { _saleDelta, _salePct } or null + return { ...it, _exclusive: exclusive, @@ -349,6 +431,9 @@ export async function renderStore($app, storeLabelRaw) { _diffVsBestDollar: diffVsBestDollar, _diffVsBestPct: diffVsBestPct, _firstSeenMs: firstSeen, + _saleDelta: sm ? sm._saleDelta : 0, + _salePct: sm ? sm._salePct : 0, + _hasSaleMeta: !!sm, }; }); @@ -492,9 +577,44 @@ export async function renderStore($app, storeLabelRaw) { return `$${esc(dollars)} higher`; } + function exclusiveAnnotHtml(it) { + const mode = String($exSort.value || "priceDesc"); + + // If sorting by sale, annotate with sale change ($ / %). If unchanged, show nothing. + if (mode === "salePct") { + const p = Number.isFinite(it._salePct) ? it._salePct : 0; + if (!p) return ""; + const abs = Math.abs(p); + if (p < 0) return `${esc(abs)}% off`; + return `+${esc(abs)}%`; + } + + if (mode === "saleAbs") { + const d = Number.isFinite(it._saleDelta) ? it._saleDelta : 0; + if (!d) return ""; + const abs = Math.round(Math.abs(d)); + if (!abs) return ""; + if (d < 0) return `$${esc(abs)} off`; + return `+$${esc(abs)}`; + } + + // Otherwise: show % off vs best other store (only when actually cheaper). + const sp = it && Number.isFinite(it._storePrice) ? it._storePrice : null; + const other = it && Number.isFinite(it._bestOther) ? it._bestOther : null; + if (sp === null || other === null || !(other > 0)) return ""; + if (!(sp < other - EPS)) return ""; + + const pct = Math.round(((other - sp) / other) * 100); + if (!Number.isFinite(pct) || pct <= 0) return ""; + return `${esc(pct)}% off`; + } + function renderCard(it) { const price = listingPriceStr(it); - const href = String(it.sampleUrl || "").trim(); + + // Link the store badge consistently (respects SKU linking / canonical SKU) + const storeHref = readLinkHrefForSkuInStore(liveAll, String(it.sku || ""), storeNorm); + const href = storeHref || String(it.sampleUrl || "").trim(); const specialBadge = it._lastStock ? `Last Stock` @@ -508,6 +628,7 @@ export async function renderStore($app, storeLabelRaw) { : ""; const diffBadge = priceBadgeHtml(it); + const exAnnot = it._exclusive || it._lastStock ? exclusiveAnnotHtml(it) : ""; const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`; return ` @@ -526,6 +647,7 @@ export async function renderStore($app, storeLabelRaw) { ${specialBadge} ${bestBadge} ${diffBadge} + ${exAnnot} ${esc(price)} ${ href @@ -630,6 +752,31 @@ export async function renderStore($app, storeLabelRaw) { function sortExclusiveInPlace(arr) { const mode = String($exSort.value || "priceDesc"); + + if (mode === "salePct") { + arr.sort((a, b) => { + const ap = Number.isFinite(a._salePct) ? a._salePct : 0; // negative = better + const bp = Number.isFinite(b._salePct) ? b._salePct : 0; + if (ap !== bp) return ap - bp; // best deal first + const an = (String(a.name) + a.sku).toLowerCase(); + const bn = (String(b.name) + b.sku).toLowerCase(); + return an.localeCompare(bn); + }); + return; + } + + if (mode === "saleAbs") { + arr.sort((a, b) => { + const ad = Number.isFinite(a._saleDelta) ? a._saleDelta : 0; // negative = better + const bd = Number.isFinite(b._saleDelta) ? b._saleDelta : 0; + if (ad !== bd) return ad - bd; // best deal first + const an = (String(a.name) + a.sku).toLowerCase(); + const bn = (String(b.name) + b.sku).toLowerCase(); + return an.localeCompare(bn); + }); + return; + } + if (mode === "priceAsc" || mode === "priceDesc") { arr.sort((a, b) => { const ap = Number.isFinite(a._storePrice) ? a._storePrice : null; @@ -638,7 +785,8 @@ export async function renderStore($app, storeLabelRaw) { ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap; const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp; - if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey; + if (aKey !== bKey) + return mode === "priceAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); return; @@ -652,7 +800,8 @@ export async function renderStore($app, storeLabelRaw) { ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad; const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd; - if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey; + if (aKey !== bKey) + return mode === "dateAsc" ? aKey - bKey : bKey - aKey; return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku); }); } From 5ef30bb652021b66828aa5772999d6091d5a749f Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:34:22 -0800 Subject: [PATCH 10/19] UX Improvements --- viz/app/store_page.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/viz/app/store_page.js b/viz/app/store_page.js index bf23190..8fed0d7 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -579,35 +579,36 @@ export async function renderStore($app, storeLabelRaw) { function exclusiveAnnotHtml(it) { const mode = String($exSort.value || "priceDesc"); - + // If sorting by sale, annotate with sale change ($ / %). If unchanged, show nothing. if (mode === "salePct") { const p = Number.isFinite(it._salePct) ? it._salePct : 0; if (!p) return ""; const abs = Math.abs(p); - if (p < 0) return `${esc(abs)}% off`; - return `+${esc(abs)}%`; + if (p < 0) return `[${esc(abs)}% Off]`; + return `[+${esc(abs)}%]`; } - + if (mode === "saleAbs") { const d = Number.isFinite(it._saleDelta) ? it._saleDelta : 0; if (!d) return ""; const abs = Math.round(Math.abs(d)); if (!abs) return ""; - if (d < 0) return `$${esc(abs)} off`; - return `+$${esc(abs)}`; + if (d < 0) return `[-$${esc(abs)}]`; + return `[+$${esc(abs)}]`; } - + // Otherwise: show % off vs best other store (only when actually cheaper). const sp = it && Number.isFinite(it._storePrice) ? it._storePrice : null; const other = it && Number.isFinite(it._bestOther) ? it._bestOther : null; if (sp === null || other === null || !(other > 0)) return ""; if (!(sp < other - EPS)) return ""; - + const pct = Math.round(((other - sp) / other) * 100); if (!Number.isFinite(pct) || pct <= 0) return ""; - return `${esc(pct)}% off`; + return `[${esc(pct)}% Off]`; } + function renderCard(it) { const price = listingPriceStr(it); From 1a5b638c58f17fd9277ea82a2e53dcd78d4b2eda Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:36:24 -0800 Subject: [PATCH 11/19] UX Improvements --- viz/app/store_page.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 8fed0d7..235999a 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -580,13 +580,13 @@ export async function renderStore($app, storeLabelRaw) { function exclusiveAnnotHtml(it) { const mode = String($exSort.value || "priceDesc"); - // If sorting by sale, annotate with sale change ($ / %). If unchanged, show nothing. + // Sale sorts: show price change for THIS store (7d recent), unchanged => nothing. if (mode === "salePct") { const p = Number.isFinite(it._salePct) ? it._salePct : 0; if (!p) return ""; const abs = Math.abs(p); - if (p < 0) return `[${esc(abs)}% Off]`; - return `[+${esc(abs)}%]`; + if (p < 0) return `${esc(abs)}% off`; + return `+${esc(abs)}%`; } if (mode === "saleAbs") { @@ -594,11 +594,12 @@ export async function renderStore($app, storeLabelRaw) { if (!d) return ""; const abs = Math.round(Math.abs(d)); if (!abs) return ""; - if (d < 0) return `[-$${esc(abs)}]`; - return `[+$${esc(abs)}]`; + if (d < 0) return `$${esc(abs)} off`; + return `+$${esc(abs)}`; } - // Otherwise: show % off vs best other store (only when actually cheaper). + // Any non-sale sort: still show % off on exclusives/last-stock when applicable. + // (% off vs best other store price) const sp = it && Number.isFinite(it._storePrice) ? it._storePrice : null; const other = it && Number.isFinite(it._bestOther) ? it._bestOther : null; if (sp === null || other === null || !(other > 0)) return ""; @@ -606,7 +607,7 @@ export async function renderStore($app, storeLabelRaw) { const pct = Math.round(((other - sp) / other) * 100); if (!Number.isFinite(pct) || pct <= 0) return ""; - return `[${esc(pct)}% Off]`; + return `${esc(pct)}% off`; } From 8531eba4e3bcedcf11acda0569457e78852c7517 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 16:38:34 -0800 Subject: [PATCH 12/19] UX Improvements --- viz/app/store_page.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/viz/app/store_page.js b/viz/app/store_page.js index 235999a..6cadaaa 100644 --- a/viz/app/store_page.js +++ b/viz/app/store_page.js @@ -598,19 +598,14 @@ export async function renderStore($app, storeLabelRaw) { return `+$${esc(abs)}`; } - // Any non-sale sort: still show % off on exclusives/last-stock when applicable. - // (% off vs best other store price) - const sp = it && Number.isFinite(it._storePrice) ? it._storePrice : null; - const other = it && Number.isFinite(it._bestOther) ? it._bestOther : null; - if (sp === null || other === null || !(other > 0)) return ""; - if (!(sp < other - EPS)) return ""; - - const pct = Math.round(((other - sp) / other) * 100); - if (!Number.isFinite(pct) || pct <= 0) return ""; - return `${esc(pct)}% off`; + // Any NON-sale sort: still show the % badge (same as Sale %) when there was a change. + const p = Number.isFinite(it._salePct) ? it._salePct : 0; + if (!p) return ""; + const abs = Math.abs(p); + if (p < 0) return `${esc(abs)}% off`; + return `+${esc(abs)}%`; } - function renderCard(it) { const price = listingPriceStr(it); From 1216ec7fe0582c896b208b04a83510ee3799bd9a Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Thu, 5 Feb 2026 17:24:19 -0800 Subject: [PATCH 13/19] UX Improvements --- viz/app/storeColors.js | 337 ++++++++++++++++++++++------------------- 1 file changed, 177 insertions(+), 160 deletions(-) diff --git a/viz/app/storeColors.js b/viz/app/storeColors.js index 49dd4c0..a336b13 100644 --- a/viz/app/storeColors.js +++ b/viz/app/storeColors.js @@ -1,166 +1,183 @@ function normalizeId(s) { - return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, ""); - } - - // Your pinned colors (exact / “roughly right”) - const OVERRIDES = { - strath: "#76B7FF", - bsw: "#E9DF7A", - kensingtonwinemarket: "#F2C200", - vessel: "#FFFFFF", - gullliquor: "#6B0F1A", - kegncork: "#111111", - legacyliquor: "#7B4A12", - vintagespirits: "#E34A2C", - - craftcellars: "#E31B23", // bright red - maltsandgrains: "#A67C52", // faded brown - - // aliases - gull: "#6B0F1A", - legacy: "#7B4A12", - vintage: "#E34A2C", - kwm: "#F2C200", - }; - - // High-contrast qualitative palette (distinct hues). - // (Avoids whites/blacks/yellows that clash w/ your overrides by filtering below.) - const PALETTE = [ - "#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD", - "#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22", - "#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5", - "#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D", - // extras to reduce wrap risk - "#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173", - "#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D", - ]; - - function uniq(arr) { - return [...new Set(arr)]; - } - - function buildUniverse(base, extra) { - const a = Array.isArray(base) ? base : []; - const b = Array.isArray(extra) ? extra : []; - return uniq([...a, ...b].map(normalizeId).filter(Boolean)); - } - - // Default known ids (keeps mapping stable even if a page only sees a subset) - const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [ - "bcl", - "bsw", - "coop", - "craftcellars", - "gullliquor", - "gull", - "kegncork", - "kwm", - "kensingtonwinemarket", - "legacy", - "legacyliquor", - "maltsandgrains", - "sierrasprings", - "strath", - "tudor", - "vessel", - "vintage", - "vintagespirits", - "willowpark", - ]); - - function isWhiteHex(c) { - return String(c || "").trim().toUpperCase() === "#FFFFFF"; - } - - export function buildStoreColorMap(extraUniverse = []) { - const universe = buildUniverse(DEFAULT_UNIVERSE, extraUniverse).sort(); - - const used = new Set(); - const map = new Map(); - - // pin overrides first - for (const id of universe) { - const c = OVERRIDES[id]; - if (c) { - map.set(id, c); - used.add(String(c).toUpperCase()); - } - } - - // filter palette to avoid exact collisions with overrides (and keep white reserved for Vessel) - const palette = PALETTE - .map((c) => String(c).toUpperCase()) - .filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111"); - - let pi = 0; - for (const id of universe) { - if (map.has(id)) continue; - if (pi >= palette.length) { - // If you ever exceed palette size, just reuse (rare). Still deterministic. - pi = 0; - } - const c = palette[pi++]; + return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, ""); +} + +// Map normalized store *labels* to canonical ids used by OVERRIDES +const ALIASES = { + strathliquor: "strath", + vesselliquor: "vessel", + tudorhouse: "tudor", + coopworldofwhisky: "coop", + + kensingtonwinemarket: "kensingtonwinemarket", + gullliquor: "gullliquor", + legacyliquor: "legacyliquor", + vintagespirits: "vintagespirits", + kegncork: "kegncork", + + // short forms + gull: "gullliquor", + legacy: "legacyliquor", + vintage: "vintagespirits", + kwm: "kensingtonwinemarket", +}; + +// Your pinned colors +const OVERRIDES = { + strath: "#76B7FF", + bsw: "#E9DF7A", + kensingtonwinemarket: "#F2C200", + vessel: "#FFFFFF", + gullliquor: "#6B0F1A", + kegncork: "#111111", + legacyliquor: "#7B4A12", + vintagespirits: "#E34A2C", + + craftcellars: "#E31B23", + maltsandgrains: "#A67C52", + + // aliases + gull: "#6B0F1A", + legacy: "#7B4A12", + vintage: "#E34A2C", + kwm: "#F2C200", +}; + +// High-contrast qualitative palette +const PALETTE = [ + "#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD", + "#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22", + "#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5", + "#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D", + "#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173", + "#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D", +]; + +function uniq(arr) { + return [...new Set(arr)]; +} + +function canonicalId(s) { + const id = normalizeId(s); + return ALIASES[id] || id; +} + +function buildUniverse(base, extra) { + const a = Array.isArray(base) ? base : []; + const b = Array.isArray(extra) ? extra : []; + return uniq([...a, ...b].map(canonicalId).filter(Boolean)); +} + +// Keep mapping stable even if page sees a subset +const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [ + "bcl", + "bsw", + "coop", + "craftcellars", + "gullliquor", + "gull", + "kegncork", + "kwm", + "kensingtonwinemarket", + "legacy", + "legacyliquor", + "maltsandgrains", + "sierrasprings", + "strath", + "tudor", + "vessel", + "vintage", + "vintagespirits", + "willowpark", +]); + +function isWhiteHex(c) { + return String(c || "").trim().toUpperCase() === "#FFFFFF"; +} + +export function buildStoreColorMap(extraUniverse = []) { + const universe = buildUniverse(DEFAULT_UNIVERSE, extraUniverse).sort(); + + const used = new Set(); + const map = new Map(); + + // Pin overrides first + for (const id of universe) { + const c = OVERRIDES[id]; + if (c) { map.set(id, c); - used.add(c); + used.add(String(c).toUpperCase()); } - - return map; } - - export function storeColor(storeKeyOrLabel, colorMap) { - const id = normalizeId(storeKeyOrLabel); - if (!id) return "#7F7F7F"; - - const forced = OVERRIDES[id]; - if (forced) return forced; - - if (colorMap && typeof colorMap.get === "function") { - const c = colorMap.get(id); - if (c) return c; - } - - // fallback: deterministic but not “no conflicts” - return PALETTE[(id.length + id.charCodeAt(0)) % PALETTE.length]; + + // Filter palette to avoid collisions and keep white/black reserved + const palette = PALETTE + .map((c) => String(c).toUpperCase()) + .filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111"); + + let pi = 0; + for (const id of universe) { + if (map.has(id)) continue; + if (pi >= palette.length) pi = 0; + const c = palette[pi++]; + map.set(id, c); + used.add(c); } - - export function datasetStrokeWidth(color) { - return isWhiteHex(color) ? 2.5 : 1.5; + + return map; +} + +export function storeColor(storeKeyOrLabel, colorMap) { + const id = canonicalId(storeKeyOrLabel); + if (!id) return "#7F7F7F"; + + const forced = OVERRIDES[id]; + if (forced) return forced; + + if (colorMap && typeof colorMap.get === "function") { + const c = colorMap.get(id); + if (c) return c; } - - export function datasetPointRadius(color) { - return isWhiteHex(color) ? 2.8 : 2.2; - } - - function clamp(v, lo, hi) { - return Math.max(lo, Math.min(hi, v)); - } - - function hexToRgb(hex) { - const m = String(hex).replace("#", ""); - if (m.length !== 6) return null; - const n = parseInt(m, 16); - return { - r: (n >> 16) & 255, - g: (n >> 8) & 255, - b: n & 255, - }; - } - - function rgbToHex({ r, g, b }) { - const h = (x) => clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0"); - return `#${h(r)}${h(g)}${h(b)}`; - } - - // lighten by mixing with white (amount 0–1) - export function lighten(hex, amount = 0.25) { - const rgb = hexToRgb(hex); - if (!rgb) return hex; - return rgbToHex({ - r: rgb.r + (255 - rgb.r) * amount, - g: rgb.g + (255 - rgb.g) * amount, - b: rgb.b + (255 - rgb.b) * amount, - }); - } - - - \ No newline at end of file + + return PALETTE[(id.length + id.charCodeAt(0)) % PALETTE.length]; +} + +export function datasetStrokeWidth(color) { + return isWhiteHex(color) ? 2.5 : 1.5; +} + +export function datasetPointRadius(color) { + return isWhiteHex(color) ? 2.8 : 2.2; +} + +function clamp(v, lo, hi) { + return Math.max(lo, Math.min(hi, v)); +} + +function hexToRgb(hex) { + const m = String(hex).replace("#", ""); + if (m.length !== 6) return null; + const n = parseInt(m, 16); + return { + r: (n >> 16) & 255, + g: (n >> 8) & 255, + b: n & 255, + }; +} + +function rgbToHex({ r, g, b }) { + const h = (x) => + clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0"); + return `#${h(r)}${h(g)}${h(b)}`; +} + +// Lighten by mixing with white (0–1) +export function lighten(hex, amount = 0.25) { + const rgb = hexToRgb(hex); + if (!rgb) return hex; + return rgbToHex({ + r: rgb.r + (255 - rgb.r) * amount, + g: rgb.g + (255 - rgb.g) * amount, + b: rgb.b + (255 - rgb.b) * amount, + }); +} From ae1d98612a21a485279ad5a8bfb8e744d7332db5 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:01:10 -0800 Subject: [PATCH 14/19] UX Improvements --- viz/app/stats_page.js | 1 + 1 file changed, 1 insertion(+) diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index 3920d2d..8d2fc99 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -773,6 +773,7 @@ export async function renderStats($app) { max: yBounds?.max, title: { display: true, text: "Avg % vs per-SKU median" }, ticks: { + stepSize: 1, // <- minimum 1% between ticks callback: (v) => `${Number(v).toFixed(0)}%`, maxTicksLimit: 12, }, From 952a6c6abab55635e5889a67158735c1ff3ba2fd Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:03:01 -0800 Subject: [PATCH 15/19] UX Improvements --- viz/app/stats_page.js | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index 8d2fc99..d75ce33 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -438,9 +438,8 @@ function computeSeriesFromRaw(raw, filter) { /* ---------------- y-axis bounds ---------------- */ -function computeYBounds(seriesByStore, defaultAbs) { - let mn = Infinity; - let mx = -Infinity; +function computeYBounds(seriesByStore, minSpan = 6, pad = 1) { + let mn = Infinity, mx = -Infinity; for (const arr of Object.values(seriesByStore || {})) { if (!Array.isArray(arr)) continue; @@ -451,13 +450,24 @@ function computeYBounds(seriesByStore, defaultAbs) { } } - if (mn === Infinity) return { min: -defaultAbs, max: defaultAbs }; + if (mn === Infinity) return { min: -minSpan / 2, max: minSpan / 2 }; - const min = Math.min(-defaultAbs, Math.floor(mn)); - const max = Math.max(defaultAbs, Math.ceil(mx)); - return { min, max }; + // pad a bit so lines aren't glued to edges + mn = Math.floor(mn - pad); + mx = Math.ceil(mx + pad); + + // enforce a minimum visible range so it doesn't get *too* tight + const span = mx - mn; + if (span < minSpan) { + const mid = (mn + mx) / 2; + mn = Math.floor(mid - minSpan / 2); + mx = Math.ceil(mid + minSpan / 2); + } + + return { min: mn, max: mx }; } + /* ---------------- prefs ---------------- */ const LS_GROUP = "stviz:v1:stats:group"; @@ -842,8 +852,7 @@ export async function renderStats($app) { maxPrice: selectedMaxPrice, }); - const abs = group === "all" ? 12 : 8; - const yBounds = computeYBounds(series.seriesByStore, abs); + const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); @@ -878,8 +887,7 @@ export async function renderStats($app) { maxPrice: selectedMaxPrice, }); - const abs = group === "all" ? 12 : 8; - const yBounds = computeYBounds(series.seriesByStore, abs); + const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); From 13089e9c7d94eb2deeaee1695bf28f691ed8ccf2 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:04:52 -0800 Subject: [PATCH 16/19] UX Improvements --- viz/app/stats_page.js | 8 +++++--- viz/style.css | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index d75ce33..9a0bcac 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -508,7 +508,7 @@ export async function renderStats($app) { const pref = loadPrefs(); $app.innerHTML = ` -
+
@@ -567,8 +567,8 @@ export async function renderStats($app) {
-
-
+
+
@@ -855,6 +855,7 @@ export async function renderStats($app) { const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); + _chart?.resize(); const short = `Loaded ${series.labels.length} day(s). Filtered SKUs: ${series.newestUsed}/${series.newestTotal}.`; onStatus(short); @@ -890,6 +891,7 @@ export async function renderStats($app) { const yBounds = computeYBounds(series.seriesByStore, group === "all" ? 8 : 6, 1); await drawOrUpdateChart(series, yBounds); + _chart?.resize(); const short = `Loaded ${series.labels.length} day(s). Filtered SKUs: ${series.newestUsed}/${series.newestTotal}.`; onStatus(short); diff --git a/viz/style.css b/viz/style.css index c8864e6..af6b48b 100644 --- a/viz/style.css +++ b/viz/style.css @@ -549,3 +549,27 @@ html { overflow-y: scroll; } .rangeDual input[type="range"]::-moz-range-track { background: transparent; } .rangeDual input[type="range"]::-moz-range-progress { background: transparent; } .rangeDual input[type="range"]::-moz-range-thumb { pointer-events: all; } + + +/* Stats page: make chart fill remaining viewport height */ +.containerFull{ + min-height: 100vh; + display: flex; + flex-direction: column; +} + +.cardFill{ + flex: 1 1 auto; + min-height: 320px; /* safety */ + display: flex; +} + +.chartFill{ + flex: 1 1 auto; + min-height: 0; /* IMPORTANT so flex children can actually shrink/grow */ +} + +.chartFill canvas{ + width: 100% !important; + height: 100% !important; +} From 372f69f99da8f72536c231bc1c45ea4d4d19e45a Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:07:09 -0800 Subject: [PATCH 17/19] UX Improvements --- viz/app/stats_page.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index 9a0bcac..cea4a8b 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -782,11 +782,22 @@ export async function renderStats($app) { min: yBounds?.min, max: yBounds?.max, title: { display: true, text: "Avg % vs per-SKU median" }, + ticks: { - stepSize: 1, // <- minimum 1% between ticks + stepSize: 1, callback: (v) => `${Number(v).toFixed(0)}%`, maxTicksLimit: 12, }, + + grid: { + drawBorder: false, + color: (ctx) => { + // Only draw for integer % values + if (!Number.isInteger(ctx.tick.value)) return "transparent"; + return ctx.tick.value === 0 ? "rgba(154,166,178,0.35)" : "rgba(154,166,178,0.18)"; + }, + lineWidth: 1, + }, }, }, }, From 12c1c87433e060930a447dcd6e5db3f61012b83f Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:09:32 -0800 Subject: [PATCH 18/19] UX Improvements --- viz/app/stats_page.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js index cea4a8b..0c0092f 100644 --- a/viz/app/stats_page.js +++ b/viz/app/stats_page.js @@ -452,6 +452,9 @@ function computeYBounds(seriesByStore, minSpan = 6, pad = 1) { if (mn === Infinity) return { min: -minSpan / 2, max: minSpan / 2 }; + mn = Math.min(mn, 0); + mx = Math.max(mx, 0); + // pad a bit so lines aren't glued to edges mn = Math.floor(mn - pad); mx = Math.ceil(mx + pad); @@ -785,17 +788,17 @@ export async function renderStats($app) { ticks: { stepSize: 1, + precision: 0, + autoSkip: false, // <- don't skip integer ticks callback: (v) => `${Number(v).toFixed(0)}%`, - maxTicksLimit: 12, }, grid: { drawBorder: false, - color: (ctx) => { - // Only draw for integer % values - if (!Number.isInteger(ctx.tick.value)) return "transparent"; - return ctx.tick.value === 0 ? "rgba(154,166,178,0.35)" : "rgba(154,166,178,0.18)"; - }, + color: (ctx) => + ctx.tick.value === 0 + ? "rgba(154,166,178,0.35)" + : "rgba(154,166,178,0.18)", lineWidth: 1, }, }, From 39323b87470c108aee9288cbf939bc4390f843d2 Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Fri, 6 Feb 2026 13:10:25 -0800 Subject: [PATCH 19/19] UX Improvements --- tools/build_email_alert.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_email_alert.js b/tools/build_email_alert.js index 8a15cd6..ecde6fc 100755 --- a/tools/build_email_alert.js +++ b/tools/build_email_alert.js @@ -299,7 +299,7 @@ function renderHtml({ title, subtitle, uniqueNews, bigSales, commitUrl, pagesUrl const name = htmlEscape(it.name || ""); const store = htmlEscape(it.storeLabel || ""); const cat = htmlEscape(it.categoryLabel || ""); - const price = htmlEscape(it.price || it.newPrice || ""); + const price = htmlEscape(it.price || ""); const url = htmlEscape(it.url || ""); return `