From 35e5956c12ebe5148d229d311392d0718fd1c09a Mon Sep 17 00:00:00 2001 From: "Brennan Wilkes (Text Groove)" Date: Mon, 2 Feb 2026 19:13:49 -0800 Subject: [PATCH] feat: New reports --- scripts/run_daily.sh | 13 +++++ tools/build_common_listings.js | 71 ++++++++++++++++++++----- tools/build_viz_commits.js | 96 +++++++++++++++++++++++++--------- viz/app/main.js | 4 ++ viz/app/search_page.js | 1 + viz/app/stats_page.js | 78 +++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 37 deletions(-) create mode 100644 viz/app/stats_page.js diff --git a/scripts/run_daily.sh b/scripts/run_daily.sh index d259fa0..cfe160d 100755 --- a/scripts/run_daily.sh +++ b/scripts/run_daily.sh @@ -106,6 +106,19 @@ fi "$NODE_BIN" tools/build_viz_commits.js "$NODE_BIN" tools/build_viz_recent.js +# Build common listings artifacts (9 files) +"$NODE_BIN" tools/build_common_listings.js --group all --top 50 --out "reports/common_listings_all_top50.json" +"$NODE_BIN" tools/build_common_listings.js --group all --top 250 --out "reports/common_listings_all_top250.json" +"$NODE_BIN" tools/build_common_listings.js --group all --top 1000 --out "reports/common_listings_all_top1000.json" + +"$NODE_BIN" tools/build_common_listings.js --group bc --top 50 --out "reports/common_listings_bc_top50.json" +"$NODE_BIN" tools/build_common_listings.js --group bc --top 250 --out "reports/common_listings_bc_top250.json" +"$NODE_BIN" tools/build_common_listings.js --group bc --top 1000 --out "reports/common_listings_bc_top1000.json" + +"$NODE_BIN" tools/build_common_listings.js --group ab --top 50 --out "reports/common_listings_ab_top50.json" +"$NODE_BIN" tools/build_common_listings.js --group ab --top 250 --out "reports/common_listings_ab_top250.json" +"$NODE_BIN" tools/build_common_listings.js --group ab --top 1000 --out "reports/common_listings_ab_top1000.json" + # Stage only data/report/viz outputs git add -A data/db reports viz/data diff --git a/tools/build_common_listings.js b/tools/build_common_listings.js index ff20343..7dc3fcf 100644 --- a/tools/build_common_listings.js +++ b/tools/build_common_listings.js @@ -6,12 +6,14 @@ - Store = storeLabel (union across categories). - Canonicalizes via sku_map. - Debug output while scanning. - - Writes: reports/common_listings.json + - Writes: reports/common_listings__top.json (or --out) Flags: --top N --min-stores N --require-all + --group all|bc|ab + --out path/to/file.json */ const fs = require("fs"); @@ -94,16 +96,31 @@ function canonicalize(k, skuMap) { /* ---------------- args ---------------- */ function parseArgs(argv) { - const out = { top: 50, minStores: 2, requireAll: false }; + const out = { + top: 50, + minStores: 2, + requireAll: false, + group: "all", // all|bc|ab + out: "", // optional explicit output path + }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || 50; else if (a === "--min-stores" && argv[i + 1]) out.minStores = Number(argv[++i]) || 2; else if (a === "--require-all") out.requireAll = true; + else if (a === "--group" && argv[i + 1]) out.group = String(argv[++i] || "all"); + else if (a === "--out" && argv[i + 1]) out.out = String(argv[++i] || ""); } return out; } +function groupStores(group, allStoresSorted) { + const bc = new Set(["gull", "strath", "bcl", "legacy", "tudor"]); + if (group === "bc") return allStoresSorted.filter((s) => bc.has(s)); + if (group === "ab") return allStoresSorted.filter((s) => !bc.has(s)); + return allStoresSorted; // "all" +} + /* ---------------- main ---------------- */ function main() { @@ -123,8 +140,8 @@ function main() { console.log(`[debug] skuMap: ${skuMap ? "loaded" : "missing"}`); console.log(`[debug] scanning ${dbFiles.length} db files`); - const storeToCanon = new Map(); // storeLabel -> Set(canonSku) - const canonAgg = new Map(); // canonSku -> { stores:Set, listings:[], cheapest } + const storeToCanon = new Map(); // storeLabel -> Set(canonSku) + const canonAgg = new Map(); // canonSku -> { stores:Set, listings:[], cheapest, perStore:Map(storeLabel -> {priceNum, item}) } let liveRows = 0; let removedRows = 0; @@ -167,7 +184,7 @@ function main() { let agg = canonAgg.get(canonSku); if (!agg) { - agg = { stores: new Set(), listings: [], cheapest: null }; + agg = { stores: new Set(), listings: [], cheapest: null, perStore: new Map() }; canonAgg.set(canonSku, agg); } @@ -195,13 +212,25 @@ function main() { agg.cheapest = { priceNum, item: listing }; } } + + // per-store numeric price (best/lowest numeric; otherwise first seen) + const prev = agg.perStore.get(storeLabel); + if (priceNum !== null) { + if (!prev || prev.priceNum === null || priceNum < prev.priceNum) { + agg.perStore.set(storeLabel, { priceNum, item: listing }); + } + } else { + if (!prev) agg.perStore.set(storeLabel, { priceNum: null, item: listing }); + } } } - const stores = [...storeToCanon.keys()].sort(); + const allStores = [...storeToCanon.keys()].sort(); + const stores = groupStores(String(args.group || "all").toLowerCase(), allStores); const storeCount = stores.length; - console.log(`[debug] stores (${storeCount}): ${stores.join(", ")}`); + console.log(`[debug] stores(all) (${allStores.length}): ${allStores.join(", ")}`); + console.log(`[debug] group="${args.group}" stores(${storeCount}): ${stores.join(", ")}`); console.log(`[debug] liveRows=${liveRows} removedRows=${removedRows} canonSkus=${canonAgg.size}`); function pickRepresentative(agg) { @@ -217,14 +246,24 @@ function main() { const rows = []; for (const [canonSku, agg] of canonAgg.entries()) { + const groupStoresPresent = stores.filter((s) => agg.stores.has(s)); + if (groupStoresPresent.length === 0) continue; + const rep = pickRepresentative(agg); const missingStores = stores.filter((s) => !agg.stores.has(s)); + const storePrices = {}; + for (const s of stores) { + const ps = agg.perStore.get(s); + storePrices[s] = ps ? ps.priceNum : null; + } + rows.push({ canonSku, - storeCount: agg.stores.size, - stores: [...agg.stores].sort(), + storeCount: groupStoresPresent.length, + stores: groupStoresPresent.sort(), missingStores, + storePrices, representative: rep ? { name: rep.name, @@ -249,7 +288,11 @@ function main() { }); } - rows.sort((a, b) => b.storeCount - a.storeCount); + // stable-ish ordering: primary by store coverage, tie-break by canonSku + rows.sort((a, b) => { + if (b.storeCount !== a.storeCount) return b.storeCount - a.storeCount; + return String(a.canonSku).localeCompare(String(b.canonSku)); + }); const filtered = args.requireAll ? rows.filter((r) => r.storeCount === storeCount) @@ -257,9 +300,11 @@ function main() { const top = filtered.slice(0, args.top); + const safeGroup = String(args.group || "all").toLowerCase(); + const payload = { generatedAt: new Date().toISOString(), - args, + args: { ...args, group: safeGroup }, storeCount, stores, totals: { @@ -271,7 +316,9 @@ function main() { rows: top, }; - const outPath = path.join(reportsDir, "common_listings.json"); + const defaultName = `common_listings_${safeGroup}_top${args.top}.json`; + const outPath = args.out ? path.resolve(repoRoot, args.out) : path.join(reportsDir, defaultName); + fs.writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n", "utf8"); console.log(`Wrote ${path.relative(repoRoot, outPath)} (${top.length} rows)`); } diff --git a/tools/build_viz_commits.js b/tools/build_viz_commits.js index 5f92a87..0d8e528 100755 --- a/tools/build_viz_commits.js +++ b/tools/build_viz_commits.js @@ -20,45 +20,44 @@ function listDbFiles(dbDir) { } } +function listCommonListingReportFiles(reportsDir) { + try { + return fs + .readdirSync(reportsDir, { withFileTypes: true }) + .filter((e) => e.isFile() && e.name.endsWith(".json")) + .map((e) => e.name) + .filter((name) => /^common_listings_.*_top\d+\.json$/i.test(name)) + .map((name) => path.join(reportsDir, name)); + } catch { + return []; + } +} + function dateOnly(iso) { const m = String(iso ?? "").match(/^(\d{4}-\d{2}-\d{2})/); return m ? m[1] : ""; } -function main() { - const repoRoot = process.cwd(); - const dbDir = path.join(repoRoot, "data", "db"); - const outDir = path.join(repoRoot, "viz", "data"); - const outFile = path.join(outDir, "db_commits.json"); - - fs.mkdirSync(outDir, { recursive: true }); - - const files = listDbFiles(dbDir).map((abs) => path.posix.join("data/db", path.basename(abs))); - +function buildCommitPayloadForFiles({ repoRoot, relFiles, maxRawPerFile, maxDaysPerFile }) { const payload = { generatedAt: new Date().toISOString(), branch: "data", files: {}, }; - // We want the viz to show ONE point per day (the most recent run that day). - // So we collapse multiple commits per day down to the newest commit for that date. - // - // With multiple runs/day, we also want to keep a long-ish daily history. - // Raw commits per day could be ~4, so grab a larger raw window and then collapse. - const MAX_RAW_PER_FILE = 2400; // ~600 days @ 4 runs/day - const MAX_DAYS_PER_FILE = 600; // daily points kept after collapsing - - for (const rel of files.sort()) { + for (const rel of relFiles.sort()) { let txt = ""; try { // %H = sha, %cI = committer date strict ISO 8601 (includes time + tz) - txt = runGit(["log", "--format=%H %cI", `-${MAX_RAW_PER_FILE}`, "--", rel]); + txt = runGit(["log", "--format=%H %cI", `-${maxRawPerFile}`, "--", rel]); } catch { continue; } - const lines = txt.split(/\r?\n/).map((s) => s.trim()).filter(Boolean); + const lines = txt + .split(/\r?\n/) + .map((s) => s.trim()) + .filter(Boolean); // git log is newest -> oldest. // Keep the FIRST commit we see for each date (that is the most recent commit for that date). @@ -79,15 +78,62 @@ function main() { let arr = [...byDate.values()].reverse(); // Keep only the newest MAX_DAYS_PER_FILE (still oldest -> newest) - if (arr.length > MAX_DAYS_PER_FILE) { - arr = arr.slice(arr.length - MAX_DAYS_PER_FILE); + if (arr.length > maxDaysPerFile) { + arr = arr.slice(arr.length - maxDaysPerFile); } payload.files[rel] = arr; } - fs.writeFileSync(outFile, JSON.stringify(payload, null, 2) + "\n", "utf8"); - process.stdout.write(`Wrote ${outFile} (${Object.keys(payload.files).length} files)\n`); + return payload; +} + +function main() { + const repoRoot = process.cwd(); + const dbDir = path.join(repoRoot, "data", "db"); + const reportsDir = path.join(repoRoot, "reports"); + const outDir = path.join(repoRoot, "viz", "data"); + + fs.mkdirSync(outDir, { recursive: true }); + + // ---- Existing output (UNCHANGED): db_commits.json ---- + const outFileDb = path.join(outDir, "db_commits.json"); + + const dbFiles = listDbFiles(dbDir).map((abs) => path.posix.join("data/db", path.basename(abs))); + + // We want the viz to show ONE point per day (the most recent run that day). + // So we collapse multiple commits per day down to the newest commit for that date. + // + // With multiple runs/day, we also want to keep a long-ish daily history. + // Raw commits per day could be ~4, so grab a larger raw window and then collapse. + const MAX_RAW_PER_FILE = 2400; // ~600 days @ 4 runs/day + const MAX_DAYS_PER_FILE = 600; // daily points kept after collapsing + + const payloadDb = buildCommitPayloadForFiles({ + repoRoot, + relFiles: dbFiles, + maxRawPerFile: MAX_RAW_PER_FILE, + maxDaysPerFile: MAX_DAYS_PER_FILE, + }); + + fs.writeFileSync(outFileDb, JSON.stringify(payloadDb, null, 2) + "\n", "utf8"); + process.stdout.write(`Wrote ${outFileDb} (${Object.keys(payloadDb.files).length} files)\n`); + + // ---- New output: common listings report commits ---- + const outFileCommon = path.join(outDir, "common_listings_commits.json"); + + const reportFilesAbs = listCommonListingReportFiles(reportsDir); + const reportFilesRel = reportFilesAbs.map((abs) => path.posix.join("reports", path.basename(abs))); + + const payloadCommon = buildCommitPayloadForFiles({ + repoRoot, + relFiles: reportFilesRel, + maxRawPerFile: MAX_RAW_PER_FILE, + maxDaysPerFile: MAX_DAYS_PER_FILE, + }); + + fs.writeFileSync(outFileCommon, JSON.stringify(payloadCommon, null, 2) + "\n", "utf8"); + process.stdout.write(`Wrote ${outFileCommon} (${Object.keys(payloadCommon.files).length} files)\n`); } main(); diff --git a/viz/app/main.js b/viz/app/main.js index c37f042..e0b8c4a 100644 --- a/viz/app/main.js +++ b/viz/app/main.js @@ -4,6 +4,7 @@ * #/item/ detail * #/link sku linker (local-write only) * #/store/ store page (in-stock only) + * #/stats statistics */ import { destroyChart } from "./item_page.js"; @@ -11,6 +12,7 @@ import { renderSearch } from "./search_page.js"; import { renderItem } from "./item_page.js"; import { renderSkuLinker } from "./linker_page.js"; import { renderStore } from "./store_page.js"; ++import { renderStats, destroyStatsChart } from "./stats_page.js"; function route() { const $app = document.getElementById("app"); @@ -18,6 +20,7 @@ function route() { // always clean up chart when navigating destroyChart(); + destroyStatsChart(); const h = location.hash || "#/"; const parts = h.replace(/^#\/?/, "").split("/").filter(Boolean); @@ -26,6 +29,7 @@ function route() { if (parts[0] === "item" && parts[1]) return renderItem($app, decodeURIComponent(parts[1])); if (parts[0] === "store" && parts[1]) return renderStore($app, decodeURIComponent(parts[1])); if (parts[0] === "link") return renderSkuLinker($app); + if (parts[0] === "stats") return renderStats($app); return renderSearch($app); } diff --git a/viz/app/search_page.js b/viz/app/search_page.js index c6ab22f..c6ef290 100644 --- a/viz/app/search_page.js +++ b/viz/app/search_page.js @@ -17,6 +17,7 @@ export function renderSearch($app) {
+ Statistics Link SKUs
diff --git a/viz/app/stats_page.js b/viz/app/stats_page.js new file mode 100644 index 0000000..9924482 --- /dev/null +++ b/viz/app/stats_page.js @@ -0,0 +1,78 @@ +import { esc } from "./dom.js"; + +let _chart = null; + +export function destroyStatsChart() { + try { + if (_chart) _chart.destroy(); + } catch {} + _chart = null; +} + +function ensureChartJs() { + if (window.Chart) return Promise.resolve(window.Chart); + + return new Promise((resolve, reject) => { + const s = document.createElement("script"); + // UMD build -> window.Chart + s.src = "https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"; + s.async = true; + s.onload = () => resolve(window.Chart); + s.onerror = () => reject(new Error("Failed to load Chart.js")); + document.head.appendChild(s); + }); +} + +export async function renderStats($app) { + destroyStatsChart(); + + $app.innerHTML = ` +
+
+
+
+

Statistics

+
Coming soon
+
+
+ ← Back +
+
+
+ +
+
+ +
+
+
+ `; + + try { + const Chart = await ensureChartJs(); + const canvas = document.getElementById("statsChart"); + if (!canvas) return; + + const ctx = canvas.getContext("2d"); + _chart = new Chart(ctx, { + type: "line", + data: { + labels: [], + datasets: [{ label: "Price", data: [] }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + animation: false, + plugins: { legend: { display: true } }, + scales: { + x: { title: { display: true, text: "Time" } }, + y: { title: { display: true, text: "Value" } }, + }, + }, + }); + } catch (e) { + const msg = esc(e?.message || String(e)); + $app.querySelector(".card").innerHTML = `
Chart unavailable: ${msg}
`; + } +}