UX Improvements

This commit is contained in:
Brennan Wilkes (Text Groove) 2026-02-10 16:45:22 -08:00
parent e9f8f805c5
commit 7a33d51c90
73 changed files with 13094 additions and 13094 deletions

View file

@ -205,14 +205,16 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
url,
tag,
ua,
{ mode = "text", method = "GET", headers = {}, body = null, cookies = true } = {}
{ mode = "text", method = "GET", headers = {}, body = null, cookies = true } = {},
) {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
const reqId = ++reqSeq;
const start = Date.now();
inflight++;
logger?.dbg?.(`REQ#${reqId} START ${tag} attempt=${attempt + 1}/${maxRetries + 1} ${url} (${inflightStr()})`);
logger?.dbg?.(
`REQ#${reqId} START ${tag} attempt=${attempt + 1}/${maxRetries + 1} ${url} (${inflightStr()})`,
);
const releaseHost = await acquireHost(url);
@ -268,9 +270,7 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
if (status >= 400) {
const bodyTxt = await safeText(res);
throw new Error(
`HTTP ${status} bodyHead=${String(bodyTxt).slice(0, 160).replace(/\s+/g, " ")}`
);
throw new Error(`HTTP ${status} bodyHead=${String(bodyTxt).slice(0, 160).replace(/\s+/g, " ")}`);
}
if (mode === "json") {
@ -298,8 +298,8 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
logger?.dbg?.(
`REQ#${reqId} FAIL ${tag} retryable=${retryable} err=${e?.message || e} host=${host} nextOkIn=${Math.max(
0,
nextOk - Date.now()
)}ms`
nextOk - Date.now(),
)}ms`,
);
if (!retryable || attempt === maxRetries) throw e;

View file

@ -16,8 +16,7 @@ const { runAllStores } = require("./tracker/run_all");
const { renderFinalReport } = require("./tracker/report");
const { ensureDir } = require("./tracker/db");
const DEFAULT_UA =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36";
const DEFAULT_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36";
function resolveDir(p, fallback) {
const v = String(p || "").trim();
@ -79,12 +78,8 @@ function filterStoresOrThrow(stores, wantedListRaw) {
}
if (missing.length) {
const avail = stores
.map((s) => `${s.key}${s.name ? ` (${s.name})` : ""}`)
.join(", ");
throw new Error(
`Unknown store(s) in --stores: ${missing.join(", ")}\nAvailable: ${avail}`
);
const avail = stores.map((s) => `${s.key}${s.name ? ` (${s.name})` : ""}`).join(", ");
throw new Error(`Unknown store(s) in --stores: ${missing.join(", ")}\nAvailable: ${avail}`);
}
// de-dupe by key (in case name+key both matched)
@ -100,9 +95,7 @@ function filterStoresOrThrow(stores, wantedListRaw) {
async function main() {
if (typeof fetch !== "function") {
throw new Error(
"Global fetch() not found. Please use Node.js 18+ (or newer). "
);
throw new Error("Global fetch() not found. Please use Node.js 18+ (or newer). ");
}
const argv = process.argv.slice(2);
@ -114,25 +107,16 @@ async function main() {
debug: args.debug,
maxPages: args.maxPages,
concurrency: args.concurrency ?? clampInt(process.env.CONCURRENCY, 6, 1, 64),
staggerMs:
args.staggerMs ?? clampInt(process.env.STAGGER_MS, 150, 0, 5000),
staggerMs: args.staggerMs ?? clampInt(process.env.STAGGER_MS, 150, 0, 5000),
maxRetries: clampInt(process.env.MAX_RETRIES, 6, 0, 20),
timeoutMs: clampInt(process.env.TIMEOUT_MS, 25000, 1000, 120000),
discoveryGuess:
args.guess ?? clampInt(process.env.DISCOVERY_GUESS, 20, 1, 5000),
discoveryStep:
args.step ?? clampInt(process.env.DISCOVERY_STEP, 5, 1, 500),
discoveryGuess: args.guess ?? clampInt(process.env.DISCOVERY_GUESS, 20, 1, 5000),
discoveryStep: args.step ?? clampInt(process.env.DISCOVERY_STEP, 5, 1, 500),
categoryConcurrency: clampInt(process.env.CATEGORY_CONCURRENCY, 5, 1, 64),
defaultUa: DEFAULT_UA,
defaultParseProducts: parseProductsSierra,
dbDir: resolveDir(
args.dataDir ?? process.env.DATA_DIR,
path.join(process.cwd(), "data", "db")
),
reportDir: resolveDir(
args.reportDir ?? process.env.REPORT_DIR,
path.join(process.cwd(), "reports")
),
dbDir: resolveDir(args.dataDir ?? process.env.DATA_DIR, path.join(process.cwd(), "data", "db")),
reportDir: resolveDir(args.reportDir ?? process.env.REPORT_DIR, path.join(process.cwd(), "reports")),
};
ensureDir(config.dbDir);
@ -146,8 +130,7 @@ async function main() {
});
const stores = createStores({ defaultUa: config.defaultUa });
const storesFilterRaw =
getFlagValue(argv, "--stores") || String(process.env.STORES || "").trim();
const storesFilterRaw = getFlagValue(argv, "--stores") || String(process.env.STORES || "").trim();
const storesToRun = filterStoresOrThrow(stores, storesFilterRaw);
if (storesFilterRaw) {
@ -180,10 +163,7 @@ async function main() {
dbDir: config.dbDir,
colorize: false,
});
const file = path.join(
config.reportDir,
`${isoTimestampFileSafe(new Date())}.txt`
);
const file = path.join(config.reportDir, `${isoTimestampFileSafe(new Date())}.txt`);
try {
fs.writeFileSync(file, reportTextPlain, "utf8");
logger.ok(`Report saved: ${logger.dim(file)}`);

View file

@ -125,21 +125,15 @@ function arcNormalizeImg(raw) {
const hasCspcId = /^\d{1,11}$/.test(rawCspcId);
const id = Number(p?.id);
const rawSku =
hasCspcId ? `id:${rawCspcId}` :
Number.isFinite(id) ? `id:${id}` :
"";
const sku =
normalizeSkuKey(rawSku, { storeLabel: ctx?.store?.name, url }) || rawSku || "";
const rawSku = hasCspcId ? `id:${rawCspcId}` : Number.isFinite(id) ? `id:${id}` : "";
const sku = normalizeSkuKey(rawSku, { storeLabel: ctx?.store?.name, url }) || rawSku || "";
const img = arcNormalizeImg(p.image || p.image_url || p.img || "");
return { name, price, url, sku, img };
}
function parseCategoryParamsFromStartUrl(startUrl) {
try {
const u = new URL(startUrl);
@ -161,7 +155,7 @@ function avoidMassRemoval(prevDb, discovered, ctx, reason) {
if (ratio >= 0.6) return false;
ctx.logger.warn?.(
`${ctx.catPrefixOut} | ARC partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`
`${ctx.catPrefixOut} | ARC partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`,
);
// Preserve prior active items not seen this run.
@ -238,12 +232,12 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
// Log early (even for empty)
ctx.logger.ok(
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(
3
)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(0, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "")
.toString()
.padEnd(3)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(0, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
ctx.http.inflightStr(),
11
)} | ${secStr(r.ms)}`
11,
)} | ${secStr(r.ms)}`,
);
if (!rawCount) break;
@ -274,12 +268,14 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
// Re-log with kept filled in (overwrite-style isnt possible; just emit a second line)
ctx.logger.ok(
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(
3
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "")
.toString()
.padEnd(
3,
)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(kept, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
ctx.http.inflightStr(),
11
)} | ${secStr(r.ms)}`
11,
)} | ${secStr(r.ms)}`,
);
// Stop condition #1: last page (short page)
@ -301,8 +297,11 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } =
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{ storeLabel: ctx.store.name },
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -311,7 +310,7 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
const elapsedMs = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsedMs)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Meta=${metaChangedItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsedMs)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Meta=${metaChangedItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -334,10 +333,17 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
report.totals.restoredCount += restoredItems.length;
report.totals.metaChangedCount += metaChangedItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {
return {
key: "arc",

View file

@ -158,7 +158,7 @@ function bclHitToItem(hit) {
// ✅ Fix: BCL appears to serve .jpg (not .jpeg) for these imagecache URLs.
// Also use https.
const img = `https://www.bcliquorstores.com/sites/default/files/imagecache/height400px/${encodeURIComponent(
skuRaw
skuRaw,
)}.jpg`;
return { name, price, url, sku, img };
@ -204,7 +204,11 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
ctx.logger.warn(`${ctx.catPrefixOut} | BCL browse fetch failed: ${e?.message || e}`);
const discovered = new Map();
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{ storeLabel: ctx.store.name },
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -226,7 +230,15 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
report.totals.updatedCount += updatedItems.length;
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
return;
}
@ -234,7 +246,9 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
const totalPages = Math.max(1, Math.ceil(total / size));
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
ctx.logger.ok(`${ctx.catPrefixOut} | Total=${total} Size=${size} Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Total=${total} Size=${size} Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
);
const pageNums = [];
for (let p = 1; p <= scanPages; p++) pageNums.push(p);
@ -259,12 +273,12 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pageNums.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageNums.length)} | items=${padLeft(
items.length,
3
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
3,
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
return items;
}
},
);
const discovered = new Map();
@ -276,9 +290,13 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
}
}
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
);
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
});
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -287,7 +305,7 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
const elapsed = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -309,7 +327,15 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {
@ -324,13 +350,15 @@ function createStore(defaultUa) {
key: "whisky",
label: "Whisky / Whiskey",
// informational only; scan uses ajax/browse
startUrl: "https://www.bcliquorstores.com/product-catalogue?category=spirits&type=whisky%20/%20whiskey&sort=featuredProducts:desc&page=1",
startUrl:
"https://www.bcliquorstores.com/product-catalogue?category=spirits&type=whisky%20/%20whiskey&sort=featuredProducts:desc&page=1",
bclType: "whisky / whiskey",
},
{
key: "rum",
label: "Rum",
startUrl: "https://www.bcliquorstores.com/product-catalogue?category=spirits&type=rum&sort=featuredProducts:desc&page=1",
startUrl:
"https://www.bcliquorstores.com/product-catalogue?category=spirits&type=rum&sort=featuredProducts:desc&page=1",
bclType: "rum",
},
],

View file

@ -82,7 +82,6 @@ function bswPickPrice(hit) {
return pick(null, false);
}
function bswHitToItem(hit) {
const name = cleanText(hit && (hit.title || hit.name || hit.product_title || hit.product_name || ""));
const handle = hit && (hit.handle || hit.product_handle || hit.slug || "");
@ -228,19 +227,24 @@ function bswPickImage(hit) {
return "";
}
async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
const t0 = Date.now();
let collectionId = Number.isFinite(ctx.cat.bswCollectionId) ? ctx.cat.bswCollectionId : null;
if (!collectionId) {
try {
const { text: html } = await ctx.http.fetchTextWithRetry(ctx.cat.startUrl, `bsw:html:${ctx.cat.key}`, ctx.store.ua);
const { text: html } = await ctx.http.fetchTextWithRetry(
ctx.cat.startUrl,
`bsw:html:${ctx.cat.key}`,
ctx.store.ua,
);
collectionId = bswExtractCollectionIdFromHtml(html);
if (collectionId) ctx.logger.ok(`${ctx.catPrefixOut} | BSW discovered collectionId=${collectionId}`);
else ctx.logger.warn(`${ctx.catPrefixOut} | BSW could not discover collectionId from HTML.`);
} catch (e) {
ctx.logger.warn(`${ctx.catPrefixOut} | BSW HTML fetch failed for collectionId discovery: ${e?.message || e}`);
ctx.logger.warn(
`${ctx.catPrefixOut} | BSW HTML fetch failed for collectionId discovery: ${e?.message || e}`,
);
}
}
@ -248,7 +252,11 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
ctx.logger.warn(`${ctx.catPrefixOut} | BSW missing collectionId; defaulting to 1 page with 0 items.`);
const discovered = new Map();
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{ storeLabel: ctx.store.name },
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -272,7 +280,15 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
report.totals.updatedCount += updatedItems.length;
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
return;
}
@ -285,16 +301,23 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
const totalPages = Math.max(1, nbPages);
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
ctx.logger.ok(`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
);
const pageIdxs = [];
for (let p = 0; p < scanPages; p++) pageIdxs.push(p);
let donePages = 0;
const perPageItems = await require("../utils/async").parallelMapStaggered(pageIdxs, ctx.config.concurrency, ctx.config.staggerMs, async (page0, idx) => {
const perPageItems = await require("../utils/async").parallelMapStaggered(
pageIdxs,
ctx.config.concurrency,
ctx.config.staggerMs,
async (page0, idx) => {
const pnum = idx + 1;
const r = page0 === 0 ? first : await bswFetchAlgoliaPage(ctx, collectionId, ruleContext, page0, hitsPerPage);
const r =
page0 === 0 ? first : await bswFetchAlgoliaPage(ctx, collectionId, ruleContext, page0, hitsPerPage);
const res0 = r?.json?.results?.[0] || null;
const hits = res0 && Array.isArray(res0.hits) ? res0.hits : [];
@ -309,12 +332,13 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(pnum, pageIdxs.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageIdxs.length)} | items=${padLeft(
items.length,
3
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
3,
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
return items;
});
},
);
const discovered = new Map();
let dups = 0;
@ -325,9 +349,13 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
}
}
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
);
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
});
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -336,7 +364,7 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
const elapsed = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -357,7 +385,15 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {

View file

@ -90,12 +90,11 @@ const r = await coopFetchText(ctx, REFERER, "coop:bootstrap", {
if (!coop.sessionKey || !coop.chainId || !coop.storeId || !coop.appVersion) {
throw new Error(
`coop bootstrap missing values: sessionKey=${!!coop.sessionKey} chainId=${!!coop.chainId} storeId=${!!coop.storeId} appVersion=${!!coop.appVersion}`
`coop bootstrap missing values: sessionKey=${!!coop.sessionKey} chainId=${!!coop.chainId} storeId=${!!coop.storeId} appVersion=${!!coop.appVersion}`,
);
}
}
async function ensureCoopSession(ctx) {
const coop = ctx.store.coop;
if (coop.sessionId) return;
@ -110,20 +109,13 @@ async function ensureCoopSession(ctx) {
headers: coopHeaders(ctx, "/worldofwhisky"),
// browser sends Content-Length: 0; easiest equivalent:
body: "",
}
},
);
const sid =
r?.json?.SessionID ||
r?.json?.sessionID ||
r?.json?.sessionId ||
r?.json?.SessionId ||
"";
const sid = r?.json?.SessionID || r?.json?.sessionID || r?.json?.sessionId || r?.json?.SessionId || "";
if (!sid) {
throw new Error(
`createSession: missing SessionID (status=${r?.status})`
);
throw new Error(`createSession: missing SessionID (status=${r?.status})`);
}
coop.sessionId = sid;
@ -157,10 +149,7 @@ function productFromApi(p) {
const url = productUrlFromId(productId);
const price =
p?.CountDetails?.PriceText ||
(Number.isFinite(p?.Price) ? `$${Number(p.Price).toFixed(2)}` : "");
const price = p?.CountDetails?.PriceText || (Number.isFinite(p?.Price) ? `$${Number(p.Price).toFixed(2)}` : "");
const upc = String(p.UPC || "").trim();
@ -207,7 +196,7 @@ async function fetchCategoryPage(ctx, categoryId, page) {
},
orderby: null,
}),
}
},
);
let r = await doReq();
@ -228,9 +217,7 @@ function avoidMassRemoval(prevDb, discovered, ctx) {
if (!prev || !curr) return;
if (curr / prev >= 0.6) return;
ctx.logger.warn(
`${ctx.catPrefixOut} | Partial scan (${curr}/${prev}); preserving DB`
);
ctx.logger.warn(`${ctx.catPrefixOut} | Partial scan (${curr}/${prev}); preserving DB`);
for (const [k, v] of prevDb.entries()) {
if (!discovered.has(k)) discovered.set(k, v);
@ -241,8 +228,7 @@ async function scanCategoryCoop(ctx, prevDb, report) {
const t0 = Date.now();
const discovered = new Map();
const maxPages =
ctx.config.maxPages === null ? 500 : Math.min(ctx.config.maxPages, 500);
const maxPages = ctx.config.maxPages === null ? 500 : Math.min(ctx.config.maxPages, 500);
let done = 0;
@ -251,15 +237,11 @@ async function scanCategoryCoop(ctx, prevDb, report) {
try {
r = await fetchCategoryPage(ctx, ctx.cat.coopCategoryId, page);
} catch (e) {
ctx.logger.warn(
`${ctx.catPrefixOut} | page ${page} failed: ${e?.message || e}`
);
ctx.logger.warn(`${ctx.catPrefixOut} | page ${page} failed: ${e?.message || e}`);
break;
}
const arr = Array.isArray(r?.json?.Products?.Result)
? r.json.Products.Result
: [];
const arr = Array.isArray(r?.json?.Products?.Result) ? r.json.Products.Result : [];
done++;
@ -272,11 +254,11 @@ async function scanCategoryCoop(ctx, prevDb, report) {
}
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${padLeft(page, 3)} | ${String(
r.status || ""
).padEnd(3)} | items=${padLeft(kept, 3)} | bytes=${kbStr(
r.bytes
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
`${ctx.catPrefixOut} | Page ${padLeft(page, 3)} | ${String(r.status || "").padEnd(
3,
)} | items=${padLeft(kept, 3)} | bytes=${kbStr(
r.bytes,
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
if (!arr.length) break;
@ -286,8 +268,9 @@ async function scanCategoryCoop(ctx, prevDb, report) {
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products: ${discovered.size}`);
const { merged, newItems, updatedItems, removedItems, restoredItems } =
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
});
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -320,7 +303,7 @@ async function scanCategoryCoop(ctx, prevDb, report) {
newItems,
updatedItems,
removedItems,
restoredItems
restoredItems,
);
}
@ -345,13 +328,55 @@ function createStore(defaultUa) {
},
categories: [
{ key: "canadian-whisky", label: "Canadian Whisky", coopSlug: "canadian_whisky", coopCategoryId: 4, startUrl: `${REFERER}#/category/canadian_whisky` },
{ key: "bourbon-whiskey", label: "Bourbon Whiskey", coopSlug: "bourbon_whiskey", coopCategoryId: 9, startUrl: `${REFERER}#/category/bourbon_whiskey` },
{ key: "scottish-single-malts", label: "Scottish Single Malts", coopSlug: "scottish_single_malts", coopCategoryId: 6, startUrl: `${REFERER}#/category/scottish_single_malts` },
{ key: "scottish-blends", label: "Scottish Whisky Blends", coopSlug: "scottish_whisky_blends", coopCategoryId: 5, startUrl: `${REFERER}#/category/scottish_whisky_blends` },
{ key: "american-whiskey", label: "American Whiskey", coopSlug: "american_whiskey", coopCategoryId: 8, startUrl: `${REFERER}#/category/american_whiskey` },
{ key: "world-whisky", label: "World Whisky", coopSlug: "world_international", coopCategoryId: 10, startUrl: `${REFERER}#/category/world_international` },
{ key: "rum", label: "Rum", coopSlug: "spirits_rum", coopCategoryId: 24, startUrl: `${REFERER}#/category/spirits_rum` },
{
key: "canadian-whisky",
label: "Canadian Whisky",
coopSlug: "canadian_whisky",
coopCategoryId: 4,
startUrl: `${REFERER}#/category/canadian_whisky`,
},
{
key: "bourbon-whiskey",
label: "Bourbon Whiskey",
coopSlug: "bourbon_whiskey",
coopCategoryId: 9,
startUrl: `${REFERER}#/category/bourbon_whiskey`,
},
{
key: "scottish-single-malts",
label: "Scottish Single Malts",
coopSlug: "scottish_single_malts",
coopCategoryId: 6,
startUrl: `${REFERER}#/category/scottish_single_malts`,
},
{
key: "scottish-blends",
label: "Scottish Whisky Blends",
coopSlug: "scottish_whisky_blends",
coopCategoryId: 5,
startUrl: `${REFERER}#/category/scottish_whisky_blends`,
},
{
key: "american-whiskey",
label: "American Whiskey",
coopSlug: "american_whiskey",
coopCategoryId: 8,
startUrl: `${REFERER}#/category/american_whiskey`,
},
{
key: "world-whisky",
label: "World Whisky",
coopSlug: "world_international",
coopCategoryId: 10,
startUrl: `${REFERER}#/category/world_international`,
},
{
key: "rum",
label: "Rum",
coopSlug: "spirits_rum",
coopCategoryId: 24,
startUrl: `${REFERER}#/category/spirits_rum`,
},
],
};
}

View file

@ -32,10 +32,7 @@ function canonicalizeCraftProductUrl(raw) {
function extractShopifyCardPrice(block) {
const b = String(block || "");
const dollars = (txt) =>
[...String(txt).matchAll(/\$\s*[\d,]+(?:\.\d{2})?/g)].map((m) =>
m[0].replace(/\s+/g, "")
);
const dollars = (txt) => [...String(txt).matchAll(/\$\s*[\d,]+(?:\.\d{2})?/g)].map((m) => m[0].replace(/\s+/g, ""));
const saleRegion = b.split(/sale price/i)[1] || "";
const saleD = dollars(saleRegion);
@ -52,14 +49,8 @@ function extractShopifyCardPrice(block) {
function parseProductsCraftCellars(html, ctx) {
const s = String(html || "");
const g1 =
s.match(
/<div\b[^>]*id=["']ProductGridContainer["'][^>]*>[\s\S]*?<\/div>/i
)?.[0] || "";
const g2 =
s.match(
/<div\b[^>]*id=["']product-grid["'][^>]*>[\s\S]*?<\/div>/i
)?.[0] || "";
const g1 = s.match(/<div\b[^>]*id=["']ProductGridContainer["'][^>]*>[\s\S]*?<\/div>/i)?.[0] || "";
const g2 = s.match(/<div\b[^>]*id=["']product-grid["'][^>]*>[\s\S]*?<\/div>/i)?.[0] || "";
const gridCandidate = g1.length > g2.length ? g1 : g2;
const grid = /\/products\//i.test(gridCandidate) ? gridCandidate : s;
@ -71,24 +62,18 @@ function parseProductsCraftCellarsInner(html, ctx) {
const s = String(html || "");
const items = [];
let blocks = [...s.matchAll(/<li\b[^>]*>[\s\S]*?<\/li>/gi)].map(
(m) => m[0]
);
let blocks = [...s.matchAll(/<li\b[^>]*>[\s\S]*?<\/li>/gi)].map((m) => m[0]);
if (blocks.length < 5) {
blocks = [
...s.matchAll(
/<div\b[^>]*class=["'][^"']*\bcard\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi
),
].map((m) => m[0]);
blocks = [...s.matchAll(/<div\b[^>]*class=["'][^"']*\bcard\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi)].map(
(m) => m[0],
);
}
const base = `https://${(ctx && ctx.store && ctx.store.host) || "craftcellars.ca"}/`;
for (const block of blocks) {
const href =
block.match(
/<a\b[^>]*href=["']([^"']*\/products\/[^"']+)["']/i
)?.[1] ||
block.match(/<a\b[^>]*href=["']([^"']*\/products\/[^"']+)["']/i)?.[1] ||
block.match(/href=["']([^"']*\/products\/[^"']+)["']/i)?.[1];
if (!href) continue;
@ -101,15 +86,11 @@ function parseProductsCraftCellarsInner(html, ctx) {
url = canonicalizeCraftProductUrl(url);
const nameHtml =
block.match(/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>\s*<[^>]*>\s*([^<]{2,200}?)\s*</i)?.[1] ||
block.match(
/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>\s*<[^>]*>\s*([^<]{2,200}?)\s*</i
/<h[23]\b[^>]*>[\s\S]*?<a\b[^>]*\/products\/[^"']+[^>]*>([\s\S]*?)<\/a>[\s\S]*?<\/h[23]>/i,
)?.[1] ||
block.match(
/<h[23]\b[^>]*>[\s\S]*?<a\b[^>]*\/products\/[^"']+[^>]*>([\s\S]*?)<\/a>[\s\S]*?<\/h[23]>/i
)?.[1] ||
block.match(
/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>([\s\S]*?)<\/a>/i
)?.[1];
block.match(/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>([\s\S]*?)<\/a>/i)?.[1];
const name = sanitizeName(stripTags(decodeHtml(nameHtml || "")));
if (!name) continue;
@ -160,23 +141,13 @@ function extractCraftSkuFromProductPageHtml(html) {
async function scanCategoryCraftCellars(ctx, prevDb, report) {
const t0 = Date.now();
const perPageDelayMs =
Math.max(
0,
cfgNum(ctx?.cat?.pageStaggerMs, cfgNum(ctx?.cat?.discoveryDelayMs, 0))
) || 0;
const perPageDelayMs = Math.max(0, cfgNum(ctx?.cat?.pageStaggerMs, cfgNum(ctx?.cat?.discoveryDelayMs, 0))) || 0;
const perJsonPageDelayMs = Math.max(
0,
cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)
);
const perJsonPageDelayMs = Math.max(0, cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs));
const htmlMap = new Map();
const maxPages =
ctx.config.maxPages === null
? 200
: Math.min(ctx.config.maxPages, 200);
const maxPages = ctx.config.maxPages === null ? 200 : Math.min(ctx.config.maxPages, 200);
let htmlPagesFetched = 0;
let emptyStreak = 0;
@ -188,7 +159,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
const { text: html } = await ctx.http.fetchTextWithRetry(
pageUrl,
`craft:html:${ctx.cat.key}:p${p}`,
ctx.store.ua
ctx.store.ua,
);
htmlPagesFetched++;
@ -215,9 +186,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
}
if (!htmlMap.size) {
ctx.logger.warn(
`${ctx.catPrefixOut} | HTML listing returned 0 items; refusing JSON-only discovery`
);
ctx.logger.warn(`${ctx.catPrefixOut} | HTML listing returned 0 items; refusing JSON-only discovery`);
}
const jsonMap = new Map();
@ -225,10 +194,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
if (htmlMap.size) {
const start = new URL(ctx.cat.startUrl);
const m = start.pathname.match(/^\/collections\/([^/]+)/i);
if (!m)
throw new Error(
`CraftCellars: couldn't extract collection handle from ${ctx.cat.startUrl}`
);
if (!m) throw new Error(`CraftCellars: couldn't extract collection handle from ${ctx.cat.startUrl}`);
const collectionHandle = m[1];
const limit = 250;
@ -236,19 +202,12 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
let jsonPagesFetched = 0;
while (true) {
if (jsonPage > 1 && perJsonPageDelayMs > 0)
await sleep(perJsonPageDelayMs);
if (jsonPage > 1 && perJsonPageDelayMs > 0) await sleep(perJsonPageDelayMs);
const url = `https://${ctx.store.host}/collections/${collectionHandle}/products.json?limit=${limit}&page=${jsonPage}`;
const r = await ctx.http.fetchJsonWithRetry(
url,
`craft:coljson:${ctx.cat.key}:p${jsonPage}`,
ctx.store.ua
);
const r = await ctx.http.fetchJsonWithRetry(url, `craft:coljson:${ctx.cat.key}:p${jsonPage}`, ctx.store.ua);
const products = Array.isArray(r?.json?.products)
? r.json.products
: [];
const products = Array.isArray(r?.json?.products) ? r.json.products : [];
jsonPagesFetched++;
if (!products.length) break;
@ -257,16 +216,11 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
const handle = String(p?.handle || "");
if (!handle) continue;
const prodUrl = canonicalizeCraftProductUrl(
`https://${ctx.store.host}/products/${handle}`
);
const prodUrl = canonicalizeCraftProductUrl(`https://${ctx.store.host}/products/${handle}`);
if (!htmlMap.has(prodUrl)) continue;
const variants = Array.isArray(p?.variants) ? p.variants : [];
const v =
variants.find((x) => x && x.available === true) ||
variants[0] ||
null;
const v = variants.find((x) => x && x.available === true) || variants[0] || null;
const sku = normalizeCspc(v?.sku || "");
const price = v?.price ? usdFromShopifyPriceStr(v.price) : "";
@ -274,13 +228,9 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
let img = "";
const images = Array.isArray(p?.images) ? p.images : [];
if (images[0]) {
img =
typeof images[0] === "string"
? images[0]
: String(images[0]?.src || images[0]?.url || "");
img = typeof images[0] === "string" ? images[0] : String(images[0]?.src || images[0]?.url || "");
}
if (!img && p?.image)
img = String(p.image?.src || p.image?.url || p.image || "");
if (!img && p?.image) img = String(p.image?.src || p.image?.url || p.image || "");
img = String(img || "").trim();
if (img.startsWith("//")) img = `https:${img}`;
@ -291,9 +241,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
if (++jsonPage > 200) break;
}
ctx.logger.ok(
`${ctx.catPrefixOut} | HTML pages=${htmlPagesFetched} JSON pages=${jsonPagesFetched}`
);
ctx.logger.ok(`${ctx.catPrefixOut} | HTML pages=${htmlPagesFetched} JSON pages=${jsonPagesFetched}`);
}
const discovered = new Map();
@ -308,17 +256,14 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
// reuse cached SKU unless we found something better this run
sku: pickBetterSku(j?.sku || "", prev?.sku || ""),
// reuse cached image if we didn't find one
img: (j?.img || it.img || prev?.img || ""),
img: j?.img || it.img || prev?.img || "",
});
}
/* ---------- NEW: product page SKU fallback (cached; only when needed) ---------- */
const perProductSkuDelayMs = Math.max(
0,
cfgNum(
ctx?.cat?.skuPageDelayMs,
cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)
)
cfgNum(ctx?.cat?.skuPageDelayMs, cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)),
);
let skuPagesFetched = 0;
@ -332,10 +277,8 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
try {
const { text } = await ctx.http.fetchTextWithRetry(
it.url,
`craft:prodpage:${ctx.cat.key}:${Buffer.from(it.url)
.toString("base64")
.slice(0, 24)}`,
ctx.store.ua
`craft:prodpage:${ctx.cat.key}:${Buffer.from(it.url).toString("base64").slice(0, 24)}`,
ctx.store.ua,
);
skuPagesFetched++;
@ -346,21 +289,11 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
}
}
ctx.logger.ok(
`${ctx.catPrefixOut} | SKU fallback pages=${skuPagesFetched}`
);
ctx.logger.ok(`${ctx.catPrefixOut} | SKU fallback pages=${skuPagesFetched}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`
);
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
const {
merged,
newItems,
updatedItems,
removedItems,
restoredItems,
} = mergeDiscoveredIntoDb(prevDb, discovered, {
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
});
@ -395,7 +328,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
newItems,
updatedItems,
removedItems,
restoredItems
restoredItems,
);
}
@ -416,8 +349,7 @@ function createStore(defaultUa) {
{
key: "whisky",
label: "Whisky",
startUrl:
"https://craftcellars.ca/collections/whisky?filter.v.availability=1",
startUrl: "https://craftcellars.ca/collections/whisky?filter.v.availability=1",
pageConcurrency: 1,
pageStaggerMs: 10000,
discoveryDelayMs: 10000,
@ -426,8 +358,7 @@ function createStore(defaultUa) {
{
key: "rum",
label: "Rum",
startUrl:
"https://craftcellars.ca/collections/rum?filter.v.availability=1",
startUrl: "https://craftcellars.ca/collections/rum?filter.v.availability=1",
pageConcurrency: 1,
pageStaggerMs: 10000,
discoveryDelayMs: 10000,

View file

@ -77,9 +77,7 @@ function extractGullSkuFromProductPage(html) {
const s = String(html || "");
// Most reliable: <span class="sku">67424</span>
const m1 = s.match(
/<span\b[^>]*class=["'][^"']*\bsku\b[^"']*["'][^>]*>\s*([0-9]{3,10})\s*<\/span>/i
);
const m1 = s.match(/<span\b[^>]*class=["'][^"']*\bsku\b[^"']*["'][^>]*>\s*([0-9]{3,10})\s*<\/span>/i);
if (m1?.[1]) return normalizeGullSku(m1[1]);
// Fallback: "SKU: 67424" text
@ -120,10 +118,7 @@ async function fetchWith429Backoff(url, { fetchFn, headers, maxRetries = 2 }) {
if (attempt >= maxRetries) throw new Error(`HTTP 429 fetching ${url}`);
// Respect Retry-After if present; otherwise progressive backoff.
const ra =
res.headers && typeof res.headers.get === "function"
? res.headers.get("retry-after")
: null;
const ra = res.headers && typeof res.headers.get === "function" ? res.headers.get("retry-after") : null;
const waitSec = ra && /^\d+$/.test(ra) ? parseInt(ra, 10) : 15 * (attempt + 1);
await new Promise((r) => setTimeout(r, waitSec * 1000));
@ -137,10 +132,7 @@ async function fetchWith429Backoff(url, { fetchFn, headers, maxRetries = 2 }) {
*
* NEW: accepts prevDb so we can skip fetch if URL already has a good SKU cached.
*/
async function hydrateGullSkus(
items,
{ fetchFn, ua, minIntervalMs = 12000, maxRetries = 2, prevDb } = {}
) {
async function hydrateGullSkus(items, { fetchFn, ua, minIntervalMs = 12000, maxRetries = 2, prevDb } = {}) {
if (!fetchFn) throw new Error("hydrateGullSkus requires opts.fetchFn");
const schedule = createMinIntervalLimiter(minIntervalMs);
@ -162,9 +154,7 @@ async function hydrateGullSkus(
if (!isGeneratedUrlSku(it.sku)) continue; // only where required
const html = await schedule(() =>
fetchWith429Backoff(it.url, { fetchFn, headers, maxRetries })
);
const html = await schedule(() => fetchWith429Backoff(it.url, { fetchFn, headers, maxRetries }));
const realSku = extractGullSkuFromProductPage(html);
if (realSku) it.sku = pickBetterSku(realSku, it.sku);
@ -178,9 +168,7 @@ function parseProductsGull(html, ctx) {
const items = [];
// split on <li class="product ...">
const parts = s.split(
/<li\b[^>]*class=["'][^"']*\bproduct\b[^"']*["'][^>]*>/i
);
const parts = s.split(/<li\b[^>]*class=["'][^"']*\bproduct\b[^"']*["'][^>]*>/i);
if (parts.length <= 1) return items;
const base = `https://${(ctx && ctx.store && ctx.store.host) || "gullliquorstore.com"}/`;
@ -191,7 +179,7 @@ function parseProductsGull(html, ctx) {
if (!looksInStock(block)) continue;
const hrefM = block.match(
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bwoocommerce-LoopProduct-link\b/i
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bwoocommerce-LoopProduct-link\b/i,
);
if (!hrefM || !hrefM[1]) continue;
@ -203,7 +191,7 @@ function parseProductsGull(html, ctx) {
}
const titleM = block.match(
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
);
const name = cleanText(decodeHtml(titleM ? titleM[1] : ""));
if (!name) continue;
@ -245,8 +233,7 @@ function createStore(defaultUa) {
{
key: "whisky",
label: "Whisky",
startUrl:
"https://gullliquorstore.com/product-category/spirits/?spirit_type=whisky",
startUrl: "https://gullliquorstore.com/product-category/spirits/?spirit_type=whisky",
discoveryStartPage: 3,
discoveryStep: 2,
pageConcurrency: 1,
@ -256,8 +243,7 @@ function createStore(defaultUa) {
{
key: "rum",
label: "Rum",
startUrl:
"https://gullliquorstore.com/product-category/spirits/?spirit_type=rum",
startUrl: "https://gullliquorstore.com/product-category/spirits/?spirit_type=rum",
discoveryStartPage: 3,
discoveryStep: 2,
pageConcurrency: 1,

View file

@ -20,7 +20,7 @@ function parseProductsKegNCork(html, ctx) {
const block = "<li" + blocks[i];
const mTitle = block.match(
/<h4\b[^>]*class=["'][^"']*\bcard-title\b[^"']*["'][^>]*>[\s\S]*?<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/i
/<h4\b[^>]*class=["'][^"']*\bcard-title\b[^"']*["'][^>]*>[\s\S]*?<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/i,
);
if (!mTitle) continue;
@ -49,7 +49,6 @@ function parseProductsKegNCork(html, ctx) {
return [...uniq.values()];
}
function createStore(defaultUa) {
return {
key: "kegncork",

View file

@ -114,7 +114,10 @@ function kwmExtractPrice(block) {
const priceDiv = kwmExtractFirstDivByClass(block, "product-price");
if (!priceDiv) return "";
const cleaned = String(priceDiv).replace(/<span\b[^>]*class=["'][^"']*\bstrike\b[^"']*["'][^>]*>[\s\S]*?<\/span>/gi, " ");
const cleaned = String(priceDiv).replace(
/<span\b[^>]*class=["'][^"']*\bstrike\b[^"']*["'][^>]*>[\s\S]*?<\/span>/gi,
" ",
);
const txt = cleanText(decodeHtml(stripTags(cleaned)));
const dollars = [...txt.matchAll(/\$\s*\d+(?:\.\d{2})?/g)];
@ -160,7 +163,6 @@ function parseProductsKWM(html, ctx) {
return [...uniq.values()];
}
function createStore(defaultUa) {
return {
key: "kwm",

View file

@ -157,7 +157,10 @@ function legacyProductToItem(p, ctx) {
const base = "https://www.legacyliquorstore.com";
// Matches observed pattern: /LL/product/spirits/<category>/<slug>
const url = new URL(`/LL/product/spirits/${encodeURIComponent(ctx.cat.key)}/${encodeURIComponent(slug)}`, base).toString();
const url = new URL(
`/LL/product/spirits/${encodeURIComponent(ctx.cat.key)}/${encodeURIComponent(slug)}`,
base,
).toString();
const nameRaw =
String(v?.fullName || "").trim() ||
@ -165,13 +168,12 @@ function legacyProductToItem(p, ctx) {
const name = String(nameRaw || "").trim();
if (!name) return null;
const price =
cad(v?.price) ||
cad(p?.priceFrom) ||
cad(p?.priceTo) ||
"";
const price = cad(v?.price) || cad(p?.priceFrom) || cad(p?.priceTo) || "";
const sku = normalizeLegacySku(v?.sku, { storeLabel: ctx.store.name, url }) || normalizeLegacySku(url, { storeLabel: ctx.store.name, url }) ||"";
const sku =
normalizeLegacySku(v?.sku, { storeLabel: ctx.store.name, url }) ||
normalizeLegacySku(url, { storeLabel: ctx.store.name, url }) ||
"";
const img = normalizeAbsUrl(v?.image || "");
return { name, price, url, sku, img };
@ -206,7 +208,11 @@ async function legacyFetchPage(ctx, pageCursor, pageLimit) {
},
};
return await ctx.http.fetchJsonWithRetry(LEGACY_GQL_URL, `legacy:${ctx.cat.key}:${pageCursor || "first"}`, ctx.store.ua, {
return await ctx.http.fetchJsonWithRetry(
LEGACY_GQL_URL,
`legacy:${ctx.cat.key}:${pageCursor || "first"}`,
ctx.store.ua,
{
method: "POST",
headers: {
Accept: "application/json",
@ -215,7 +221,8 @@ async function legacyFetchPage(ctx, pageCursor, pageLimit) {
Referer: "https://www.legacyliquorstore.com/",
},
body: JSON.stringify(body),
});
},
);
}
async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
@ -257,8 +264,8 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(done, done)} | ${String(r.status || "").padEnd(3)} | ${pctStr(done, done)} | kept=${padLeft(
kept,
3
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
3,
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
if (!next || !arr.length) break;
@ -266,13 +273,15 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
cursor = next;
}
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
});
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
const elapsed = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -293,7 +302,15 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {

View file

@ -37,16 +37,18 @@ function parseProductsMaltsAndGrains(html, ctx) {
const cats = [];
for (const m of String(classAttr || "").matchAll(/\bproduct_cat-([a-z0-9_-]+)\b/gi)) {
const v = String(m[1] || "").trim().toLowerCase();
const v = String(m[1] || "")
.trim()
.toLowerCase();
if (v) cats.push(v);
}
let href =
block.match(
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b/i
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b/i,
)?.[1] ||
block.match(
/<a\b[^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b[^"']*["'][^>]*href=["']([^"']+)["']/i
/<a\b[^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b[^"']*["'][^>]*href=["']([^"']+)["']/i,
)?.[2] ||
block.match(/<a\b[^>]*href=["']([^"']*\/product\/[^"']+)["']/i)?.[1];
@ -61,7 +63,7 @@ function parseProductsMaltsAndGrains(html, ctx) {
if (!/^https?:\/\//i.test(url)) continue;
const mTitle = block.match(
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
);
const name = mTitle && mTitle[1] ? cleanText(decodeHtml(stripTags(mTitle[1]))) : "";
if (!name) continue;
@ -71,7 +73,7 @@ function parseProductsMaltsAndGrains(html, ctx) {
const sku = normalizeCspc(
block.match(/\bdata-product_sku=["']([^"']+)["']/i)?.[1] ||
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
""
"",
);
const img = extractFirstImgUrl(block, base);
@ -84,7 +86,6 @@ function parseProductsMaltsAndGrains(html, ctx) {
return [...uniq.values()];
}
function createStore(defaultUa) {
return {
key: "maltsandgrains",

View file

@ -10,7 +10,7 @@ const { mergeDiscoveredIntoDb } = require("../tracker/merge");
const { addCategoryResultToReport } = require("../tracker/report");
function allowSierraUrlRumWhisky(item) {
const u = (item && item.url) ? String(item.url) : "";
const u = item && item.url ? String(item.url) : "";
const s = u.toLowerCase();
if (!/^https?:\/\/sierraspringsliquor\.ca\//.test(s)) return false;
return /\b(rum|whisk(?:e)?y)\b/.test(s);
@ -48,26 +48,22 @@ function parseWooStoreProductsJson(payload, ctx) {
if (!Array.isArray(data)) return items;
for (const p of data) {
const url = (p && p.permalink) ? String(p.permalink) : "";
const url = p && p.permalink ? String(p.permalink) : "";
if (!url) continue;
const name = (p && p.name) ? cleanText(decodeHtml(String(p.name))) : "";
const name = p && p.name ? cleanText(decodeHtml(String(p.name))) : "";
if (!name) continue;
const price = formatWooStorePrice(p.prices);
const rawSku =
(typeof p?.sku === "string" && p.sku.trim()) ? p.sku.trim()
: (p && (p.id ?? p.id === 0)) ? String(p.id)
: "";
typeof p?.sku === "string" && p.sku.trim() ? p.sku.trim() : p && (p.id ?? p.id === 0) ? String(p.id) : "";
const taggedSku = /^\d{1,11}$/.test(rawSku) ? `id:${rawSku}` : rawSku;
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
const img =
(p.images && Array.isArray(p.images) && p.images[0] && p.images[0].src)
? String(p.images[0].src)
: null;
p.images && Array.isArray(p.images) && p.images[0] && p.images[0].src ? String(p.images[0].src) : null;
const item = { name, price, url, sku, img };
@ -96,16 +92,18 @@ function parseWooProductsHtml(html, ctx) {
if (/class=["'][^"']*\bproduct-category\b/i.test(chunk)) continue;
const endIdx = chunk.search(/<\/li>/i);
const block = (endIdx >= 0 ? chunk.slice(0, endIdx + 5) : chunk);
const block = endIdx >= 0 ? chunk.slice(0, endIdx + 5) : chunk;
const hrefs = [...block.matchAll(/<a\b[^>]*href=["']([^"']+)["']/gi)].map(m => m[1]);
const href = hrefs.find(h => !/add-to-cart=|\/cart\/|\/checkout\//i.test(h)) || "";
const hrefs = [...block.matchAll(/<a\b[^>]*href=["']([^"']+)["']/gi)].map((m) => m[1]);
const href = hrefs.find((h) => !/add-to-cart=|\/cart\/|\/checkout\//i.test(h)) || "";
if (!href) continue;
const url = new URL(decodeHtml(href), base).toString();
const nameHtml =
block.match(/<h2\b[^>]*class=["'][^"']*woocommerce-loop-product__title[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i)?.[1] ||
block.match(
/<h2\b[^>]*class=["'][^"']*woocommerce-loop-product__title[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
)?.[1] ||
block.match(/<h3\b[^>]*>([\s\S]*?)<\/h3>/i)?.[1] ||
"";
const name = cleanText(decodeHtml(nameHtml));
@ -156,10 +154,10 @@ function parseProductsSierra(body, ctx) {
if (blocks.length > 1) {
const items = [];
for (let i = 1; i < blocks.length; i++) {
const block = "<div class=\"tmb" + blocks[i];
const block = '<div class="tmb' + blocks[i];
const titleMatch = block.match(
/<h3\b[^>]*class=["'][^"']*t-entry-title[^"']*["'][^>]*>\s*<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>\s*<\/h3>/i
/<h3\b[^>]*class=["'][^"']*t-entry-title[^"']*["'][^>]*>\s*<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>\s*<\/h3>/i,
);
if (!titleMatch) continue;
@ -174,9 +172,7 @@ function parseProductsSierra(body, ctx) {
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
"";
const taggedSku = /^\d{1,11}$/.test(String(rawSku).trim())
? `id:${String(rawSku).trim()}`
: rawSku;
const taggedSku = /^\d{1,11}$/.test(String(rawSku).trim()) ? `id:${String(rawSku).trim()}` : rawSku;
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
const img = extractFirstImgUrl(block, base);
@ -202,9 +198,7 @@ function parseProductsSierra(body, ctx) {
function extractProductCatTermId(html) {
const s = String(html || "");
// Typical body classes contain: "tax-product_cat term-<slug> term-1131 ..."
const m =
s.match(/tax-product_cat[^"']{0,400}\bterm-(\d{1,10})\b/i) ||
s.match(/\bterm-(\d{1,10})\b/i);
const m = s.match(/tax-product_cat[^"']{0,400}\bterm-(\d{1,10})\b/i) || s.match(/\bterm-(\d{1,10})\b/i);
if (!m) return null;
const n = Number(m[1]);
return Number.isFinite(n) ? n : null;
@ -222,7 +216,9 @@ async function getWooCategoryIdForCat(ctx) {
const id = extractProductCatTermId(text);
if (!id) {
ctx.logger.warn(`${ctx.catPrefixOut} | Could not infer product_cat term id from category page; falling back to HTML parsing only.`);
ctx.logger.warn(
`${ctx.catPrefixOut} | Could not infer product_cat term id from category page; falling back to HTML parsing only.`,
);
ctx.cat._wooCategoryId = null;
return null;
}
@ -260,18 +256,15 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
const { text, status, bytes, ms, finalUrl } = await ctx.http.fetchTextWithRetry(
pageUrl,
`page:${ctx.store.key}:${ctx.cat.key}:${page}`,
ctx.store.ua
ctx.store.ua,
);
// IMPORTANT:
// Parse WITHOUT allowUrl so pagination is based on real API page size
const ctxNoFilter =
typeof ctx?.cat?.allowUrl === "function"
? { ...ctx, cat: { ...ctx.cat, allowUrl: null } }
: ctx;
typeof ctx?.cat?.allowUrl === "function" ? { ...ctx, cat: { ...ctx.cat, allowUrl: null } } : ctx;
const itemsAll =
(ctx.store.parseProducts || ctx.config.defaultParseProducts)(text, ctxNoFilter, finalUrl);
const itemsAll = (ctx.store.parseProducts || ctx.config.defaultParseProducts)(text, ctxNoFilter, finalUrl);
const rawCount = itemsAll.length;
@ -284,7 +277,7 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
}
logger.ok(
`${ctx.catPrefixOut} | Page ${String(page).padStart(3, " ")} | ${String(status).padStart(3, " ")} | raw=${String(rawCount).padStart(3, " ")} kept=${String(items.length).padStart(3, " ")} | bytes=${String(bytes || 0).padStart(8, " ")} | ${(ms / 1000).toFixed(1).padStart(6, " ")}s`
`${ctx.catPrefixOut} | Page ${String(page).padStart(3, " ")} | ${String(status).padStart(3, " ")} | raw=${String(rawCount).padStart(3, " ")} kept=${String(items.length).padStart(3, " ")} | bytes=${String(bytes || 0).padStart(8, " ")} | ${(ms / 1000).toFixed(1).padStart(6, " ")}s`,
);
// Stop only when the API page itself is empty
@ -300,14 +293,11 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
const {
merged,
newItems,
updatedItems,
removedItems,
restoredItems,
metaChangedItems,
} = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{ storeLabel: ctx.store.name },
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -344,7 +334,7 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
newItems,
updatedItems,
removedItems,
restoredItems
restoredItems,
);
}

View file

@ -51,10 +51,7 @@ function normalizePrice(str) {
function pickPriceFromArticle(articleHtml) {
const a = String(articleHtml || "");
const noMember = a.replace(
/<div\b[^>]*class=["'][^"']*\bwhiskyfolk-price\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi,
" "
);
const noMember = a.replace(/<div\b[^>]*class=["'][^"']*\bwhiskyfolk-price\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, " ");
const ins = noMember.match(/<ins\b[^>]*>[\s\S]*?(\$[\s\S]{0,32}?)<\/ins>/i);
if (ins && ins[1]) return normalizePrice(ins[1]);
@ -62,9 +59,7 @@ function pickPriceFromArticle(articleHtml) {
const reg = noMember.match(/class=["'][^"']*\bregular-price-card\b[^"']*["'][^>]*>\s*([^<]+)/i);
if (reg && reg[1]) return normalizePrice(reg[1]);
const priceDiv = noMember.match(
/<div\b[^>]*class=["'][^"']*\bproduct-price\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/i
);
const priceDiv = noMember.match(/<div\b[^>]*class=["'][^"']*\bproduct-price\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/i);
const scope = priceDiv && priceDiv[1] ? priceDiv[1] : noMember;
return normalizePrice(scope);
@ -157,7 +152,6 @@ function parseProductFromArticle(articleHtml) {
productId,
img,
};
}
/* ---------------- Store API paging ---------------- */
@ -185,12 +179,16 @@ function buildStoreApiBaseUrlFromCategoryUrl(startUrl) {
}
function hasCategorySlug(p, wanted) {
const w = String(wanted || "").trim().toLowerCase();
const w = String(wanted || "")
.trim()
.toLowerCase();
if (!w) return true;
const cats = Array.isArray(p?.categories) ? p.categories : [];
for (const c of cats) {
const slug = String(c?.slug || "").trim().toLowerCase();
const slug = String(c?.slug || "")
.trim()
.toLowerCase();
if (slug === w) return true;
}
return false;
@ -304,7 +302,7 @@ function avoidMassRemoval(prevDb, discovered, ctx, reason) {
if (ratio >= 0.6) return false;
ctx.logger.warn?.(
`${ctx.catPrefixOut} | Strath partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`
`${ctx.catPrefixOut} | Strath partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`,
);
if (prevDb && typeof prevDb.entries === "function") {
@ -353,8 +351,8 @@ async function scanCategoryStrath(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(1, 1)} | ${String(listingStatus || "").padEnd(3)} | ${pctStr(1, 1)} | items=${padLeft(
listingItems,
3
)} | bytes=${kbStr(listingBytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(listingMs)}`
3,
)} | bytes=${kbStr(listingBytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(listingMs)}`,
);
const apiBase = buildStoreApiBaseUrlFromCategoryUrl(listingFinalUrl || ctx.cat.startUrl);
@ -362,7 +360,9 @@ async function scanCategoryStrath(ctx, prevDb, report) {
const perPage = 100;
const maxPagesCap = ctx.config.maxPages === null ? 5000 : ctx.config.maxPages;
const wantedSlug = String(ctx.cat.apiCategorySlug || "").trim().toLowerCase();
const wantedSlug = String(ctx.cat.apiCategorySlug || "")
.trim()
.toLowerCase();
let donePages = 0;
let emptyMatchPages = 0;
@ -399,7 +399,6 @@ async function scanCategoryStrath(ctx, prevDb, report) {
const sku = normalizeProductSku(p);
const productId = normalizeProductId(p);
const prev = discovered.get(url) || null;
const apiImg = normalizeProductImage(p) || "";
@ -411,7 +410,6 @@ async function scanCategoryStrath(ctx, prevDb, report) {
const newSku = sku || fallbackSku;
const mergedSku = pickBetterSku(newSku, prev && prev.sku);
discovered.set(url, {
name,
price,
@ -426,8 +424,8 @@ async function scanCategoryStrath(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(3)} | kept=${padLeft(
kept,
3
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
3,
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
if (wantedSlug) {
@ -458,7 +456,7 @@ async function scanCategoryStrath(ctx, prevDb, report) {
const elapsed = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -479,7 +477,15 @@ async function scanCategoryStrath(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {

View file

@ -176,7 +176,6 @@ function parseDisplayPriceFromHtml(html) {
return Number.isFinite(n) ? n : null;
}
function pickAnySkuFromProduct(p) {
const vs = Array.isArray(p?.variants) ? p.variants : [];
for (const v of vs) {
@ -353,7 +352,7 @@ async function supplementImageFromSku(ctx, skuProbe) {
const v = pickInStockVariantWithFallback(prod);
const img = normalizeAbsUrl(
firstNonEmptyStr(v?.image, prod?.gulpImages, prod?.posImages, prod?.customImages, prod?.imageIds)
firstNonEmptyStr(v?.image, prod?.gulpImages, prod?.posImages, prod?.customImages, prod?.imageIds),
);
return img ? { img } : null;
@ -369,9 +368,7 @@ function parseSkuFromHtml(html) {
const s = String(html || "");
// 1) Visible block: <div class="sku ...">SKU: 67433</div>
const m1 =
s.match(/>\s*SKU:\s*([A-Za-z0-9._-]+)\s*</i) ||
s.match(/\bSKU:\s*([A-Za-z0-9._-]+)\b/i);
const m1 = s.match(/>\s*SKU:\s*([A-Za-z0-9._-]+)\s*</i) || s.match(/\bSKU:\s*([A-Za-z0-9._-]+)\b/i);
if (m1 && m1[1]) return String(m1[1]).trim();
// 2) Embedded SAPPER preloaded JSON has variants with `"sku":"67433"`
@ -448,7 +445,6 @@ async function tudorDetailFromProductPage(ctx, url) {
return out;
}
/* ---------------- item builder (fast, no extra calls) ---------------- */
function tudorItemFromProductFast(p, ctx) {
@ -469,9 +465,7 @@ function tudorItemFromProductFast(p, ctx) {
const skuRaw = String(v?.sku || "").trim() || pickAnySkuFromProduct(p);
const sku = normalizeTudorSku(skuRaw);
const img = normalizeAbsUrl(
firstNonEmptyStr(v?.image, p?.gulpImages, p?.posImages, p?.customImages, p?.imageIds)
);
const img = normalizeAbsUrl(firstNonEmptyStr(v?.image, p?.gulpImages, p?.posImages, p?.customImages, p?.imageIds));
// NEW: keep lightweight variant snapshot so repair can match HTML SKU -> exact GQL variant price
const variants = Array.isArray(p?.variants)
@ -492,9 +486,7 @@ async function tudorRepairItem(ctx, it) {
// Determine if we need HTML for precision:
// - Missing/synthetic SKU (existing behavior)
// - OR multi-variant product where fast-path may choose the wrong variant for this URL
const inStockVariants = Array.isArray(it._variants)
? it._variants.filter((v) => Number(v?.quantity) > 0)
: [];
const inStockVariants = Array.isArray(it._variants) ? it._variants.filter((v) => Number(v?.quantity) > 0) : [];
const hasMultiInStock = inStockVariants.length >= 2;
@ -513,7 +505,9 @@ async function tudorRepairItem(ctx, it) {
// Price precision:
// - Best: match HTML SKU to a GQL variant sku => exact numeric variant price
// - Fallback: use displayed HTML price
const htmlSkuDigits = String(d?.sku || "").replace(/^id:/i, "").trim();
const htmlSkuDigits = String(d?.sku || "")
.replace(/^id:/i, "")
.trim();
if (htmlSkuDigits && inStockVariants.length) {
const match = inStockVariants.find((v) => String(v?.sku || "").trim() === htmlSkuDigits);
@ -542,7 +536,6 @@ async function tudorRepairItem(ctx, it) {
return it;
}
/* ---------------- scanner ---------------- */
async function scanCategoryTudor(ctx, prevDb, report) {
@ -586,8 +579,8 @@ async function scanCategoryTudor(ctx, prevDb, report) {
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(page, maxPages)} | 200 | items=${padLeft(
kept,
3
)} | bytes=${kbStr(0)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
3,
)} | bytes=${kbStr(0)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
);
cursor = prod?.nextPageCursor || null;
@ -623,7 +616,7 @@ async function scanCategoryTudor(ctx, prevDb, report) {
}
ctx.logger.ok(
`${ctx.catPrefixOut} | Unique products: ${discovered.size} | detail(html=${htmlUsed}/${htmlBudget}, gql=${gqlUsed}/${gqlBudget})`
`${ctx.catPrefixOut} | Unique products: ${discovered.size} | detail(html=${htmlUsed}/${htmlBudget}, gql=${gqlUsed}/${gqlBudget})`,
);
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
@ -654,7 +647,15 @@ async function scanCategoryTudor(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
/* ---------------- store ---------------- */

View file

@ -162,7 +162,8 @@ function createStore(defaultUa) {
{
key: "rum-cane-spirit",
label: "Rum / Cane Spirit",
startUrl: "https://vesselliquor.com/collections/rum-cane-spirit?sort_by=title-ascending&filter.v.availability=1",
startUrl:
"https://vesselliquor.com/collections/rum-cane-spirit?sort_by=title-ascending&filter.v.availability=1",
discoveryStartPage: 20,
discoveryStep: 10,
},

View file

@ -110,9 +110,13 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
ctx.logger.warn(`${ctx.catPrefixOut} | Vintage API fetch failed: ${e?.message || e}`);
const discovered = new Map();
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{
storeLabel: ctx.store.name,
});
},
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -134,7 +138,15 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
report.totals.updatedCount += updatedItems.length;
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
return;
}
@ -142,7 +154,7 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
ctx.logger.ok(
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
);
const pages = [];
@ -167,14 +179,14 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
donePages++;
ctx.logger.ok(
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pages.length)} | ${String(r.status || "").padEnd(
3
3,
)} | ${pctStr(donePages, pages.length)} | items=${padLeft(items.length, 3)} | bytes=${kbStr(
r.bytes
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
r.bytes,
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
);
return items;
}
},
);
const discovered = new Map();
@ -186,7 +198,9 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
}
}
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
ctx.logger.ok(
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
);
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
storeLabel: ctx.store.name,
@ -197,7 +211,7 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
const elapsed = Date.now() - t0;
ctx.logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -218,7 +232,15 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
report.totals.removedCount += removedItems.length;
report.totals.restoredCount += restoredItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
function createStore(defaultUa) {

View file

@ -47,9 +47,8 @@ function extractWillowCardPrice(block) {
const current =
b.match(
/grid-product__price--current[\s\S]*?<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i
)?.[1] ||
b.match(/<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i)?.[1];
/grid-product__price--current[\s\S]*?<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i,
)?.[1] || b.match(/<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i)?.[1];
if (current) return current.replace(/\s+/g, "");
@ -101,10 +100,7 @@ function parseProductsWillowPark(html, ctx, finalUrl) {
const img = extractFirstImgUrl(block, base);
const pid = block.match(/\bdata-product-id=["'](\d+)["']/i)?.[1] || "";
const sku =
extractSkuFromUrlOrHref(href) ||
extractSkuFromUrlOrHref(url) ||
extractSkuFromWillowBlock(block);
const sku = extractSkuFromUrlOrHref(href) || extractSkuFromUrlOrHref(url) || extractSkuFromWillowBlock(block);
items.push({ name, price, url, sku, img, pid });
}
@ -164,9 +160,7 @@ function extractStorefrontTokenFromHtml(html) {
}
// 2) meta name="shopify-checkout-api-token"
const m = s.match(
/<meta[^>]+name=["']shopify-checkout-api-token["'][^>]+content=["']([^"']+)["']/i
)?.[1];
const m = s.match(/<meta[^>]+name=["']shopify-checkout-api-token["'][^>]+content=["']([^"']+)["']/i)?.[1];
return String(m || "").trim();
}
@ -213,7 +207,6 @@ function normalizeWillowGqlSku(rawSku) {
return s;
}
async function willowFetchSkuByPid(ctx, pid) {
const id = String(pid || "").trim();
if (!id) return "";

View file

@ -54,7 +54,9 @@ function progCell(v) {
}
function logProgressLine(logger, ctx, action, statusRaw, statusOk, progVal, rest) {
logger.ok(`${ctx.catPrefixOut} | ${actionCell(action)} | ${statusCell(logger, statusRaw, statusOk)} | ${progCell(progVal)} | ${rest}`);
logger.ok(
`${ctx.catPrefixOut} | ${actionCell(action)} | ${statusCell(logger, statusRaw, statusOk)} | ${progCell(progVal)} | ${rest}`,
);
}
function makeCatPrefixers(stores, logger) {
@ -168,7 +170,7 @@ async function probePage(ctx, baseUrl, pageNum, state) {
r.ok ? "OK" : "MISS",
Boolean(r.ok),
prog,
`items=${padLeftV(r.items, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
`items=${padLeftV(r.items, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
);
return r;
@ -213,12 +215,20 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
// Fetch page 1 ONCE and try to extract total pages from pagination.
const url1 = makePageUrlForCtx(ctx, baseUrl, 1);
const t0 = Date.now();
const { text: html1, ms, status, bytes, finalUrl } = await ctx.http.fetchTextWithRetry(url1, "discover", ctx.store.ua);
const {
text: html1,
ms,
status,
bytes,
finalUrl,
} = await ctx.http.fetchTextWithRetry(url1, "discover", ctx.store.ua);
const pMs = Date.now() - t0;
if (typeof ctx.store.isEmptyListingPage === "function") {
if (ctx.store.isEmptyListingPage(html1, ctx, url1)) {
ctx.logger.warn(`${ctx.store.name} | ${ctx.cat.label} | Page 1 did not look like a listing. Defaulting to 1.`);
ctx.logger.warn(
`${ctx.store.name} | ${ctx.cat.label} | Page 1 did not look like a listing. Defaulting to 1.`,
);
return 1;
}
}
@ -233,7 +243,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
items1 > 0 ? "OK" : "MISS",
items1 > 0,
discoverProg(state),
`items=${padLeftV(items1, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms || pMs)}`
`items=${padLeftV(items1, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms || pMs)}`,
);
if (items1 <= 0) {
@ -246,8 +256,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
// Shopify collections with filters often lie about pagination.
// If page 1 looks full, don't trust a tiny extracted count.
if (extracted && extracted >= 1) {
const looksTruncated =
extracted <= 2 && items1 >= 40; // Shopify default page size ≈ 48
const looksTruncated = extracted <= 2 && items1 >= 40; // Shopify default page size ≈ 48
if (!looksTruncated) {
ctx.logger.ok(`${ctx.catPrefixOut} | Total pages (from pagination): ${extracted}`);
@ -255,7 +264,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
}
ctx.logger.warn(
`${ctx.catPrefixOut} | Pagination says ${extracted} but page looks full; falling back to probe`
`${ctx.catPrefixOut} | Pagination says ${extracted} but page looks full; falling back to probe`,
);
}
@ -271,7 +280,9 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
if (!pr.ok) return await binaryFindLastOk(ctx, baseUrl, lastOk, probe, state);
lastOk = probe;
if (lastOk > 5000) {
ctx.logger.warn(`${ctx.store.name} | ${ctx.cat.label} | Discovery hit safety cap at ${lastOk}. Using that as total pages.`);
ctx.logger.warn(
`${ctx.store.name} | ${ctx.cat.label} | Discovery hit safety cap at ${lastOk}. Using that as total pages.`,
);
return lastOk;
}
}
@ -293,7 +304,9 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
const totalPages = await discoverTotalPagesFast(ctx, ctx.baseUrl, guess, step);
const scanPages = config.maxPages === null ? totalPages : Math.min(config.maxPages, totalPages);
logger.ok(`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
logger.ok(
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
);
const pages = [];
for (let p = 1; p <= scanPages; p++) pages.push(makePageUrlForCtx(ctx, ctx.baseUrl, p));
@ -306,11 +319,13 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
const perPageItems = await parallelMapStaggered(pages, pageConc, pageStagger, async (pageUrl, idx) => {
const pnum = idx + 1;
const { text: html, ms, bytes, status, finalUrl } = await ctx.http.fetchTextWithRetry(
pageUrl,
`page:${ctx.store.key}:${ctx.cat.key}:${pnum}`,
ctx.store.ua
);
const {
text: html,
ms,
bytes,
status,
finalUrl,
} = await ctx.http.fetchTextWithRetry(pageUrl, `page:${ctx.store.key}:${ctx.cat.key}:${pnum}`, ctx.store.ua);
const parser = ctx.store.parseProducts || config.defaultParseProducts;
const itemsRaw = parser(html, ctx, finalUrl);
@ -328,7 +343,7 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
status ? String(status) : "",
status >= 200 && status < 400,
pctStr(donePages, pages.length),
`items=${padLeft(items.length, 3)} | bytes=${kbStr(bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
`items=${padLeft(items.length, 3)} | bytes=${kbStr(bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
);
return items;
@ -349,8 +364,11 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } =
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
prevDb,
discovered,
{ storeLabel: ctx.store.name },
);
const dbObj = buildDbObject(ctx, merged);
writeJsonAtomic(ctx.dbFile, dbObj);
@ -359,7 +377,7 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
const elapsed = Date.now() - t0;
logger.ok(
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
);
report.categories.push({
@ -382,7 +400,15 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
report.totals.restoredCount += restoredItems.length;
report.totals.metaChangedCount += metaChangedItems.length;
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
addCategoryResultToReport(
report,
ctx.store.name,
ctx.cat.label,
newItems,
updatedItems,
removedItems,
restoredItems,
);
}
module.exports = { makeCatPrefixers, buildCategoryContext, loadCategoryDb, discoverAndScanCategory };

View file

@ -110,7 +110,8 @@ function buildCheapestSkuIndexFromAllDbs(dbDir, { skuMap } = {}) {
const skuKey = normalizeSkuKey(it?.sku || "", { storeLabel, url: it?.url || "" });
if (!skuKey) continue;
const canon = skuMap && typeof skuMap.canonicalSku === "function" ? skuMap.canonicalSku(skuKey) : skuKey;
const canon =
skuMap && typeof skuMap.canonicalSku === "function" ? skuMap.canonicalSku(skuKey) : skuKey;
const p = priceToNumber(it?.price || "");
if (!Number.isFinite(p) || p <= 0) continue;

View file

@ -12,7 +12,6 @@ function normImg(v) {
return s;
}
function dbStoreLabel(prevDb) {
return String(prevDb?.storeLabel || prevDb?.store || "").trim();
}
@ -21,7 +20,7 @@ function mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel } = {}) {
const effectiveStoreLabel = String(storeLabel || dbStoreLabel(prevDb)).trim();
if (!effectiveStoreLabel) {
throw new Error(
"mergeDiscoveredIntoDb: missing storeLabel; refusing to generate synthetic SKUs with fallback 'store'"
"mergeDiscoveredIntoDb: missing storeLabel; refusing to generate synthetic SKUs with fallback 'store'",
);
}

View file

@ -31,10 +31,23 @@ function createReport() {
function addCategoryResultToReport(report, storeName, catLabel, newItems, updatedItems, removedItems, restoredItems) {
const reportCatLabel = `${storeName} | ${catLabel}`;
for (const it of newItems) report.newItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
for (const it of newItems)
report.newItems.push({
catLabel: reportCatLabel,
name: it.name,
price: it.price || "",
sku: it.sku || "",
url: it.url,
});
for (const it of restoredItems)
report.restoredItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
report.restoredItems.push({
catLabel: reportCatLabel,
name: it.name,
price: it.price || "",
sku: it.sku || "",
url: it.url,
});
for (const u of updatedItems) {
report.updatedItems.push({
@ -48,7 +61,13 @@ function addCategoryResultToReport(report, storeName, catLabel, newItems, update
}
for (const it of removedItems)
report.removedItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
report.removedItems.push({
catLabel: reportCatLabel,
name: it.name,
price: it.price || "",
sku: it.sku || "",
url: it.url,
});
}
function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout && process.stdout.isTTY) } = {}) {
@ -64,7 +83,10 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
const durMs = endedAt - report.startedAt;
const storesSet = new Set(report.categories.map((c) => c.store));
const totalUnique = report.categories.reduce((acc, c) => acc + (Number.isFinite(c.discoveredUnique) ? c.discoveredUnique : 0), 0);
const totalUnique = report.categories.reduce(
(acc, c) => acc + (Number.isFinite(c.discoveredUnique) ? c.discoveredUnique : 0),
0,
);
let out = "";
const ln = (s = "") => {
@ -76,8 +98,8 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
ln(
paint("[OK] ", C.green) +
`Totals | Stores=${storesSet.size} | Categories=${report.categories.length} | Unique=${totalUnique} | New=${report.totals.newCount} | Restored=${report.totals.restoredCount} | Removed=${report.totals.removedCount} | PriceChanges=${report.totals.updatedCount} | Runtime=${secStr(
durMs
)}`
durMs,
)}`,
);
ln("");
@ -94,11 +116,13 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
}));
const catW = Math.min(48, Math.max(...rows.map((r) => r.cat.length), 8));
ln(`${padRight("Store | Category", catW)} ${padLeft("Pages", 5)} ${padLeft("Unique", 6)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)} ${padLeft("Sec", 7)}`);
ln(
`${padRight("Store | Category", catW)} ${padLeft("Pages", 5)} ${padLeft("Unique", 6)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)} ${padLeft("Sec", 7)}`,
);
ln(`${"-".repeat(catW)} ----- ------ ---- ---- ---- ---- -------`);
for (const r of rows) {
ln(
`${padRight(r.cat, catW)} ${padLeft(r.pages, 5)} ${padLeft(r.uniq, 6)} ${padLeft(r.newC, 4)} ${padLeft(r.resC, 4)} ${padLeft(r.remC, 4)} ${padLeft(r.updC, 4)} ${secStr(r.ms)}`
`${padRight(r.cat, catW)} ${padLeft(r.pages, 5)} ${padLeft(r.uniq, 6)} ${padLeft(r.newC, 4)} ${padLeft(r.resC, 4)} ${padLeft(r.remC, 4)} ${padLeft(r.updC, 4)} ${secStr(r.ms)}`,
);
}
ln("");
@ -108,7 +132,7 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
...report.newItems.map((x) => x.catLabel.length),
...report.restoredItems.map((x) => x.catLabel.length),
...report.updatedItems.map((x) => x.catLabel.length),
...report.removedItems.map((x) => x.catLabel.length)
...report.removedItems.map((x) => x.catLabel.length),
);
function storeFromCatLabel(catLabel) {
@ -163,7 +187,9 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
const sku = String(it.sku || "");
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
ln(`${paint("+", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`);
ln(
`${paint("+", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -174,11 +200,15 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
if (report.restoredItems.length) {
ln(paint(`RESTORED (${report.restoredItems.length})`, C.bold + C.green));
for (const it of report.restoredItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
for (const it of report.restoredItems.sort((a, b) =>
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
)) {
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
const sku = String(it.sku || "");
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
ln(`${paint("R", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`);
ln(
`${paint("R", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -193,7 +223,9 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
const sku = String(it.sku || "");
const availTag = availableAtInline(it.catLabel, sku, it.url);
ln(`${paint("-", C.yellow)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${availTag}`);
ln(
`${paint("-", C.yellow)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${availTag}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -235,7 +267,7 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
const cheapTag = cheaperAtInline(u.catLabel, sku, u.url, newRaw || "");
ln(
`${paint("~", C.cyan)} ${padRight(u.catLabel, reportLabelW)} | ${paint(u.name, C.bold)}${skuInline(sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}${cheapTag}`
`${paint("~", C.cyan)} ${padRight(u.catLabel, reportLabelW)} | ${paint(u.name, C.bold)}${skuInline(sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}${cheapTag}`,
);
ln(` ${paint(u.url, C.dim)}`);
}

View file

@ -3,12 +3,7 @@
const { createReport } = require("./report");
const { setTimeout: sleep } = require("timers/promises");
const {
makeCatPrefixers,
buildCategoryContext,
loadCategoryDb,
discoverAndScanCategory,
} = require("./category_scan");
const { makeCatPrefixers, buildCategoryContext, loadCategoryDb, discoverAndScanCategory } = require("./category_scan");
// Some sites will intermittently 403/429. We don't want a single category/store
// to abort the entire run. Log and continue.
@ -25,11 +20,9 @@ async function runAllStores(stores, { config, logger, http }) {
logger.info(`Debug=on`);
logger.info(
`Concurrency=${config.concurrency} StaggerMs=${config.staggerMs} Retries=${config.maxRetries} TimeoutMs=${config.timeoutMs}`
);
logger.info(
`DiscoveryGuess=${config.discoveryGuess} DiscoveryStep=${config.discoveryStep}`
`Concurrency=${config.concurrency} StaggerMs=${config.staggerMs} Retries=${config.maxRetries} TimeoutMs=${config.timeoutMs}`,
);
logger.info(`DiscoveryGuess=${config.discoveryGuess} DiscoveryStep=${config.discoveryStep}`);
logger.info(`MaxPages=${config.maxPages === null ? "none" : config.maxPages}`);
logger.info(`CategoryConcurrency=${config.categoryConcurrency}`);

View file

@ -31,10 +31,7 @@ function escapeRe(s) {
}
function extractHtmlAttr(html, attrName) {
const re = new RegExp(
`\\b${escapeRe(attrName)}\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+))`,
"i"
);
const re = new RegExp(`\\b${escapeRe(attrName)}\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+))`, "i");
const m = re.exec(html);
if (!m) return "";
return m[1] ?? m[2] ?? m[3] ?? "";
@ -130,7 +127,6 @@ function extractFirstImgUrl(html, baseUrl) {
return "";
}
module.exports = {
stripTags,
cleanText,

View file

@ -1,7 +1,9 @@
"use strict";
function normPrice(p) {
return String(p || "").trim().replace(/\s+/g, "");
return String(p || "")
.trim()
.replace(/\s+/g, "");
}
function priceToNumber(p) {

View file

@ -16,7 +16,6 @@ function idToCspc6(idDigits) {
return s.padStart(6, "0");
}
function normalizeCspc(v) {
const m = String(v ?? "").match(/\b(\d{6})\b/);
return m ? m[1] : "";

View file

@ -61,7 +61,6 @@ function isUpcSku(k) {
return /^\d{12,14}$/.test(s); // keep legacy support
}
function compareSku(a, b) {
a = String(a || "").trim();
b = String(b || "").trim();
@ -71,12 +70,10 @@ function compareSku(a, b) {
const bu = isUnknownSkuKey(b);
if (au !== bu) return au ? 1 : -1; // real first
const aUpc = isUpcSku(a);
const bUpc = isUpcSku(b);
if (aUpc !== bUpc) return aUpc ? 1 : -1; // UPCs after other "real" keys
const an = isNumericSku(a);
const bn = isNumericSku(b);
if (an && bn) {

View file

@ -10,7 +10,10 @@ function ts(d = new Date()) {
function isoTimestampFileSafe(d = new Date()) {
// 2026-01-16T21-27-01Z
return d.toISOString().replace(/:/g, "-").replace(/\.\d{3}Z$/, "Z");
return d
.toISOString()
.replace(/:/g, "-")
.replace(/\.\d{3}Z$/, "Z");
}
module.exports = { ts, isoTimestampFileSafe };

View file

@ -47,4 +47,10 @@ function makePageUrlShopifyQueryPage(baseUrl, pageNum) {
return u.toString();
}
module.exports = { normalizeBaseUrl, makePageUrl, makePageUrlForCtx, makePageUrlQueryParam, makePageUrlShopifyQueryPage };
module.exports = {
normalizeBaseUrl,
makePageUrl,
makePageUrlForCtx,
makePageUrlQueryParam,
makePageUrlShopifyQueryPage,
};

View file

@ -102,17 +102,7 @@ function canonicalize(k, skuMap) {
/* ---------------- grouping ---------------- */
const BC_STORE_KEYS = new Set([
"gull",
"strath",
"bcl",
"legacy",
"legacyliquor",
"tudor",
"vessel",
"vintage",
"arc"
]);
const BC_STORE_KEYS = new Set(["gull", "strath", "bcl", "legacy", "legacyliquor", "tudor", "vessel", "vintage", "arc"]);
function groupAllowsStore(group, storeKey) {
const k = String(storeKey || "").toLowerCase();

View file

@ -489,7 +489,7 @@ function main() {
fs.writeFileSync(htmlPath, html, "utf8");
fs.writeFileSync(subjPath, subject + "\n", "utf8");
fs.writeFileSync(sendPath, (shouldSend ? "1\n" : "0\n"), "utf8");
fs.writeFileSync(sendPath, shouldSend ? "1\n" : "0\n", "utf8");
writeGithubOutput({
should_send: shouldSend ? 1 : 0,

View file

@ -80,14 +80,7 @@ function keySkuForItem(it, storeLabel) {
}
// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file).
function computeFirstSeenForDbFile({
repoRoot,
relDbFile,
storeLabel,
wantSkuKeys,
commitsArr,
nowIso,
}) {
function computeFirstSeenForDbFile({ repoRoot, relDbFile, storeLabel, wantSkuKeys, commitsArr, nowIso }) {
const out = new Map();
const want = new Set(wantSkuKeys);
@ -226,9 +219,7 @@ function main() {
};
fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8");
process.stdout.write(
`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`
);
process.stdout.write(`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`);
}
module.exports = { main };

View file

@ -36,7 +36,10 @@ function gitFileExistsAtSha(sha, filePath) {
function gitListTreeFiles(sha, dirRel) {
try {
const out = runGit(["ls-tree", "-r", "--name-only", sha, dirRel]);
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
return out
.split(/\r?\n/)
.map((s) => s.trim())
.filter(Boolean);
} catch {
return [];
}
@ -237,10 +240,16 @@ function listChangedDbFiles(fromSha, toSha) {
try {
if (toSha === "WORKTREE") {
const out = runGit(["diff", "--name-only", fromSha, "--", "data/db"]);
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
return out
.split(/\r?\n/)
.map((s) => s.trim())
.filter(Boolean);
}
const out = runGit(["diff", "--name-only", fromSha, toSha, "--", "data/db"]);
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
return out
.split(/\r?\n/)
.map((s) => s.trim())
.filter(Boolean);
} catch {
return [];
}
@ -249,7 +258,10 @@ function listChangedDbFiles(fromSha, toSha) {
function logDbCommitsSince(sinceIso) {
try {
const out = runGit(["log", `--since=${sinceIso}`, "--format=%H %cI", "--", "data/db"]);
const lines = out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
const lines = out
.split(/\r?\n/)
.map((s) => s.trim())
.filter(Boolean);
const arr = [];
for (const line of lines) {
const m = line.match(/^([0-9a-f]{7,40})\s+(.+)$/i);
@ -315,11 +327,7 @@ function main() {
}
function isSmwsBottle(storeLabel, it) {
const hay = [
storeLabel,
it?.name,
it?.url,
]
const hay = [storeLabel, it?.name, it?.url]
.map((x) => String(x || ""))
.join(" | ")
.toLowerCase();
@ -348,39 +356,24 @@ function main() {
}
const nextExists =
toSha === "WORKTREE"
? fs.existsSync(path.join(repoRoot, file))
: gitFileExistsAtSha(toSha, file);
toSha === "WORKTREE" ? fs.existsSync(path.join(repoRoot, file)) : gitFileExistsAtSha(toSha, file);
if (!nextExists) continue;
if (!prevObj && !nextObj) continue;
const storeLabel = String(
nextObj?.storeLabel ||
nextObj?.store ||
prevObj?.storeLabel ||
prevObj?.store ||
""
nextObj?.storeLabel || nextObj?.store || prevObj?.storeLabel || prevObj?.store || "",
);
const categoryLabel = String(
nextObj?.categoryLabel ||
nextObj?.category ||
prevObj?.categoryLabel ||
prevObj?.category ||
""
nextObj?.categoryLabel || nextObj?.category || prevObj?.categoryLabel || prevObj?.category || "",
);
const isNewStoreFile =
Boolean(fromSha) &&
!gitFileExistsAtSha(fromSha, file) &&
(toSha === "WORKTREE"
? fs.existsSync(path.join(repoRoot, file))
: gitFileExistsAtSha(toSha, file));
(toSha === "WORKTREE" ? fs.existsSync(path.join(repoRoot, file)) : gitFileExistsAtSha(toSha, file));
let { newItems, restoredItems, removedItems, priceChanges } = diffDb(
prevObj,
nextObj
);
let { newItems, restoredItems, removedItems, priceChanges } = diffDb(prevObj, nextObj);
if (isNewStoreFile) {
newItems = [];

View file

@ -25,7 +25,10 @@ function gitShowText(sha, filePath) {
function gitListDbFiles(sha, dbDirRel) {
const out = runGit(["ls-tree", "-r", "--name-only", sha, dbDirRel]);
const lines = out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
const lines = out
.split(/\r?\n/)
.map((s) => s.trim())
.filter(Boolean);
return new Set(lines);
}
@ -154,7 +157,7 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
ln(paint("========== DIFF REPORT ==========", C.bold));
ln(`${paint("From", C.bold)} ${fromSha} ${paint("to", C.bold)} ${toSha}`);
ln(
`${paint("Totals", C.bold)} | Categories=${diffReport.categories.length} | New=${diffReport.totals.newCount} | Restored=${diffReport.totals.restoredCount} | Removed=${diffReport.totals.removedCount} | PriceChanges=${diffReport.totals.updatedCount}`
`${paint("Totals", C.bold)} | Categories=${diffReport.categories.length} | New=${diffReport.totals.newCount} | Restored=${diffReport.totals.restoredCount} | Removed=${diffReport.totals.removedCount} | PriceChanges=${diffReport.totals.updatedCount}`,
);
ln("");
@ -162,14 +165,24 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
const catW = Math.min(56, Math.max(...rows.map((r) => r.catLabel.length), 12));
ln(paint("Per-category summary:", C.bold));
ln(`${padRight("Store | Category", catW)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)}`);
ln(
`${padRight("Store | Category", catW)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)}`,
);
ln(`${"-".repeat(catW)} ---- ---- ---- ----`);
for (const r of rows) {
ln(`${padRight(r.catLabel, catW)} ${padLeft(r.newCount, 4)} ${padLeft(r.restoredCount, 4)} ${padLeft(r.removedCount, 4)} ${padLeft(r.updatedCount, 4)}`);
ln(
`${padRight(r.catLabel, catW)} ${padLeft(r.newCount, 4)} ${padLeft(r.restoredCount, 4)} ${padLeft(r.removedCount, 4)} ${padLeft(r.updatedCount, 4)}`,
);
}
ln("");
const labelW = Math.max(16, ...diffReport.newItems.map((x) => x.catLabel.length), ...diffReport.restoredItems.map((x) => x.catLabel.length), ...diffReport.removedItems.map((x) => x.catLabel.length), ...diffReport.updatedItems.map((x) => x.catLabel.length));
const labelW = Math.max(
16,
...diffReport.newItems.map((x) => x.catLabel.length),
...diffReport.restoredItems.map((x) => x.catLabel.length),
...diffReport.removedItems.map((x) => x.catLabel.length),
...diffReport.updatedItems.map((x) => x.catLabel.length),
);
const skuInline = (sku) => {
const s = normalizeCspc(sku);
@ -180,7 +193,9 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
ln(paint(`NEW (${diffReport.newItems.length})`, C.bold + C.green));
for (const it of diffReport.newItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
ln(`${paint("+", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
ln(
`${paint("+", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -188,9 +203,13 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
if (diffReport.restoredItems.length) {
ln(paint(`RESTORED (${diffReport.restoredItems.length})`, C.bold + C.green));
for (const it of diffReport.restoredItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
for (const it of diffReport.restoredItems.sort((a, b) =>
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
)) {
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
ln(`${paint("R", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
ln(
`${paint("R", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -198,9 +217,13 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
if (diffReport.removedItems.length) {
ln(paint(`REMOVED (${diffReport.removedItems.length})`, C.bold + C.yellow));
for (const it of diffReport.removedItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
for (const it of diffReport.removedItems.sort((a, b) =>
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
)) {
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
ln(`${paint("-", C.yellow)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
ln(
`${paint("-", C.yellow)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
);
ln(` ${paint(it.url, C.dim)}`);
}
ln("");
@ -209,7 +232,9 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
if (diffReport.updatedItems.length) {
ln(paint(`PRICE CHANGES (${diffReport.updatedItems.length})`, C.bold + C.cyan));
for (const u of diffReport.updatedItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
for (const u of diffReport.updatedItems.sort((a, b) =>
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
)) {
const oldRaw = u.oldPrice || "";
const newRaw = u.newPrice || "";
@ -231,7 +256,7 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
} else newP = paint(newP, C.cyan);
ln(
`${paint("~", C.cyan)} ${padRight(u.catLabel, labelW)} | ${paint(u.name, C.bold)}${skuInline(u.sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}`
`${paint("~", C.cyan)} ${padRight(u.catLabel, labelW)} | ${paint(u.name, C.bold)}${skuInline(u.sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}`,
);
ln(` ${paint(u.url, C.dim)}`);
}
@ -248,7 +273,9 @@ async function main() {
const { fromSha, toSha, dbDir, outFile, flags } = parseArgs(process.argv.slice(2));
if (!fromSha || !toSha) {
console.error(`Usage: ${path.basename(process.argv[1])} <fromSha> <toSha> [--db-dir data/db] [--out reports/<file>.txt] [--no-color]`);
console.error(
`Usage: ${path.basename(process.argv[1])} <fromSha> <toSha> [--db-dir data/db] [--out reports/<file>.txt] [--no-color]`,
);
process.exitCode = 2;
return;
}
@ -273,8 +300,16 @@ async function main() {
const prevObj = parseJsonOrNull(gitShowText(fromSha, file));
const nextObj = parseJsonOrNull(gitShowText(toSha, file));
const storeLabel = String(nextObj?.storeLabel || prevObj?.storeLabel || nextObj?.store || prevObj?.store || "?");
const catLabel = String(nextObj?.categoryLabel || prevObj?.categoryLabel || nextObj?.category || prevObj?.category || path.basename(file));
const storeLabel = String(
nextObj?.storeLabel || prevObj?.storeLabel || nextObj?.store || prevObj?.store || "?",
);
const catLabel = String(
nextObj?.categoryLabel ||
prevObj?.categoryLabel ||
nextObj?.category ||
prevObj?.category ||
path.basename(file),
);
const catLabelFull = `${storeLabel} | ${catLabel}`;
const { newItems, restoredItems, removedItems, updatedItems } = buildDiffForDb(prevObj, nextObj);
@ -301,9 +336,7 @@ async function main() {
const reportText = renderDiffReport(diffReport, { fromSha, toSha, colorize });
process.stdout.write(reportText);
const outPath = outFile
? (path.isAbsolute(outFile) ? outFile : path.join(process.cwd(), outFile))
: "";
const outPath = outFile ? (path.isAbsolute(outFile) ? outFile : path.join(process.cwd(), outFile)) : "";
if (outPath) {
fs.mkdirSync(path.dirname(outPath), { recursive: true });

View file

@ -41,17 +41,13 @@ function parseArgs(argv) {
if (a === "--ab" && argv[i + 1]) out.ab = argv[++i];
else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i];
else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i];
else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top;
else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep;
else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore;
else if (a === "--min-contain" && argv[i + 1]) out.minContain = Number(argv[++i]) || out.minContain;
else if (a === "--include-missing") out.includeMissing = true;
else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base);
else if (a === "--no-cross-group") out.requireCrossGroup = false;
else if (a === "--debug") out.debug = true;
else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN;
else if (a === "--debug-payload") out.debugPayload = true;
@ -66,7 +62,14 @@ function parseArgs(argv) {
function extractRows(payload) {
if (Array.isArray(payload)) return payload;
const candidates = [payload?.rows, payload?.data?.rows, payload?.data, payload?.items, payload?.list, payload?.results];
const candidates = [
payload?.rows,
payload?.data?.rows,
payload?.data,
payload?.items,
payload?.list,
payload?.results,
];
for (const x of candidates) if (Array.isArray(x)) return x;
return [];
}
@ -185,7 +188,8 @@ function compareSku(a, b) {
const aNum = /^\d+$/.test(a);
const bNum = /^\d+$/.test(b);
if (aNum && bNum) {
const na = Number(a), nb = Number(b);
const na = Number(a),
nb = Number(b);
if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1;
}
return a < b ? -1 : 1;
@ -252,16 +256,42 @@ function tokenizeQuery(q) {
}
const SIM_STOP_TOKENS = new Set([
"the","a","an","and","of","to","in","for","with",
"year","years","yr","yrs","old",
"whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof",
"the",
"a",
"an",
"and",
"of",
"to",
"in",
"for",
"with",
"year",
"years",
"yr",
"yrs",
"old",
"whisky",
"whiskey",
"scotch",
"single",
"malt",
"cask",
"finish",
"edition",
"release",
"batch",
"strength",
"abv",
"proof",
"anniversary",
]);
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
function numKey(t) {
const s = String(t || "").trim().toLowerCase();
const s = String(t || "")
.trim()
.toLowerCase();
if (!s) return "";
if (/^\d+$/.test(s)) return s;
const m = s.match(ORDINAL_RE);
@ -302,7 +332,9 @@ function filterSimTokens(tokens) {
const arr = Array.isArray(tokens) ? tokens : [];
for (let i = 0; i < arr.length; i++) {
let t = String(arr[i] || "").trim().toLowerCase();
let t = String(arr[i] || "")
.trim()
.toLowerCase();
if (!t) continue;
if (!/[a-z0-9]/i.test(t)) continue;
if (VOL_INLINE_RE.test(t)) continue;
@ -316,7 +348,9 @@ function filterSimTokens(tokens) {
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
if (/^\d+(?:\.\d+)?$/.test(t)) {
const next = String(arr[i + 1] || "").trim().toLowerCase();
const next = String(arr[i + 1] || "")
.trim()
.toLowerCase();
const nextNorm = SIM_EQUIV.get(next) || next;
if (VOL_UNIT.has(nextNorm)) {
i++;
@ -358,7 +392,8 @@ function tokenContainmentScore(aTokens, bTokens) {
function levenshtein(a, b) {
a = String(a || "");
b = String(b || "");
const n = a.length, m = b.length;
const n = a.length,
m = b.length;
if (!n) return m;
if (!m) return n;
@ -422,17 +457,13 @@ function similarityScore(aName, bName) {
const maxLen = Math.max(1, Math.max(a.length, b.length));
const levSim = 1 - d / maxLen;
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
const smallN = Math.min(aToks.length, bToks.length);
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
const numGate = numberMismatchPenalty(aToks, bToks);
let s =
numGate *
(firstMatch * 3.0 +
overlapTail * 2.2 * gate +
levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain)));
let s = numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain));
if (ageMatch) s *= 2.2;
else if (ageMismatch) s *= 0.18;
@ -444,8 +475,13 @@ function similarityScore(aName, bName) {
/* ---------------- debug helpers ---------------- */
function eprintln(...args) { console.error(...args); }
function truncate(s, n) { s = String(s || ""); return s.length <= n ? s : s.slice(0, n - 1) + "…"; }
function eprintln(...args) {
console.error(...args);
}
function truncate(s, n) {
s = String(s || "");
return s.length <= n ? s : s.slice(0, n - 1) + "…";
}
/* ---------------- main ---------------- */
@ -498,7 +534,9 @@ function main() {
if (args.debug) {
eprintln("[rank_discrepency] inputs:", {
abPath, bcPath, metaPath: metaPath || "(none)",
abPath,
bcPath,
metaPath: metaPath || "(none)",
linkCount: Array.isArray(meta?.links) ? meta.links.length : 0,
ignoreCount: Array.isArray(meta?.ignores) ? meta.ignores.length : 0,
ignoreSetSize: ignoreSet.size,
@ -509,8 +547,17 @@ function main() {
top: args.top,
includeMissing: args.includeMissing,
});
eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, abKeys: abMap.size, bcKeys: bcMap.size });
eprintln("[rank_discrepency] name coverage:", { totalSkus: allSkus.length, named: namedCount, unnamed: allSkus.length - namedCount });
eprintln("[rank_discrepency] extracted rows:", {
abRows: abBuilt.rowsLen,
bcRows: bcBuilt.rowsLen,
abKeys: abMap.size,
bcKeys: bcMap.size,
});
eprintln("[rank_discrepency] name coverage:", {
totalSkus: allSkus.length,
named: namedCount,
unnamed: allSkus.length - namedCount,
});
}
if (args.debugPayload) {
@ -544,11 +591,15 @@ function main() {
if (args.debug) {
eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length });
eprintln("[rank_discrepency] top discrep sample:",
eprintln(
"[rank_discrepency] top discrep sample:",
diffs.slice(0, 5).map((d) => ({
sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC,
sku: d.canonSku,
discrep: d.discrep,
rankAB: d.rankAB,
rankBC: d.rankBC,
name: truncate(allNames.get(String(d.canonSku)) || "", 80),
}))
})),
);
}
@ -583,7 +634,9 @@ function main() {
side: aInAB ? "AB" : "BC",
nameA: truncate(nameA, 120),
minContain: args.minContain,
top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })),
top5: scored
.slice(0, 5)
.map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })),
});
}
@ -601,7 +654,10 @@ function main() {
const groupA = canonicalSku(skuA);
const aRaw = tokenizeQuery(nameA);
let best = 0, bestSku = "", bestName = "", bestContain = 0;
let best = 0,
bestSku = "",
bestName = "",
bestContain = 0;
let bestWasIgnored = false;
for (const skuB of pool) {
@ -669,7 +725,9 @@ function main() {
for (const d of filtered) {
if (args.dumpScores) {
eprintln("[rank_discrepency] emit", JSON.stringify({
eprintln(
"[rank_discrepency] emit",
JSON.stringify({
sku: d.canonSku,
discrep: d.discrep,
rankAB: d.rankAB,
@ -678,7 +736,8 @@ function main() {
bestContain: d.bestContain,
bestSku: d.bestSku,
bestName: truncate(d.bestName, 120),
}));
}),
);
}
console.log(args.base + encodeURIComponent(String(d.canonSku)));
}

View file

@ -20,9 +20,7 @@ const REPO = process.env.REPO || "";
if (!ISSUE_NUMBER) die("Missing ISSUE_NUMBER");
if (!REPO) die("Missing REPO");
const m = ISSUE_BODY.match(
/<!--\s*stviz-sku-edits:BEGIN\s*-->\s*([\s\S]*?)\s*<!--\s*stviz-sku-edits:END\s*-->/
);
const m = ISSUE_BODY.match(/<!--\s*stviz-sku-edits:BEGIN\s*-->\s*([\s\S]*?)\s*<!--\s*stviz-sku-edits:END\s*-->/);
if (!m) die("No stviz payload found in issue body.");
let payload;
@ -204,24 +202,13 @@ function makePrettyObjBlock(objIndent, obj) {
}
if (skuA && skuB) {
return (
`${a}{\n` +
`${b}"skuA": ${JSON.stringify(skuA)},\n` +
`${b}"skuB": ${JSON.stringify(skuB)}\n` +
`${a}}`
);
return `${a}{\n` + `${b}"skuA": ${JSON.stringify(skuA)},\n` + `${b}"skuB": ${JSON.stringify(skuB)}\n` + `${a}}`;
}
return `${a}{}`;
}
function applyInsertionsToArrayText({
src,
propName,
incoming,
keyFn,
normalizeFn,
}) {
function applyInsertionsToArrayText({ src, propName, incoming, keyFn, normalizeFn }) {
const span = findJsonArraySpan(src, propName);
if (!span) die(`Could not find "${propName}" array in ${filePath}`);
@ -266,8 +253,7 @@ function applyInsertionsToArrayText({
let newInner;
if (wasInlineEmpty) {
// "links": [] -> pretty multiline
newInner =
"\n" + addBlocks.join(",\n") + "\n" + span.fieldIndent;
newInner = "\n" + addBlocks.join(",\n") + "\n" + span.fieldIndent;
} else {
// Keep existing whitespace EXACTLY; append before trailing whitespace
const m = inner.match(/\s*$/);
@ -280,7 +266,6 @@ function applyInsertionsToArrayText({
return before + newInner + after;
}
/* ---------------- Apply edits ---------------- */
const filePath = path.join("data", "sku_links.json");
@ -379,13 +364,12 @@ function extractPrUrl(out) {
// Create PR and capture URL/number without relying on unsupported flags
const prCreateOut = sh(
`gh -R "${REPO}" pr create --base data --head "${branch}" --title "${prTitle}" --body "${prBody}"`
`gh -R "${REPO}" pr create --base data --head "${branch}" --title "${prTitle}" --body "${prBody}"`,
);
const prUrl = extractPrUrl(prCreateOut);
const prNumber = sh(`gh -R "${REPO}" pr view "${prUrl}" --json number --jq .number`);
sh(
`gh -R "${REPO}" issue close "${ISSUE_NUMBER}" -c "Processed by STVIZ automation. Opened PR #${prNumber}: ${prUrl}"`
`gh -R "${REPO}" issue close "${ISSUE_NUMBER}" -c "Processed by STVIZ automation. Opened PR #${prNumber}: ${prUrl}"`,
);

View file

@ -24,7 +24,9 @@ export function inferGithubOwnerRepo() {
export function isLocalWriteMode() {
const h = String(location.hostname || "").toLowerCase();
return (location.protocol === "http:" || location.protocol === "https:") && (h === "127.0.0.1" || h === "localhost");
return (
(location.protocol === "http:" || location.protocol === "https:") && (h === "127.0.0.1" || h === "localhost")
);
}
/* ---- Local disk-backed SKU link API (only on viz/serve.js) ---- */

View file

@ -1,5 +1,8 @@
export function esc(s) {
return String(s ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
return String(s ?? "").replace(
/[&<>"']/g,
(c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c],
);
}
export function normImg(s) {
@ -56,4 +59,3 @@ export function esc(s) {
if (!img) return `<div class="thumbPlaceholder"></div>`;
return `<img referrerpolicy="no-referrer" class="${esc(cls)}" src="${esc(img)}" alt="" loading="lazy" onerror="this.style.display='none'" />`;
}

View file

@ -73,7 +73,6 @@ function weightedMeanByDuration(pointsMap, sortedDates) {
return wtot ? wsum / wtot : null;
}
function meanFinite(arr) {
if (!Array.isArray(arr)) return null;
let sum = 0,
@ -126,7 +125,14 @@ const StaticMarkerLinesPlugin = {
scalesObj.y ||
scales.find((s) => s && s.axis === "y") ||
scales.find((s) => s && typeof s.getPixelForValue === "function" && s.isHorizontal === false) ||
scales.find((s) => s && typeof s.getPixelForValue === "function" && String(s.id || "").toLowerCase().includes("y"));
scales.find(
(s) =>
s &&
typeof s.getPixelForValue === "function" &&
String(s.id || "")
.toLowerCase()
.includes("y"),
);
const area = chart?.chartArea;
if (!y || !area) return;
@ -139,9 +145,7 @@ const StaticMarkerLinesPlugin = {
const strokeStyle = String(opts?.color || "#7f8da3"); // light grey-blue
// "marker on Y axis" text
const font =
opts?.font ||
"600 11px system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif";
const font = opts?.font || "600 11px system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif";
const labelColor = String(opts?.labelColor || "#556274");
const axisInset = Number.isFinite(opts?.axisInset) ? opts.axisInset : 2;
@ -189,7 +193,6 @@ if (text) {
ctx.textAlign = "center";
ctx.fillText(text, axisCenterX, py);
}
}
ctx.restore();
@ -603,11 +606,7 @@ export async function renderItem($app, skuInput) {
const inflightFetch = new Map(); // ck -> Promise
const today = dateOnly(idx.generatedAt || new Date().toISOString());
const skuKeys = [...skuGroup];
const wantRealSkus = new Set(
skuKeys
.map((s) => String(s || "").trim())
.filter((x) => x)
);
const wantRealSkus = new Set(skuKeys.map((s) => String(s || "").trim()).filter((x) => x));
// Tuning knobs:
// - keep compute modest: only a few stores processed simultaneously
@ -946,7 +945,7 @@ export async function renderItem($app, skuInput) {
const t = (storeMins[0].min + storeMins[1].min + storeMins[2].min) / 3;
if (Number.isFinite(t)) markers.push({ y: Math.round(t), text: "Target" });
}
const markerYs = markers.map(m => Number(m.y)).filter(Number.isFinite);
const markerYs = markers.map((m) => Number(m.y)).filter(Number.isFinite);
// helper: approximate font px size from a CSS font string (Chart uses one)
function fontPx(font) {
@ -1020,15 +1019,12 @@ export async function renderItem($app, skuInput) {
// derive a "collision window" from tick label height
// Chart.js puts resolved font on ticks.font (v3+), otherwise fall back
const tickFont =
this?.options?.ticks?.font ||
this?.ctx?.font ||
"12px system-ui";
const tickFont = this?.options?.ticks?.font || this?.ctx?.font || "12px system-ui";
const h = fontPx(
typeof tickFont === "string"
? tickFont
: `${tickFont?.size || 12}px ${tickFont?.family || "system-ui"}`
: `${tickFont?.size || 12}px ${tickFont?.family || "system-ui"}`,
);
// hide if within 55% of label height (tweak 0.450.75)

View file

@ -12,7 +12,9 @@ function isSoftSkuKey(k) {
}
function isUnknownSkuKey2(k) {
return String(k || "").trim().startsWith("u:");
return String(k || "")
.trim()
.startsWith("u:");
}
function isBCStoreLabel(label) {
@ -40,12 +42,7 @@ function skuIsBC(allRows, skuKey) {
function isABStoreLabel(label) {
const s = String(label || "").toLowerCase();
return (
s.includes("alberta") ||
s.includes("calgary") ||
s.includes("edmonton") ||
/\bab\b/.test(s)
);
return s.includes("alberta") || s.includes("calgary") || s.includes("edmonton") || /\bab\b/.test(s);
}
function skuIsAB(allRows, skuKey) {

View file

@ -48,7 +48,7 @@ export function buildPricePenaltyForPair({ allAgg, rules, kPerGroup = 6 }) {
if (!(gap >= 0)) return 1.0;
if (gap <= 0.35) return 1.0;
if (gap <= 0.50) {
if (gap <= 0.5) {
const t = (gap - 0.35) / 0.15; // 0..1
return 1.0 - 0.25 * t; // 1.00 -> 0.75
}
@ -75,4 +75,3 @@ export function buildPricePenaltyForPair({ allAgg, rules, kPerGroup = 6 }) {
return gapToMultiplier(gap);
};
}

View file

@ -3,9 +3,33 @@ import { tokenizeQuery, normSearchText } from "../sku.js";
// Ignore ultra-common / low-signal tokens in bottle names.
const SIM_STOP_TOKENS = new Set([
"the","a","an","and","of","to","in","for","with",
"year","years","yr","yrs","old",
"whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof",
"the",
"a",
"an",
"and",
"of",
"to",
"in",
"for",
"with",
"year",
"years",
"yr",
"yrs",
"old",
"whisky",
"whiskey",
"scotch",
"single",
"malt",
"cask",
"finish",
"edition",
"release",
"batch",
"strength",
"abv",
"proof",
"anniversary",
]);
@ -22,7 +46,9 @@ export function smwsKeyFromName(name) {
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
export function numKey(t) {
const s = String(t || "").trim().toLowerCase();
const s = String(t || "")
.trim()
.toLowerCase();
if (!s) return "";
if (/^\d+$/.test(s)) return s;
const m = s.match(ORDINAL_RE);
@ -68,7 +94,9 @@ export function filterSimTokens(tokens) {
for (let i = 0; i < arr.length; i++) {
const raw = arr[i];
let t = String(raw || "").trim().toLowerCase();
let t = String(raw || "")
.trim()
.toLowerCase();
if (!t) continue;
if (!/[a-z0-9]/i.test(t)) continue;
@ -84,7 +112,9 @@ export function filterSimTokens(tokens) {
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
if (/^\d+(?:\.\d+)?$/.test(t)) {
const next = String(arr[i + 1] || "").trim().toLowerCase();
const next = String(arr[i + 1] || "")
.trim()
.toLowerCase();
const nextNorm = SIM_EQUIV.get(next) || next;
if (VOL_UNIT.has(nextNorm)) {
i++;
@ -113,7 +143,8 @@ export function numberMismatchPenalty(aTokens, bTokens) {
export function levenshtein(a, b) {
a = String(a || "");
b = String(b || "");
const n = a.length, m = b.length;
const n = a.length,
m = b.length;
if (!n) return m;
if (!m) return n;
@ -190,18 +221,14 @@ export function similarityScore(aName, bName) {
const maxLen = Math.max(1, Math.max(a.length, b.length));
const levSim = 1 - d / maxLen;
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
const smallN = Math.min(aToks.length, bToks.length);
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
const numGate = numberMismatchPenalty(aToks, bToks);
let s =
numGate *
(firstMatch * 3.0 +
overlapTail * 2.2 * gate +
levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain)));
let s = numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain));
if (ageMatch) s *= 2.2;
else if (ageMismatch) s *= 0.18;
@ -244,15 +271,9 @@ export function fastSimilarityScore(aTokens, bTokens, aNormName, bNormName) {
const denom = Math.max(1, Math.max(aTail.length, bTail.length));
const overlapTail = inter / denom;
const pref =
firstMatch &&
a.slice(0, 10) &&
b.slice(0, 10) &&
a.slice(0, 10) === b.slice(0, 10)
? 0.2
: 0;
const pref = firstMatch && a.slice(0, 10) && b.slice(0, 10) && a.slice(0, 10) === b.slice(0, 10) ? 0.2 : 0;
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
const smallN = Math.min(aTokF.length, bTokF.length);
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;

View file

@ -101,8 +101,8 @@ export function buildSizePenaltyForPair({ allRows, allAgg, rules }) {
return function sizePenaltyForPair(aSku, bSku) {
const aCanon = String(rules.canonicalSku(String(aSku || "")) || "");
const bCanon = String(rules.canonicalSku(String(bSku || "")) || "");
const A = aCanon ? (CANON_SIZE_CACHE.get(aCanon) || new Set()) : new Set();
const B = bCanon ? (CANON_SIZE_CACHE.get(bCanon) || new Set()) : new Set();
const A = aCanon ? CANON_SIZE_CACHE.get(aCanon) || new Set() : new Set();
const B = bCanon ? CANON_SIZE_CACHE.get(bCanon) || new Set() : new Set();
return sizePenalty(A, B);
};
}

View file

@ -40,4 +40,3 @@ function canonKeyForSku(rules, skuKey) {
return false;
};
}

View file

@ -51,7 +51,6 @@ export function topSuggestions(allAgg, limit, otherPinnedSku, mappedSkus) {
return scored.slice(0, limit).map((x) => x.it);
}
// viz/app/linker/suggestions.js
// (requires fnv1a32u(str) helper to exist in this file)
@ -65,7 +64,7 @@ export function recommendSimilar(
sizePenaltyFn,
pricePenaltyFn,
sameStoreFn,
sameGroupFn
sameGroupFn,
) {
if (!pinned || !pinned.name) return topSuggestions(allAgg, limit, otherPinnedSku, mappedSkus);
@ -134,11 +133,7 @@ export function recommendSimilar(
if (k && k === pinnedSmws) {
const stores = it.stores ? it.stores.size : 0;
const hasPrice = it.cheapestPriceNum != null ? 1 : 0;
pushTopK(
cheap,
{ it, s: 1e9 + stores * 10 + hasPrice, itNorm: "", itRawToks: null },
MAX_CHEAP_KEEP
);
pushTopK(cheap, { it, s: 1e9 + stores * 10 + hasPrice, itNorm: "", itRawToks: null }, MAX_CHEAP_KEEP);
continue;
}
}
@ -161,7 +156,7 @@ export function recommendSimilar(
// Soft first-token mismatch penalty (never blocks)
if (!firstMatch) {
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
let mult = 0.10 + 0.95 * contain;
let mult = 0.1 + 0.95 * contain;
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
s0 *= Math.min(1.0, mult);
}
@ -211,7 +206,7 @@ export function recommendSimilar(
if (!firstMatch) {
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
let mult = 0.10 + 0.95 * contain;
let mult = 0.1 + 0.95 * contain;
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
s *= Math.min(1.0, mult);
if (s <= 0) continue;
@ -266,9 +261,6 @@ export function recommendSimilar(
return fallback.slice(0, limit).map((x) => x.it);
}
export function computeInitialPairsFast(
allAgg,
mappedSkus,
@ -277,7 +269,7 @@ export function recommendSimilar(
sameStoreFn,
sameGroupFn, // ✅ NEW
sizePenaltyFn,
pricePenaltyFn
pricePenaltyFn,
) {
const itemsAll = allAgg.filter((it) => !!it);
@ -382,7 +374,7 @@ export function recommendSimilar(
// --- Improved general pairing logic ---
const seeds = topSuggestions(work, Math.min(220, work.length), "", mappedSkus).filter(
(it) => !used.has(String(it?.sku || ""))
(it) => !used.has(String(it?.sku || "")),
);
// Build token buckets over normalized names
@ -470,7 +462,7 @@ export function recommendSimilar(
if (!firstMatch) {
const smallN = Math.min(aFilt.length || 0, bFilt.length || 0);
let mult = 0.10 + 0.95 * contain;
let mult = 0.1 + 0.95 * contain;
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
s *= Math.min(1.0, mult);
}
@ -505,7 +497,7 @@ export function recommendSimilar(
if (!x.firstMatch) {
const smallN = Math.min(aFilt.length || 0, (x.bFilt || []).length || 0);
let mult = 0.10 + 0.95 * x.contain;
let mult = 0.1 + 0.95 * x.contain;
if (smallN <= 3 && x.contain < 0.78) mult *= 0.22;
s *= Math.min(1.0, mult);
if (s <= 0) continue;
@ -526,7 +518,7 @@ export function recommendSimilar(
else s *= 0.15;
}
if (String(aSku).startsWith("u:") || String(bSku).startsWith("u:")) s *= 1.10;
if (String(aSku).startsWith("u:") || String(bSku).startsWith("u:")) s *= 1.1;
if (s > bestS) {
bestS = s;
@ -534,7 +526,7 @@ export function recommendSimilar(
}
}
if (!bestB || bestS < 0.50) continue;
if (!bestB || bestS < 0.5) continue;
const bSku = String(bestB.sku || "");
if (!bSku || used.has(bSku)) continue;
@ -592,11 +584,6 @@ export function recommendSimilar(
return out.slice(0, limitPairs);
}
function fnv1a32u(str) {
let h = 0x811c9dc5;
str = String(str || "");

View file

@ -1,12 +1,6 @@
// viz/app/linker_page.js
import { esc, renderThumbHtml } from "./dom.js";
import {
tokenizeQuery,
matchesAllTokens,
displaySku,
keySkuForRow,
normSearchText,
} from "./sku.js";
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, normSearchText } from "./sku.js";
import { loadIndex } from "./state.js";
import { aggregateBySku } from "./catalog.js";
import { loadSkuRules, clearSkuRulesCache } from "./mapping.js";
@ -32,11 +26,7 @@ import { buildSizePenaltyForPair } from "./linker/size.js";
import { pickPreferredCanonical } from "./linker/canonical_pref.js";
import { smwsKeyFromName } from "./linker/similarity.js";
import { buildPricePenaltyForPair } from "./linker/price.js";
import {
topSuggestions,
recommendSimilar,
computeInitialPairsFast,
} from "./linker/suggestions.js";
import { topSuggestions, recommendSimilar, computeInitialPairsFast } from "./linker/suggestions.js";
/* ---------------- Page ---------------- */
@ -163,16 +153,12 @@ export async function renderSkuLinker($app) {
sameStoreCanon,
sameGroup, // ✅ NEW: hard-block already-linked pairs (incl SMWS stage)
sizePenaltyForPair,
pricePenaltyForPair
pricePenaltyForPair,
);
return initialPairs;
}
let pinnedL = null;
let pinnedR = null;
@ -192,7 +178,7 @@ export async function renderSkuLinker($app) {
const storeBadge = href
? `<a class="badge" href="${esc(href)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
store
store,
)}${esc(plus)}</a>`
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
@ -253,7 +239,7 @@ export async function renderSkuLinker($app) {
sizePenaltyForPair,
pricePenaltyForPair,
sameStoreCanon,
sameGroup
sameGroup,
);
const pairs = getInitialPairsIfNeeded();
@ -261,9 +247,7 @@ export async function renderSkuLinker($app) {
const list = side === "L" ? pairs.map((p) => p.a) : pairs.map((p) => p.b);
return list.filter(
(it) =>
it &&
it.sku !== otherSku &&
(!mappedSkus.has(String(it.sku)) || smwsKeyFromName(it.name || ""))
it && it.sku !== otherSku && (!mappedSkus.has(String(it.sku)) || smwsKeyFromName(it.name || "")),
);
}
@ -271,7 +255,6 @@ export async function renderSkuLinker($app) {
}
function attachHandlers($root, side) {
for (const el of Array.from($root.querySelectorAll(".thumbInternalLink"))) {
el.addEventListener("click", (e) => {
e.preventDefault();
@ -458,7 +441,7 @@ export async function renderSkuLinker($app) {
ignores: editsToSend.ignores,
},
null,
2
2,
);
const title = `[stviz] sku link updates (${editsToSend.links.length} link, ${editsToSend.ignores.length} ignore)`;
@ -507,10 +490,7 @@ export async function renderSkuLinker($app) {
it = allAgg.find((x) => String(x?.sku || "") === canonWant);
if (it) return it;
return (
allAgg.find((x) => String(rules.canonicalSku(String(x?.sku || "")) || "") === canonWant) ||
null
);
return allAgg.find((x) => String(rules.canonicalSku(String(x?.sku || "")) || "") === canonWant) || null;
}
function updateAll() {
@ -647,7 +627,7 @@ export async function renderSkuLinker($app) {
for (let i = 0; i < uniq.length; i++) {
const w = uniq[i];
$status.textContent = `Writing (${i + 1}/${uniq.length}): ${displaySku(
w.fromSku
w.fromSku,
)} ${displaySku(w.toSku)} `;
await apiWriteSkuLink(w.fromSku, w.toSku);
}
@ -764,7 +744,7 @@ export async function renderSkuLinker($app) {
// ✅ NEW: always include rules.links (meta can be incomplete)
const merged = [
...((rules0 && Array.isArray(rules0.links)) ? rules0.links : []),
...(rules0 && Array.isArray(rules0.links) ? rules0.links : []),
...(Array.isArray(links) ? links : []),
];
@ -775,7 +755,4 @@ export async function renderSkuLinker($app) {
return s;
}
}

View file

@ -114,7 +114,7 @@ export function addPendingLink(fromSku, toSku) {
[
...pending.links.map((x) => linkKey(x.fromSku, x.toSku)),
...submitted.links.map((x) => linkKey(x.fromSku, x.toSku)),
].filter(Boolean)
].filter(Boolean),
);
if (seen.has(k)) return false;
@ -137,7 +137,7 @@ export function addPendingIgnore(skuA, skuB) {
[
...pending.ignores.map((x) => pairKey(x.skuA, x.skuB)),
...submitted.ignores.map((x) => pairKey(x.skuA, x.skuB)),
].filter(Boolean)
].filter(Boolean),
);
if (seen.has(k)) return false;
@ -164,9 +164,7 @@ export function applyPendingToMeta(meta) {
// merge links (dedupe by from→to)
const seenL = new Set(
base.links
.map((x) => linkKey(String(x?.fromSku || "").trim(), String(x?.toSku || "").trim()))
.filter(Boolean)
base.links.map((x) => linkKey(String(x?.fromSku || "").trim(), String(x?.toSku || "").trim())).filter(Boolean),
);
for (const x of overlay.links) {
const k = linkKey(x.fromSku, x.toSku);
@ -179,7 +177,7 @@ export function applyPendingToMeta(meta) {
const seenI = new Set(
base.ignores
.map((x) => pairKey(String(x?.skuA || x?.a || "").trim(), String(x?.skuB || x?.b || "").trim()))
.filter(Boolean)
.filter(Boolean),
);
for (const x of overlay.ignores) {
const k = pairKey(x.skuA, x.skuB);

View file

@ -1,18 +1,9 @@
import { esc, renderThumbHtml, prettyTs } from "./dom.js";
import {
tokenizeQuery,
matchesAllTokens,
displaySku,
keySkuForRow,
parsePriceToNumber,
} from "./sku.js";
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, parsePriceToNumber } from "./sku.js";
import { loadIndex, loadRecent, loadSavedQuery, saveQuery } from "./state.js";
import { aggregateBySku } from "./catalog.js";
import { loadSkuRules } from "./mapping.js";
import {
smwsDistilleryCodesForQueryPrefix,
smwsDistilleryCodeFromName,
} from "./smws.js";
import { smwsDistilleryCodesForQueryPrefix, smwsDistilleryCodeFromName } from "./smws.js";
export function renderSearch($app) {
$app.innerHTML = `
@ -126,13 +117,9 @@ export function renderSearch($app) {
$stores.innerHTML = stores
.map((s, i) => {
const btn = `<a class="storeBtn" href="#/store/${encodeURIComponent(
s
)}">${esc(s)}</a>`;
const btn = `<a class="storeBtn" href="#/store/${encodeURIComponent(s)}">${esc(s)}</a>`;
const brk =
i === breakAt - 1 && stores.length > 1
? `<span class="storeBreak" aria-hidden="true"></span>`
: "";
i === breakAt - 1 && stores.length > 1 ? `<span class="storeBreak" aria-hidden="true"></span>` : "";
return btn + brk;
})
.join("");
@ -156,15 +143,13 @@ export function renderSearch($app) {
const href = urlForAgg(it, store) || String(it.sampleUrl || "").trim();
const storeBadge = href
? `<a class="badge" href="${esc(
href
href,
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
store
store,
)}${esc(plus)}</a>`
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
const skuLink = `#/link/?left=${encodeURIComponent(
String(it.sku || "")
)}`;
const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`;
return `
<div class="item" data-sku="${esc(it.sku)}">
@ -176,9 +161,9 @@ export function renderSearch($app) {
<div class="itemTop">
<div class="itemName">${esc(it.name || "(no name)")}</div>
<a class="badge mono skuLink" href="${esc(
skuLink
skuLink,
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
displaySku(it.sku)
displaySku(it.sku),
)}</a>
</div>
<div class="metaRow">
@ -241,7 +226,10 @@ export function renderSearch($app) {
const storeLabelRaw = String(r?.storeLabel || r?.store || "").trim();
const bestStoreRaw = String(agg?.cheapestStoreLabel || "").trim();
const normStore = (s) => String(s || "").trim().toLowerCase();
const normStore = (s) =>
String(s || "")
.trim()
.toLowerCase();
// Normalize kind
let kind = String(r?.kind || "");
@ -254,27 +242,16 @@ export function renderSearch($app) {
}
}
const pctOff =
kind === "price_down"
? salePctOff(r?.oldPrice || "", r?.newPrice || "")
: null;
const pctUp =
kind === "price_up"
? pctChange(r?.oldPrice || "", r?.newPrice || "")
: null;
const pctOff = kind === "price_down" ? salePctOff(r?.oldPrice || "", r?.newPrice || "") : null;
const pctUp = kind === "price_up" ? pctChange(r?.oldPrice || "", r?.newPrice || "") : null;
const isNew = kind === "new";
const storeCount = agg?.stores?.size || 0;
const isNewUnique = isNew && storeCount <= 1;
// Cheapest checks (use aggregate index)
const newPriceNum =
kind === "price_down" || kind === "price_up"
? parsePriceToNumber(r?.newPrice || "")
: null;
const bestPriceNum = Number.isFinite(agg?.cheapestPriceNum)
? agg.cheapestPriceNum
: null;
const newPriceNum = kind === "price_down" || kind === "price_up" ? parsePriceToNumber(r?.newPrice || "") : null;
const bestPriceNum = Number.isFinite(agg?.cheapestPriceNum) ? agg.cheapestPriceNum : null;
const EPS = 0.01;
const priceMatchesBest =
@ -283,14 +260,10 @@ export function renderSearch($app) {
: false;
const storeIsBest =
normStore(storeLabelRaw) &&
normStore(bestStoreRaw) &&
normStore(storeLabelRaw) === normStore(bestStoreRaw);
normStore(storeLabelRaw) && normStore(bestStoreRaw) && normStore(storeLabelRaw) === normStore(bestStoreRaw);
const saleIsCheapestHere =
kind === "price_down" && storeIsBest && priceMatchesBest;
const saleIsTiedCheapest =
kind === "price_down" && !storeIsBest && priceMatchesBest;
const saleIsCheapestHere = kind === "price_down" && storeIsBest && priceMatchesBest;
const saleIsTiedCheapest = kind === "price_down" && !storeIsBest && priceMatchesBest;
const saleIsCheapest = saleIsCheapestHere || saleIsTiedCheapest;
// Bucketed scoring (higher = earlier)
@ -343,8 +316,7 @@ export function renderSearch($app) {
return;
}
const canon =
typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x;
const canon = typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x;
const nowMs = Date.now();
const cutoffMs = nowMs - 3 * 24 * 60 * 60 * 1000;
@ -400,9 +372,7 @@ export function renderSearch($app) {
!best ||
meta.score > best.meta.score ||
(meta.score === best.meta.score && meta.tie > best.meta.tie) ||
(meta.score === best.meta.score &&
meta.tie === best.meta.tie &&
ms > best.ms)
(meta.score === best.meta.score && meta.tie === best.meta.tie && ms > best.ms)
) {
best = { r, meta, ms };
}
@ -455,22 +425,18 @@ export function renderSearch($app) {
const href = String(r.url || "").trim();
const storeBadge = href
? `<a class="badge" href="${esc(
href
href,
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
(r.storeLabel || r.store || "") + plus
(r.storeLabel || r.store || "") + plus,
)}</a>`
: `<span class="badge">${esc(
(r.storeLabel || r.store || "") + plus
)}</span>`;
: `<span class="badge">${esc((r.storeLabel || r.store || "") + plus)}</span>`;
const dateBadge = when
? `<span class="badge mono">${esc(when)}</span>`
: "";
const dateBadge = when ? `<span class="badge mono">${esc(when)}</span>` : "";
const offBadge =
meta.kind === "price_down" && meta.pctOff !== null
? `<span class="badge" style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);">[${esc(
meta.pctOff
meta.pctOff,
)}% Off]</span>`
: "";
@ -491,9 +457,9 @@ export function renderSearch($app) {
<div class="itemTop">
<div class="itemName">${esc(r.name || "(no name)")}</div>
<a class="badge mono skuLink" href="${esc(
skuLink
skuLink,
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
displaySku(sku)
displaySku(sku),
)}</a>
</div>
<div class="metaRow">
@ -527,9 +493,7 @@ export function renderSearch($app) {
const tokens = tokenizeQuery($q.value);
if (!tokens.length) return;
const matches = allAgg.filter((it) =>
matchesAllTokens(it.searchText, tokens)
);
const matches = allAgg.filter((it) => matchesAllTokens(it.searchText, tokens));
const wantCodes = new Set(smwsDistilleryCodesForQueryPrefix($q.value));
if (!wantCodes.size) {
@ -571,15 +535,11 @@ export function renderSearch($app) {
if (tokens.length) {
applySearch();
} else {
return loadRecent().then((recent) =>
renderRecent(recent, rules.canonicalSku)
);
return loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku));
}
})
.catch((e) => {
$results.innerHTML = `<div class="small">Failed to load: ${esc(
e.message
)}</div>`;
$results.innerHTML = `<div class="small">Failed to load: ${esc(e.message)}</div>`;
});
$clearSearch.addEventListener("click", () => {
@ -588,9 +548,7 @@ export function renderSearch($app) {
saveQuery("");
}
loadSkuRules()
.then((rules) =>
loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku))
)
.then((rules) => loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku)))
.catch(() => {
$results.innerHTML = `<div class="small">Type to search…</div>`;
});
@ -605,9 +563,7 @@ export function renderSearch($app) {
const tokens = tokenizeQuery($q.value);
if (!tokens.length) {
loadSkuRules()
.then((rules) =>
loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku))
)
.then((rules) => loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku)))
.catch(() => {
$results.innerHTML = `<div class="small">Type to search…</div>`;
});

View file

@ -57,4 +57,3 @@ export function parsePriceToNumber(v) {
for (const t of tokens) if (!hayNorm.includes(t)) return false;
return true;
}

View file

@ -1,10 +1,5 @@
import { esc } from "./dom.js";
import {
fetchJson,
inferGithubOwnerRepo,
githubFetchFileAtSha,
githubListCommits,
} from "./api.js";
import { fetchJson, inferGithubOwnerRepo, githubFetchFileAtSha, githubListCommits } from "./api.js";
import { buildStoreColorMap, storeColor, datasetStrokeWidth, lighten } from "./storeColors.js";
let _chart = null;
@ -47,8 +42,7 @@ function ensureChartJs() {
return new Promise((resolve, reject) => {
const s = document.createElement("script");
// UMD build -> window.Chart
s.src =
"https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js";
s.src = "https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js";
s.async = true;
s.onload = () => resolve(window.Chart);
s.onerror = () => reject(new Error("Failed to load Chart.js"));
@ -117,15 +111,7 @@ function matchesAllTokens(haystack, tokens) {
function rowSearchText(r) {
const rep = r?.representative || {};
return [
r?.canonSku,
rep?.name,
rep?.skuRaw,
rep?.skuKey,
rep?.categoryLabel,
rep?.storeLabel,
rep?.storeKey,
]
return [r?.canonSku, rep?.name, rep?.skuRaw, rep?.skuKey, rep?.categoryLabel, rep?.storeLabel, rep?.storeKey]
.map((x) => String(x || "").trim())
.filter(Boolean)
.join(" | ")
@ -286,9 +272,7 @@ async function loadCommitsFallback({ owner, repo, branch, relPath }) {
const byDate = new Map();
for (const c of apiCommits) {
const sha = String(c?.sha || "");
const ts = String(
c?.commit?.committer?.date || c?.commit?.author?.date || ""
);
const ts = String(c?.commit?.committer?.date || c?.commit?.author?.date || "");
const d = dateOnly(ts);
if (!sha || !d) continue;
if (!byDate.has(d)) byDate.set(d, { sha, date: d, ts });
@ -310,13 +294,10 @@ async function loadRawSeries({ group, size, onStatus }) {
const manifest = await loadCommonCommitsManifest();
let commits = Array.isArray(manifest?.files?.[rel])
? manifest.files[rel]
: null;
let commits = Array.isArray(manifest?.files?.[rel]) ? manifest.files[rel] : null;
if (!commits || !commits.length) {
if (typeof onStatus === "function")
onStatus(`Commits manifest missing for ${rel}; using GitHub API fallback…`);
if (typeof onStatus === "function") onStatus(`Commits manifest missing for ${rel}; using GitHub API fallback…`);
commits = await loadCommitsFallback({ owner, repo, branch, relPath: rel });
}
@ -328,11 +309,7 @@ async function loadRawSeries({ group, size, onStatus }) {
const cacheKey = `${group}:${size}`;
const cached = RAW_SERIES_CACHE.get(cacheKey);
if (
cached &&
cached.latestSha === latestSha &&
cached.labels?.length === commits.length
) {
if (cached && cached.latestSha === latestSha && cached.labels?.length === commits.length) {
return cached;
}
@ -340,21 +317,15 @@ async function loadRawSeries({ group, size, onStatus }) {
const limitNet = makeLimiter(NET_CONCURRENCY);
if (typeof onStatus === "function") onStatus(`Loading stores…`);
const newestReport = await limitNet(() =>
githubFetchFileAtSha({ owner, repo, sha: latestSha, path: rel })
);
const newestReport = await limitNet(() => githubFetchFileAtSha({ owner, repo, sha: latestSha, path: rel }));
const stores = Array.isArray(newestReport?.stores)
? newestReport.stores.map(String)
: [];
if (!stores.length)
throw new Error(`No stores found in ${rel} at ${latestSha.slice(0, 7)}`);
const stores = Array.isArray(newestReport?.stores) ? newestReport.stores.map(String) : [];
if (!stores.length) throw new Error(`No stores found in ${rel} at ${latestSha.slice(0, 7)}`);
const labels = commits.map((c) => String(c.date || "")).filter(Boolean);
const shaByIdx = commits.map((c) => String(c.sha || ""));
if (typeof onStatus === "function")
onStatus(`Loading ${labels.length} day(s)…`);
if (typeof onStatus === "function") onStatus(`Loading ${labels.length} day(s)…`);
const reportsByIdx = new Array(shaByIdx.length).fill(null);
@ -373,15 +344,12 @@ async function loadRawSeries({ group, size, onStatus }) {
reportsByIdx[idx] = null;
} finally {
done++;
if (
typeof onStatus === "function" &&
(done % 10 === 0 || done === shaByIdx.length)
) {
if (typeof onStatus === "function" && (done % 10 === 0 || done === shaByIdx.length)) {
onStatus(`Loading ${labels.length} day(s)… ${done}/${labels.length}`);
}
}
})
)
}),
),
);
const out = { latestSha, labels, stores, commits, reportsByIdx };
@ -440,7 +408,8 @@ function computeSeriesFromRaw(raw, filter) {
/* ---------------- y-axis bounds ---------------- */
function computeYBounds(seriesByStore, minSpan = 6, pad = 1) {
let mn = Infinity, mx = -Infinity;
let mn = Infinity,
mx = -Infinity;
for (const arr of Object.values(seriesByStore || {})) {
if (!Array.isArray(arr)) continue;
@ -471,7 +440,6 @@ function computeYBounds(seriesByStore, minSpan = 6, pad = 1) {
return { min: mn, max: mx };
}
/* ---------------- prefs ---------------- */
const LS_GROUP = "stviz:v1:stats:group";
@ -669,9 +637,7 @@ export async function renderStats($app) {
function updatePriceLabel() {
if (!$priceLabel) return;
$priceLabel.textContent = `${formatDollars(selectedMinPrice)} ${formatDollars(
selectedMaxPrice
)}`;
$priceLabel.textContent = `${formatDollars(selectedMinPrice)} ${formatDollars(selectedMaxPrice)}`;
}
function saveFilterPrefs(group, size) {
@ -719,7 +685,8 @@ export async function renderStats($app) {
const order = stores
.map((s) => ({ s, v: lastFiniteFromEnd(seriesByStore[s]) }))
.sort((a, b) => {
const av = a.v, bv = b.v;
const av = a.v,
bv = b.v;
if (av === null && bv === null) return displayStoreName(a.s).localeCompare(displayStoreName(b.s));
if (av === null) return 1;
if (bv === null) return -1;
@ -797,9 +764,7 @@ export async function renderStats($app) {
grid: {
drawBorder: false,
color: (ctx) =>
ctx.tick.value === 0
? "rgba(154,166,178,0.35)"
: "rgba(154,166,178,0.18)",
ctx.tick.value === 0 ? "rgba(154,166,178,0.35)" : "rgba(154,166,178,0.18)",
lineWidth: 1,
},
},
@ -827,8 +792,7 @@ export async function renderStats($app) {
// floor is ALWAYS 0 now
boundMin = 0;
boundMax =
Number.isFinite(b.max) && b.max > 0 ? Math.ceil(b.max) : 1000;
boundMax = Number.isFinite(b.max) && b.max > 0 ? Math.ceil(b.max) : 1000;
const saved = loadFilterPrefs(group, size);
if ($q) $q.value = saved.q || "";
@ -850,8 +814,7 @@ export async function renderStats($app) {
selectedMinPrice = clampAndRound(wantMin);
selectedMaxPrice = clampAndRound(wantMax);
if (selectedMinPrice > selectedMaxPrice)
selectedMinPrice = selectedMaxPrice;
if (selectedMinPrice > selectedMaxPrice) selectedMinPrice = selectedMaxPrice;
}
setSliderFromPrice($minR, selectedMinPrice);

View file

@ -1,5 +1,7 @@
function normalizeId(s) {
return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, "");
return String(s || "")
.toLowerCase()
.replace(/[^a-z0-9]+/g, "");
}
// Map normalized store *labels* to canonical ids used by OVERRIDES
@ -45,12 +47,36 @@ const OVERRIDES = {
// High-contrast qualitative palette
const PALETTE = [
"#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD",
"#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22",
"#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5",
"#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D",
"#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173",
"#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D",
"#1F77B4",
"#FF7F0E",
"#2CA02C",
"#D62728",
"#9467BD",
"#8C564B",
"#E377C2",
"#7F7F7F",
"#17BECF",
"#BCBD22",
"#AEC7E8",
"#FFBB78",
"#98DF8A",
"#FF9896",
"#C5B0D5",
"#C49C94",
"#F7B6D2",
"#C7C7C7",
"#9EDAE5",
"#DBDB8D",
"#393B79",
"#637939",
"#8C6D31",
"#843C39",
"#7B4173",
"#3182BD",
"#31A354",
"#756BB1",
"#636363",
"#E6550D",
];
function uniq(arr) {
@ -89,11 +115,15 @@ const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [
"vintage",
"vintagespirits",
"willowpark",
"arc"
"arc",
]);
function isWhiteHex(c) {
return String(c || "").trim().toUpperCase() === "#FFFFFF";
return (
String(c || "")
.trim()
.toUpperCase() === "#FFFFFF"
);
}
export function buildStoreColorMap(extraUniverse = []) {
@ -112,9 +142,9 @@ export function buildStoreColorMap(extraUniverse = []) {
}
// Filter palette to avoid collisions and keep white/black reserved
const palette = PALETTE
.map((c) => String(c).toUpperCase())
.filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111");
const palette = PALETTE.map((c) => String(c).toUpperCase()).filter(
(c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111",
);
let pi = 0;
for (const id of universe) {
@ -167,8 +197,7 @@ function hexToRgb(hex) {
}
function rgbToHex({ r, g, b }) {
const h = (x) =>
clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0");
const h = (x) => clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0");
return `#${h(r)}${h(g)}${h(b)}`;
}

View file

@ -1,17 +1,13 @@
import { esc, renderThumbHtml } from "./dom.js";
import {
tokenizeQuery,
matchesAllTokens,
displaySku,
keySkuForRow,
parsePriceToNumber,
} from "./sku.js";
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, parsePriceToNumber } from "./sku.js";
import { loadIndex, loadRecent } from "./state.js";
import { aggregateBySku } from "./catalog.js";
import { loadSkuRules } from "./mapping.js";
function normStoreLabel(s) {
return String(s || "").trim().toLowerCase();
return String(s || "")
.trim()
.toLowerCase();
}
function abbrevStoreLabel(s) {
@ -38,9 +34,7 @@ function readLinkHrefForSkuInStore(listingsLive, canonSku, storeLabelNorm) {
const store = normStoreLabel(r.storeLabel || r.store || "");
if (store !== storeLabelNorm) continue;
const skuKey = String(
rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r)
);
const skuKey = String(rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r));
if (skuKey !== canonSku) continue;
const u = String(r.url || "").trim();
@ -60,8 +54,7 @@ let rulesCache = null;
export async function renderStore($app, storeLabelRaw) {
const storeLabel = String(storeLabelRaw || "").trim();
const storeLabelShort =
abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
const storeLabelShort = abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
$app.innerHTML = `
<div class="container">
@ -184,8 +177,7 @@ export async function renderStore($app, storeLabelRaw) {
// Persist max price per store (clamped later once bounds known)
const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`;
const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE);
let savedMaxPrice =
savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
let savedMaxPrice = savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null;
// Persist exclusives sort per store
@ -264,15 +256,13 @@ export async function renderStore($app, storeLabelRaw) {
if (!r) return null;
const kind = normalizeKindForPrice(r);
if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change")
return null;
if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") return null;
const oldStr = String(r?.oldPrice || "").trim();
const newStr = String(r?.newPrice || "").trim();
const oldN = parsePriceToNumber(oldStr);
const newN = parsePriceToNumber(newStr);
if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0))
return null;
if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) return null;
const delta = newN - oldN; // negative = down
const pct = Math.round(((newN - oldN) / oldN) * 100); // negative = down
@ -368,9 +358,7 @@ export async function renderStore($app, storeLabelRaw) {
}
// Store-specific live rows only (in-stock for that store)
const rowsStoreLive = liveAll.filter(
(r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm
);
const rowsStoreLive = liveAll.filter((r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm);
// Aggregate in this store, grouped by canonical SKU (so mappings count as same bottle)
let items = aggregateBySku(rowsStoreLive, rules.canonicalSku);
@ -383,35 +371,23 @@ export async function renderStore($app, storeLabelRaw) {
const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]);
const everStoreSet = everStoresBySku.get(sku) || liveStoreSet;
const soloLiveHere =
liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
const soloLiveHere = liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
const lastStock = soloLiveHere && everStoreSet.size > 1;
const exclusive = soloLiveHere && !lastStock;
const storePrice = Number.isFinite(it.cheapestPriceNum)
? it.cheapestPriceNum
: null;
const storePrice = Number.isFinite(it.cheapestPriceNum) ? it.cheapestPriceNum : null;
const bestAll = bestAllPrice(sku);
const other = bestOtherPrice(sku, storeNorm);
const isBest =
storePrice !== null && bestAll !== null
? storePrice <= bestAll + EPS
: false;
const isBest = storePrice !== null && bestAll !== null ? storePrice <= bestAll + EPS : false;
const diffVsOtherDollar =
storePrice !== null && other !== null ? storePrice - other : null;
const diffVsOtherDollar = storePrice !== null && other !== null ? storePrice - other : null;
const diffVsOtherPct =
storePrice !== null && other !== null && other > 0
? ((storePrice - other) / other) * 100
: null;
storePrice !== null && other !== null && other > 0 ? ((storePrice - other) / other) * 100 : null;
const diffVsBestDollar =
storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
const diffVsBestDollar = storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
const diffVsBestPct =
storePrice !== null && bestAll !== null && bestAll > 0
? ((storePrice - bestAll) / bestAll) * 100
: null;
storePrice !== null && bestAll !== null && bestAll > 0 ? ((storePrice - bestAll) / bestAll) * 100 : null;
const firstSeenMs = firstSeenBySkuInStore.get(sku);
const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null;
@ -495,9 +471,7 @@ export async function renderStore($app, storeLabelRaw) {
return `$${Math.round(p)}`;
}
let selectedMaxPrice = clampAndRound(
savedMaxPrice !== null ? savedMaxPrice : boundMax
);
let selectedMaxPrice = clampAndRound(savedMaxPrice !== null ? savedMaxPrice : boundMax);
function setSliderFromPrice(p) {
const t = tFromPrice(p);
@ -536,8 +510,7 @@ export async function renderStore($app, storeLabelRaw) {
// ---- Listing display price: keep cents (no rounding) ----
function listingPriceStr(it) {
const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null;
if (p === null)
return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
if (p === null) return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
return `$${p.toFixed(2)}`;
}
@ -620,9 +593,7 @@ export async function renderStore($app, storeLabelRaw) {
: "";
const bestBadge =
!it._exclusive && !it._lastStock && it._isBest
? `<span class="badge badgeBest">Best Price</span>`
: "";
!it._exclusive && !it._lastStock && it._isBest ? `<span class="badge badgeBest">Best Price</span>` : "";
const diffBadge = priceBadgeHtml(it);
const exAnnot = it._exclusive || it._lastStock ? exclusiveAnnotHtml(it) : "";
@ -636,9 +607,7 @@ export async function renderStore($app, storeLabelRaw) {
<div class="itemTop">
<div class="itemName">${esc(it.name || "(no name)")}</div>
<a class="badge mono skuLink" target="_blank" rel="noopener noreferrer"
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(
displaySku(it.sku)
)}</a>
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(displaySku(it.sku))}</a>
</div>
<div class="metaRow">
${specialBadge}
@ -649,9 +618,9 @@ export async function renderStore($app, storeLabelRaw) {
${
href
? `<a class="badge" href="${esc(
href
href,
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
storeLabelShort
storeLabelShort,
)}</a>`
: ``
}
@ -686,9 +655,7 @@ export async function renderStore($app, storeLabelRaw) {
}
if (pageMax !== null) {
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(
selectedMaxPrice
)}).`;
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(selectedMaxPrice)}).`;
return;
}
@ -703,28 +670,14 @@ export async function renderStore($app, storeLabelRaw) {
shownCompare = 0;
}
const sliceEx = filteredExclusive.slice(
shownExclusive,
shownExclusive + PAGE_EACH
);
const sliceCo = filteredCompare.slice(
shownCompare,
shownCompare + PAGE_EACH
);
const sliceEx = filteredExclusive.slice(shownExclusive, shownExclusive + PAGE_EACH);
const sliceCo = filteredCompare.slice(shownCompare, shownCompare + PAGE_EACH);
shownExclusive += sliceEx.length;
shownCompare += sliceCo.length;
if (sliceEx.length)
$resultsExclusive.insertAdjacentHTML(
"beforeend",
sliceEx.map(renderCard).join("")
);
if (sliceCo.length)
$resultsCompare.insertAdjacentHTML(
"beforeend",
sliceCo.map(renderCard).join("")
);
if (sliceEx.length) $resultsExclusive.insertAdjacentHTML("beforeend", sliceEx.map(renderCard).join(""));
if (sliceCo.length) $resultsCompare.insertAdjacentHTML("beforeend", sliceCo.map(renderCard).join(""));
const total = totalFiltered();
const shown = totalShown();
@ -778,12 +731,9 @@ export async function renderStore($app, storeLabelRaw) {
arr.sort((a, b) => {
const ap = Number.isFinite(a._storePrice) ? a._storePrice : null;
const bp = Number.isFinite(b._storePrice) ? b._storePrice : null;
const aKey =
ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
const bKey =
bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
if (aKey !== bKey)
return mode === "priceAsc" ? aKey - bKey : bKey - aKey;
const aKey = ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey;
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
});
return;
@ -793,12 +743,9 @@ export async function renderStore($app, storeLabelRaw) {
arr.sort((a, b) => {
const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null;
const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null;
const aKey =
ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
const bKey =
bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
if (aKey !== bKey)
return mode === "dateAsc" ? aKey - bKey : bKey - aKey;
const aKey = ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey;
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
});
}
@ -857,7 +804,7 @@ export async function renderStore($app, storeLabelRaw) {
if (totalShown() >= totalFiltered()) return;
renderNext(false);
},
{ root: null, rootMargin: "600px 0px", threshold: 0.01 }
{ root: null, rootMargin: "600px 0px", threshold: 0.01 },
);
io.observe($sentinel);