mirror of
https://github.com/samsonjs/spirit-tracker.git
synced 2026-03-25 09:25:51 +00:00
UX Improvements
This commit is contained in:
parent
e9f8f805c5
commit
7a33d51c90
73 changed files with 13094 additions and 13094 deletions
|
|
@ -205,14 +205,16 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
|
||||||
url,
|
url,
|
||||||
tag,
|
tag,
|
||||||
ua,
|
ua,
|
||||||
{ mode = "text", method = "GET", headers = {}, body = null, cookies = true } = {}
|
{ mode = "text", method = "GET", headers = {}, body = null, cookies = true } = {},
|
||||||
) {
|
) {
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
const reqId = ++reqSeq;
|
const reqId = ++reqSeq;
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
|
|
||||||
inflight++;
|
inflight++;
|
||||||
logger?.dbg?.(`REQ#${reqId} START ${tag} attempt=${attempt + 1}/${maxRetries + 1} ${url} (${inflightStr()})`);
|
logger?.dbg?.(
|
||||||
|
`REQ#${reqId} START ${tag} attempt=${attempt + 1}/${maxRetries + 1} ${url} (${inflightStr()})`,
|
||||||
|
);
|
||||||
|
|
||||||
const releaseHost = await acquireHost(url);
|
const releaseHost = await acquireHost(url);
|
||||||
|
|
||||||
|
|
@ -268,9 +270,7 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
|
||||||
|
|
||||||
if (status >= 400) {
|
if (status >= 400) {
|
||||||
const bodyTxt = await safeText(res);
|
const bodyTxt = await safeText(res);
|
||||||
throw new Error(
|
throw new Error(`HTTP ${status} bodyHead=${String(bodyTxt).slice(0, 160).replace(/\s+/g, " ")}`);
|
||||||
`HTTP ${status} bodyHead=${String(bodyTxt).slice(0, 160).replace(/\s+/g, " ")}`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mode === "json") {
|
if (mode === "json") {
|
||||||
|
|
@ -298,8 +298,8 @@ function createHttpClient({ maxRetries, timeoutMs, defaultUa, logger }) {
|
||||||
logger?.dbg?.(
|
logger?.dbg?.(
|
||||||
`REQ#${reqId} FAIL ${tag} retryable=${retryable} err=${e?.message || e} host=${host} nextOkIn=${Math.max(
|
`REQ#${reqId} FAIL ${tag} retryable=${retryable} err=${e?.message || e} host=${host} nextOkIn=${Math.max(
|
||||||
0,
|
0,
|
||||||
nextOk - Date.now()
|
nextOk - Date.now(),
|
||||||
)}ms`
|
)}ms`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!retryable || attempt === maxRetries) throw e;
|
if (!retryable || attempt === maxRetries) throw e;
|
||||||
|
|
|
||||||
42
src/main.js
42
src/main.js
|
|
@ -16,8 +16,7 @@ const { runAllStores } = require("./tracker/run_all");
|
||||||
const { renderFinalReport } = require("./tracker/report");
|
const { renderFinalReport } = require("./tracker/report");
|
||||||
const { ensureDir } = require("./tracker/db");
|
const { ensureDir } = require("./tracker/db");
|
||||||
|
|
||||||
const DEFAULT_UA =
|
const DEFAULT_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36";
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36";
|
|
||||||
|
|
||||||
function resolveDir(p, fallback) {
|
function resolveDir(p, fallback) {
|
||||||
const v = String(p || "").trim();
|
const v = String(p || "").trim();
|
||||||
|
|
@ -79,12 +78,8 @@ function filterStoresOrThrow(stores, wantedListRaw) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (missing.length) {
|
if (missing.length) {
|
||||||
const avail = stores
|
const avail = stores.map((s) => `${s.key}${s.name ? ` (${s.name})` : ""}`).join(", ");
|
||||||
.map((s) => `${s.key}${s.name ? ` (${s.name})` : ""}`)
|
throw new Error(`Unknown store(s) in --stores: ${missing.join(", ")}\nAvailable: ${avail}`);
|
||||||
.join(", ");
|
|
||||||
throw new Error(
|
|
||||||
`Unknown store(s) in --stores: ${missing.join(", ")}\nAvailable: ${avail}`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// de-dupe by key (in case name+key both matched)
|
// de-dupe by key (in case name+key both matched)
|
||||||
|
|
@ -100,9 +95,7 @@ function filterStoresOrThrow(stores, wantedListRaw) {
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
if (typeof fetch !== "function") {
|
if (typeof fetch !== "function") {
|
||||||
throw new Error(
|
throw new Error("Global fetch() not found. Please use Node.js 18+ (or newer). ");
|
||||||
"Global fetch() not found. Please use Node.js 18+ (or newer). "
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const argv = process.argv.slice(2);
|
const argv = process.argv.slice(2);
|
||||||
|
|
@ -114,25 +107,16 @@ async function main() {
|
||||||
debug: args.debug,
|
debug: args.debug,
|
||||||
maxPages: args.maxPages,
|
maxPages: args.maxPages,
|
||||||
concurrency: args.concurrency ?? clampInt(process.env.CONCURRENCY, 6, 1, 64),
|
concurrency: args.concurrency ?? clampInt(process.env.CONCURRENCY, 6, 1, 64),
|
||||||
staggerMs:
|
staggerMs: args.staggerMs ?? clampInt(process.env.STAGGER_MS, 150, 0, 5000),
|
||||||
args.staggerMs ?? clampInt(process.env.STAGGER_MS, 150, 0, 5000),
|
|
||||||
maxRetries: clampInt(process.env.MAX_RETRIES, 6, 0, 20),
|
maxRetries: clampInt(process.env.MAX_RETRIES, 6, 0, 20),
|
||||||
timeoutMs: clampInt(process.env.TIMEOUT_MS, 25000, 1000, 120000),
|
timeoutMs: clampInt(process.env.TIMEOUT_MS, 25000, 1000, 120000),
|
||||||
discoveryGuess:
|
discoveryGuess: args.guess ?? clampInt(process.env.DISCOVERY_GUESS, 20, 1, 5000),
|
||||||
args.guess ?? clampInt(process.env.DISCOVERY_GUESS, 20, 1, 5000),
|
discoveryStep: args.step ?? clampInt(process.env.DISCOVERY_STEP, 5, 1, 500),
|
||||||
discoveryStep:
|
|
||||||
args.step ?? clampInt(process.env.DISCOVERY_STEP, 5, 1, 500),
|
|
||||||
categoryConcurrency: clampInt(process.env.CATEGORY_CONCURRENCY, 5, 1, 64),
|
categoryConcurrency: clampInt(process.env.CATEGORY_CONCURRENCY, 5, 1, 64),
|
||||||
defaultUa: DEFAULT_UA,
|
defaultUa: DEFAULT_UA,
|
||||||
defaultParseProducts: parseProductsSierra,
|
defaultParseProducts: parseProductsSierra,
|
||||||
dbDir: resolveDir(
|
dbDir: resolveDir(args.dataDir ?? process.env.DATA_DIR, path.join(process.cwd(), "data", "db")),
|
||||||
args.dataDir ?? process.env.DATA_DIR,
|
reportDir: resolveDir(args.reportDir ?? process.env.REPORT_DIR, path.join(process.cwd(), "reports")),
|
||||||
path.join(process.cwd(), "data", "db")
|
|
||||||
),
|
|
||||||
reportDir: resolveDir(
|
|
||||||
args.reportDir ?? process.env.REPORT_DIR,
|
|
||||||
path.join(process.cwd(), "reports")
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
ensureDir(config.dbDir);
|
ensureDir(config.dbDir);
|
||||||
|
|
@ -146,8 +130,7 @@ async function main() {
|
||||||
});
|
});
|
||||||
const stores = createStores({ defaultUa: config.defaultUa });
|
const stores = createStores({ defaultUa: config.defaultUa });
|
||||||
|
|
||||||
const storesFilterRaw =
|
const storesFilterRaw = getFlagValue(argv, "--stores") || String(process.env.STORES || "").trim();
|
||||||
getFlagValue(argv, "--stores") || String(process.env.STORES || "").trim();
|
|
||||||
|
|
||||||
const storesToRun = filterStoresOrThrow(stores, storesFilterRaw);
|
const storesToRun = filterStoresOrThrow(stores, storesFilterRaw);
|
||||||
if (storesFilterRaw) {
|
if (storesFilterRaw) {
|
||||||
|
|
@ -180,10 +163,7 @@ async function main() {
|
||||||
dbDir: config.dbDir,
|
dbDir: config.dbDir,
|
||||||
colorize: false,
|
colorize: false,
|
||||||
});
|
});
|
||||||
const file = path.join(
|
const file = path.join(config.reportDir, `${isoTimestampFileSafe(new Date())}.txt`);
|
||||||
config.reportDir,
|
|
||||||
`${isoTimestampFileSafe(new Date())}.txt`
|
|
||||||
);
|
|
||||||
try {
|
try {
|
||||||
fs.writeFileSync(file, reportTextPlain, "utf8");
|
fs.writeFileSync(file, reportTextPlain, "utf8");
|
||||||
logger.ok(`Report saved: ${logger.dim(file)}`);
|
logger.ok(`Report saved: ${logger.dim(file)}`);
|
||||||
|
|
|
||||||
|
|
@ -125,21 +125,15 @@ function arcNormalizeImg(raw) {
|
||||||
const hasCspcId = /^\d{1,11}$/.test(rawCspcId);
|
const hasCspcId = /^\d{1,11}$/.test(rawCspcId);
|
||||||
|
|
||||||
const id = Number(p?.id);
|
const id = Number(p?.id);
|
||||||
const rawSku =
|
const rawSku = hasCspcId ? `id:${rawCspcId}` : Number.isFinite(id) ? `id:${id}` : "";
|
||||||
hasCspcId ? `id:${rawCspcId}` :
|
|
||||||
Number.isFinite(id) ? `id:${id}` :
|
|
||||||
"";
|
|
||||||
|
|
||||||
const sku =
|
|
||||||
normalizeSkuKey(rawSku, { storeLabel: ctx?.store?.name, url }) || rawSku || "";
|
|
||||||
|
|
||||||
|
const sku = normalizeSkuKey(rawSku, { storeLabel: ctx?.store?.name, url }) || rawSku || "";
|
||||||
|
|
||||||
const img = arcNormalizeImg(p.image || p.image_url || p.img || "");
|
const img = arcNormalizeImg(p.image || p.image_url || p.img || "");
|
||||||
|
|
||||||
return { name, price, url, sku, img };
|
return { name, price, url, sku, img };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function parseCategoryParamsFromStartUrl(startUrl) {
|
function parseCategoryParamsFromStartUrl(startUrl) {
|
||||||
try {
|
try {
|
||||||
const u = new URL(startUrl);
|
const u = new URL(startUrl);
|
||||||
|
|
@ -161,7 +155,7 @@ function avoidMassRemoval(prevDb, discovered, ctx, reason) {
|
||||||
if (ratio >= 0.6) return false;
|
if (ratio >= 0.6) return false;
|
||||||
|
|
||||||
ctx.logger.warn?.(
|
ctx.logger.warn?.(
|
||||||
`${ctx.catPrefixOut} | ARC partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`
|
`${ctx.catPrefixOut} | ARC partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Preserve prior active items not seen this run.
|
// Preserve prior active items not seen this run.
|
||||||
|
|
@ -238,12 +232,12 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
// Log early (even for empty)
|
// Log early (even for empty)
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(
|
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "")
|
||||||
3
|
.toString()
|
||||||
)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(0, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
|
.padEnd(3)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(0, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
|
||||||
ctx.http.inflightStr(),
|
ctx.http.inflightStr(),
|
||||||
11
|
11,
|
||||||
)} | ${secStr(r.ms)}`
|
)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!rawCount) break;
|
if (!rawCount) break;
|
||||||
|
|
@ -274,12 +268,14 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
// Re-log with kept filled in (overwrite-style isn’t possible; just emit a second line)
|
// Re-log with kept filled in (overwrite-style isn’t possible; just emit a second line)
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(
|
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "")
|
||||||
3
|
.toString()
|
||||||
|
.padEnd(
|
||||||
|
3,
|
||||||
)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(kept, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
|
)} | raw=${padLeft(rawCount, 3)} kept=${padLeft(kept, 3)} | bytes=${kbStr(r.bytes)} | ${padRight(
|
||||||
ctx.http.inflightStr(),
|
ctx.http.inflightStr(),
|
||||||
11
|
11,
|
||||||
)} | ${secStr(r.ms)}`
|
)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Stop condition #1: last page (short page)
|
// Stop condition #1: last page (short page)
|
||||||
|
|
@ -301,8 +297,11 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } =
|
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
|
||||||
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
prevDb,
|
||||||
|
discovered,
|
||||||
|
{ storeLabel: ctx.store.name },
|
||||||
|
);
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -311,7 +310,7 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsedMs = Date.now() - t0;
|
const elapsedMs = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsedMs)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Meta=${metaChangedItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsedMs)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Meta=${metaChangedItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -334,10 +333,17 @@ async function scanCategoryArcApi(ctx, prevDb, report) {
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
report.totals.metaChangedCount += metaChangedItems.length;
|
report.totals.metaChangedCount += metaChangedItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
return {
|
return {
|
||||||
key: "arc",
|
key: "arc",
|
||||||
|
|
|
||||||
|
|
@ -158,7 +158,7 @@ function bclHitToItem(hit) {
|
||||||
// ✅ Fix: BCL appears to serve .jpg (not .jpeg) for these imagecache URLs.
|
// ✅ Fix: BCL appears to serve .jpg (not .jpeg) for these imagecache URLs.
|
||||||
// Also use https.
|
// Also use https.
|
||||||
const img = `https://www.bcliquorstores.com/sites/default/files/imagecache/height400px/${encodeURIComponent(
|
const img = `https://www.bcliquorstores.com/sites/default/files/imagecache/height400px/${encodeURIComponent(
|
||||||
skuRaw
|
skuRaw,
|
||||||
)}.jpg`;
|
)}.jpg`;
|
||||||
|
|
||||||
return { name, price, url, sku, img };
|
return { name, price, url, sku, img };
|
||||||
|
|
@ -204,7 +204,11 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
ctx.logger.warn(`${ctx.catPrefixOut} | BCL browse fetch failed: ${e?.message || e}`);
|
ctx.logger.warn(`${ctx.catPrefixOut} | BCL browse fetch failed: ${e?.message || e}`);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
|
||||||
|
prevDb,
|
||||||
|
discovered,
|
||||||
|
{ storeLabel: ctx.store.name },
|
||||||
|
);
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
||||||
|
|
@ -226,7 +230,15 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
report.totals.updatedCount += updatedItems.length;
|
report.totals.updatedCount += updatedItems.length;
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -234,7 +246,9 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
const totalPages = Math.max(1, Math.ceil(total / size));
|
const totalPages = Math.max(1, Math.ceil(total / size));
|
||||||
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Total=${total} Size=${size} Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
|
ctx.logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Total=${total} Size=${size} Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const pageNums = [];
|
const pageNums = [];
|
||||||
for (let p = 1; p <= scanPages; p++) pageNums.push(p);
|
for (let p = 1; p <= scanPages; p++) pageNums.push(p);
|
||||||
|
|
@ -259,12 +273,12 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pageNums.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageNums.length)} | items=${padLeft(
|
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pageNums.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageNums.length)} | items=${padLeft(
|
||||||
items.length,
|
items.length,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
}
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
|
|
@ -276,9 +290,13 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
|
ctx.logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
|
storeLabel: ctx.store.name,
|
||||||
|
});
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -287,7 +305,7 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -309,7 +327,15 @@ async function scanCategoryBCLAjax(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
|
|
@ -324,13 +350,15 @@ function createStore(defaultUa) {
|
||||||
key: "whisky",
|
key: "whisky",
|
||||||
label: "Whisky / Whiskey",
|
label: "Whisky / Whiskey",
|
||||||
// informational only; scan uses ajax/browse
|
// informational only; scan uses ajax/browse
|
||||||
startUrl: "https://www.bcliquorstores.com/product-catalogue?category=spirits&type=whisky%20/%20whiskey&sort=featuredProducts:desc&page=1",
|
startUrl:
|
||||||
|
"https://www.bcliquorstores.com/product-catalogue?category=spirits&type=whisky%20/%20whiskey&sort=featuredProducts:desc&page=1",
|
||||||
bclType: "whisky / whiskey",
|
bclType: "whisky / whiskey",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: "rum",
|
key: "rum",
|
||||||
label: "Rum",
|
label: "Rum",
|
||||||
startUrl: "https://www.bcliquorstores.com/product-catalogue?category=spirits&type=rum&sort=featuredProducts:desc&page=1",
|
startUrl:
|
||||||
|
"https://www.bcliquorstores.com/product-catalogue?category=spirits&type=rum&sort=featuredProducts:desc&page=1",
|
||||||
bclType: "rum",
|
bclType: "rum",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
|
||||||
|
|
@ -82,7 +82,6 @@ function bswPickPrice(hit) {
|
||||||
return pick(null, false);
|
return pick(null, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function bswHitToItem(hit) {
|
function bswHitToItem(hit) {
|
||||||
const name = cleanText(hit && (hit.title || hit.name || hit.product_title || hit.product_name || ""));
|
const name = cleanText(hit && (hit.title || hit.name || hit.product_title || hit.product_name || ""));
|
||||||
const handle = hit && (hit.handle || hit.product_handle || hit.slug || "");
|
const handle = hit && (hit.handle || hit.product_handle || hit.slug || "");
|
||||||
|
|
@ -228,19 +227,24 @@ function bswPickImage(hit) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
|
|
||||||
let collectionId = Number.isFinite(ctx.cat.bswCollectionId) ? ctx.cat.bswCollectionId : null;
|
let collectionId = Number.isFinite(ctx.cat.bswCollectionId) ? ctx.cat.bswCollectionId : null;
|
||||||
if (!collectionId) {
|
if (!collectionId) {
|
||||||
try {
|
try {
|
||||||
const { text: html } = await ctx.http.fetchTextWithRetry(ctx.cat.startUrl, `bsw:html:${ctx.cat.key}`, ctx.store.ua);
|
const { text: html } = await ctx.http.fetchTextWithRetry(
|
||||||
|
ctx.cat.startUrl,
|
||||||
|
`bsw:html:${ctx.cat.key}`,
|
||||||
|
ctx.store.ua,
|
||||||
|
);
|
||||||
collectionId = bswExtractCollectionIdFromHtml(html);
|
collectionId = bswExtractCollectionIdFromHtml(html);
|
||||||
if (collectionId) ctx.logger.ok(`${ctx.catPrefixOut} | BSW discovered collectionId=${collectionId}`);
|
if (collectionId) ctx.logger.ok(`${ctx.catPrefixOut} | BSW discovered collectionId=${collectionId}`);
|
||||||
else ctx.logger.warn(`${ctx.catPrefixOut} | BSW could not discover collectionId from HTML.`);
|
else ctx.logger.warn(`${ctx.catPrefixOut} | BSW could not discover collectionId from HTML.`);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
ctx.logger.warn(`${ctx.catPrefixOut} | BSW HTML fetch failed for collectionId discovery: ${e?.message || e}`);
|
ctx.logger.warn(
|
||||||
|
`${ctx.catPrefixOut} | BSW HTML fetch failed for collectionId discovery: ${e?.message || e}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -248,7 +252,11 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
ctx.logger.warn(`${ctx.catPrefixOut} | BSW missing collectionId; defaulting to 1 page with 0 items.`);
|
ctx.logger.warn(`${ctx.catPrefixOut} | BSW missing collectionId; defaulting to 1 page with 0 items.`);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
|
||||||
|
prevDb,
|
||||||
|
discovered,
|
||||||
|
{ storeLabel: ctx.store.name },
|
||||||
|
);
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
||||||
|
|
@ -272,7 +280,15 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
report.totals.updatedCount += updatedItems.length;
|
report.totals.updatedCount += updatedItems.length;
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -285,16 +301,23 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
|
|
||||||
const totalPages = Math.max(1, nbPages);
|
const totalPages = Math.max(1, nbPages);
|
||||||
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
|
ctx.logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const pageIdxs = [];
|
const pageIdxs = [];
|
||||||
for (let p = 0; p < scanPages; p++) pageIdxs.push(p);
|
for (let p = 0; p < scanPages; p++) pageIdxs.push(p);
|
||||||
|
|
||||||
let donePages = 0;
|
let donePages = 0;
|
||||||
|
|
||||||
const perPageItems = await require("../utils/async").parallelMapStaggered(pageIdxs, ctx.config.concurrency, ctx.config.staggerMs, async (page0, idx) => {
|
const perPageItems = await require("../utils/async").parallelMapStaggered(
|
||||||
|
pageIdxs,
|
||||||
|
ctx.config.concurrency,
|
||||||
|
ctx.config.staggerMs,
|
||||||
|
async (page0, idx) => {
|
||||||
const pnum = idx + 1;
|
const pnum = idx + 1;
|
||||||
const r = page0 === 0 ? first : await bswFetchAlgoliaPage(ctx, collectionId, ruleContext, page0, hitsPerPage);
|
const r =
|
||||||
|
page0 === 0 ? first : await bswFetchAlgoliaPage(ctx, collectionId, ruleContext, page0, hitsPerPage);
|
||||||
|
|
||||||
const res0 = r?.json?.results?.[0] || null;
|
const res0 = r?.json?.results?.[0] || null;
|
||||||
const hits = res0 && Array.isArray(res0.hits) ? res0.hits : [];
|
const hits = res0 && Array.isArray(res0.hits) ? res0.hits : [];
|
||||||
|
|
@ -309,12 +332,13 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(pnum, pageIdxs.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageIdxs.length)} | items=${padLeft(
|
`${ctx.catPrefixOut} | Page ${pageStr(pnum, pageIdxs.length)} | ${String(r.status || "").padEnd(3)} | ${pctStr(donePages, pageIdxs.length)} | items=${padLeft(
|
||||||
items.length,
|
items.length,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
});
|
},
|
||||||
|
);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
let dups = 0;
|
let dups = 0;
|
||||||
|
|
@ -325,9 +349,13 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
|
ctx.logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
|
storeLabel: ctx.store.name,
|
||||||
|
});
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -336,7 +364,7 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -357,7 +385,15 @@ async function scanCategoryBSWAlgolia(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
|
|
|
||||||
|
|
@ -90,12 +90,11 @@ const r = await coopFetchText(ctx, REFERER, "coop:bootstrap", {
|
||||||
|
|
||||||
if (!coop.sessionKey || !coop.chainId || !coop.storeId || !coop.appVersion) {
|
if (!coop.sessionKey || !coop.chainId || !coop.storeId || !coop.appVersion) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`coop bootstrap missing values: sessionKey=${!!coop.sessionKey} chainId=${!!coop.chainId} storeId=${!!coop.storeId} appVersion=${!!coop.appVersion}`
|
`coop bootstrap missing values: sessionKey=${!!coop.sessionKey} chainId=${!!coop.chainId} storeId=${!!coop.storeId} appVersion=${!!coop.appVersion}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async function ensureCoopSession(ctx) {
|
async function ensureCoopSession(ctx) {
|
||||||
const coop = ctx.store.coop;
|
const coop = ctx.store.coop;
|
||||||
if (coop.sessionId) return;
|
if (coop.sessionId) return;
|
||||||
|
|
@ -110,20 +109,13 @@ async function ensureCoopSession(ctx) {
|
||||||
headers: coopHeaders(ctx, "/worldofwhisky"),
|
headers: coopHeaders(ctx, "/worldofwhisky"),
|
||||||
// browser sends Content-Length: 0; easiest equivalent:
|
// browser sends Content-Length: 0; easiest equivalent:
|
||||||
body: "",
|
body: "",
|
||||||
}
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
const sid =
|
const sid = r?.json?.SessionID || r?.json?.sessionID || r?.json?.sessionId || r?.json?.SessionId || "";
|
||||||
r?.json?.SessionID ||
|
|
||||||
r?.json?.sessionID ||
|
|
||||||
r?.json?.sessionId ||
|
|
||||||
r?.json?.SessionId ||
|
|
||||||
"";
|
|
||||||
|
|
||||||
if (!sid) {
|
if (!sid) {
|
||||||
throw new Error(
|
throw new Error(`createSession: missing SessionID (status=${r?.status})`);
|
||||||
`createSession: missing SessionID (status=${r?.status})`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
coop.sessionId = sid;
|
coop.sessionId = sid;
|
||||||
|
|
@ -157,10 +149,7 @@ function productFromApi(p) {
|
||||||
|
|
||||||
const url = productUrlFromId(productId);
|
const url = productUrlFromId(productId);
|
||||||
|
|
||||||
const price =
|
const price = p?.CountDetails?.PriceText || (Number.isFinite(p?.Price) ? `$${Number(p.Price).toFixed(2)}` : "");
|
||||||
p?.CountDetails?.PriceText ||
|
|
||||||
(Number.isFinite(p?.Price) ? `$${Number(p.Price).toFixed(2)}` : "");
|
|
||||||
|
|
||||||
|
|
||||||
const upc = String(p.UPC || "").trim();
|
const upc = String(p.UPC || "").trim();
|
||||||
|
|
||||||
|
|
@ -207,7 +196,7 @@ async function fetchCategoryPage(ctx, categoryId, page) {
|
||||||
},
|
},
|
||||||
orderby: null,
|
orderby: null,
|
||||||
}),
|
}),
|
||||||
}
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
let r = await doReq();
|
let r = await doReq();
|
||||||
|
|
@ -228,9 +217,7 @@ function avoidMassRemoval(prevDb, discovered, ctx) {
|
||||||
if (!prev || !curr) return;
|
if (!prev || !curr) return;
|
||||||
if (curr / prev >= 0.6) return;
|
if (curr / prev >= 0.6) return;
|
||||||
|
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(`${ctx.catPrefixOut} | Partial scan (${curr}/${prev}); preserving DB`);
|
||||||
`${ctx.catPrefixOut} | Partial scan (${curr}/${prev}); preserving DB`
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const [k, v] of prevDb.entries()) {
|
for (const [k, v] of prevDb.entries()) {
|
||||||
if (!discovered.has(k)) discovered.set(k, v);
|
if (!discovered.has(k)) discovered.set(k, v);
|
||||||
|
|
@ -241,8 +228,7 @@ async function scanCategoryCoop(ctx, prevDb, report) {
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
|
|
||||||
const maxPages =
|
const maxPages = ctx.config.maxPages === null ? 500 : Math.min(ctx.config.maxPages, 500);
|
||||||
ctx.config.maxPages === null ? 500 : Math.min(ctx.config.maxPages, 500);
|
|
||||||
|
|
||||||
let done = 0;
|
let done = 0;
|
||||||
|
|
||||||
|
|
@ -251,15 +237,11 @@ async function scanCategoryCoop(ctx, prevDb, report) {
|
||||||
try {
|
try {
|
||||||
r = await fetchCategoryPage(ctx, ctx.cat.coopCategoryId, page);
|
r = await fetchCategoryPage(ctx, ctx.cat.coopCategoryId, page);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(`${ctx.catPrefixOut} | page ${page} failed: ${e?.message || e}`);
|
||||||
`${ctx.catPrefixOut} | page ${page} failed: ${e?.message || e}`
|
|
||||||
);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
const arr = Array.isArray(r?.json?.Products?.Result)
|
const arr = Array.isArray(r?.json?.Products?.Result) ? r.json.Products.Result : [];
|
||||||
? r.json.Products.Result
|
|
||||||
: [];
|
|
||||||
|
|
||||||
done++;
|
done++;
|
||||||
|
|
||||||
|
|
@ -272,11 +254,11 @@ async function scanCategoryCoop(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${padLeft(page, 3)} | ${String(
|
`${ctx.catPrefixOut} | Page ${padLeft(page, 3)} | ${String(r.status || "").padEnd(
|
||||||
r.status || ""
|
3,
|
||||||
).padEnd(3)} | items=${padLeft(kept, 3)} | bytes=${kbStr(
|
)} | items=${padLeft(kept, 3)} | bytes=${kbStr(
|
||||||
r.bytes
|
r.bytes,
|
||||||
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!arr.length) break;
|
if (!arr.length) break;
|
||||||
|
|
@ -286,8 +268,9 @@ async function scanCategoryCoop(ctx, prevDb, report) {
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products: ${discovered.size}`);
|
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products: ${discovered.size}`);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } =
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
storeLabel: ctx.store.name,
|
||||||
|
});
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -320,7 +303,7 @@ async function scanCategoryCoop(ctx, prevDb, report) {
|
||||||
newItems,
|
newItems,
|
||||||
updatedItems,
|
updatedItems,
|
||||||
removedItems,
|
removedItems,
|
||||||
restoredItems
|
restoredItems,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -345,13 +328,55 @@ function createStore(defaultUa) {
|
||||||
},
|
},
|
||||||
|
|
||||||
categories: [
|
categories: [
|
||||||
{ key: "canadian-whisky", label: "Canadian Whisky", coopSlug: "canadian_whisky", coopCategoryId: 4, startUrl: `${REFERER}#/category/canadian_whisky` },
|
{
|
||||||
{ key: "bourbon-whiskey", label: "Bourbon Whiskey", coopSlug: "bourbon_whiskey", coopCategoryId: 9, startUrl: `${REFERER}#/category/bourbon_whiskey` },
|
key: "canadian-whisky",
|
||||||
{ key: "scottish-single-malts", label: "Scottish Single Malts", coopSlug: "scottish_single_malts", coopCategoryId: 6, startUrl: `${REFERER}#/category/scottish_single_malts` },
|
label: "Canadian Whisky",
|
||||||
{ key: "scottish-blends", label: "Scottish Whisky Blends", coopSlug: "scottish_whisky_blends", coopCategoryId: 5, startUrl: `${REFERER}#/category/scottish_whisky_blends` },
|
coopSlug: "canadian_whisky",
|
||||||
{ key: "american-whiskey", label: "American Whiskey", coopSlug: "american_whiskey", coopCategoryId: 8, startUrl: `${REFERER}#/category/american_whiskey` },
|
coopCategoryId: 4,
|
||||||
{ key: "world-whisky", label: "World Whisky", coopSlug: "world_international", coopCategoryId: 10, startUrl: `${REFERER}#/category/world_international` },
|
startUrl: `${REFERER}#/category/canadian_whisky`,
|
||||||
{ key: "rum", label: "Rum", coopSlug: "spirits_rum", coopCategoryId: 24, startUrl: `${REFERER}#/category/spirits_rum` },
|
},
|
||||||
|
{
|
||||||
|
key: "bourbon-whiskey",
|
||||||
|
label: "Bourbon Whiskey",
|
||||||
|
coopSlug: "bourbon_whiskey",
|
||||||
|
coopCategoryId: 9,
|
||||||
|
startUrl: `${REFERER}#/category/bourbon_whiskey`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "scottish-single-malts",
|
||||||
|
label: "Scottish Single Malts",
|
||||||
|
coopSlug: "scottish_single_malts",
|
||||||
|
coopCategoryId: 6,
|
||||||
|
startUrl: `${REFERER}#/category/scottish_single_malts`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "scottish-blends",
|
||||||
|
label: "Scottish Whisky Blends",
|
||||||
|
coopSlug: "scottish_whisky_blends",
|
||||||
|
coopCategoryId: 5,
|
||||||
|
startUrl: `${REFERER}#/category/scottish_whisky_blends`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "american-whiskey",
|
||||||
|
label: "American Whiskey",
|
||||||
|
coopSlug: "american_whiskey",
|
||||||
|
coopCategoryId: 8,
|
||||||
|
startUrl: `${REFERER}#/category/american_whiskey`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "world-whisky",
|
||||||
|
label: "World Whisky",
|
||||||
|
coopSlug: "world_international",
|
||||||
|
coopCategoryId: 10,
|
||||||
|
startUrl: `${REFERER}#/category/world_international`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "rum",
|
||||||
|
label: "Rum",
|
||||||
|
coopSlug: "spirits_rum",
|
||||||
|
coopCategoryId: 24,
|
||||||
|
startUrl: `${REFERER}#/category/spirits_rum`,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -32,10 +32,7 @@ function canonicalizeCraftProductUrl(raw) {
|
||||||
|
|
||||||
function extractShopifyCardPrice(block) {
|
function extractShopifyCardPrice(block) {
|
||||||
const b = String(block || "");
|
const b = String(block || "");
|
||||||
const dollars = (txt) =>
|
const dollars = (txt) => [...String(txt).matchAll(/\$\s*[\d,]+(?:\.\d{2})?/g)].map((m) => m[0].replace(/\s+/g, ""));
|
||||||
[...String(txt).matchAll(/\$\s*[\d,]+(?:\.\d{2})?/g)].map((m) =>
|
|
||||||
m[0].replace(/\s+/g, "")
|
|
||||||
);
|
|
||||||
|
|
||||||
const saleRegion = b.split(/sale price/i)[1] || "";
|
const saleRegion = b.split(/sale price/i)[1] || "";
|
||||||
const saleD = dollars(saleRegion);
|
const saleD = dollars(saleRegion);
|
||||||
|
|
@ -52,14 +49,8 @@ function extractShopifyCardPrice(block) {
|
||||||
function parseProductsCraftCellars(html, ctx) {
|
function parseProductsCraftCellars(html, ctx) {
|
||||||
const s = String(html || "");
|
const s = String(html || "");
|
||||||
|
|
||||||
const g1 =
|
const g1 = s.match(/<div\b[^>]*id=["']ProductGridContainer["'][^>]*>[\s\S]*?<\/div>/i)?.[0] || "";
|
||||||
s.match(
|
const g2 = s.match(/<div\b[^>]*id=["']product-grid["'][^>]*>[\s\S]*?<\/div>/i)?.[0] || "";
|
||||||
/<div\b[^>]*id=["']ProductGridContainer["'][^>]*>[\s\S]*?<\/div>/i
|
|
||||||
)?.[0] || "";
|
|
||||||
const g2 =
|
|
||||||
s.match(
|
|
||||||
/<div\b[^>]*id=["']product-grid["'][^>]*>[\s\S]*?<\/div>/i
|
|
||||||
)?.[0] || "";
|
|
||||||
|
|
||||||
const gridCandidate = g1.length > g2.length ? g1 : g2;
|
const gridCandidate = g1.length > g2.length ? g1 : g2;
|
||||||
const grid = /\/products\//i.test(gridCandidate) ? gridCandidate : s;
|
const grid = /\/products\//i.test(gridCandidate) ? gridCandidate : s;
|
||||||
|
|
@ -71,24 +62,18 @@ function parseProductsCraftCellarsInner(html, ctx) {
|
||||||
const s = String(html || "");
|
const s = String(html || "");
|
||||||
const items = [];
|
const items = [];
|
||||||
|
|
||||||
let blocks = [...s.matchAll(/<li\b[^>]*>[\s\S]*?<\/li>/gi)].map(
|
let blocks = [...s.matchAll(/<li\b[^>]*>[\s\S]*?<\/li>/gi)].map((m) => m[0]);
|
||||||
(m) => m[0]
|
|
||||||
);
|
|
||||||
if (blocks.length < 5) {
|
if (blocks.length < 5) {
|
||||||
blocks = [
|
blocks = [...s.matchAll(/<div\b[^>]*class=["'][^"']*\bcard\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi)].map(
|
||||||
...s.matchAll(
|
(m) => m[0],
|
||||||
/<div\b[^>]*class=["'][^"']*\bcard\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi
|
);
|
||||||
),
|
|
||||||
].map((m) => m[0]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const base = `https://${(ctx && ctx.store && ctx.store.host) || "craftcellars.ca"}/`;
|
const base = `https://${(ctx && ctx.store && ctx.store.host) || "craftcellars.ca"}/`;
|
||||||
|
|
||||||
for (const block of blocks) {
|
for (const block of blocks) {
|
||||||
const href =
|
const href =
|
||||||
block.match(
|
block.match(/<a\b[^>]*href=["']([^"']*\/products\/[^"']+)["']/i)?.[1] ||
|
||||||
/<a\b[^>]*href=["']([^"']*\/products\/[^"']+)["']/i
|
|
||||||
)?.[1] ||
|
|
||||||
block.match(/href=["']([^"']*\/products\/[^"']+)["']/i)?.[1];
|
block.match(/href=["']([^"']*\/products\/[^"']+)["']/i)?.[1];
|
||||||
if (!href) continue;
|
if (!href) continue;
|
||||||
|
|
||||||
|
|
@ -101,15 +86,11 @@ function parseProductsCraftCellarsInner(html, ctx) {
|
||||||
url = canonicalizeCraftProductUrl(url);
|
url = canonicalizeCraftProductUrl(url);
|
||||||
|
|
||||||
const nameHtml =
|
const nameHtml =
|
||||||
|
block.match(/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>\s*<[^>]*>\s*([^<]{2,200}?)\s*</i)?.[1] ||
|
||||||
block.match(
|
block.match(
|
||||||
/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>\s*<[^>]*>\s*([^<]{2,200}?)\s*</i
|
/<h[23]\b[^>]*>[\s\S]*?<a\b[^>]*\/products\/[^"']+[^>]*>([\s\S]*?)<\/a>[\s\S]*?<\/h[23]>/i,
|
||||||
)?.[1] ||
|
)?.[1] ||
|
||||||
block.match(
|
block.match(/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>([\s\S]*?)<\/a>/i)?.[1];
|
||||||
/<h[23]\b[^>]*>[\s\S]*?<a\b[^>]*\/products\/[^"']+[^>]*>([\s\S]*?)<\/a>[\s\S]*?<\/h[23]>/i
|
|
||||||
)?.[1] ||
|
|
||||||
block.match(
|
|
||||||
/<a\b[^>]*href=["'][^"']*\/products\/[^"']+["'][^>]*>([\s\S]*?)<\/a>/i
|
|
||||||
)?.[1];
|
|
||||||
|
|
||||||
const name = sanitizeName(stripTags(decodeHtml(nameHtml || "")));
|
const name = sanitizeName(stripTags(decodeHtml(nameHtml || "")));
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
|
|
@ -160,23 +141,13 @@ function extractCraftSkuFromProductPageHtml(html) {
|
||||||
async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
|
|
||||||
const perPageDelayMs =
|
const perPageDelayMs = Math.max(0, cfgNum(ctx?.cat?.pageStaggerMs, cfgNum(ctx?.cat?.discoveryDelayMs, 0))) || 0;
|
||||||
Math.max(
|
|
||||||
0,
|
|
||||||
cfgNum(ctx?.cat?.pageStaggerMs, cfgNum(ctx?.cat?.discoveryDelayMs, 0))
|
|
||||||
) || 0;
|
|
||||||
|
|
||||||
const perJsonPageDelayMs = Math.max(
|
const perJsonPageDelayMs = Math.max(0, cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs));
|
||||||
0,
|
|
||||||
cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)
|
|
||||||
);
|
|
||||||
|
|
||||||
const htmlMap = new Map();
|
const htmlMap = new Map();
|
||||||
|
|
||||||
const maxPages =
|
const maxPages = ctx.config.maxPages === null ? 200 : Math.min(ctx.config.maxPages, 200);
|
||||||
ctx.config.maxPages === null
|
|
||||||
? 200
|
|
||||||
: Math.min(ctx.config.maxPages, 200);
|
|
||||||
|
|
||||||
let htmlPagesFetched = 0;
|
let htmlPagesFetched = 0;
|
||||||
let emptyStreak = 0;
|
let emptyStreak = 0;
|
||||||
|
|
@ -188,7 +159,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
const { text: html } = await ctx.http.fetchTextWithRetry(
|
const { text: html } = await ctx.http.fetchTextWithRetry(
|
||||||
pageUrl,
|
pageUrl,
|
||||||
`craft:html:${ctx.cat.key}:p${p}`,
|
`craft:html:${ctx.cat.key}:p${p}`,
|
||||||
ctx.store.ua
|
ctx.store.ua,
|
||||||
);
|
);
|
||||||
htmlPagesFetched++;
|
htmlPagesFetched++;
|
||||||
|
|
||||||
|
|
@ -215,9 +186,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!htmlMap.size) {
|
if (!htmlMap.size) {
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(`${ctx.catPrefixOut} | HTML listing returned 0 items; refusing JSON-only discovery`);
|
||||||
`${ctx.catPrefixOut} | HTML listing returned 0 items; refusing JSON-only discovery`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const jsonMap = new Map();
|
const jsonMap = new Map();
|
||||||
|
|
@ -225,10 +194,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
if (htmlMap.size) {
|
if (htmlMap.size) {
|
||||||
const start = new URL(ctx.cat.startUrl);
|
const start = new URL(ctx.cat.startUrl);
|
||||||
const m = start.pathname.match(/^\/collections\/([^/]+)/i);
|
const m = start.pathname.match(/^\/collections\/([^/]+)/i);
|
||||||
if (!m)
|
if (!m) throw new Error(`CraftCellars: couldn't extract collection handle from ${ctx.cat.startUrl}`);
|
||||||
throw new Error(
|
|
||||||
`CraftCellars: couldn't extract collection handle from ${ctx.cat.startUrl}`
|
|
||||||
);
|
|
||||||
const collectionHandle = m[1];
|
const collectionHandle = m[1];
|
||||||
|
|
||||||
const limit = 250;
|
const limit = 250;
|
||||||
|
|
@ -236,19 +202,12 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
let jsonPagesFetched = 0;
|
let jsonPagesFetched = 0;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
if (jsonPage > 1 && perJsonPageDelayMs > 0)
|
if (jsonPage > 1 && perJsonPageDelayMs > 0) await sleep(perJsonPageDelayMs);
|
||||||
await sleep(perJsonPageDelayMs);
|
|
||||||
|
|
||||||
const url = `https://${ctx.store.host}/collections/${collectionHandle}/products.json?limit=${limit}&page=${jsonPage}`;
|
const url = `https://${ctx.store.host}/collections/${collectionHandle}/products.json?limit=${limit}&page=${jsonPage}`;
|
||||||
const r = await ctx.http.fetchJsonWithRetry(
|
const r = await ctx.http.fetchJsonWithRetry(url, `craft:coljson:${ctx.cat.key}:p${jsonPage}`, ctx.store.ua);
|
||||||
url,
|
|
||||||
`craft:coljson:${ctx.cat.key}:p${jsonPage}`,
|
|
||||||
ctx.store.ua
|
|
||||||
);
|
|
||||||
|
|
||||||
const products = Array.isArray(r?.json?.products)
|
const products = Array.isArray(r?.json?.products) ? r.json.products : [];
|
||||||
? r.json.products
|
|
||||||
: [];
|
|
||||||
jsonPagesFetched++;
|
jsonPagesFetched++;
|
||||||
|
|
||||||
if (!products.length) break;
|
if (!products.length) break;
|
||||||
|
|
@ -257,16 +216,11 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
const handle = String(p?.handle || "");
|
const handle = String(p?.handle || "");
|
||||||
if (!handle) continue;
|
if (!handle) continue;
|
||||||
|
|
||||||
const prodUrl = canonicalizeCraftProductUrl(
|
const prodUrl = canonicalizeCraftProductUrl(`https://${ctx.store.host}/products/${handle}`);
|
||||||
`https://${ctx.store.host}/products/${handle}`
|
|
||||||
);
|
|
||||||
if (!htmlMap.has(prodUrl)) continue;
|
if (!htmlMap.has(prodUrl)) continue;
|
||||||
|
|
||||||
const variants = Array.isArray(p?.variants) ? p.variants : [];
|
const variants = Array.isArray(p?.variants) ? p.variants : [];
|
||||||
const v =
|
const v = variants.find((x) => x && x.available === true) || variants[0] || null;
|
||||||
variants.find((x) => x && x.available === true) ||
|
|
||||||
variants[0] ||
|
|
||||||
null;
|
|
||||||
|
|
||||||
const sku = normalizeCspc(v?.sku || "");
|
const sku = normalizeCspc(v?.sku || "");
|
||||||
const price = v?.price ? usdFromShopifyPriceStr(v.price) : "";
|
const price = v?.price ? usdFromShopifyPriceStr(v.price) : "";
|
||||||
|
|
@ -274,13 +228,9 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
let img = "";
|
let img = "";
|
||||||
const images = Array.isArray(p?.images) ? p.images : [];
|
const images = Array.isArray(p?.images) ? p.images : [];
|
||||||
if (images[0]) {
|
if (images[0]) {
|
||||||
img =
|
img = typeof images[0] === "string" ? images[0] : String(images[0]?.src || images[0]?.url || "");
|
||||||
typeof images[0] === "string"
|
|
||||||
? images[0]
|
|
||||||
: String(images[0]?.src || images[0]?.url || "");
|
|
||||||
}
|
}
|
||||||
if (!img && p?.image)
|
if (!img && p?.image) img = String(p.image?.src || p.image?.url || p.image || "");
|
||||||
img = String(p.image?.src || p.image?.url || p.image || "");
|
|
||||||
img = String(img || "").trim();
|
img = String(img || "").trim();
|
||||||
if (img.startsWith("//")) img = `https:${img}`;
|
if (img.startsWith("//")) img = `https:${img}`;
|
||||||
|
|
||||||
|
|
@ -291,9 +241,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
if (++jsonPage > 200) break;
|
if (++jsonPage > 200) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(`${ctx.catPrefixOut} | HTML pages=${htmlPagesFetched} JSON pages=${jsonPagesFetched}`);
|
||||||
`${ctx.catPrefixOut} | HTML pages=${htmlPagesFetched} JSON pages=${jsonPagesFetched}`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
|
|
@ -308,17 +256,14 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
// reuse cached SKU unless we found something better this run
|
// reuse cached SKU unless we found something better this run
|
||||||
sku: pickBetterSku(j?.sku || "", prev?.sku || ""),
|
sku: pickBetterSku(j?.sku || "", prev?.sku || ""),
|
||||||
// reuse cached image if we didn't find one
|
// reuse cached image if we didn't find one
|
||||||
img: (j?.img || it.img || prev?.img || ""),
|
img: j?.img || it.img || prev?.img || "",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------- NEW: product page SKU fallback (cached; only when needed) ---------- */
|
/* ---------- NEW: product page SKU fallback (cached; only when needed) ---------- */
|
||||||
const perProductSkuDelayMs = Math.max(
|
const perProductSkuDelayMs = Math.max(
|
||||||
0,
|
0,
|
||||||
cfgNum(
|
cfgNum(ctx?.cat?.skuPageDelayMs, cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)),
|
||||||
ctx?.cat?.skuPageDelayMs,
|
|
||||||
cfgNum(ctx?.cat?.jsonPageDelayMs, perPageDelayMs)
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
let skuPagesFetched = 0;
|
let skuPagesFetched = 0;
|
||||||
|
|
@ -332,10 +277,8 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
try {
|
try {
|
||||||
const { text } = await ctx.http.fetchTextWithRetry(
|
const { text } = await ctx.http.fetchTextWithRetry(
|
||||||
it.url,
|
it.url,
|
||||||
`craft:prodpage:${ctx.cat.key}:${Buffer.from(it.url)
|
`craft:prodpage:${ctx.cat.key}:${Buffer.from(it.url).toString("base64").slice(0, 24)}`,
|
||||||
.toString("base64")
|
ctx.store.ua,
|
||||||
.slice(0, 24)}`,
|
|
||||||
ctx.store.ua
|
|
||||||
);
|
);
|
||||||
skuPagesFetched++;
|
skuPagesFetched++;
|
||||||
|
|
||||||
|
|
@ -346,21 +289,11 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(`${ctx.catPrefixOut} | SKU fallback pages=${skuPagesFetched}`);
|
||||||
`${ctx.catPrefixOut} | SKU fallback pages=${skuPagesFetched}`
|
|
||||||
);
|
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
||||||
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`
|
|
||||||
);
|
|
||||||
|
|
||||||
const {
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
merged,
|
|
||||||
newItems,
|
|
||||||
updatedItems,
|
|
||||||
removedItems,
|
|
||||||
restoredItems,
|
|
||||||
} = mergeDiscoveredIntoDb(prevDb, discovered, {
|
|
||||||
storeLabel: ctx.store.name,
|
storeLabel: ctx.store.name,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -395,7 +328,7 @@ async function scanCategoryCraftCellars(ctx, prevDb, report) {
|
||||||
newItems,
|
newItems,
|
||||||
updatedItems,
|
updatedItems,
|
||||||
removedItems,
|
removedItems,
|
||||||
restoredItems
|
restoredItems,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -416,8 +349,7 @@ function createStore(defaultUa) {
|
||||||
{
|
{
|
||||||
key: "whisky",
|
key: "whisky",
|
||||||
label: "Whisky",
|
label: "Whisky",
|
||||||
startUrl:
|
startUrl: "https://craftcellars.ca/collections/whisky?filter.v.availability=1",
|
||||||
"https://craftcellars.ca/collections/whisky?filter.v.availability=1",
|
|
||||||
pageConcurrency: 1,
|
pageConcurrency: 1,
|
||||||
pageStaggerMs: 10000,
|
pageStaggerMs: 10000,
|
||||||
discoveryDelayMs: 10000,
|
discoveryDelayMs: 10000,
|
||||||
|
|
@ -426,8 +358,7 @@ function createStore(defaultUa) {
|
||||||
{
|
{
|
||||||
key: "rum",
|
key: "rum",
|
||||||
label: "Rum",
|
label: "Rum",
|
||||||
startUrl:
|
startUrl: "https://craftcellars.ca/collections/rum?filter.v.availability=1",
|
||||||
"https://craftcellars.ca/collections/rum?filter.v.availability=1",
|
|
||||||
pageConcurrency: 1,
|
pageConcurrency: 1,
|
||||||
pageStaggerMs: 10000,
|
pageStaggerMs: 10000,
|
||||||
discoveryDelayMs: 10000,
|
discoveryDelayMs: 10000,
|
||||||
|
|
|
||||||
|
|
@ -77,9 +77,7 @@ function extractGullSkuFromProductPage(html) {
|
||||||
const s = String(html || "");
|
const s = String(html || "");
|
||||||
|
|
||||||
// Most reliable: <span class="sku">67424</span>
|
// Most reliable: <span class="sku">67424</span>
|
||||||
const m1 = s.match(
|
const m1 = s.match(/<span\b[^>]*class=["'][^"']*\bsku\b[^"']*["'][^>]*>\s*([0-9]{3,10})\s*<\/span>/i);
|
||||||
/<span\b[^>]*class=["'][^"']*\bsku\b[^"']*["'][^>]*>\s*([0-9]{3,10})\s*<\/span>/i
|
|
||||||
);
|
|
||||||
if (m1?.[1]) return normalizeGullSku(m1[1]);
|
if (m1?.[1]) return normalizeGullSku(m1[1]);
|
||||||
|
|
||||||
// Fallback: "SKU: 67424" text
|
// Fallback: "SKU: 67424" text
|
||||||
|
|
@ -120,10 +118,7 @@ async function fetchWith429Backoff(url, { fetchFn, headers, maxRetries = 2 }) {
|
||||||
if (attempt >= maxRetries) throw new Error(`HTTP 429 fetching ${url}`);
|
if (attempt >= maxRetries) throw new Error(`HTTP 429 fetching ${url}`);
|
||||||
|
|
||||||
// Respect Retry-After if present; otherwise progressive backoff.
|
// Respect Retry-After if present; otherwise progressive backoff.
|
||||||
const ra =
|
const ra = res.headers && typeof res.headers.get === "function" ? res.headers.get("retry-after") : null;
|
||||||
res.headers && typeof res.headers.get === "function"
|
|
||||||
? res.headers.get("retry-after")
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const waitSec = ra && /^\d+$/.test(ra) ? parseInt(ra, 10) : 15 * (attempt + 1);
|
const waitSec = ra && /^\d+$/.test(ra) ? parseInt(ra, 10) : 15 * (attempt + 1);
|
||||||
await new Promise((r) => setTimeout(r, waitSec * 1000));
|
await new Promise((r) => setTimeout(r, waitSec * 1000));
|
||||||
|
|
@ -137,10 +132,7 @@ async function fetchWith429Backoff(url, { fetchFn, headers, maxRetries = 2 }) {
|
||||||
*
|
*
|
||||||
* NEW: accepts prevDb so we can skip fetch if URL already has a good SKU cached.
|
* NEW: accepts prevDb so we can skip fetch if URL already has a good SKU cached.
|
||||||
*/
|
*/
|
||||||
async function hydrateGullSkus(
|
async function hydrateGullSkus(items, { fetchFn, ua, minIntervalMs = 12000, maxRetries = 2, prevDb } = {}) {
|
||||||
items,
|
|
||||||
{ fetchFn, ua, minIntervalMs = 12000, maxRetries = 2, prevDb } = {}
|
|
||||||
) {
|
|
||||||
if (!fetchFn) throw new Error("hydrateGullSkus requires opts.fetchFn");
|
if (!fetchFn) throw new Error("hydrateGullSkus requires opts.fetchFn");
|
||||||
|
|
||||||
const schedule = createMinIntervalLimiter(minIntervalMs);
|
const schedule = createMinIntervalLimiter(minIntervalMs);
|
||||||
|
|
@ -162,9 +154,7 @@ async function hydrateGullSkus(
|
||||||
|
|
||||||
if (!isGeneratedUrlSku(it.sku)) continue; // only where required
|
if (!isGeneratedUrlSku(it.sku)) continue; // only where required
|
||||||
|
|
||||||
const html = await schedule(() =>
|
const html = await schedule(() => fetchWith429Backoff(it.url, { fetchFn, headers, maxRetries }));
|
||||||
fetchWith429Backoff(it.url, { fetchFn, headers, maxRetries })
|
|
||||||
);
|
|
||||||
|
|
||||||
const realSku = extractGullSkuFromProductPage(html);
|
const realSku = extractGullSkuFromProductPage(html);
|
||||||
if (realSku) it.sku = pickBetterSku(realSku, it.sku);
|
if (realSku) it.sku = pickBetterSku(realSku, it.sku);
|
||||||
|
|
@ -178,9 +168,7 @@ function parseProductsGull(html, ctx) {
|
||||||
const items = [];
|
const items = [];
|
||||||
|
|
||||||
// split on <li class="product ...">
|
// split on <li class="product ...">
|
||||||
const parts = s.split(
|
const parts = s.split(/<li\b[^>]*class=["'][^"']*\bproduct\b[^"']*["'][^>]*>/i);
|
||||||
/<li\b[^>]*class=["'][^"']*\bproduct\b[^"']*["'][^>]*>/i
|
|
||||||
);
|
|
||||||
if (parts.length <= 1) return items;
|
if (parts.length <= 1) return items;
|
||||||
|
|
||||||
const base = `https://${(ctx && ctx.store && ctx.store.host) || "gullliquorstore.com"}/`;
|
const base = `https://${(ctx && ctx.store && ctx.store.host) || "gullliquorstore.com"}/`;
|
||||||
|
|
@ -191,7 +179,7 @@ function parseProductsGull(html, ctx) {
|
||||||
if (!looksInStock(block)) continue;
|
if (!looksInStock(block)) continue;
|
||||||
|
|
||||||
const hrefM = block.match(
|
const hrefM = block.match(
|
||||||
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bwoocommerce-LoopProduct-link\b/i
|
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bwoocommerce-LoopProduct-link\b/i,
|
||||||
);
|
);
|
||||||
if (!hrefM || !hrefM[1]) continue;
|
if (!hrefM || !hrefM[1]) continue;
|
||||||
|
|
||||||
|
|
@ -203,7 +191,7 @@ function parseProductsGull(html, ctx) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const titleM = block.match(
|
const titleM = block.match(
|
||||||
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i
|
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
|
||||||
);
|
);
|
||||||
const name = cleanText(decodeHtml(titleM ? titleM[1] : ""));
|
const name = cleanText(decodeHtml(titleM ? titleM[1] : ""));
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
|
|
@ -245,8 +233,7 @@ function createStore(defaultUa) {
|
||||||
{
|
{
|
||||||
key: "whisky",
|
key: "whisky",
|
||||||
label: "Whisky",
|
label: "Whisky",
|
||||||
startUrl:
|
startUrl: "https://gullliquorstore.com/product-category/spirits/?spirit_type=whisky",
|
||||||
"https://gullliquorstore.com/product-category/spirits/?spirit_type=whisky",
|
|
||||||
discoveryStartPage: 3,
|
discoveryStartPage: 3,
|
||||||
discoveryStep: 2,
|
discoveryStep: 2,
|
||||||
pageConcurrency: 1,
|
pageConcurrency: 1,
|
||||||
|
|
@ -256,8 +243,7 @@ function createStore(defaultUa) {
|
||||||
{
|
{
|
||||||
key: "rum",
|
key: "rum",
|
||||||
label: "Rum",
|
label: "Rum",
|
||||||
startUrl:
|
startUrl: "https://gullliquorstore.com/product-category/spirits/?spirit_type=rum",
|
||||||
"https://gullliquorstore.com/product-category/spirits/?spirit_type=rum",
|
|
||||||
discoveryStartPage: 3,
|
discoveryStartPage: 3,
|
||||||
discoveryStep: 2,
|
discoveryStep: 2,
|
||||||
pageConcurrency: 1,
|
pageConcurrency: 1,
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ function parseProductsKegNCork(html, ctx) {
|
||||||
const block = "<li" + blocks[i];
|
const block = "<li" + blocks[i];
|
||||||
|
|
||||||
const mTitle = block.match(
|
const mTitle = block.match(
|
||||||
/<h4\b[^>]*class=["'][^"']*\bcard-title\b[^"']*["'][^>]*>[\s\S]*?<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/i
|
/<h4\b[^>]*class=["'][^"']*\bcard-title\b[^"']*["'][^>]*>[\s\S]*?<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/i,
|
||||||
);
|
);
|
||||||
if (!mTitle) continue;
|
if (!mTitle) continue;
|
||||||
|
|
||||||
|
|
@ -49,7 +49,6 @@ function parseProductsKegNCork(html, ctx) {
|
||||||
return [...uniq.values()];
|
return [...uniq.values()];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
return {
|
return {
|
||||||
key: "kegncork",
|
key: "kegncork",
|
||||||
|
|
|
||||||
|
|
@ -114,7 +114,10 @@ function kwmExtractPrice(block) {
|
||||||
const priceDiv = kwmExtractFirstDivByClass(block, "product-price");
|
const priceDiv = kwmExtractFirstDivByClass(block, "product-price");
|
||||||
if (!priceDiv) return "";
|
if (!priceDiv) return "";
|
||||||
|
|
||||||
const cleaned = String(priceDiv).replace(/<span\b[^>]*class=["'][^"']*\bstrike\b[^"']*["'][^>]*>[\s\S]*?<\/span>/gi, " ");
|
const cleaned = String(priceDiv).replace(
|
||||||
|
/<span\b[^>]*class=["'][^"']*\bstrike\b[^"']*["'][^>]*>[\s\S]*?<\/span>/gi,
|
||||||
|
" ",
|
||||||
|
);
|
||||||
|
|
||||||
const txt = cleanText(decodeHtml(stripTags(cleaned)));
|
const txt = cleanText(decodeHtml(stripTags(cleaned)));
|
||||||
const dollars = [...txt.matchAll(/\$\s*\d+(?:\.\d{2})?/g)];
|
const dollars = [...txt.matchAll(/\$\s*\d+(?:\.\d{2})?/g)];
|
||||||
|
|
@ -160,7 +163,6 @@ function parseProductsKWM(html, ctx) {
|
||||||
return [...uniq.values()];
|
return [...uniq.values()];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
return {
|
return {
|
||||||
key: "kwm",
|
key: "kwm",
|
||||||
|
|
|
||||||
|
|
@ -157,7 +157,10 @@ function legacyProductToItem(p, ctx) {
|
||||||
|
|
||||||
const base = "https://www.legacyliquorstore.com";
|
const base = "https://www.legacyliquorstore.com";
|
||||||
// Matches observed pattern: /LL/product/spirits/<category>/<slug>
|
// Matches observed pattern: /LL/product/spirits/<category>/<slug>
|
||||||
const url = new URL(`/LL/product/spirits/${encodeURIComponent(ctx.cat.key)}/${encodeURIComponent(slug)}`, base).toString();
|
const url = new URL(
|
||||||
|
`/LL/product/spirits/${encodeURIComponent(ctx.cat.key)}/${encodeURIComponent(slug)}`,
|
||||||
|
base,
|
||||||
|
).toString();
|
||||||
|
|
||||||
const nameRaw =
|
const nameRaw =
|
||||||
String(v?.fullName || "").trim() ||
|
String(v?.fullName || "").trim() ||
|
||||||
|
|
@ -165,13 +168,12 @@ function legacyProductToItem(p, ctx) {
|
||||||
const name = String(nameRaw || "").trim();
|
const name = String(nameRaw || "").trim();
|
||||||
if (!name) return null;
|
if (!name) return null;
|
||||||
|
|
||||||
const price =
|
const price = cad(v?.price) || cad(p?.priceFrom) || cad(p?.priceTo) || "";
|
||||||
cad(v?.price) ||
|
|
||||||
cad(p?.priceFrom) ||
|
|
||||||
cad(p?.priceTo) ||
|
|
||||||
"";
|
|
||||||
|
|
||||||
const sku = normalizeLegacySku(v?.sku, { storeLabel: ctx.store.name, url }) || normalizeLegacySku(url, { storeLabel: ctx.store.name, url }) ||"";
|
const sku =
|
||||||
|
normalizeLegacySku(v?.sku, { storeLabel: ctx.store.name, url }) ||
|
||||||
|
normalizeLegacySku(url, { storeLabel: ctx.store.name, url }) ||
|
||||||
|
"";
|
||||||
const img = normalizeAbsUrl(v?.image || "");
|
const img = normalizeAbsUrl(v?.image || "");
|
||||||
|
|
||||||
return { name, price, url, sku, img };
|
return { name, price, url, sku, img };
|
||||||
|
|
@ -206,7 +208,11 @@ async function legacyFetchPage(ctx, pageCursor, pageLimit) {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
return await ctx.http.fetchJsonWithRetry(LEGACY_GQL_URL, `legacy:${ctx.cat.key}:${pageCursor || "first"}`, ctx.store.ua, {
|
return await ctx.http.fetchJsonWithRetry(
|
||||||
|
LEGACY_GQL_URL,
|
||||||
|
`legacy:${ctx.cat.key}:${pageCursor || "first"}`,
|
||||||
|
ctx.store.ua,
|
||||||
|
{
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
Accept: "application/json",
|
Accept: "application/json",
|
||||||
|
|
@ -215,7 +221,8 @@ async function legacyFetchPage(ctx, pageCursor, pageLimit) {
|
||||||
Referer: "https://www.legacyliquorstore.com/",
|
Referer: "https://www.legacyliquorstore.com/",
|
||||||
},
|
},
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
});
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
|
async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
|
||||||
|
|
@ -257,8 +264,8 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(done, done)} | ${String(r.status || "").padEnd(3)} | ${pctStr(done, done)} | kept=${padLeft(
|
`${ctx.catPrefixOut} | Page ${pageStr(done, done)} | ${String(r.status || "").padEnd(3)} | ${pctStr(done, done)} | kept=${padLeft(
|
||||||
kept,
|
kept,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!next || !arr.length) break;
|
if (!next || !arr.length) break;
|
||||||
|
|
@ -266,13 +273,15 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
|
||||||
cursor = next;
|
cursor = next;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
|
storeLabel: ctx.store.name,
|
||||||
|
});
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -293,7 +302,15 @@ async function scanCategoryLegacyLiquor(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
|
|
|
||||||
|
|
@ -37,16 +37,18 @@ function parseProductsMaltsAndGrains(html, ctx) {
|
||||||
|
|
||||||
const cats = [];
|
const cats = [];
|
||||||
for (const m of String(classAttr || "").matchAll(/\bproduct_cat-([a-z0-9_-]+)\b/gi)) {
|
for (const m of String(classAttr || "").matchAll(/\bproduct_cat-([a-z0-9_-]+)\b/gi)) {
|
||||||
const v = String(m[1] || "").trim().toLowerCase();
|
const v = String(m[1] || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (v) cats.push(v);
|
if (v) cats.push(v);
|
||||||
}
|
}
|
||||||
|
|
||||||
let href =
|
let href =
|
||||||
block.match(
|
block.match(
|
||||||
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b/i
|
/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b/i,
|
||||||
)?.[1] ||
|
)?.[1] ||
|
||||||
block.match(
|
block.match(
|
||||||
/<a\b[^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b[^"']*["'][^>]*href=["']([^"']+)["']/i
|
/<a\b[^>]*class=["'][^"']*\b(woocommerce-LoopProduct-link|woocommerce-loop-product__link|ast-loop-product__link)\b[^"']*["'][^>]*href=["']([^"']+)["']/i,
|
||||||
)?.[2] ||
|
)?.[2] ||
|
||||||
block.match(/<a\b[^>]*href=["']([^"']*\/product\/[^"']+)["']/i)?.[1];
|
block.match(/<a\b[^>]*href=["']([^"']*\/product\/[^"']+)["']/i)?.[1];
|
||||||
|
|
||||||
|
|
@ -61,7 +63,7 @@ function parseProductsMaltsAndGrains(html, ctx) {
|
||||||
if (!/^https?:\/\//i.test(url)) continue;
|
if (!/^https?:\/\//i.test(url)) continue;
|
||||||
|
|
||||||
const mTitle = block.match(
|
const mTitle = block.match(
|
||||||
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i
|
/<h2\b[^>]*class=["'][^"']*\bwoocommerce-loop-product__title\b[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
|
||||||
);
|
);
|
||||||
const name = mTitle && mTitle[1] ? cleanText(decodeHtml(stripTags(mTitle[1]))) : "";
|
const name = mTitle && mTitle[1] ? cleanText(decodeHtml(stripTags(mTitle[1]))) : "";
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
|
|
@ -71,7 +73,7 @@ function parseProductsMaltsAndGrains(html, ctx) {
|
||||||
const sku = normalizeCspc(
|
const sku = normalizeCspc(
|
||||||
block.match(/\bdata-product_sku=["']([^"']+)["']/i)?.[1] ||
|
block.match(/\bdata-product_sku=["']([^"']+)["']/i)?.[1] ||
|
||||||
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
|
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
|
||||||
""
|
"",
|
||||||
);
|
);
|
||||||
|
|
||||||
const img = extractFirstImgUrl(block, base);
|
const img = extractFirstImgUrl(block, base);
|
||||||
|
|
@ -84,7 +86,6 @@ function parseProductsMaltsAndGrains(html, ctx) {
|
||||||
return [...uniq.values()];
|
return [...uniq.values()];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
return {
|
return {
|
||||||
key: "maltsandgrains",
|
key: "maltsandgrains",
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ const { mergeDiscoveredIntoDb } = require("../tracker/merge");
|
||||||
const { addCategoryResultToReport } = require("../tracker/report");
|
const { addCategoryResultToReport } = require("../tracker/report");
|
||||||
|
|
||||||
function allowSierraUrlRumWhisky(item) {
|
function allowSierraUrlRumWhisky(item) {
|
||||||
const u = (item && item.url) ? String(item.url) : "";
|
const u = item && item.url ? String(item.url) : "";
|
||||||
const s = u.toLowerCase();
|
const s = u.toLowerCase();
|
||||||
if (!/^https?:\/\/sierraspringsliquor\.ca\//.test(s)) return false;
|
if (!/^https?:\/\/sierraspringsliquor\.ca\//.test(s)) return false;
|
||||||
return /\b(rum|whisk(?:e)?y)\b/.test(s);
|
return /\b(rum|whisk(?:e)?y)\b/.test(s);
|
||||||
|
|
@ -48,26 +48,22 @@ function parseWooStoreProductsJson(payload, ctx) {
|
||||||
if (!Array.isArray(data)) return items;
|
if (!Array.isArray(data)) return items;
|
||||||
|
|
||||||
for (const p of data) {
|
for (const p of data) {
|
||||||
const url = (p && p.permalink) ? String(p.permalink) : "";
|
const url = p && p.permalink ? String(p.permalink) : "";
|
||||||
if (!url) continue;
|
if (!url) continue;
|
||||||
|
|
||||||
const name = (p && p.name) ? cleanText(decodeHtml(String(p.name))) : "";
|
const name = p && p.name ? cleanText(decodeHtml(String(p.name))) : "";
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
|
|
||||||
const price = formatWooStorePrice(p.prices);
|
const price = formatWooStorePrice(p.prices);
|
||||||
|
|
||||||
const rawSku =
|
const rawSku =
|
||||||
(typeof p?.sku === "string" && p.sku.trim()) ? p.sku.trim()
|
typeof p?.sku === "string" && p.sku.trim() ? p.sku.trim() : p && (p.id ?? p.id === 0) ? String(p.id) : "";
|
||||||
: (p && (p.id ?? p.id === 0)) ? String(p.id)
|
|
||||||
: "";
|
|
||||||
|
|
||||||
const taggedSku = /^\d{1,11}$/.test(rawSku) ? `id:${rawSku}` : rawSku;
|
const taggedSku = /^\d{1,11}$/.test(rawSku) ? `id:${rawSku}` : rawSku;
|
||||||
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
|
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
|
||||||
|
|
||||||
const img =
|
const img =
|
||||||
(p.images && Array.isArray(p.images) && p.images[0] && p.images[0].src)
|
p.images && Array.isArray(p.images) && p.images[0] && p.images[0].src ? String(p.images[0].src) : null;
|
||||||
? String(p.images[0].src)
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const item = { name, price, url, sku, img };
|
const item = { name, price, url, sku, img };
|
||||||
|
|
||||||
|
|
@ -96,16 +92,18 @@ function parseWooProductsHtml(html, ctx) {
|
||||||
if (/class=["'][^"']*\bproduct-category\b/i.test(chunk)) continue;
|
if (/class=["'][^"']*\bproduct-category\b/i.test(chunk)) continue;
|
||||||
|
|
||||||
const endIdx = chunk.search(/<\/li>/i);
|
const endIdx = chunk.search(/<\/li>/i);
|
||||||
const block = (endIdx >= 0 ? chunk.slice(0, endIdx + 5) : chunk);
|
const block = endIdx >= 0 ? chunk.slice(0, endIdx + 5) : chunk;
|
||||||
|
|
||||||
const hrefs = [...block.matchAll(/<a\b[^>]*href=["']([^"']+)["']/gi)].map(m => m[1]);
|
const hrefs = [...block.matchAll(/<a\b[^>]*href=["']([^"']+)["']/gi)].map((m) => m[1]);
|
||||||
const href = hrefs.find(h => !/add-to-cart=|\/cart\/|\/checkout\//i.test(h)) || "";
|
const href = hrefs.find((h) => !/add-to-cart=|\/cart\/|\/checkout\//i.test(h)) || "";
|
||||||
if (!href) continue;
|
if (!href) continue;
|
||||||
|
|
||||||
const url = new URL(decodeHtml(href), base).toString();
|
const url = new URL(decodeHtml(href), base).toString();
|
||||||
|
|
||||||
const nameHtml =
|
const nameHtml =
|
||||||
block.match(/<h2\b[^>]*class=["'][^"']*woocommerce-loop-product__title[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i)?.[1] ||
|
block.match(
|
||||||
|
/<h2\b[^>]*class=["'][^"']*woocommerce-loop-product__title[^"']*["'][^>]*>([\s\S]*?)<\/h2>/i,
|
||||||
|
)?.[1] ||
|
||||||
block.match(/<h3\b[^>]*>([\s\S]*?)<\/h3>/i)?.[1] ||
|
block.match(/<h3\b[^>]*>([\s\S]*?)<\/h3>/i)?.[1] ||
|
||||||
"";
|
"";
|
||||||
const name = cleanText(decodeHtml(nameHtml));
|
const name = cleanText(decodeHtml(nameHtml));
|
||||||
|
|
@ -156,10 +154,10 @@ function parseProductsSierra(body, ctx) {
|
||||||
if (blocks.length > 1) {
|
if (blocks.length > 1) {
|
||||||
const items = [];
|
const items = [];
|
||||||
for (let i = 1; i < blocks.length; i++) {
|
for (let i = 1; i < blocks.length; i++) {
|
||||||
const block = "<div class=\"tmb" + blocks[i];
|
const block = '<div class="tmb' + blocks[i];
|
||||||
|
|
||||||
const titleMatch = block.match(
|
const titleMatch = block.match(
|
||||||
/<h3\b[^>]*class=["'][^"']*t-entry-title[^"']*["'][^>]*>\s*<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>\s*<\/h3>/i
|
/<h3\b[^>]*class=["'][^"']*t-entry-title[^"']*["'][^>]*>\s*<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>\s*<\/h3>/i,
|
||||||
);
|
);
|
||||||
if (!titleMatch) continue;
|
if (!titleMatch) continue;
|
||||||
|
|
||||||
|
|
@ -174,9 +172,7 @@ function parseProductsSierra(body, ctx) {
|
||||||
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
|
block.match(/\bSKU[:\s]*([0-9]{6})\b/i)?.[1] ||
|
||||||
"";
|
"";
|
||||||
|
|
||||||
const taggedSku = /^\d{1,11}$/.test(String(rawSku).trim())
|
const taggedSku = /^\d{1,11}$/.test(String(rawSku).trim()) ? `id:${String(rawSku).trim()}` : rawSku;
|
||||||
? `id:${String(rawSku).trim()}`
|
|
||||||
: rawSku;
|
|
||||||
|
|
||||||
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
|
const sku = normalizeSkuKey(taggedSku, { storeLabel: ctx?.store?.name, url });
|
||||||
const img = extractFirstImgUrl(block, base);
|
const img = extractFirstImgUrl(block, base);
|
||||||
|
|
@ -202,9 +198,7 @@ function parseProductsSierra(body, ctx) {
|
||||||
function extractProductCatTermId(html) {
|
function extractProductCatTermId(html) {
|
||||||
const s = String(html || "");
|
const s = String(html || "");
|
||||||
// Typical body classes contain: "tax-product_cat term-<slug> term-1131 ..."
|
// Typical body classes contain: "tax-product_cat term-<slug> term-1131 ..."
|
||||||
const m =
|
const m = s.match(/tax-product_cat[^"']{0,400}\bterm-(\d{1,10})\b/i) || s.match(/\bterm-(\d{1,10})\b/i);
|
||||||
s.match(/tax-product_cat[^"']{0,400}\bterm-(\d{1,10})\b/i) ||
|
|
||||||
s.match(/\bterm-(\d{1,10})\b/i);
|
|
||||||
if (!m) return null;
|
if (!m) return null;
|
||||||
const n = Number(m[1]);
|
const n = Number(m[1]);
|
||||||
return Number.isFinite(n) ? n : null;
|
return Number.isFinite(n) ? n : null;
|
||||||
|
|
@ -222,7 +216,9 @@ async function getWooCategoryIdForCat(ctx) {
|
||||||
const id = extractProductCatTermId(text);
|
const id = extractProductCatTermId(text);
|
||||||
|
|
||||||
if (!id) {
|
if (!id) {
|
||||||
ctx.logger.warn(`${ctx.catPrefixOut} | Could not infer product_cat term id from category page; falling back to HTML parsing only.`);
|
ctx.logger.warn(
|
||||||
|
`${ctx.catPrefixOut} | Could not infer product_cat term id from category page; falling back to HTML parsing only.`,
|
||||||
|
);
|
||||||
ctx.cat._wooCategoryId = null;
|
ctx.cat._wooCategoryId = null;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
@ -260,18 +256,15 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
|
||||||
const { text, status, bytes, ms, finalUrl } = await ctx.http.fetchTextWithRetry(
|
const { text, status, bytes, ms, finalUrl } = await ctx.http.fetchTextWithRetry(
|
||||||
pageUrl,
|
pageUrl,
|
||||||
`page:${ctx.store.key}:${ctx.cat.key}:${page}`,
|
`page:${ctx.store.key}:${ctx.cat.key}:${page}`,
|
||||||
ctx.store.ua
|
ctx.store.ua,
|
||||||
);
|
);
|
||||||
|
|
||||||
// IMPORTANT:
|
// IMPORTANT:
|
||||||
// Parse WITHOUT allowUrl so pagination is based on real API page size
|
// Parse WITHOUT allowUrl so pagination is based on real API page size
|
||||||
const ctxNoFilter =
|
const ctxNoFilter =
|
||||||
typeof ctx?.cat?.allowUrl === "function"
|
typeof ctx?.cat?.allowUrl === "function" ? { ...ctx, cat: { ...ctx.cat, allowUrl: null } } : ctx;
|
||||||
? { ...ctx, cat: { ...ctx.cat, allowUrl: null } }
|
|
||||||
: ctx;
|
|
||||||
|
|
||||||
const itemsAll =
|
const itemsAll = (ctx.store.parseProducts || ctx.config.defaultParseProducts)(text, ctxNoFilter, finalUrl);
|
||||||
(ctx.store.parseProducts || ctx.config.defaultParseProducts)(text, ctxNoFilter, finalUrl);
|
|
||||||
|
|
||||||
const rawCount = itemsAll.length;
|
const rawCount = itemsAll.length;
|
||||||
|
|
||||||
|
|
@ -284,7 +277,7 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.ok(
|
logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${String(page).padStart(3, " ")} | ${String(status).padStart(3, " ")} | raw=${String(rawCount).padStart(3, " ")} kept=${String(items.length).padStart(3, " ")} | bytes=${String(bytes || 0).padStart(8, " ")} | ${(ms / 1000).toFixed(1).padStart(6, " ")}s`
|
`${ctx.catPrefixOut} | Page ${String(page).padStart(3, " ")} | ${String(status).padStart(3, " ")} | raw=${String(rawCount).padStart(3, " ")} kept=${String(items.length).padStart(3, " ")} | bytes=${String(bytes || 0).padStart(8, " ")} | ${(ms / 1000).toFixed(1).padStart(6, " ")}s`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Stop only when the API page itself is empty
|
// Stop only when the API page itself is empty
|
||||||
|
|
@ -300,14 +293,11 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
||||||
|
|
||||||
const {
|
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
|
||||||
merged,
|
prevDb,
|
||||||
newItems,
|
discovered,
|
||||||
updatedItems,
|
{ storeLabel: ctx.store.name },
|
||||||
removedItems,
|
);
|
||||||
restoredItems,
|
|
||||||
metaChangedItems,
|
|
||||||
} = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -344,7 +334,7 @@ async function scanCategoryWooStoreApi(ctx, prevDb, report) {
|
||||||
newItems,
|
newItems,
|
||||||
updatedItems,
|
updatedItems,
|
||||||
removedItems,
|
removedItems,
|
||||||
restoredItems
|
restoredItems,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -51,10 +51,7 @@ function normalizePrice(str) {
|
||||||
|
|
||||||
function pickPriceFromArticle(articleHtml) {
|
function pickPriceFromArticle(articleHtml) {
|
||||||
const a = String(articleHtml || "");
|
const a = String(articleHtml || "");
|
||||||
const noMember = a.replace(
|
const noMember = a.replace(/<div\b[^>]*class=["'][^"']*\bwhiskyfolk-price\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, " ");
|
||||||
/<div\b[^>]*class=["'][^"']*\bwhiskyfolk-price\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi,
|
|
||||||
" "
|
|
||||||
);
|
|
||||||
|
|
||||||
const ins = noMember.match(/<ins\b[^>]*>[\s\S]*?(\$[\s\S]{0,32}?)<\/ins>/i);
|
const ins = noMember.match(/<ins\b[^>]*>[\s\S]*?(\$[\s\S]{0,32}?)<\/ins>/i);
|
||||||
if (ins && ins[1]) return normalizePrice(ins[1]);
|
if (ins && ins[1]) return normalizePrice(ins[1]);
|
||||||
|
|
@ -62,9 +59,7 @@ function pickPriceFromArticle(articleHtml) {
|
||||||
const reg = noMember.match(/class=["'][^"']*\bregular-price-card\b[^"']*["'][^>]*>\s*([^<]+)/i);
|
const reg = noMember.match(/class=["'][^"']*\bregular-price-card\b[^"']*["'][^>]*>\s*([^<]+)/i);
|
||||||
if (reg && reg[1]) return normalizePrice(reg[1]);
|
if (reg && reg[1]) return normalizePrice(reg[1]);
|
||||||
|
|
||||||
const priceDiv = noMember.match(
|
const priceDiv = noMember.match(/<div\b[^>]*class=["'][^"']*\bproduct-price\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/i);
|
||||||
/<div\b[^>]*class=["'][^"']*\bproduct-price\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/i
|
|
||||||
);
|
|
||||||
const scope = priceDiv && priceDiv[1] ? priceDiv[1] : noMember;
|
const scope = priceDiv && priceDiv[1] ? priceDiv[1] : noMember;
|
||||||
|
|
||||||
return normalizePrice(scope);
|
return normalizePrice(scope);
|
||||||
|
|
@ -157,7 +152,6 @@ function parseProductFromArticle(articleHtml) {
|
||||||
productId,
|
productId,
|
||||||
img,
|
img,
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------------- Store API paging ---------------- */
|
/* ---------------- Store API paging ---------------- */
|
||||||
|
|
@ -185,12 +179,16 @@ function buildStoreApiBaseUrlFromCategoryUrl(startUrl) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasCategorySlug(p, wanted) {
|
function hasCategorySlug(p, wanted) {
|
||||||
const w = String(wanted || "").trim().toLowerCase();
|
const w = String(wanted || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (!w) return true;
|
if (!w) return true;
|
||||||
|
|
||||||
const cats = Array.isArray(p?.categories) ? p.categories : [];
|
const cats = Array.isArray(p?.categories) ? p.categories : [];
|
||||||
for (const c of cats) {
|
for (const c of cats) {
|
||||||
const slug = String(c?.slug || "").trim().toLowerCase();
|
const slug = String(c?.slug || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (slug === w) return true;
|
if (slug === w) return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -304,7 +302,7 @@ function avoidMassRemoval(prevDb, discovered, ctx, reason) {
|
||||||
if (ratio >= 0.6) return false;
|
if (ratio >= 0.6) return false;
|
||||||
|
|
||||||
ctx.logger.warn?.(
|
ctx.logger.warn?.(
|
||||||
`${ctx.catPrefixOut} | Strath partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`
|
`${ctx.catPrefixOut} | Strath partial scan (${discSize}/${prevSize}); preserving DB to avoid removals (${reason}).`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (prevDb && typeof prevDb.entries === "function") {
|
if (prevDb && typeof prevDb.entries === "function") {
|
||||||
|
|
@ -353,8 +351,8 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(1, 1)} | ${String(listingStatus || "").padEnd(3)} | ${pctStr(1, 1)} | items=${padLeft(
|
`${ctx.catPrefixOut} | Page ${pageStr(1, 1)} | ${String(listingStatus || "").padEnd(3)} | ${pctStr(1, 1)} | items=${padLeft(
|
||||||
listingItems,
|
listingItems,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(listingBytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(listingMs)}`
|
)} | bytes=${kbStr(listingBytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(listingMs)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
const apiBase = buildStoreApiBaseUrlFromCategoryUrl(listingFinalUrl || ctx.cat.startUrl);
|
const apiBase = buildStoreApiBaseUrlFromCategoryUrl(listingFinalUrl || ctx.cat.startUrl);
|
||||||
|
|
@ -362,7 +360,9 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
const perPage = 100;
|
const perPage = 100;
|
||||||
const maxPagesCap = ctx.config.maxPages === null ? 5000 : ctx.config.maxPages;
|
const maxPagesCap = ctx.config.maxPages === null ? 5000 : ctx.config.maxPages;
|
||||||
|
|
||||||
const wantedSlug = String(ctx.cat.apiCategorySlug || "").trim().toLowerCase();
|
const wantedSlug = String(ctx.cat.apiCategorySlug || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
|
||||||
let donePages = 0;
|
let donePages = 0;
|
||||||
let emptyMatchPages = 0;
|
let emptyMatchPages = 0;
|
||||||
|
|
@ -399,7 +399,6 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
const sku = normalizeProductSku(p);
|
const sku = normalizeProductSku(p);
|
||||||
const productId = normalizeProductId(p);
|
const productId = normalizeProductId(p);
|
||||||
|
|
||||||
|
|
||||||
const prev = discovered.get(url) || null;
|
const prev = discovered.get(url) || null;
|
||||||
|
|
||||||
const apiImg = normalizeProductImage(p) || "";
|
const apiImg = normalizeProductImage(p) || "";
|
||||||
|
|
@ -411,7 +410,6 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
const newSku = sku || fallbackSku;
|
const newSku = sku || fallbackSku;
|
||||||
const mergedSku = pickBetterSku(newSku, prev && prev.sku);
|
const mergedSku = pickBetterSku(newSku, prev && prev.sku);
|
||||||
|
|
||||||
|
|
||||||
discovered.set(url, {
|
discovered.set(url, {
|
||||||
name,
|
name,
|
||||||
price,
|
price,
|
||||||
|
|
@ -426,8 +424,8 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(3)} | kept=${padLeft(
|
`${ctx.catPrefixOut} | API Page ${pageStr(donePages, donePages)} | ${(r?.status || "").toString().padEnd(3)} | kept=${padLeft(
|
||||||
kept,
|
kept,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | bytes=${kbStr(r.bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (wantedSlug) {
|
if (wantedSlug) {
|
||||||
|
|
@ -458,7 +456,7 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -479,7 +477,15 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
|
|
|
||||||
|
|
@ -176,7 +176,6 @@ function parseDisplayPriceFromHtml(html) {
|
||||||
return Number.isFinite(n) ? n : null;
|
return Number.isFinite(n) ? n : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function pickAnySkuFromProduct(p) {
|
function pickAnySkuFromProduct(p) {
|
||||||
const vs = Array.isArray(p?.variants) ? p.variants : [];
|
const vs = Array.isArray(p?.variants) ? p.variants : [];
|
||||||
for (const v of vs) {
|
for (const v of vs) {
|
||||||
|
|
@ -353,7 +352,7 @@ async function supplementImageFromSku(ctx, skuProbe) {
|
||||||
|
|
||||||
const v = pickInStockVariantWithFallback(prod);
|
const v = pickInStockVariantWithFallback(prod);
|
||||||
const img = normalizeAbsUrl(
|
const img = normalizeAbsUrl(
|
||||||
firstNonEmptyStr(v?.image, prod?.gulpImages, prod?.posImages, prod?.customImages, prod?.imageIds)
|
firstNonEmptyStr(v?.image, prod?.gulpImages, prod?.posImages, prod?.customImages, prod?.imageIds),
|
||||||
);
|
);
|
||||||
|
|
||||||
return img ? { img } : null;
|
return img ? { img } : null;
|
||||||
|
|
@ -369,9 +368,7 @@ function parseSkuFromHtml(html) {
|
||||||
const s = String(html || "");
|
const s = String(html || "");
|
||||||
|
|
||||||
// 1) Visible block: <div class="sku ...">SKU: 67433</div>
|
// 1) Visible block: <div class="sku ...">SKU: 67433</div>
|
||||||
const m1 =
|
const m1 = s.match(/>\s*SKU:\s*([A-Za-z0-9._-]+)\s*</i) || s.match(/\bSKU:\s*([A-Za-z0-9._-]+)\b/i);
|
||||||
s.match(/>\s*SKU:\s*([A-Za-z0-9._-]+)\s*</i) ||
|
|
||||||
s.match(/\bSKU:\s*([A-Za-z0-9._-]+)\b/i);
|
|
||||||
if (m1 && m1[1]) return String(m1[1]).trim();
|
if (m1 && m1[1]) return String(m1[1]).trim();
|
||||||
|
|
||||||
// 2) Embedded SAPPER preloaded JSON has variants with `"sku":"67433"`
|
// 2) Embedded SAPPER preloaded JSON has variants with `"sku":"67433"`
|
||||||
|
|
@ -448,7 +445,6 @@ async function tudorDetailFromProductPage(ctx, url) {
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ---------------- item builder (fast, no extra calls) ---------------- */
|
/* ---------------- item builder (fast, no extra calls) ---------------- */
|
||||||
|
|
||||||
function tudorItemFromProductFast(p, ctx) {
|
function tudorItemFromProductFast(p, ctx) {
|
||||||
|
|
@ -469,9 +465,7 @@ function tudorItemFromProductFast(p, ctx) {
|
||||||
const skuRaw = String(v?.sku || "").trim() || pickAnySkuFromProduct(p);
|
const skuRaw = String(v?.sku || "").trim() || pickAnySkuFromProduct(p);
|
||||||
const sku = normalizeTudorSku(skuRaw);
|
const sku = normalizeTudorSku(skuRaw);
|
||||||
|
|
||||||
const img = normalizeAbsUrl(
|
const img = normalizeAbsUrl(firstNonEmptyStr(v?.image, p?.gulpImages, p?.posImages, p?.customImages, p?.imageIds));
|
||||||
firstNonEmptyStr(v?.image, p?.gulpImages, p?.posImages, p?.customImages, p?.imageIds)
|
|
||||||
);
|
|
||||||
|
|
||||||
// NEW: keep lightweight variant snapshot so repair can match HTML SKU -> exact GQL variant price
|
// NEW: keep lightweight variant snapshot so repair can match HTML SKU -> exact GQL variant price
|
||||||
const variants = Array.isArray(p?.variants)
|
const variants = Array.isArray(p?.variants)
|
||||||
|
|
@ -492,9 +486,7 @@ async function tudorRepairItem(ctx, it) {
|
||||||
// Determine if we need HTML for precision:
|
// Determine if we need HTML for precision:
|
||||||
// - Missing/synthetic SKU (existing behavior)
|
// - Missing/synthetic SKU (existing behavior)
|
||||||
// - OR multi-variant product where fast-path may choose the wrong variant for this URL
|
// - OR multi-variant product where fast-path may choose the wrong variant for this URL
|
||||||
const inStockVariants = Array.isArray(it._variants)
|
const inStockVariants = Array.isArray(it._variants) ? it._variants.filter((v) => Number(v?.quantity) > 0) : [];
|
||||||
? it._variants.filter((v) => Number(v?.quantity) > 0)
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const hasMultiInStock = inStockVariants.length >= 2;
|
const hasMultiInStock = inStockVariants.length >= 2;
|
||||||
|
|
||||||
|
|
@ -513,7 +505,9 @@ async function tudorRepairItem(ctx, it) {
|
||||||
// Price precision:
|
// Price precision:
|
||||||
// - Best: match HTML SKU to a GQL variant sku => exact numeric variant price
|
// - Best: match HTML SKU to a GQL variant sku => exact numeric variant price
|
||||||
// - Fallback: use displayed HTML price
|
// - Fallback: use displayed HTML price
|
||||||
const htmlSkuDigits = String(d?.sku || "").replace(/^id:/i, "").trim();
|
const htmlSkuDigits = String(d?.sku || "")
|
||||||
|
.replace(/^id:/i, "")
|
||||||
|
.trim();
|
||||||
|
|
||||||
if (htmlSkuDigits && inStockVariants.length) {
|
if (htmlSkuDigits && inStockVariants.length) {
|
||||||
const match = inStockVariants.find((v) => String(v?.sku || "").trim() === htmlSkuDigits);
|
const match = inStockVariants.find((v) => String(v?.sku || "").trim() === htmlSkuDigits);
|
||||||
|
|
@ -542,7 +536,6 @@ async function tudorRepairItem(ctx, it) {
|
||||||
return it;
|
return it;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ---------------- scanner ---------------- */
|
/* ---------------- scanner ---------------- */
|
||||||
|
|
||||||
async function scanCategoryTudor(ctx, prevDb, report) {
|
async function scanCategoryTudor(ctx, prevDb, report) {
|
||||||
|
|
@ -586,8 +579,8 @@ async function scanCategoryTudor(ctx, prevDb, report) {
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(page, maxPages)} | 200 | items=${padLeft(
|
`${ctx.catPrefixOut} | Page ${pageStr(page, maxPages)} | 200 | items=${padLeft(
|
||||||
kept,
|
kept,
|
||||||
3
|
3,
|
||||||
)} | bytes=${kbStr(0)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
|
)} | bytes=${kbStr(0)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
cursor = prod?.nextPageCursor || null;
|
cursor = prod?.nextPageCursor || null;
|
||||||
|
|
@ -623,7 +616,7 @@ async function scanCategoryTudor(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Unique products: ${discovered.size} | detail(html=${htmlUsed}/${htmlBudget}, gql=${gqlUsed}/${gqlBudget})`
|
`${ctx.catPrefixOut} | Unique products: ${discovered.size} | detail(html=${htmlUsed}/${htmlBudget}, gql=${gqlUsed}/${gqlBudget})`,
|
||||||
);
|
);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
|
|
@ -654,7 +647,15 @@ async function scanCategoryTudor(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------------- store ---------------- */
|
/* ---------------- store ---------------- */
|
||||||
|
|
|
||||||
|
|
@ -162,7 +162,8 @@ function createStore(defaultUa) {
|
||||||
{
|
{
|
||||||
key: "rum-cane-spirit",
|
key: "rum-cane-spirit",
|
||||||
label: "Rum / Cane Spirit",
|
label: "Rum / Cane Spirit",
|
||||||
startUrl: "https://vesselliquor.com/collections/rum-cane-spirit?sort_by=title-ascending&filter.v.availability=1",
|
startUrl:
|
||||||
|
"https://vesselliquor.com/collections/rum-cane-spirit?sort_by=title-ascending&filter.v.availability=1",
|
||||||
discoveryStartPage: 20,
|
discoveryStartPage: 20,
|
||||||
discoveryStep: 10,
|
discoveryStep: 10,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -110,9 +110,13 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
ctx.logger.warn(`${ctx.catPrefixOut} | Vintage API fetch failed: ${e?.message || e}`);
|
ctx.logger.warn(`${ctx.catPrefixOut} | Vintage API fetch failed: ${e?.message || e}`);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(
|
||||||
|
prevDb,
|
||||||
|
discovered,
|
||||||
|
{
|
||||||
storeLabel: ctx.store.name,
|
storeLabel: ctx.store.name,
|
||||||
});
|
},
|
||||||
|
);
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
||||||
|
|
@ -134,7 +138,15 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
report.totals.updatedCount += updatedItems.length;
|
report.totals.updatedCount += updatedItems.length;
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -142,7 +154,7 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
const scanPages = ctx.config.maxPages === null ? totalPages : Math.min(ctx.config.maxPages, totalPages);
|
||||||
|
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`
|
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
const pages = [];
|
const pages = [];
|
||||||
|
|
@ -167,14 +179,14 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
donePages++;
|
donePages++;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pages.length)} | ${String(r.status || "").padEnd(
|
`${ctx.catPrefixOut} | Page ${pageStr(idx + 1, pages.length)} | ${String(r.status || "").padEnd(
|
||||||
3
|
3,
|
||||||
)} | ${pctStr(donePages, pages.length)} | items=${padLeft(items.length, 3)} | bytes=${kbStr(
|
)} | ${pctStr(donePages, pages.length)} | items=${padLeft(items.length, 3)} | bytes=${kbStr(
|
||||||
r.bytes
|
r.bytes,
|
||||||
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`
|
)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
}
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
const discovered = new Map();
|
const discovered = new Map();
|
||||||
|
|
@ -186,7 +198,9 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
|
ctx.logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
storeLabel: ctx.store.name,
|
storeLabel: ctx.store.name,
|
||||||
|
|
@ -197,7 +211,7 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
ctx.logger.ok(
|
ctx.logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -218,7 +232,15 @@ async function scanCategoryVintageApi(ctx, prevDb, report) {
|
||||||
report.totals.removedCount += removedItems.length;
|
report.totals.removedCount += removedItems.length;
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStore(defaultUa) {
|
function createStore(defaultUa) {
|
||||||
|
|
|
||||||
|
|
@ -47,9 +47,8 @@ function extractWillowCardPrice(block) {
|
||||||
|
|
||||||
const current =
|
const current =
|
||||||
b.match(
|
b.match(
|
||||||
/grid-product__price--current[\s\S]*?<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i
|
/grid-product__price--current[\s\S]*?<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i,
|
||||||
)?.[1] ||
|
)?.[1] || b.match(/<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i)?.[1];
|
||||||
b.match(/<span\b[^>]*class=["']visually-hidden["'][^>]*>\s*(\$\s*[\d,]+\.\d{2})\s*<\/span>/i)?.[1];
|
|
||||||
|
|
||||||
if (current) return current.replace(/\s+/g, "");
|
if (current) return current.replace(/\s+/g, "");
|
||||||
|
|
||||||
|
|
@ -101,10 +100,7 @@ function parseProductsWillowPark(html, ctx, finalUrl) {
|
||||||
const img = extractFirstImgUrl(block, base);
|
const img = extractFirstImgUrl(block, base);
|
||||||
const pid = block.match(/\bdata-product-id=["'](\d+)["']/i)?.[1] || "";
|
const pid = block.match(/\bdata-product-id=["'](\d+)["']/i)?.[1] || "";
|
||||||
|
|
||||||
const sku =
|
const sku = extractSkuFromUrlOrHref(href) || extractSkuFromUrlOrHref(url) || extractSkuFromWillowBlock(block);
|
||||||
extractSkuFromUrlOrHref(href) ||
|
|
||||||
extractSkuFromUrlOrHref(url) ||
|
|
||||||
extractSkuFromWillowBlock(block);
|
|
||||||
|
|
||||||
items.push({ name, price, url, sku, img, pid });
|
items.push({ name, price, url, sku, img, pid });
|
||||||
}
|
}
|
||||||
|
|
@ -164,9 +160,7 @@ function extractStorefrontTokenFromHtml(html) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2) meta name="shopify-checkout-api-token"
|
// 2) meta name="shopify-checkout-api-token"
|
||||||
const m = s.match(
|
const m = s.match(/<meta[^>]+name=["']shopify-checkout-api-token["'][^>]+content=["']([^"']+)["']/i)?.[1];
|
||||||
/<meta[^>]+name=["']shopify-checkout-api-token["'][^>]+content=["']([^"']+)["']/i
|
|
||||||
)?.[1];
|
|
||||||
return String(m || "").trim();
|
return String(m || "").trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -213,7 +207,6 @@ function normalizeWillowGqlSku(rawSku) {
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async function willowFetchSkuByPid(ctx, pid) {
|
async function willowFetchSkuByPid(ctx, pid) {
|
||||||
const id = String(pid || "").trim();
|
const id = String(pid || "").trim();
|
||||||
if (!id) return "";
|
if (!id) return "";
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,9 @@ function progCell(v) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function logProgressLine(logger, ctx, action, statusRaw, statusOk, progVal, rest) {
|
function logProgressLine(logger, ctx, action, statusRaw, statusOk, progVal, rest) {
|
||||||
logger.ok(`${ctx.catPrefixOut} | ${actionCell(action)} | ${statusCell(logger, statusRaw, statusOk)} | ${progCell(progVal)} | ${rest}`);
|
logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | ${actionCell(action)} | ${statusCell(logger, statusRaw, statusOk)} | ${progCell(progVal)} | ${rest}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function makeCatPrefixers(stores, logger) {
|
function makeCatPrefixers(stores, logger) {
|
||||||
|
|
@ -168,7 +170,7 @@ async function probePage(ctx, baseUrl, pageNum, state) {
|
||||||
r.ok ? "OK" : "MISS",
|
r.ok ? "OK" : "MISS",
|
||||||
Boolean(r.ok),
|
Boolean(r.ok),
|
||||||
prog,
|
prog,
|
||||||
`items=${padLeftV(r.items, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
|
`items=${padLeftV(r.items, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return r;
|
return r;
|
||||||
|
|
@ -213,12 +215,20 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
|
||||||
// Fetch page 1 ONCE and try to extract total pages from pagination.
|
// Fetch page 1 ONCE and try to extract total pages from pagination.
|
||||||
const url1 = makePageUrlForCtx(ctx, baseUrl, 1);
|
const url1 = makePageUrlForCtx(ctx, baseUrl, 1);
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
const { text: html1, ms, status, bytes, finalUrl } = await ctx.http.fetchTextWithRetry(url1, "discover", ctx.store.ua);
|
const {
|
||||||
|
text: html1,
|
||||||
|
ms,
|
||||||
|
status,
|
||||||
|
bytes,
|
||||||
|
finalUrl,
|
||||||
|
} = await ctx.http.fetchTextWithRetry(url1, "discover", ctx.store.ua);
|
||||||
const pMs = Date.now() - t0;
|
const pMs = Date.now() - t0;
|
||||||
|
|
||||||
if (typeof ctx.store.isEmptyListingPage === "function") {
|
if (typeof ctx.store.isEmptyListingPage === "function") {
|
||||||
if (ctx.store.isEmptyListingPage(html1, ctx, url1)) {
|
if (ctx.store.isEmptyListingPage(html1, ctx, url1)) {
|
||||||
ctx.logger.warn(`${ctx.store.name} | ${ctx.cat.label} | Page 1 did not look like a listing. Defaulting to 1.`);
|
ctx.logger.warn(
|
||||||
|
`${ctx.store.name} | ${ctx.cat.label} | Page 1 did not look like a listing. Defaulting to 1.`,
|
||||||
|
);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -233,7 +243,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
|
||||||
items1 > 0 ? "OK" : "MISS",
|
items1 > 0 ? "OK" : "MISS",
|
||||||
items1 > 0,
|
items1 > 0,
|
||||||
discoverProg(state),
|
discoverProg(state),
|
||||||
`items=${padLeftV(items1, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms || pMs)}`
|
`items=${padLeftV(items1, 3)} | bytes=${padLeftV("", 8)} | ${padRightV(ctx.http.inflightStr(), 11)} | ${secStr(ms || pMs)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (items1 <= 0) {
|
if (items1 <= 0) {
|
||||||
|
|
@ -246,8 +256,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
|
||||||
// Shopify collections with filters often lie about pagination.
|
// Shopify collections with filters often lie about pagination.
|
||||||
// If page 1 looks full, don't trust a tiny extracted count.
|
// If page 1 looks full, don't trust a tiny extracted count.
|
||||||
if (extracted && extracted >= 1) {
|
if (extracted && extracted >= 1) {
|
||||||
const looksTruncated =
|
const looksTruncated = extracted <= 2 && items1 >= 40; // Shopify default page size ≈ 48
|
||||||
extracted <= 2 && items1 >= 40; // Shopify default page size ≈ 48
|
|
||||||
|
|
||||||
if (!looksTruncated) {
|
if (!looksTruncated) {
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Total pages (from pagination): ${extracted}`);
|
ctx.logger.ok(`${ctx.catPrefixOut} | Total pages (from pagination): ${extracted}`);
|
||||||
|
|
@ -255,7 +264,7 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(
|
||||||
`${ctx.catPrefixOut} | Pagination says ${extracted} but page looks full; falling back to probe`
|
`${ctx.catPrefixOut} | Pagination says ${extracted} but page looks full; falling back to probe`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -271,7 +280,9 @@ async function discoverTotalPagesFast(ctx, baseUrl, guess, step) {
|
||||||
if (!pr.ok) return await binaryFindLastOk(ctx, baseUrl, lastOk, probe, state);
|
if (!pr.ok) return await binaryFindLastOk(ctx, baseUrl, lastOk, probe, state);
|
||||||
lastOk = probe;
|
lastOk = probe;
|
||||||
if (lastOk > 5000) {
|
if (lastOk > 5000) {
|
||||||
ctx.logger.warn(`${ctx.store.name} | ${ctx.cat.label} | Discovery hit safety cap at ${lastOk}. Using that as total pages.`);
|
ctx.logger.warn(
|
||||||
|
`${ctx.store.name} | ${ctx.cat.label} | Discovery hit safety cap at ${lastOk}. Using that as total pages.`,
|
||||||
|
);
|
||||||
return lastOk;
|
return lastOk;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -293,7 +304,9 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
const totalPages = await discoverTotalPagesFast(ctx, ctx.baseUrl, guess, step);
|
const totalPages = await discoverTotalPagesFast(ctx, ctx.baseUrl, guess, step);
|
||||||
const scanPages = config.maxPages === null ? totalPages : Math.min(config.maxPages, totalPages);
|
const scanPages = config.maxPages === null ? totalPages : Math.min(config.maxPages, totalPages);
|
||||||
|
|
||||||
logger.ok(`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`);
|
logger.ok(
|
||||||
|
`${ctx.catPrefixOut} | Pages: ${scanPages}${scanPages !== totalPages ? ` (cap from ${totalPages})` : ""}`,
|
||||||
|
);
|
||||||
|
|
||||||
const pages = [];
|
const pages = [];
|
||||||
for (let p = 1; p <= scanPages; p++) pages.push(makePageUrlForCtx(ctx, ctx.baseUrl, p));
|
for (let p = 1; p <= scanPages; p++) pages.push(makePageUrlForCtx(ctx, ctx.baseUrl, p));
|
||||||
|
|
@ -306,11 +319,13 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
const perPageItems = await parallelMapStaggered(pages, pageConc, pageStagger, async (pageUrl, idx) => {
|
const perPageItems = await parallelMapStaggered(pages, pageConc, pageStagger, async (pageUrl, idx) => {
|
||||||
const pnum = idx + 1;
|
const pnum = idx + 1;
|
||||||
|
|
||||||
const { text: html, ms, bytes, status, finalUrl } = await ctx.http.fetchTextWithRetry(
|
const {
|
||||||
pageUrl,
|
text: html,
|
||||||
`page:${ctx.store.key}:${ctx.cat.key}:${pnum}`,
|
ms,
|
||||||
ctx.store.ua
|
bytes,
|
||||||
);
|
status,
|
||||||
|
finalUrl,
|
||||||
|
} = await ctx.http.fetchTextWithRetry(pageUrl, `page:${ctx.store.key}:${ctx.cat.key}:${pnum}`, ctx.store.ua);
|
||||||
|
|
||||||
const parser = ctx.store.parseProducts || config.defaultParseProducts;
|
const parser = ctx.store.parseProducts || config.defaultParseProducts;
|
||||||
const itemsRaw = parser(html, ctx, finalUrl);
|
const itemsRaw = parser(html, ctx, finalUrl);
|
||||||
|
|
@ -328,7 +343,7 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
status ? String(status) : "",
|
status ? String(status) : "",
|
||||||
status >= 200 && status < 400,
|
status >= 200 && status < 400,
|
||||||
pctStr(donePages, pages.length),
|
pctStr(donePages, pages.length),
|
||||||
`items=${padLeft(items.length, 3)} | bytes=${kbStr(bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`
|
`items=${padLeft(items.length, 3)} | bytes=${kbStr(bytes)} | ${padRight(ctx.http.inflightStr(), 11)} | ${secStr(ms)}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
|
|
@ -349,8 +364,11 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
|
|
||||||
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
|
logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}${dups ? ` (${dups} dups)` : ""}`);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } =
|
const { merged, newItems, updatedItems, removedItems, restoredItems, metaChangedItems } = mergeDiscoveredIntoDb(
|
||||||
mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
prevDb,
|
||||||
|
discovered,
|
||||||
|
{ storeLabel: ctx.store.name },
|
||||||
|
);
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
@ -359,7 +377,7 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
|
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
logger.ok(
|
logger.ok(
|
||||||
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`
|
`${ctx.catPrefixOut} | Done in ${secStr(elapsed)}. New=${newItems.length} Updated=${updatedItems.length} Removed=${removedItems.length} Restored=${restoredItems.length} Total(DB)=${merged.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
report.categories.push({
|
report.categories.push({
|
||||||
|
|
@ -382,7 +400,15 @@ async function discoverAndScanCategory(ctx, prevDb, report) {
|
||||||
report.totals.restoredCount += restoredItems.length;
|
report.totals.restoredCount += restoredItems.length;
|
||||||
report.totals.metaChangedCount += metaChangedItems.length;
|
report.totals.metaChangedCount += metaChangedItems.length;
|
||||||
|
|
||||||
addCategoryResultToReport(report, ctx.store.name, ctx.cat.label, newItems, updatedItems, removedItems, restoredItems);
|
addCategoryResultToReport(
|
||||||
|
report,
|
||||||
|
ctx.store.name,
|
||||||
|
ctx.cat.label,
|
||||||
|
newItems,
|
||||||
|
updatedItems,
|
||||||
|
removedItems,
|
||||||
|
restoredItems,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { makeCatPrefixers, buildCategoryContext, loadCategoryDb, discoverAndScanCategory };
|
module.exports = { makeCatPrefixers, buildCategoryContext, loadCategoryDb, discoverAndScanCategory };
|
||||||
|
|
|
||||||
|
|
@ -110,7 +110,8 @@ function buildCheapestSkuIndexFromAllDbs(dbDir, { skuMap } = {}) {
|
||||||
const skuKey = normalizeSkuKey(it?.sku || "", { storeLabel, url: it?.url || "" });
|
const skuKey = normalizeSkuKey(it?.sku || "", { storeLabel, url: it?.url || "" });
|
||||||
if (!skuKey) continue;
|
if (!skuKey) continue;
|
||||||
|
|
||||||
const canon = skuMap && typeof skuMap.canonicalSku === "function" ? skuMap.canonicalSku(skuKey) : skuKey;
|
const canon =
|
||||||
|
skuMap && typeof skuMap.canonicalSku === "function" ? skuMap.canonicalSku(skuKey) : skuKey;
|
||||||
|
|
||||||
const p = priceToNumber(it?.price || "");
|
const p = priceToNumber(it?.price || "");
|
||||||
if (!Number.isFinite(p) || p <= 0) continue;
|
if (!Number.isFinite(p) || p <= 0) continue;
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ function normImg(v) {
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function dbStoreLabel(prevDb) {
|
function dbStoreLabel(prevDb) {
|
||||||
return String(prevDb?.storeLabel || prevDb?.store || "").trim();
|
return String(prevDb?.storeLabel || prevDb?.store || "").trim();
|
||||||
}
|
}
|
||||||
|
|
@ -21,7 +20,7 @@ function mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel } = {}) {
|
||||||
const effectiveStoreLabel = String(storeLabel || dbStoreLabel(prevDb)).trim();
|
const effectiveStoreLabel = String(storeLabel || dbStoreLabel(prevDb)).trim();
|
||||||
if (!effectiveStoreLabel) {
|
if (!effectiveStoreLabel) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"mergeDiscoveredIntoDb: missing storeLabel; refusing to generate synthetic SKUs with fallback 'store'"
|
"mergeDiscoveredIntoDb: missing storeLabel; refusing to generate synthetic SKUs with fallback 'store'",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,23 @@ function createReport() {
|
||||||
function addCategoryResultToReport(report, storeName, catLabel, newItems, updatedItems, removedItems, restoredItems) {
|
function addCategoryResultToReport(report, storeName, catLabel, newItems, updatedItems, removedItems, restoredItems) {
|
||||||
const reportCatLabel = `${storeName} | ${catLabel}`;
|
const reportCatLabel = `${storeName} | ${catLabel}`;
|
||||||
|
|
||||||
for (const it of newItems) report.newItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
|
for (const it of newItems)
|
||||||
|
report.newItems.push({
|
||||||
|
catLabel: reportCatLabel,
|
||||||
|
name: it.name,
|
||||||
|
price: it.price || "",
|
||||||
|
sku: it.sku || "",
|
||||||
|
url: it.url,
|
||||||
|
});
|
||||||
|
|
||||||
for (const it of restoredItems)
|
for (const it of restoredItems)
|
||||||
report.restoredItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
|
report.restoredItems.push({
|
||||||
|
catLabel: reportCatLabel,
|
||||||
|
name: it.name,
|
||||||
|
price: it.price || "",
|
||||||
|
sku: it.sku || "",
|
||||||
|
url: it.url,
|
||||||
|
});
|
||||||
|
|
||||||
for (const u of updatedItems) {
|
for (const u of updatedItems) {
|
||||||
report.updatedItems.push({
|
report.updatedItems.push({
|
||||||
|
|
@ -48,7 +61,13 @@ function addCategoryResultToReport(report, storeName, catLabel, newItems, update
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const it of removedItems)
|
for (const it of removedItems)
|
||||||
report.removedItems.push({ catLabel: reportCatLabel, name: it.name, price: it.price || "", sku: it.sku || "", url: it.url });
|
report.removedItems.push({
|
||||||
|
catLabel: reportCatLabel,
|
||||||
|
name: it.name,
|
||||||
|
price: it.price || "",
|
||||||
|
sku: it.sku || "",
|
||||||
|
url: it.url,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout && process.stdout.isTTY) } = {}) {
|
function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout && process.stdout.isTTY) } = {}) {
|
||||||
|
|
@ -64,7 +83,10 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
const durMs = endedAt - report.startedAt;
|
const durMs = endedAt - report.startedAt;
|
||||||
|
|
||||||
const storesSet = new Set(report.categories.map((c) => c.store));
|
const storesSet = new Set(report.categories.map((c) => c.store));
|
||||||
const totalUnique = report.categories.reduce((acc, c) => acc + (Number.isFinite(c.discoveredUnique) ? c.discoveredUnique : 0), 0);
|
const totalUnique = report.categories.reduce(
|
||||||
|
(acc, c) => acc + (Number.isFinite(c.discoveredUnique) ? c.discoveredUnique : 0),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
let out = "";
|
let out = "";
|
||||||
const ln = (s = "") => {
|
const ln = (s = "") => {
|
||||||
|
|
@ -76,8 +98,8 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
ln(
|
ln(
|
||||||
paint("[OK] ", C.green) +
|
paint("[OK] ", C.green) +
|
||||||
`Totals | Stores=${storesSet.size} | Categories=${report.categories.length} | Unique=${totalUnique} | New=${report.totals.newCount} | Restored=${report.totals.restoredCount} | Removed=${report.totals.removedCount} | PriceChanges=${report.totals.updatedCount} | Runtime=${secStr(
|
`Totals | Stores=${storesSet.size} | Categories=${report.categories.length} | Unique=${totalUnique} | New=${report.totals.newCount} | Restored=${report.totals.restoredCount} | Removed=${report.totals.removedCount} | PriceChanges=${report.totals.updatedCount} | Runtime=${secStr(
|
||||||
durMs
|
durMs,
|
||||||
)}`
|
)}`,
|
||||||
);
|
);
|
||||||
ln("");
|
ln("");
|
||||||
|
|
||||||
|
|
@ -94,11 +116,13 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const catW = Math.min(48, Math.max(...rows.map((r) => r.cat.length), 8));
|
const catW = Math.min(48, Math.max(...rows.map((r) => r.cat.length), 8));
|
||||||
ln(`${padRight("Store | Category", catW)} ${padLeft("Pages", 5)} ${padLeft("Unique", 6)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)} ${padLeft("Sec", 7)}`);
|
ln(
|
||||||
|
`${padRight("Store | Category", catW)} ${padLeft("Pages", 5)} ${padLeft("Unique", 6)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)} ${padLeft("Sec", 7)}`,
|
||||||
|
);
|
||||||
ln(`${"-".repeat(catW)} ----- ------ ---- ---- ---- ---- -------`);
|
ln(`${"-".repeat(catW)} ----- ------ ---- ---- ---- ---- -------`);
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
ln(
|
ln(
|
||||||
`${padRight(r.cat, catW)} ${padLeft(r.pages, 5)} ${padLeft(r.uniq, 6)} ${padLeft(r.newC, 4)} ${padLeft(r.resC, 4)} ${padLeft(r.remC, 4)} ${padLeft(r.updC, 4)} ${secStr(r.ms)}`
|
`${padRight(r.cat, catW)} ${padLeft(r.pages, 5)} ${padLeft(r.uniq, 6)} ${padLeft(r.newC, 4)} ${padLeft(r.resC, 4)} ${padLeft(r.remC, 4)} ${padLeft(r.updC, 4)} ${secStr(r.ms)}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -108,7 +132,7 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
...report.newItems.map((x) => x.catLabel.length),
|
...report.newItems.map((x) => x.catLabel.length),
|
||||||
...report.restoredItems.map((x) => x.catLabel.length),
|
...report.restoredItems.map((x) => x.catLabel.length),
|
||||||
...report.updatedItems.map((x) => x.catLabel.length),
|
...report.updatedItems.map((x) => x.catLabel.length),
|
||||||
...report.removedItems.map((x) => x.catLabel.length)
|
...report.removedItems.map((x) => x.catLabel.length),
|
||||||
);
|
);
|
||||||
|
|
||||||
function storeFromCatLabel(catLabel) {
|
function storeFromCatLabel(catLabel) {
|
||||||
|
|
@ -163,7 +187,9 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
const sku = String(it.sku || "");
|
const sku = String(it.sku || "");
|
||||||
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
|
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
|
||||||
ln(`${paint("+", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`);
|
ln(
|
||||||
|
`${paint("+", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -174,11 +200,15 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
|
|
||||||
if (report.restoredItems.length) {
|
if (report.restoredItems.length) {
|
||||||
ln(paint(`RESTORED (${report.restoredItems.length})`, C.bold + C.green));
|
ln(paint(`RESTORED (${report.restoredItems.length})`, C.bold + C.green));
|
||||||
for (const it of report.restoredItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
for (const it of report.restoredItems.sort((a, b) =>
|
||||||
|
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
|
||||||
|
)) {
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
const sku = String(it.sku || "");
|
const sku = String(it.sku || "");
|
||||||
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
|
const cheapTag = cheaperAtInline(it.catLabel, sku, it.url, it.price || "");
|
||||||
ln(`${paint("R", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`);
|
ln(
|
||||||
|
`${paint("R", C.green)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${cheapTag}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -193,7 +223,9 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
const sku = String(it.sku || "");
|
const sku = String(it.sku || "");
|
||||||
const availTag = availableAtInline(it.catLabel, sku, it.url);
|
const availTag = availableAtInline(it.catLabel, sku, it.url);
|
||||||
ln(`${paint("-", C.yellow)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${availTag}`);
|
ln(
|
||||||
|
`${paint("-", C.yellow)} ${padRight(it.catLabel, reportLabelW)} | ${paint(it.name, C.bold)}${skuInline(sku)} ${price}${availTag}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -235,7 +267,7 @@ function renderFinalReport(report, { dbDir, colorize = Boolean(process.stdout &&
|
||||||
const cheapTag = cheaperAtInline(u.catLabel, sku, u.url, newRaw || "");
|
const cheapTag = cheaperAtInline(u.catLabel, sku, u.url, newRaw || "");
|
||||||
|
|
||||||
ln(
|
ln(
|
||||||
`${paint("~", C.cyan)} ${padRight(u.catLabel, reportLabelW)} | ${paint(u.name, C.bold)}${skuInline(sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}${cheapTag}`
|
`${paint("~", C.cyan)} ${padRight(u.catLabel, reportLabelW)} | ${paint(u.name, C.bold)}${skuInline(sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}${cheapTag}`,
|
||||||
);
|
);
|
||||||
ln(` ${paint(u.url, C.dim)}`);
|
ln(` ${paint(u.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,7 @@
|
||||||
const { createReport } = require("./report");
|
const { createReport } = require("./report");
|
||||||
const { setTimeout: sleep } = require("timers/promises");
|
const { setTimeout: sleep } = require("timers/promises");
|
||||||
|
|
||||||
const {
|
const { makeCatPrefixers, buildCategoryContext, loadCategoryDb, discoverAndScanCategory } = require("./category_scan");
|
||||||
makeCatPrefixers,
|
|
||||||
buildCategoryContext,
|
|
||||||
loadCategoryDb,
|
|
||||||
discoverAndScanCategory,
|
|
||||||
} = require("./category_scan");
|
|
||||||
|
|
||||||
// Some sites will intermittently 403/429. We don't want a single category/store
|
// Some sites will intermittently 403/429. We don't want a single category/store
|
||||||
// to abort the entire run. Log and continue.
|
// to abort the entire run. Log and continue.
|
||||||
|
|
@ -25,11 +20,9 @@ async function runAllStores(stores, { config, logger, http }) {
|
||||||
|
|
||||||
logger.info(`Debug=on`);
|
logger.info(`Debug=on`);
|
||||||
logger.info(
|
logger.info(
|
||||||
`Concurrency=${config.concurrency} StaggerMs=${config.staggerMs} Retries=${config.maxRetries} TimeoutMs=${config.timeoutMs}`
|
`Concurrency=${config.concurrency} StaggerMs=${config.staggerMs} Retries=${config.maxRetries} TimeoutMs=${config.timeoutMs}`,
|
||||||
);
|
|
||||||
logger.info(
|
|
||||||
`DiscoveryGuess=${config.discoveryGuess} DiscoveryStep=${config.discoveryStep}`
|
|
||||||
);
|
);
|
||||||
|
logger.info(`DiscoveryGuess=${config.discoveryGuess} DiscoveryStep=${config.discoveryStep}`);
|
||||||
logger.info(`MaxPages=${config.maxPages === null ? "none" : config.maxPages}`);
|
logger.info(`MaxPages=${config.maxPages === null ? "none" : config.maxPages}`);
|
||||||
logger.info(`CategoryConcurrency=${config.categoryConcurrency}`);
|
logger.info(`CategoryConcurrency=${config.categoryConcurrency}`);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,7 @@ function escapeRe(s) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractHtmlAttr(html, attrName) {
|
function extractHtmlAttr(html, attrName) {
|
||||||
const re = new RegExp(
|
const re = new RegExp(`\\b${escapeRe(attrName)}\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+))`, "i");
|
||||||
`\\b${escapeRe(attrName)}\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+))`,
|
|
||||||
"i"
|
|
||||||
);
|
|
||||||
const m = re.exec(html);
|
const m = re.exec(html);
|
||||||
if (!m) return "";
|
if (!m) return "";
|
||||||
return m[1] ?? m[2] ?? m[3] ?? "";
|
return m[1] ?? m[2] ?? m[3] ?? "";
|
||||||
|
|
@ -130,7 +127,6 @@ function extractFirstImgUrl(html, baseUrl) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
stripTags,
|
stripTags,
|
||||||
cleanText,
|
cleanText,
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
function normPrice(p) {
|
function normPrice(p) {
|
||||||
return String(p || "").trim().replace(/\s+/g, "");
|
return String(p || "")
|
||||||
|
.trim()
|
||||||
|
.replace(/\s+/g, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
function priceToNumber(p) {
|
function priceToNumber(p) {
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,6 @@ function idToCspc6(idDigits) {
|
||||||
return s.padStart(6, "0");
|
return s.padStart(6, "0");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function normalizeCspc(v) {
|
function normalizeCspc(v) {
|
||||||
const m = String(v ?? "").match(/\b(\d{6})\b/);
|
const m = String(v ?? "").match(/\b(\d{6})\b/);
|
||||||
return m ? m[1] : "";
|
return m ? m[1] : "";
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,6 @@ function isUpcSku(k) {
|
||||||
return /^\d{12,14}$/.test(s); // keep legacy support
|
return /^\d{12,14}$/.test(s); // keep legacy support
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function compareSku(a, b) {
|
function compareSku(a, b) {
|
||||||
a = String(a || "").trim();
|
a = String(a || "").trim();
|
||||||
b = String(b || "").trim();
|
b = String(b || "").trim();
|
||||||
|
|
@ -71,12 +70,10 @@ function compareSku(a, b) {
|
||||||
const bu = isUnknownSkuKey(b);
|
const bu = isUnknownSkuKey(b);
|
||||||
if (au !== bu) return au ? 1 : -1; // real first
|
if (au !== bu) return au ? 1 : -1; // real first
|
||||||
|
|
||||||
|
|
||||||
const aUpc = isUpcSku(a);
|
const aUpc = isUpcSku(a);
|
||||||
const bUpc = isUpcSku(b);
|
const bUpc = isUpcSku(b);
|
||||||
if (aUpc !== bUpc) return aUpc ? 1 : -1; // UPCs after other "real" keys
|
if (aUpc !== bUpc) return aUpc ? 1 : -1; // UPCs after other "real" keys
|
||||||
|
|
||||||
|
|
||||||
const an = isNumericSku(a);
|
const an = isNumericSku(a);
|
||||||
const bn = isNumericSku(b);
|
const bn = isNumericSku(b);
|
||||||
if (an && bn) {
|
if (an && bn) {
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,10 @@ function ts(d = new Date()) {
|
||||||
|
|
||||||
function isoTimestampFileSafe(d = new Date()) {
|
function isoTimestampFileSafe(d = new Date()) {
|
||||||
// 2026-01-16T21-27-01Z
|
// 2026-01-16T21-27-01Z
|
||||||
return d.toISOString().replace(/:/g, "-").replace(/\.\d{3}Z$/, "Z");
|
return d
|
||||||
|
.toISOString()
|
||||||
|
.replace(/:/g, "-")
|
||||||
|
.replace(/\.\d{3}Z$/, "Z");
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { ts, isoTimestampFileSafe };
|
module.exports = { ts, isoTimestampFileSafe };
|
||||||
|
|
|
||||||
|
|
@ -47,4 +47,10 @@ function makePageUrlShopifyQueryPage(baseUrl, pageNum) {
|
||||||
return u.toString();
|
return u.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { normalizeBaseUrl, makePageUrl, makePageUrlForCtx, makePageUrlQueryParam, makePageUrlShopifyQueryPage };
|
module.exports = {
|
||||||
|
normalizeBaseUrl,
|
||||||
|
makePageUrl,
|
||||||
|
makePageUrlForCtx,
|
||||||
|
makePageUrlQueryParam,
|
||||||
|
makePageUrlShopifyQueryPage,
|
||||||
|
};
|
||||||
|
|
|
||||||
|
|
@ -102,17 +102,7 @@ function canonicalize(k, skuMap) {
|
||||||
|
|
||||||
/* ---------------- grouping ---------------- */
|
/* ---------------- grouping ---------------- */
|
||||||
|
|
||||||
const BC_STORE_KEYS = new Set([
|
const BC_STORE_KEYS = new Set(["gull", "strath", "bcl", "legacy", "legacyliquor", "tudor", "vessel", "vintage", "arc"]);
|
||||||
"gull",
|
|
||||||
"strath",
|
|
||||||
"bcl",
|
|
||||||
"legacy",
|
|
||||||
"legacyliquor",
|
|
||||||
"tudor",
|
|
||||||
"vessel",
|
|
||||||
"vintage",
|
|
||||||
"arc"
|
|
||||||
]);
|
|
||||||
|
|
||||||
function groupAllowsStore(group, storeKey) {
|
function groupAllowsStore(group, storeKey) {
|
||||||
const k = String(storeKey || "").toLowerCase();
|
const k = String(storeKey || "").toLowerCase();
|
||||||
|
|
|
||||||
|
|
@ -489,7 +489,7 @@ function main() {
|
||||||
|
|
||||||
fs.writeFileSync(htmlPath, html, "utf8");
|
fs.writeFileSync(htmlPath, html, "utf8");
|
||||||
fs.writeFileSync(subjPath, subject + "\n", "utf8");
|
fs.writeFileSync(subjPath, subject + "\n", "utf8");
|
||||||
fs.writeFileSync(sendPath, (shouldSend ? "1\n" : "0\n"), "utf8");
|
fs.writeFileSync(sendPath, shouldSend ? "1\n" : "0\n", "utf8");
|
||||||
|
|
||||||
writeGithubOutput({
|
writeGithubOutput({
|
||||||
should_send: shouldSend ? 1 : 0,
|
should_send: shouldSend ? 1 : 0,
|
||||||
|
|
|
||||||
|
|
@ -80,14 +80,7 @@ function keySkuForItem(it, storeLabel) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file).
|
// Returns Map(skuKey -> firstSeenAtISO) for this dbFile (store/category file).
|
||||||
function computeFirstSeenForDbFile({
|
function computeFirstSeenForDbFile({ repoRoot, relDbFile, storeLabel, wantSkuKeys, commitsArr, nowIso }) {
|
||||||
repoRoot,
|
|
||||||
relDbFile,
|
|
||||||
storeLabel,
|
|
||||||
wantSkuKeys,
|
|
||||||
commitsArr,
|
|
||||||
nowIso,
|
|
||||||
}) {
|
|
||||||
const out = new Map();
|
const out = new Map();
|
||||||
const want = new Set(wantSkuKeys);
|
const want = new Set(wantSkuKeys);
|
||||||
|
|
||||||
|
|
@ -226,9 +219,7 @@ function main() {
|
||||||
};
|
};
|
||||||
|
|
||||||
fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8");
|
fs.writeFileSync(outFile, JSON.stringify(outObj, null, 2) + "\n", "utf8");
|
||||||
process.stdout.write(
|
process.stdout.write(`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`);
|
||||||
`Wrote ${path.relative(repoRoot, outFile)} (${items.length} rows)\n`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { main };
|
module.exports = { main };
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,10 @@ function gitFileExistsAtSha(sha, filePath) {
|
||||||
function gitListTreeFiles(sha, dirRel) {
|
function gitListTreeFiles(sha, dirRel) {
|
||||||
try {
|
try {
|
||||||
const out = runGit(["ls-tree", "-r", "--name-only", sha, dirRel]);
|
const out = runGit(["ls-tree", "-r", "--name-only", sha, dirRel]);
|
||||||
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
|
return out
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
@ -237,10 +240,16 @@ function listChangedDbFiles(fromSha, toSha) {
|
||||||
try {
|
try {
|
||||||
if (toSha === "WORKTREE") {
|
if (toSha === "WORKTREE") {
|
||||||
const out = runGit(["diff", "--name-only", fromSha, "--", "data/db"]);
|
const out = runGit(["diff", "--name-only", fromSha, "--", "data/db"]);
|
||||||
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
|
return out
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
}
|
}
|
||||||
const out = runGit(["diff", "--name-only", fromSha, toSha, "--", "data/db"]);
|
const out = runGit(["diff", "--name-only", fromSha, toSha, "--", "data/db"]);
|
||||||
return out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
|
return out
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
@ -249,7 +258,10 @@ function listChangedDbFiles(fromSha, toSha) {
|
||||||
function logDbCommitsSince(sinceIso) {
|
function logDbCommitsSince(sinceIso) {
|
||||||
try {
|
try {
|
||||||
const out = runGit(["log", `--since=${sinceIso}`, "--format=%H %cI", "--", "data/db"]);
|
const out = runGit(["log", `--since=${sinceIso}`, "--format=%H %cI", "--", "data/db"]);
|
||||||
const lines = out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
|
const lines = out
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
const arr = [];
|
const arr = [];
|
||||||
for (const line of lines) {
|
for (const line of lines) {
|
||||||
const m = line.match(/^([0-9a-f]{7,40})\s+(.+)$/i);
|
const m = line.match(/^([0-9a-f]{7,40})\s+(.+)$/i);
|
||||||
|
|
@ -315,11 +327,7 @@ function main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function isSmwsBottle(storeLabel, it) {
|
function isSmwsBottle(storeLabel, it) {
|
||||||
const hay = [
|
const hay = [storeLabel, it?.name, it?.url]
|
||||||
storeLabel,
|
|
||||||
it?.name,
|
|
||||||
it?.url,
|
|
||||||
]
|
|
||||||
.map((x) => String(x || ""))
|
.map((x) => String(x || ""))
|
||||||
.join(" | ")
|
.join(" | ")
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
|
|
@ -348,39 +356,24 @@ function main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextExists =
|
const nextExists =
|
||||||
toSha === "WORKTREE"
|
toSha === "WORKTREE" ? fs.existsSync(path.join(repoRoot, file)) : gitFileExistsAtSha(toSha, file);
|
||||||
? fs.existsSync(path.join(repoRoot, file))
|
|
||||||
: gitFileExistsAtSha(toSha, file);
|
|
||||||
if (!nextExists) continue;
|
if (!nextExists) continue;
|
||||||
|
|
||||||
if (!prevObj && !nextObj) continue;
|
if (!prevObj && !nextObj) continue;
|
||||||
|
|
||||||
const storeLabel = String(
|
const storeLabel = String(
|
||||||
nextObj?.storeLabel ||
|
nextObj?.storeLabel || nextObj?.store || prevObj?.storeLabel || prevObj?.store || "",
|
||||||
nextObj?.store ||
|
|
||||||
prevObj?.storeLabel ||
|
|
||||||
prevObj?.store ||
|
|
||||||
""
|
|
||||||
);
|
);
|
||||||
const categoryLabel = String(
|
const categoryLabel = String(
|
||||||
nextObj?.categoryLabel ||
|
nextObj?.categoryLabel || nextObj?.category || prevObj?.categoryLabel || prevObj?.category || "",
|
||||||
nextObj?.category ||
|
|
||||||
prevObj?.categoryLabel ||
|
|
||||||
prevObj?.category ||
|
|
||||||
""
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const isNewStoreFile =
|
const isNewStoreFile =
|
||||||
Boolean(fromSha) &&
|
Boolean(fromSha) &&
|
||||||
!gitFileExistsAtSha(fromSha, file) &&
|
!gitFileExistsAtSha(fromSha, file) &&
|
||||||
(toSha === "WORKTREE"
|
(toSha === "WORKTREE" ? fs.existsSync(path.join(repoRoot, file)) : gitFileExistsAtSha(toSha, file));
|
||||||
? fs.existsSync(path.join(repoRoot, file))
|
|
||||||
: gitFileExistsAtSha(toSha, file));
|
|
||||||
|
|
||||||
let { newItems, restoredItems, removedItems, priceChanges } = diffDb(
|
let { newItems, restoredItems, removedItems, priceChanges } = diffDb(prevObj, nextObj);
|
||||||
prevObj,
|
|
||||||
nextObj
|
|
||||||
);
|
|
||||||
|
|
||||||
if (isNewStoreFile) {
|
if (isNewStoreFile) {
|
||||||
newItems = [];
|
newItems = [];
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,10 @@ function gitShowText(sha, filePath) {
|
||||||
|
|
||||||
function gitListDbFiles(sha, dbDirRel) {
|
function gitListDbFiles(sha, dbDirRel) {
|
||||||
const out = runGit(["ls-tree", "-r", "--name-only", sha, dbDirRel]);
|
const out = runGit(["ls-tree", "-r", "--name-only", sha, dbDirRel]);
|
||||||
const lines = out.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
|
const lines = out
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
return new Set(lines);
|
return new Set(lines);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -154,7 +157,7 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
ln(paint("========== DIFF REPORT ==========", C.bold));
|
ln(paint("========== DIFF REPORT ==========", C.bold));
|
||||||
ln(`${paint("From", C.bold)} ${fromSha} ${paint("to", C.bold)} ${toSha}`);
|
ln(`${paint("From", C.bold)} ${fromSha} ${paint("to", C.bold)} ${toSha}`);
|
||||||
ln(
|
ln(
|
||||||
`${paint("Totals", C.bold)} | Categories=${diffReport.categories.length} | New=${diffReport.totals.newCount} | Restored=${diffReport.totals.restoredCount} | Removed=${diffReport.totals.removedCount} | PriceChanges=${diffReport.totals.updatedCount}`
|
`${paint("Totals", C.bold)} | Categories=${diffReport.categories.length} | New=${diffReport.totals.newCount} | Restored=${diffReport.totals.restoredCount} | Removed=${diffReport.totals.removedCount} | PriceChanges=${diffReport.totals.updatedCount}`,
|
||||||
);
|
);
|
||||||
ln("");
|
ln("");
|
||||||
|
|
||||||
|
|
@ -162,14 +165,24 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
const catW = Math.min(56, Math.max(...rows.map((r) => r.catLabel.length), 12));
|
const catW = Math.min(56, Math.max(...rows.map((r) => r.catLabel.length), 12));
|
||||||
|
|
||||||
ln(paint("Per-category summary:", C.bold));
|
ln(paint("Per-category summary:", C.bold));
|
||||||
ln(`${padRight("Store | Category", catW)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)}`);
|
ln(
|
||||||
|
`${padRight("Store | Category", catW)} ${padLeft("New", 4)} ${padLeft("Res", 4)} ${padLeft("Rem", 4)} ${padLeft("Upd", 4)}`,
|
||||||
|
);
|
||||||
ln(`${"-".repeat(catW)} ---- ---- ---- ----`);
|
ln(`${"-".repeat(catW)} ---- ---- ---- ----`);
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
ln(`${padRight(r.catLabel, catW)} ${padLeft(r.newCount, 4)} ${padLeft(r.restoredCount, 4)} ${padLeft(r.removedCount, 4)} ${padLeft(r.updatedCount, 4)}`);
|
ln(
|
||||||
|
`${padRight(r.catLabel, catW)} ${padLeft(r.newCount, 4)} ${padLeft(r.restoredCount, 4)} ${padLeft(r.removedCount, 4)} ${padLeft(r.updatedCount, 4)}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
||||||
const labelW = Math.max(16, ...diffReport.newItems.map((x) => x.catLabel.length), ...diffReport.restoredItems.map((x) => x.catLabel.length), ...diffReport.removedItems.map((x) => x.catLabel.length), ...diffReport.updatedItems.map((x) => x.catLabel.length));
|
const labelW = Math.max(
|
||||||
|
16,
|
||||||
|
...diffReport.newItems.map((x) => x.catLabel.length),
|
||||||
|
...diffReport.restoredItems.map((x) => x.catLabel.length),
|
||||||
|
...diffReport.removedItems.map((x) => x.catLabel.length),
|
||||||
|
...diffReport.updatedItems.map((x) => x.catLabel.length),
|
||||||
|
);
|
||||||
|
|
||||||
const skuInline = (sku) => {
|
const skuInline = (sku) => {
|
||||||
const s = normalizeCspc(sku);
|
const s = normalizeCspc(sku);
|
||||||
|
|
@ -180,7 +193,9 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
ln(paint(`NEW (${diffReport.newItems.length})`, C.bold + C.green));
|
ln(paint(`NEW (${diffReport.newItems.length})`, C.bold + C.green));
|
||||||
for (const it of diffReport.newItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
for (const it of diffReport.newItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
ln(`${paint("+", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
|
ln(
|
||||||
|
`${paint("+", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -188,9 +203,13 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
|
|
||||||
if (diffReport.restoredItems.length) {
|
if (diffReport.restoredItems.length) {
|
||||||
ln(paint(`RESTORED (${diffReport.restoredItems.length})`, C.bold + C.green));
|
ln(paint(`RESTORED (${diffReport.restoredItems.length})`, C.bold + C.green));
|
||||||
for (const it of diffReport.restoredItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
for (const it of diffReport.restoredItems.sort((a, b) =>
|
||||||
|
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
|
||||||
|
)) {
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
ln(`${paint("R", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
|
ln(
|
||||||
|
`${paint("R", C.green)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -198,9 +217,13 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
|
|
||||||
if (diffReport.removedItems.length) {
|
if (diffReport.removedItems.length) {
|
||||||
ln(paint(`REMOVED (${diffReport.removedItems.length})`, C.bold + C.yellow));
|
ln(paint(`REMOVED (${diffReport.removedItems.length})`, C.bold + C.yellow));
|
||||||
for (const it of diffReport.removedItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
for (const it of diffReport.removedItems.sort((a, b) =>
|
||||||
|
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
|
||||||
|
)) {
|
||||||
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
const price = it.price ? paint(it.price, C.cyan) : paint("(no price)", C.gray);
|
||||||
ln(`${paint("-", C.yellow)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`);
|
ln(
|
||||||
|
`${paint("-", C.yellow)} ${padRight(it.catLabel, labelW)} | ${paint(it.name, C.bold)}${skuInline(it.sku)} ${price}`,
|
||||||
|
);
|
||||||
ln(` ${paint(it.url, C.dim)}`);
|
ln(` ${paint(it.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
ln("");
|
ln("");
|
||||||
|
|
@ -209,7 +232,9 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
if (diffReport.updatedItems.length) {
|
if (diffReport.updatedItems.length) {
|
||||||
ln(paint(`PRICE CHANGES (${diffReport.updatedItems.length})`, C.bold + C.cyan));
|
ln(paint(`PRICE CHANGES (${diffReport.updatedItems.length})`, C.bold + C.cyan));
|
||||||
|
|
||||||
for (const u of diffReport.updatedItems.sort((a, b) => (a.catLabel + a.name).localeCompare(b.catLabel + b.name))) {
|
for (const u of diffReport.updatedItems.sort((a, b) =>
|
||||||
|
(a.catLabel + a.name).localeCompare(b.catLabel + b.name),
|
||||||
|
)) {
|
||||||
const oldRaw = u.oldPrice || "";
|
const oldRaw = u.oldPrice || "";
|
||||||
const newRaw = u.newPrice || "";
|
const newRaw = u.newPrice || "";
|
||||||
|
|
||||||
|
|
@ -231,7 +256,7 @@ function renderDiffReport(diffReport, { fromSha, toSha, colorize }) {
|
||||||
} else newP = paint(newP, C.cyan);
|
} else newP = paint(newP, C.cyan);
|
||||||
|
|
||||||
ln(
|
ln(
|
||||||
`${paint("~", C.cyan)} ${padRight(u.catLabel, labelW)} | ${paint(u.name, C.bold)}${skuInline(u.sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}`
|
`${paint("~", C.cyan)} ${padRight(u.catLabel, labelW)} | ${paint(u.name, C.bold)}${skuInline(u.sku)} ${oldP} ${paint("->", C.gray)} ${newP}${offTag}`,
|
||||||
);
|
);
|
||||||
ln(` ${paint(u.url, C.dim)}`);
|
ln(` ${paint(u.url, C.dim)}`);
|
||||||
}
|
}
|
||||||
|
|
@ -248,7 +273,9 @@ async function main() {
|
||||||
const { fromSha, toSha, dbDir, outFile, flags } = parseArgs(process.argv.slice(2));
|
const { fromSha, toSha, dbDir, outFile, flags } = parseArgs(process.argv.slice(2));
|
||||||
|
|
||||||
if (!fromSha || !toSha) {
|
if (!fromSha || !toSha) {
|
||||||
console.error(`Usage: ${path.basename(process.argv[1])} <fromSha> <toSha> [--db-dir data/db] [--out reports/<file>.txt] [--no-color]`);
|
console.error(
|
||||||
|
`Usage: ${path.basename(process.argv[1])} <fromSha> <toSha> [--db-dir data/db] [--out reports/<file>.txt] [--no-color]`,
|
||||||
|
);
|
||||||
process.exitCode = 2;
|
process.exitCode = 2;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -273,8 +300,16 @@ async function main() {
|
||||||
const prevObj = parseJsonOrNull(gitShowText(fromSha, file));
|
const prevObj = parseJsonOrNull(gitShowText(fromSha, file));
|
||||||
const nextObj = parseJsonOrNull(gitShowText(toSha, file));
|
const nextObj = parseJsonOrNull(gitShowText(toSha, file));
|
||||||
|
|
||||||
const storeLabel = String(nextObj?.storeLabel || prevObj?.storeLabel || nextObj?.store || prevObj?.store || "?");
|
const storeLabel = String(
|
||||||
const catLabel = String(nextObj?.categoryLabel || prevObj?.categoryLabel || nextObj?.category || prevObj?.category || path.basename(file));
|
nextObj?.storeLabel || prevObj?.storeLabel || nextObj?.store || prevObj?.store || "?",
|
||||||
|
);
|
||||||
|
const catLabel = String(
|
||||||
|
nextObj?.categoryLabel ||
|
||||||
|
prevObj?.categoryLabel ||
|
||||||
|
nextObj?.category ||
|
||||||
|
prevObj?.category ||
|
||||||
|
path.basename(file),
|
||||||
|
);
|
||||||
const catLabelFull = `${storeLabel} | ${catLabel}`;
|
const catLabelFull = `${storeLabel} | ${catLabel}`;
|
||||||
|
|
||||||
const { newItems, restoredItems, removedItems, updatedItems } = buildDiffForDb(prevObj, nextObj);
|
const { newItems, restoredItems, removedItems, updatedItems } = buildDiffForDb(prevObj, nextObj);
|
||||||
|
|
@ -301,9 +336,7 @@ async function main() {
|
||||||
const reportText = renderDiffReport(diffReport, { fromSha, toSha, colorize });
|
const reportText = renderDiffReport(diffReport, { fromSha, toSha, colorize });
|
||||||
process.stdout.write(reportText);
|
process.stdout.write(reportText);
|
||||||
|
|
||||||
const outPath = outFile
|
const outPath = outFile ? (path.isAbsolute(outFile) ? outFile : path.join(process.cwd(), outFile)) : "";
|
||||||
? (path.isAbsolute(outFile) ? outFile : path.join(process.cwd(), outFile))
|
|
||||||
: "";
|
|
||||||
|
|
||||||
if (outPath) {
|
if (outPath) {
|
||||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||||
|
|
|
||||||
|
|
@ -41,17 +41,13 @@ function parseArgs(argv) {
|
||||||
if (a === "--ab" && argv[i + 1]) out.ab = argv[++i];
|
if (a === "--ab" && argv[i + 1]) out.ab = argv[++i];
|
||||||
else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i];
|
else if (a === "--bc" && argv[i + 1]) out.bc = argv[++i];
|
||||||
else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i];
|
else if (a === "--meta" && argv[i + 1]) out.meta = argv[++i];
|
||||||
|
|
||||||
else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top;
|
else if (a === "--top" && argv[i + 1]) out.top = Number(argv[++i]) || out.top;
|
||||||
else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep;
|
else if (a === "--min" && argv[i + 1]) out.minDiscrep = Number(argv[++i]) || out.minDiscrep;
|
||||||
else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore;
|
else if (a === "--min-score" && argv[i + 1]) out.minScore = Number(argv[++i]) || out.minScore;
|
||||||
else if (a === "--min-contain" && argv[i + 1]) out.minContain = Number(argv[++i]) || out.minContain;
|
else if (a === "--min-contain" && argv[i + 1]) out.minContain = Number(argv[++i]) || out.minContain;
|
||||||
|
|
||||||
else if (a === "--include-missing") out.includeMissing = true;
|
else if (a === "--include-missing") out.includeMissing = true;
|
||||||
else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base);
|
else if (a === "--base" && argv[i + 1]) out.base = String(argv[++i] || out.base);
|
||||||
|
|
||||||
else if (a === "--no-cross-group") out.requireCrossGroup = false;
|
else if (a === "--no-cross-group") out.requireCrossGroup = false;
|
||||||
|
|
||||||
else if (a === "--debug") out.debug = true;
|
else if (a === "--debug") out.debug = true;
|
||||||
else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN;
|
else if (a === "--debug-n" && argv[i + 1]) out.debugN = Number(argv[++i]) || out.debugN;
|
||||||
else if (a === "--debug-payload") out.debugPayload = true;
|
else if (a === "--debug-payload") out.debugPayload = true;
|
||||||
|
|
@ -66,7 +62,14 @@ function parseArgs(argv) {
|
||||||
|
|
||||||
function extractRows(payload) {
|
function extractRows(payload) {
|
||||||
if (Array.isArray(payload)) return payload;
|
if (Array.isArray(payload)) return payload;
|
||||||
const candidates = [payload?.rows, payload?.data?.rows, payload?.data, payload?.items, payload?.list, payload?.results];
|
const candidates = [
|
||||||
|
payload?.rows,
|
||||||
|
payload?.data?.rows,
|
||||||
|
payload?.data,
|
||||||
|
payload?.items,
|
||||||
|
payload?.list,
|
||||||
|
payload?.results,
|
||||||
|
];
|
||||||
for (const x of candidates) if (Array.isArray(x)) return x;
|
for (const x of candidates) if (Array.isArray(x)) return x;
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
@ -185,7 +188,8 @@ function compareSku(a, b) {
|
||||||
const aNum = /^\d+$/.test(a);
|
const aNum = /^\d+$/.test(a);
|
||||||
const bNum = /^\d+$/.test(b);
|
const bNum = /^\d+$/.test(b);
|
||||||
if (aNum && bNum) {
|
if (aNum && bNum) {
|
||||||
const na = Number(a), nb = Number(b);
|
const na = Number(a),
|
||||||
|
nb = Number(b);
|
||||||
if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1;
|
if (Number.isFinite(na) && Number.isFinite(nb) && na !== nb) return na < nb ? -1 : 1;
|
||||||
}
|
}
|
||||||
return a < b ? -1 : 1;
|
return a < b ? -1 : 1;
|
||||||
|
|
@ -252,16 +256,42 @@ function tokenizeQuery(q) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const SIM_STOP_TOKENS = new Set([
|
const SIM_STOP_TOKENS = new Set([
|
||||||
"the","a","an","and","of","to","in","for","with",
|
"the",
|
||||||
"year","years","yr","yrs","old",
|
"a",
|
||||||
"whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof",
|
"an",
|
||||||
|
"and",
|
||||||
|
"of",
|
||||||
|
"to",
|
||||||
|
"in",
|
||||||
|
"for",
|
||||||
|
"with",
|
||||||
|
"year",
|
||||||
|
"years",
|
||||||
|
"yr",
|
||||||
|
"yrs",
|
||||||
|
"old",
|
||||||
|
"whisky",
|
||||||
|
"whiskey",
|
||||||
|
"scotch",
|
||||||
|
"single",
|
||||||
|
"malt",
|
||||||
|
"cask",
|
||||||
|
"finish",
|
||||||
|
"edition",
|
||||||
|
"release",
|
||||||
|
"batch",
|
||||||
|
"strength",
|
||||||
|
"abv",
|
||||||
|
"proof",
|
||||||
"anniversary",
|
"anniversary",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
|
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
|
||||||
|
|
||||||
function numKey(t) {
|
function numKey(t) {
|
||||||
const s = String(t || "").trim().toLowerCase();
|
const s = String(t || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (!s) return "";
|
if (!s) return "";
|
||||||
if (/^\d+$/.test(s)) return s;
|
if (/^\d+$/.test(s)) return s;
|
||||||
const m = s.match(ORDINAL_RE);
|
const m = s.match(ORDINAL_RE);
|
||||||
|
|
@ -302,7 +332,9 @@ function filterSimTokens(tokens) {
|
||||||
const arr = Array.isArray(tokens) ? tokens : [];
|
const arr = Array.isArray(tokens) ? tokens : [];
|
||||||
|
|
||||||
for (let i = 0; i < arr.length; i++) {
|
for (let i = 0; i < arr.length; i++) {
|
||||||
let t = String(arr[i] || "").trim().toLowerCase();
|
let t = String(arr[i] || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (!t) continue;
|
if (!t) continue;
|
||||||
if (!/[a-z0-9]/i.test(t)) continue;
|
if (!/[a-z0-9]/i.test(t)) continue;
|
||||||
if (VOL_INLINE_RE.test(t)) continue;
|
if (VOL_INLINE_RE.test(t)) continue;
|
||||||
|
|
@ -316,7 +348,9 @@ function filterSimTokens(tokens) {
|
||||||
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
|
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
|
||||||
|
|
||||||
if (/^\d+(?:\.\d+)?$/.test(t)) {
|
if (/^\d+(?:\.\d+)?$/.test(t)) {
|
||||||
const next = String(arr[i + 1] || "").trim().toLowerCase();
|
const next = String(arr[i + 1] || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
const nextNorm = SIM_EQUIV.get(next) || next;
|
const nextNorm = SIM_EQUIV.get(next) || next;
|
||||||
if (VOL_UNIT.has(nextNorm)) {
|
if (VOL_UNIT.has(nextNorm)) {
|
||||||
i++;
|
i++;
|
||||||
|
|
@ -358,7 +392,8 @@ function tokenContainmentScore(aTokens, bTokens) {
|
||||||
function levenshtein(a, b) {
|
function levenshtein(a, b) {
|
||||||
a = String(a || "");
|
a = String(a || "");
|
||||||
b = String(b || "");
|
b = String(b || "");
|
||||||
const n = a.length, m = b.length;
|
const n = a.length,
|
||||||
|
m = b.length;
|
||||||
if (!n) return m;
|
if (!n) return m;
|
||||||
if (!m) return n;
|
if (!m) return n;
|
||||||
|
|
||||||
|
|
@ -422,17 +457,13 @@ function similarityScore(aName, bName) {
|
||||||
const maxLen = Math.max(1, Math.max(a.length, b.length));
|
const maxLen = Math.max(1, Math.max(a.length, b.length));
|
||||||
const levSim = 1 - d / maxLen;
|
const levSim = 1 - d / maxLen;
|
||||||
|
|
||||||
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
|
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
|
||||||
const smallN = Math.min(aToks.length, bToks.length);
|
const smallN = Math.min(aToks.length, bToks.length);
|
||||||
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
||||||
|
|
||||||
const numGate = numberMismatchPenalty(aToks, bToks);
|
const numGate = numberMismatchPenalty(aToks, bToks);
|
||||||
|
|
||||||
let s =
|
let s = numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain));
|
||||||
numGate *
|
|
||||||
(firstMatch * 3.0 +
|
|
||||||
overlapTail * 2.2 * gate +
|
|
||||||
levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain)));
|
|
||||||
|
|
||||||
if (ageMatch) s *= 2.2;
|
if (ageMatch) s *= 2.2;
|
||||||
else if (ageMismatch) s *= 0.18;
|
else if (ageMismatch) s *= 0.18;
|
||||||
|
|
@ -444,8 +475,13 @@ function similarityScore(aName, bName) {
|
||||||
|
|
||||||
/* ---------------- debug helpers ---------------- */
|
/* ---------------- debug helpers ---------------- */
|
||||||
|
|
||||||
function eprintln(...args) { console.error(...args); }
|
function eprintln(...args) {
|
||||||
function truncate(s, n) { s = String(s || ""); return s.length <= n ? s : s.slice(0, n - 1) + "…"; }
|
console.error(...args);
|
||||||
|
}
|
||||||
|
function truncate(s, n) {
|
||||||
|
s = String(s || "");
|
||||||
|
return s.length <= n ? s : s.slice(0, n - 1) + "…";
|
||||||
|
}
|
||||||
|
|
||||||
/* ---------------- main ---------------- */
|
/* ---------------- main ---------------- */
|
||||||
|
|
||||||
|
|
@ -498,7 +534,9 @@ function main() {
|
||||||
|
|
||||||
if (args.debug) {
|
if (args.debug) {
|
||||||
eprintln("[rank_discrepency] inputs:", {
|
eprintln("[rank_discrepency] inputs:", {
|
||||||
abPath, bcPath, metaPath: metaPath || "(none)",
|
abPath,
|
||||||
|
bcPath,
|
||||||
|
metaPath: metaPath || "(none)",
|
||||||
linkCount: Array.isArray(meta?.links) ? meta.links.length : 0,
|
linkCount: Array.isArray(meta?.links) ? meta.links.length : 0,
|
||||||
ignoreCount: Array.isArray(meta?.ignores) ? meta.ignores.length : 0,
|
ignoreCount: Array.isArray(meta?.ignores) ? meta.ignores.length : 0,
|
||||||
ignoreSetSize: ignoreSet.size,
|
ignoreSetSize: ignoreSet.size,
|
||||||
|
|
@ -509,8 +547,17 @@ function main() {
|
||||||
top: args.top,
|
top: args.top,
|
||||||
includeMissing: args.includeMissing,
|
includeMissing: args.includeMissing,
|
||||||
});
|
});
|
||||||
eprintln("[rank_discrepency] extracted rows:", { abRows: abBuilt.rowsLen, bcRows: bcBuilt.rowsLen, abKeys: abMap.size, bcKeys: bcMap.size });
|
eprintln("[rank_discrepency] extracted rows:", {
|
||||||
eprintln("[rank_discrepency] name coverage:", { totalSkus: allSkus.length, named: namedCount, unnamed: allSkus.length - namedCount });
|
abRows: abBuilt.rowsLen,
|
||||||
|
bcRows: bcBuilt.rowsLen,
|
||||||
|
abKeys: abMap.size,
|
||||||
|
bcKeys: bcMap.size,
|
||||||
|
});
|
||||||
|
eprintln("[rank_discrepency] name coverage:", {
|
||||||
|
totalSkus: allSkus.length,
|
||||||
|
named: namedCount,
|
||||||
|
unnamed: allSkus.length - namedCount,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (args.debugPayload) {
|
if (args.debugPayload) {
|
||||||
|
|
@ -544,11 +591,15 @@ function main() {
|
||||||
|
|
||||||
if (args.debug) {
|
if (args.debug) {
|
||||||
eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length });
|
eprintln("[rank_discrepency] diffs:", { unionKeys: keys.size, diffsAfterMin: diffs.length });
|
||||||
eprintln("[rank_discrepency] top discrep sample:",
|
eprintln(
|
||||||
|
"[rank_discrepency] top discrep sample:",
|
||||||
diffs.slice(0, 5).map((d) => ({
|
diffs.slice(0, 5).map((d) => ({
|
||||||
sku: d.canonSku, discrep: d.discrep, rankAB: d.rankAB, rankBC: d.rankBC,
|
sku: d.canonSku,
|
||||||
|
discrep: d.discrep,
|
||||||
|
rankAB: d.rankAB,
|
||||||
|
rankBC: d.rankBC,
|
||||||
name: truncate(allNames.get(String(d.canonSku)) || "", 80),
|
name: truncate(allNames.get(String(d.canonSku)) || "", 80),
|
||||||
}))
|
})),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -583,7 +634,9 @@ function main() {
|
||||||
side: aInAB ? "AB" : "BC",
|
side: aInAB ? "AB" : "BC",
|
||||||
nameA: truncate(nameA, 120),
|
nameA: truncate(nameA, 120),
|
||||||
minContain: args.minContain,
|
minContain: args.minContain,
|
||||||
top5: scored.slice(0, 5).map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })),
|
top5: scored
|
||||||
|
.slice(0, 5)
|
||||||
|
.map((x) => ({ sku: x.skuB, score: x.s, contain: x.contain, name: truncate(x.nameB, 120) })),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -601,7 +654,10 @@ function main() {
|
||||||
const groupA = canonicalSku(skuA);
|
const groupA = canonicalSku(skuA);
|
||||||
const aRaw = tokenizeQuery(nameA);
|
const aRaw = tokenizeQuery(nameA);
|
||||||
|
|
||||||
let best = 0, bestSku = "", bestName = "", bestContain = 0;
|
let best = 0,
|
||||||
|
bestSku = "",
|
||||||
|
bestName = "",
|
||||||
|
bestContain = 0;
|
||||||
let bestWasIgnored = false;
|
let bestWasIgnored = false;
|
||||||
|
|
||||||
for (const skuB of pool) {
|
for (const skuB of pool) {
|
||||||
|
|
@ -669,7 +725,9 @@ function main() {
|
||||||
|
|
||||||
for (const d of filtered) {
|
for (const d of filtered) {
|
||||||
if (args.dumpScores) {
|
if (args.dumpScores) {
|
||||||
eprintln("[rank_discrepency] emit", JSON.stringify({
|
eprintln(
|
||||||
|
"[rank_discrepency] emit",
|
||||||
|
JSON.stringify({
|
||||||
sku: d.canonSku,
|
sku: d.canonSku,
|
||||||
discrep: d.discrep,
|
discrep: d.discrep,
|
||||||
rankAB: d.rankAB,
|
rankAB: d.rankAB,
|
||||||
|
|
@ -678,7 +736,8 @@ function main() {
|
||||||
bestContain: d.bestContain,
|
bestContain: d.bestContain,
|
||||||
bestSku: d.bestSku,
|
bestSku: d.bestSku,
|
||||||
bestName: truncate(d.bestName, 120),
|
bestName: truncate(d.bestName, 120),
|
||||||
}));
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
console.log(args.base + encodeURIComponent(String(d.canonSku)));
|
console.log(args.base + encodeURIComponent(String(d.canonSku)));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,9 +20,7 @@ const REPO = process.env.REPO || "";
|
||||||
if (!ISSUE_NUMBER) die("Missing ISSUE_NUMBER");
|
if (!ISSUE_NUMBER) die("Missing ISSUE_NUMBER");
|
||||||
if (!REPO) die("Missing REPO");
|
if (!REPO) die("Missing REPO");
|
||||||
|
|
||||||
const m = ISSUE_BODY.match(
|
const m = ISSUE_BODY.match(/<!--\s*stviz-sku-edits:BEGIN\s*-->\s*([\s\S]*?)\s*<!--\s*stviz-sku-edits:END\s*-->/);
|
||||||
/<!--\s*stviz-sku-edits:BEGIN\s*-->\s*([\s\S]*?)\s*<!--\s*stviz-sku-edits:END\s*-->/
|
|
||||||
);
|
|
||||||
if (!m) die("No stviz payload found in issue body.");
|
if (!m) die("No stviz payload found in issue body.");
|
||||||
|
|
||||||
let payload;
|
let payload;
|
||||||
|
|
@ -204,24 +202,13 @@ function makePrettyObjBlock(objIndent, obj) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (skuA && skuB) {
|
if (skuA && skuB) {
|
||||||
return (
|
return `${a}{\n` + `${b}"skuA": ${JSON.stringify(skuA)},\n` + `${b}"skuB": ${JSON.stringify(skuB)}\n` + `${a}}`;
|
||||||
`${a}{\n` +
|
|
||||||
`${b}"skuA": ${JSON.stringify(skuA)},\n` +
|
|
||||||
`${b}"skuB": ${JSON.stringify(skuB)}\n` +
|
|
||||||
`${a}}`
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return `${a}{}`;
|
return `${a}{}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyInsertionsToArrayText({
|
function applyInsertionsToArrayText({ src, propName, incoming, keyFn, normalizeFn }) {
|
||||||
src,
|
|
||||||
propName,
|
|
||||||
incoming,
|
|
||||||
keyFn,
|
|
||||||
normalizeFn,
|
|
||||||
}) {
|
|
||||||
const span = findJsonArraySpan(src, propName);
|
const span = findJsonArraySpan(src, propName);
|
||||||
if (!span) die(`Could not find "${propName}" array in ${filePath}`);
|
if (!span) die(`Could not find "${propName}" array in ${filePath}`);
|
||||||
|
|
||||||
|
|
@ -266,8 +253,7 @@ function applyInsertionsToArrayText({
|
||||||
let newInner;
|
let newInner;
|
||||||
if (wasInlineEmpty) {
|
if (wasInlineEmpty) {
|
||||||
// "links": [] -> pretty multiline
|
// "links": [] -> pretty multiline
|
||||||
newInner =
|
newInner = "\n" + addBlocks.join(",\n") + "\n" + span.fieldIndent;
|
||||||
"\n" + addBlocks.join(",\n") + "\n" + span.fieldIndent;
|
|
||||||
} else {
|
} else {
|
||||||
// Keep existing whitespace EXACTLY; append before trailing whitespace
|
// Keep existing whitespace EXACTLY; append before trailing whitespace
|
||||||
const m = inner.match(/\s*$/);
|
const m = inner.match(/\s*$/);
|
||||||
|
|
@ -280,7 +266,6 @@ function applyInsertionsToArrayText({
|
||||||
return before + newInner + after;
|
return before + newInner + after;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ---------------- Apply edits ---------------- */
|
/* ---------------- Apply edits ---------------- */
|
||||||
|
|
||||||
const filePath = path.join("data", "sku_links.json");
|
const filePath = path.join("data", "sku_links.json");
|
||||||
|
|
@ -379,13 +364,12 @@ function extractPrUrl(out) {
|
||||||
|
|
||||||
// Create PR and capture URL/number without relying on unsupported flags
|
// Create PR and capture URL/number without relying on unsupported flags
|
||||||
const prCreateOut = sh(
|
const prCreateOut = sh(
|
||||||
`gh -R "${REPO}" pr create --base data --head "${branch}" --title "${prTitle}" --body "${prBody}"`
|
`gh -R "${REPO}" pr create --base data --head "${branch}" --title "${prTitle}" --body "${prBody}"`,
|
||||||
);
|
);
|
||||||
const prUrl = extractPrUrl(prCreateOut);
|
const prUrl = extractPrUrl(prCreateOut);
|
||||||
|
|
||||||
|
|
||||||
const prNumber = sh(`gh -R "${REPO}" pr view "${prUrl}" --json number --jq .number`);
|
const prNumber = sh(`gh -R "${REPO}" pr view "${prUrl}" --json number --jq .number`);
|
||||||
|
|
||||||
sh(
|
sh(
|
||||||
`gh -R "${REPO}" issue close "${ISSUE_NUMBER}" -c "Processed by STVIZ automation. Opened PR #${prNumber}: ${prUrl}"`
|
`gh -R "${REPO}" issue close "${ISSUE_NUMBER}" -c "Processed by STVIZ automation. Opened PR #${prNumber}: ${prUrl}"`,
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,9 @@ export function inferGithubOwnerRepo() {
|
||||||
|
|
||||||
export function isLocalWriteMode() {
|
export function isLocalWriteMode() {
|
||||||
const h = String(location.hostname || "").toLowerCase();
|
const h = String(location.hostname || "").toLowerCase();
|
||||||
return (location.protocol === "http:" || location.protocol === "https:") && (h === "127.0.0.1" || h === "localhost");
|
return (
|
||||||
|
(location.protocol === "http:" || location.protocol === "https:") && (h === "127.0.0.1" || h === "localhost")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---- Local disk-backed SKU link API (only on viz/serve.js) ---- */
|
/* ---- Local disk-backed SKU link API (only on viz/serve.js) ---- */
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
export function esc(s) {
|
export function esc(s) {
|
||||||
return String(s ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
return String(s ?? "").replace(
|
||||||
|
/[&<>"']/g,
|
||||||
|
(c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c],
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function normImg(s) {
|
export function normImg(s) {
|
||||||
|
|
@ -56,4 +59,3 @@ export function esc(s) {
|
||||||
if (!img) return `<div class="thumbPlaceholder"></div>`;
|
if (!img) return `<div class="thumbPlaceholder"></div>`;
|
||||||
return `<img referrerpolicy="no-referrer" class="${esc(cls)}" src="${esc(img)}" alt="" loading="lazy" onerror="this.style.display='none'" />`;
|
return `<img referrerpolicy="no-referrer" class="${esc(cls)}" src="${esc(img)}" alt="" loading="lazy" onerror="this.style.display='none'" />`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -73,7 +73,6 @@ function weightedMeanByDuration(pointsMap, sortedDates) {
|
||||||
return wtot ? wsum / wtot : null;
|
return wtot ? wsum / wtot : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function meanFinite(arr) {
|
function meanFinite(arr) {
|
||||||
if (!Array.isArray(arr)) return null;
|
if (!Array.isArray(arr)) return null;
|
||||||
let sum = 0,
|
let sum = 0,
|
||||||
|
|
@ -126,7 +125,14 @@ const StaticMarkerLinesPlugin = {
|
||||||
scalesObj.y ||
|
scalesObj.y ||
|
||||||
scales.find((s) => s && s.axis === "y") ||
|
scales.find((s) => s && s.axis === "y") ||
|
||||||
scales.find((s) => s && typeof s.getPixelForValue === "function" && s.isHorizontal === false) ||
|
scales.find((s) => s && typeof s.getPixelForValue === "function" && s.isHorizontal === false) ||
|
||||||
scales.find((s) => s && typeof s.getPixelForValue === "function" && String(s.id || "").toLowerCase().includes("y"));
|
scales.find(
|
||||||
|
(s) =>
|
||||||
|
s &&
|
||||||
|
typeof s.getPixelForValue === "function" &&
|
||||||
|
String(s.id || "")
|
||||||
|
.toLowerCase()
|
||||||
|
.includes("y"),
|
||||||
|
);
|
||||||
|
|
||||||
const area = chart?.chartArea;
|
const area = chart?.chartArea;
|
||||||
if (!y || !area) return;
|
if (!y || !area) return;
|
||||||
|
|
@ -139,9 +145,7 @@ const StaticMarkerLinesPlugin = {
|
||||||
const strokeStyle = String(opts?.color || "#7f8da3"); // light grey-blue
|
const strokeStyle = String(opts?.color || "#7f8da3"); // light grey-blue
|
||||||
|
|
||||||
// "marker on Y axis" text
|
// "marker on Y axis" text
|
||||||
const font =
|
const font = opts?.font || "600 11px system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif";
|
||||||
opts?.font ||
|
|
||||||
"600 11px system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif";
|
|
||||||
const labelColor = String(opts?.labelColor || "#556274");
|
const labelColor = String(opts?.labelColor || "#556274");
|
||||||
const axisInset = Number.isFinite(opts?.axisInset) ? opts.axisInset : 2;
|
const axisInset = Number.isFinite(opts?.axisInset) ? opts.axisInset : 2;
|
||||||
|
|
||||||
|
|
@ -189,7 +193,6 @@ if (text) {
|
||||||
ctx.textAlign = "center";
|
ctx.textAlign = "center";
|
||||||
ctx.fillText(text, axisCenterX, py);
|
ctx.fillText(text, axisCenterX, py);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.restore();
|
ctx.restore();
|
||||||
|
|
@ -603,11 +606,7 @@ export async function renderItem($app, skuInput) {
|
||||||
const inflightFetch = new Map(); // ck -> Promise
|
const inflightFetch = new Map(); // ck -> Promise
|
||||||
const today = dateOnly(idx.generatedAt || new Date().toISOString());
|
const today = dateOnly(idx.generatedAt || new Date().toISOString());
|
||||||
const skuKeys = [...skuGroup];
|
const skuKeys = [...skuGroup];
|
||||||
const wantRealSkus = new Set(
|
const wantRealSkus = new Set(skuKeys.map((s) => String(s || "").trim()).filter((x) => x));
|
||||||
skuKeys
|
|
||||||
.map((s) => String(s || "").trim())
|
|
||||||
.filter((x) => x)
|
|
||||||
);
|
|
||||||
|
|
||||||
// Tuning knobs:
|
// Tuning knobs:
|
||||||
// - keep compute modest: only a few stores processed simultaneously
|
// - keep compute modest: only a few stores processed simultaneously
|
||||||
|
|
@ -946,7 +945,7 @@ export async function renderItem($app, skuInput) {
|
||||||
const t = (storeMins[0].min + storeMins[1].min + storeMins[2].min) / 3;
|
const t = (storeMins[0].min + storeMins[1].min + storeMins[2].min) / 3;
|
||||||
if (Number.isFinite(t)) markers.push({ y: Math.round(t), text: "Target" });
|
if (Number.isFinite(t)) markers.push({ y: Math.round(t), text: "Target" });
|
||||||
}
|
}
|
||||||
const markerYs = markers.map(m => Number(m.y)).filter(Number.isFinite);
|
const markerYs = markers.map((m) => Number(m.y)).filter(Number.isFinite);
|
||||||
|
|
||||||
// helper: approximate font px size from a CSS font string (Chart uses one)
|
// helper: approximate font px size from a CSS font string (Chart uses one)
|
||||||
function fontPx(font) {
|
function fontPx(font) {
|
||||||
|
|
@ -1020,15 +1019,12 @@ export async function renderItem($app, skuInput) {
|
||||||
|
|
||||||
// derive a "collision window" from tick label height
|
// derive a "collision window" from tick label height
|
||||||
// Chart.js puts resolved font on ticks.font (v3+), otherwise fall back
|
// Chart.js puts resolved font on ticks.font (v3+), otherwise fall back
|
||||||
const tickFont =
|
const tickFont = this?.options?.ticks?.font || this?.ctx?.font || "12px system-ui";
|
||||||
this?.options?.ticks?.font ||
|
|
||||||
this?.ctx?.font ||
|
|
||||||
"12px system-ui";
|
|
||||||
|
|
||||||
const h = fontPx(
|
const h = fontPx(
|
||||||
typeof tickFont === "string"
|
typeof tickFont === "string"
|
||||||
? tickFont
|
? tickFont
|
||||||
: `${tickFont?.size || 12}px ${tickFont?.family || "system-ui"}`
|
: `${tickFont?.size || 12}px ${tickFont?.family || "system-ui"}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// hide if within 55% of label height (tweak 0.45–0.75)
|
// hide if within 55% of label height (tweak 0.45–0.75)
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,9 @@ function isSoftSkuKey(k) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function isUnknownSkuKey2(k) {
|
function isUnknownSkuKey2(k) {
|
||||||
return String(k || "").trim().startsWith("u:");
|
return String(k || "")
|
||||||
|
.trim()
|
||||||
|
.startsWith("u:");
|
||||||
}
|
}
|
||||||
|
|
||||||
function isBCStoreLabel(label) {
|
function isBCStoreLabel(label) {
|
||||||
|
|
@ -40,12 +42,7 @@ function skuIsBC(allRows, skuKey) {
|
||||||
|
|
||||||
function isABStoreLabel(label) {
|
function isABStoreLabel(label) {
|
||||||
const s = String(label || "").toLowerCase();
|
const s = String(label || "").toLowerCase();
|
||||||
return (
|
return s.includes("alberta") || s.includes("calgary") || s.includes("edmonton") || /\bab\b/.test(s);
|
||||||
s.includes("alberta") ||
|
|
||||||
s.includes("calgary") ||
|
|
||||||
s.includes("edmonton") ||
|
|
||||||
/\bab\b/.test(s)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function skuIsAB(allRows, skuKey) {
|
function skuIsAB(allRows, skuKey) {
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ export function buildPricePenaltyForPair({ allAgg, rules, kPerGroup = 6 }) {
|
||||||
if (!(gap >= 0)) return 1.0;
|
if (!(gap >= 0)) return 1.0;
|
||||||
if (gap <= 0.35) return 1.0;
|
if (gap <= 0.35) return 1.0;
|
||||||
|
|
||||||
if (gap <= 0.50) {
|
if (gap <= 0.5) {
|
||||||
const t = (gap - 0.35) / 0.15; // 0..1
|
const t = (gap - 0.35) / 0.15; // 0..1
|
||||||
return 1.0 - 0.25 * t; // 1.00 -> 0.75
|
return 1.0 - 0.25 * t; // 1.00 -> 0.75
|
||||||
}
|
}
|
||||||
|
|
@ -75,4 +75,3 @@ export function buildPricePenaltyForPair({ allAgg, rules, kPerGroup = 6 }) {
|
||||||
return gapToMultiplier(gap);
|
return gapToMultiplier(gap);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3,9 +3,33 @@ import { tokenizeQuery, normSearchText } from "../sku.js";
|
||||||
|
|
||||||
// Ignore ultra-common / low-signal tokens in bottle names.
|
// Ignore ultra-common / low-signal tokens in bottle names.
|
||||||
const SIM_STOP_TOKENS = new Set([
|
const SIM_STOP_TOKENS = new Set([
|
||||||
"the","a","an","and","of","to","in","for","with",
|
"the",
|
||||||
"year","years","yr","yrs","old",
|
"a",
|
||||||
"whisky","whiskey","scotch","single","malt","cask","finish","edition","release","batch","strength","abv","proof",
|
"an",
|
||||||
|
"and",
|
||||||
|
"of",
|
||||||
|
"to",
|
||||||
|
"in",
|
||||||
|
"for",
|
||||||
|
"with",
|
||||||
|
"year",
|
||||||
|
"years",
|
||||||
|
"yr",
|
||||||
|
"yrs",
|
||||||
|
"old",
|
||||||
|
"whisky",
|
||||||
|
"whiskey",
|
||||||
|
"scotch",
|
||||||
|
"single",
|
||||||
|
"malt",
|
||||||
|
"cask",
|
||||||
|
"finish",
|
||||||
|
"edition",
|
||||||
|
"release",
|
||||||
|
"batch",
|
||||||
|
"strength",
|
||||||
|
"abv",
|
||||||
|
"proof",
|
||||||
"anniversary",
|
"anniversary",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -22,7 +46,9 @@ export function smwsKeyFromName(name) {
|
||||||
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
|
const ORDINAL_RE = /^(\d+)(st|nd|rd|th)$/i;
|
||||||
|
|
||||||
export function numKey(t) {
|
export function numKey(t) {
|
||||||
const s = String(t || "").trim().toLowerCase();
|
const s = String(t || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (!s) return "";
|
if (!s) return "";
|
||||||
if (/^\d+$/.test(s)) return s;
|
if (/^\d+$/.test(s)) return s;
|
||||||
const m = s.match(ORDINAL_RE);
|
const m = s.match(ORDINAL_RE);
|
||||||
|
|
@ -68,7 +94,9 @@ export function filterSimTokens(tokens) {
|
||||||
|
|
||||||
for (let i = 0; i < arr.length; i++) {
|
for (let i = 0; i < arr.length; i++) {
|
||||||
const raw = arr[i];
|
const raw = arr[i];
|
||||||
let t = String(raw || "").trim().toLowerCase();
|
let t = String(raw || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
if (!t) continue;
|
if (!t) continue;
|
||||||
|
|
||||||
if (!/[a-z0-9]/i.test(t)) continue;
|
if (!/[a-z0-9]/i.test(t)) continue;
|
||||||
|
|
@ -84,7 +112,9 @@ export function filterSimTokens(tokens) {
|
||||||
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
|
if (VOL_UNIT.has(t) || t === "abv" || t === "proof") continue;
|
||||||
|
|
||||||
if (/^\d+(?:\.\d+)?$/.test(t)) {
|
if (/^\d+(?:\.\d+)?$/.test(t)) {
|
||||||
const next = String(arr[i + 1] || "").trim().toLowerCase();
|
const next = String(arr[i + 1] || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
const nextNorm = SIM_EQUIV.get(next) || next;
|
const nextNorm = SIM_EQUIV.get(next) || next;
|
||||||
if (VOL_UNIT.has(nextNorm)) {
|
if (VOL_UNIT.has(nextNorm)) {
|
||||||
i++;
|
i++;
|
||||||
|
|
@ -113,7 +143,8 @@ export function numberMismatchPenalty(aTokens, bTokens) {
|
||||||
export function levenshtein(a, b) {
|
export function levenshtein(a, b) {
|
||||||
a = String(a || "");
|
a = String(a || "");
|
||||||
b = String(b || "");
|
b = String(b || "");
|
||||||
const n = a.length, m = b.length;
|
const n = a.length,
|
||||||
|
m = b.length;
|
||||||
if (!n) return m;
|
if (!n) return m;
|
||||||
if (!m) return n;
|
if (!m) return n;
|
||||||
|
|
||||||
|
|
@ -190,18 +221,14 @@ export function similarityScore(aName, bName) {
|
||||||
const maxLen = Math.max(1, Math.max(a.length, b.length));
|
const maxLen = Math.max(1, Math.max(a.length, b.length));
|
||||||
const levSim = 1 - d / maxLen;
|
const levSim = 1 - d / maxLen;
|
||||||
|
|
||||||
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
|
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
|
||||||
|
|
||||||
const smallN = Math.min(aToks.length, bToks.length);
|
const smallN = Math.min(aToks.length, bToks.length);
|
||||||
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
||||||
|
|
||||||
const numGate = numberMismatchPenalty(aToks, bToks);
|
const numGate = numberMismatchPenalty(aToks, bToks);
|
||||||
|
|
||||||
let s =
|
let s = numGate * (firstMatch * 3.0 + overlapTail * 2.2 * gate + levSim * (firstMatch ? 1.0 : 0.1 + 0.7 * contain));
|
||||||
numGate *
|
|
||||||
(firstMatch * 3.0 +
|
|
||||||
overlapTail * 2.2 * gate +
|
|
||||||
levSim * (firstMatch ? 1.0 : (0.10 + 0.70 * contain)));
|
|
||||||
|
|
||||||
if (ageMatch) s *= 2.2;
|
if (ageMatch) s *= 2.2;
|
||||||
else if (ageMismatch) s *= 0.18;
|
else if (ageMismatch) s *= 0.18;
|
||||||
|
|
@ -244,15 +271,9 @@ export function fastSimilarityScore(aTokens, bTokens, aNormName, bNormName) {
|
||||||
const denom = Math.max(1, Math.max(aTail.length, bTail.length));
|
const denom = Math.max(1, Math.max(aTail.length, bTail.length));
|
||||||
const overlapTail = inter / denom;
|
const overlapTail = inter / denom;
|
||||||
|
|
||||||
const pref =
|
const pref = firstMatch && a.slice(0, 10) && b.slice(0, 10) && a.slice(0, 10) === b.slice(0, 10) ? 0.2 : 0;
|
||||||
firstMatch &&
|
|
||||||
a.slice(0, 10) &&
|
|
||||||
b.slice(0, 10) &&
|
|
||||||
a.slice(0, 10) === b.slice(0, 10)
|
|
||||||
? 0.2
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
let gate = firstMatch ? 1.0 : Math.min(0.80, 0.06 + 0.95 * contain);
|
let gate = firstMatch ? 1.0 : Math.min(0.8, 0.06 + 0.95 * contain);
|
||||||
const smallN = Math.min(aTokF.length, bTokF.length);
|
const smallN = Math.min(aTokF.length, bTokF.length);
|
||||||
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
if (!firstMatch && smallN <= 3 && contain < 0.78) gate *= 0.18;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -101,8 +101,8 @@ export function buildSizePenaltyForPair({ allRows, allAgg, rules }) {
|
||||||
return function sizePenaltyForPair(aSku, bSku) {
|
return function sizePenaltyForPair(aSku, bSku) {
|
||||||
const aCanon = String(rules.canonicalSku(String(aSku || "")) || "");
|
const aCanon = String(rules.canonicalSku(String(aSku || "")) || "");
|
||||||
const bCanon = String(rules.canonicalSku(String(bSku || "")) || "");
|
const bCanon = String(rules.canonicalSku(String(bSku || "")) || "");
|
||||||
const A = aCanon ? (CANON_SIZE_CACHE.get(aCanon) || new Set()) : new Set();
|
const A = aCanon ? CANON_SIZE_CACHE.get(aCanon) || new Set() : new Set();
|
||||||
const B = bCanon ? (CANON_SIZE_CACHE.get(bCanon) || new Set()) : new Set();
|
const B = bCanon ? CANON_SIZE_CACHE.get(bCanon) || new Set() : new Set();
|
||||||
return sizePenalty(A, B);
|
return sizePenalty(A, B);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -40,4 +40,3 @@ function canonKeyForSku(rules, skuKey) {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -51,7 +51,6 @@ export function topSuggestions(allAgg, limit, otherPinnedSku, mappedSkus) {
|
||||||
return scored.slice(0, limit).map((x) => x.it);
|
return scored.slice(0, limit).map((x) => x.it);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// viz/app/linker/suggestions.js
|
// viz/app/linker/suggestions.js
|
||||||
// (requires fnv1a32u(str) helper to exist in this file)
|
// (requires fnv1a32u(str) helper to exist in this file)
|
||||||
|
|
||||||
|
|
@ -65,7 +64,7 @@ export function recommendSimilar(
|
||||||
sizePenaltyFn,
|
sizePenaltyFn,
|
||||||
pricePenaltyFn,
|
pricePenaltyFn,
|
||||||
sameStoreFn,
|
sameStoreFn,
|
||||||
sameGroupFn
|
sameGroupFn,
|
||||||
) {
|
) {
|
||||||
if (!pinned || !pinned.name) return topSuggestions(allAgg, limit, otherPinnedSku, mappedSkus);
|
if (!pinned || !pinned.name) return topSuggestions(allAgg, limit, otherPinnedSku, mappedSkus);
|
||||||
|
|
||||||
|
|
@ -134,11 +133,7 @@ export function recommendSimilar(
|
||||||
if (k && k === pinnedSmws) {
|
if (k && k === pinnedSmws) {
|
||||||
const stores = it.stores ? it.stores.size : 0;
|
const stores = it.stores ? it.stores.size : 0;
|
||||||
const hasPrice = it.cheapestPriceNum != null ? 1 : 0;
|
const hasPrice = it.cheapestPriceNum != null ? 1 : 0;
|
||||||
pushTopK(
|
pushTopK(cheap, { it, s: 1e9 + stores * 10 + hasPrice, itNorm: "", itRawToks: null }, MAX_CHEAP_KEEP);
|
||||||
cheap,
|
|
||||||
{ it, s: 1e9 + stores * 10 + hasPrice, itNorm: "", itRawToks: null },
|
|
||||||
MAX_CHEAP_KEEP
|
|
||||||
);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -161,7 +156,7 @@ export function recommendSimilar(
|
||||||
// Soft first-token mismatch penalty (never blocks)
|
// Soft first-token mismatch penalty (never blocks)
|
||||||
if (!firstMatch) {
|
if (!firstMatch) {
|
||||||
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
|
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
|
||||||
let mult = 0.10 + 0.95 * contain;
|
let mult = 0.1 + 0.95 * contain;
|
||||||
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
||||||
s0 *= Math.min(1.0, mult);
|
s0 *= Math.min(1.0, mult);
|
||||||
}
|
}
|
||||||
|
|
@ -211,7 +206,7 @@ export function recommendSimilar(
|
||||||
|
|
||||||
if (!firstMatch) {
|
if (!firstMatch) {
|
||||||
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
|
const smallN = Math.min(pinToks.length || 0, itToks.length || 0);
|
||||||
let mult = 0.10 + 0.95 * contain;
|
let mult = 0.1 + 0.95 * contain;
|
||||||
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
||||||
s *= Math.min(1.0, mult);
|
s *= Math.min(1.0, mult);
|
||||||
if (s <= 0) continue;
|
if (s <= 0) continue;
|
||||||
|
|
@ -266,9 +261,6 @@ export function recommendSimilar(
|
||||||
return fallback.slice(0, limit).map((x) => x.it);
|
return fallback.slice(0, limit).map((x) => x.it);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
export function computeInitialPairsFast(
|
export function computeInitialPairsFast(
|
||||||
allAgg,
|
allAgg,
|
||||||
mappedSkus,
|
mappedSkus,
|
||||||
|
|
@ -277,7 +269,7 @@ export function recommendSimilar(
|
||||||
sameStoreFn,
|
sameStoreFn,
|
||||||
sameGroupFn, // ✅ NEW
|
sameGroupFn, // ✅ NEW
|
||||||
sizePenaltyFn,
|
sizePenaltyFn,
|
||||||
pricePenaltyFn
|
pricePenaltyFn,
|
||||||
) {
|
) {
|
||||||
const itemsAll = allAgg.filter((it) => !!it);
|
const itemsAll = allAgg.filter((it) => !!it);
|
||||||
|
|
||||||
|
|
@ -382,7 +374,7 @@ export function recommendSimilar(
|
||||||
// --- Improved general pairing logic ---
|
// --- Improved general pairing logic ---
|
||||||
|
|
||||||
const seeds = topSuggestions(work, Math.min(220, work.length), "", mappedSkus).filter(
|
const seeds = topSuggestions(work, Math.min(220, work.length), "", mappedSkus).filter(
|
||||||
(it) => !used.has(String(it?.sku || ""))
|
(it) => !used.has(String(it?.sku || "")),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Build token buckets over normalized names
|
// Build token buckets over normalized names
|
||||||
|
|
@ -470,7 +462,7 @@ export function recommendSimilar(
|
||||||
|
|
||||||
if (!firstMatch) {
|
if (!firstMatch) {
|
||||||
const smallN = Math.min(aFilt.length || 0, bFilt.length || 0);
|
const smallN = Math.min(aFilt.length || 0, bFilt.length || 0);
|
||||||
let mult = 0.10 + 0.95 * contain;
|
let mult = 0.1 + 0.95 * contain;
|
||||||
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
if (smallN <= 3 && contain < 0.78) mult *= 0.22;
|
||||||
s *= Math.min(1.0, mult);
|
s *= Math.min(1.0, mult);
|
||||||
}
|
}
|
||||||
|
|
@ -505,7 +497,7 @@ export function recommendSimilar(
|
||||||
|
|
||||||
if (!x.firstMatch) {
|
if (!x.firstMatch) {
|
||||||
const smallN = Math.min(aFilt.length || 0, (x.bFilt || []).length || 0);
|
const smallN = Math.min(aFilt.length || 0, (x.bFilt || []).length || 0);
|
||||||
let mult = 0.10 + 0.95 * x.contain;
|
let mult = 0.1 + 0.95 * x.contain;
|
||||||
if (smallN <= 3 && x.contain < 0.78) mult *= 0.22;
|
if (smallN <= 3 && x.contain < 0.78) mult *= 0.22;
|
||||||
s *= Math.min(1.0, mult);
|
s *= Math.min(1.0, mult);
|
||||||
if (s <= 0) continue;
|
if (s <= 0) continue;
|
||||||
|
|
@ -526,7 +518,7 @@ export function recommendSimilar(
|
||||||
else s *= 0.15;
|
else s *= 0.15;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (String(aSku).startsWith("u:") || String(bSku).startsWith("u:")) s *= 1.10;
|
if (String(aSku).startsWith("u:") || String(bSku).startsWith("u:")) s *= 1.1;
|
||||||
|
|
||||||
if (s > bestS) {
|
if (s > bestS) {
|
||||||
bestS = s;
|
bestS = s;
|
||||||
|
|
@ -534,7 +526,7 @@ export function recommendSimilar(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!bestB || bestS < 0.50) continue;
|
if (!bestB || bestS < 0.5) continue;
|
||||||
|
|
||||||
const bSku = String(bestB.sku || "");
|
const bSku = String(bestB.sku || "");
|
||||||
if (!bSku || used.has(bSku)) continue;
|
if (!bSku || used.has(bSku)) continue;
|
||||||
|
|
@ -592,11 +584,6 @@ export function recommendSimilar(
|
||||||
return out.slice(0, limitPairs);
|
return out.slice(0, limitPairs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function fnv1a32u(str) {
|
function fnv1a32u(str) {
|
||||||
let h = 0x811c9dc5;
|
let h = 0x811c9dc5;
|
||||||
str = String(str || "");
|
str = String(str || "");
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,6 @@
|
||||||
// viz/app/linker_page.js
|
// viz/app/linker_page.js
|
||||||
import { esc, renderThumbHtml } from "./dom.js";
|
import { esc, renderThumbHtml } from "./dom.js";
|
||||||
import {
|
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, normSearchText } from "./sku.js";
|
||||||
tokenizeQuery,
|
|
||||||
matchesAllTokens,
|
|
||||||
displaySku,
|
|
||||||
keySkuForRow,
|
|
||||||
normSearchText,
|
|
||||||
} from "./sku.js";
|
|
||||||
import { loadIndex } from "./state.js";
|
import { loadIndex } from "./state.js";
|
||||||
import { aggregateBySku } from "./catalog.js";
|
import { aggregateBySku } from "./catalog.js";
|
||||||
import { loadSkuRules, clearSkuRulesCache } from "./mapping.js";
|
import { loadSkuRules, clearSkuRulesCache } from "./mapping.js";
|
||||||
|
|
@ -32,11 +26,7 @@ import { buildSizePenaltyForPair } from "./linker/size.js";
|
||||||
import { pickPreferredCanonical } from "./linker/canonical_pref.js";
|
import { pickPreferredCanonical } from "./linker/canonical_pref.js";
|
||||||
import { smwsKeyFromName } from "./linker/similarity.js";
|
import { smwsKeyFromName } from "./linker/similarity.js";
|
||||||
import { buildPricePenaltyForPair } from "./linker/price.js";
|
import { buildPricePenaltyForPair } from "./linker/price.js";
|
||||||
import {
|
import { topSuggestions, recommendSimilar, computeInitialPairsFast } from "./linker/suggestions.js";
|
||||||
topSuggestions,
|
|
||||||
recommendSimilar,
|
|
||||||
computeInitialPairsFast,
|
|
||||||
} from "./linker/suggestions.js";
|
|
||||||
|
|
||||||
/* ---------------- Page ---------------- */
|
/* ---------------- Page ---------------- */
|
||||||
|
|
||||||
|
|
@ -163,16 +153,12 @@ export async function renderSkuLinker($app) {
|
||||||
sameStoreCanon,
|
sameStoreCanon,
|
||||||
sameGroup, // ✅ NEW: hard-block already-linked pairs (incl SMWS stage)
|
sameGroup, // ✅ NEW: hard-block already-linked pairs (incl SMWS stage)
|
||||||
sizePenaltyForPair,
|
sizePenaltyForPair,
|
||||||
pricePenaltyForPair
|
pricePenaltyForPair,
|
||||||
);
|
);
|
||||||
|
|
||||||
return initialPairs;
|
return initialPairs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
let pinnedL = null;
|
let pinnedL = null;
|
||||||
let pinnedR = null;
|
let pinnedR = null;
|
||||||
|
|
||||||
|
|
@ -192,7 +178,7 @@ export async function renderSkuLinker($app) {
|
||||||
|
|
||||||
const storeBadge = href
|
const storeBadge = href
|
||||||
? `<a class="badge" href="${esc(href)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
? `<a class="badge" href="${esc(href)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
store
|
store,
|
||||||
)}${esc(plus)}</a>`
|
)}${esc(plus)}</a>`
|
||||||
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
|
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
|
||||||
|
|
||||||
|
|
@ -253,7 +239,7 @@ export async function renderSkuLinker($app) {
|
||||||
sizePenaltyForPair,
|
sizePenaltyForPair,
|
||||||
pricePenaltyForPair,
|
pricePenaltyForPair,
|
||||||
sameStoreCanon,
|
sameStoreCanon,
|
||||||
sameGroup
|
sameGroup,
|
||||||
);
|
);
|
||||||
|
|
||||||
const pairs = getInitialPairsIfNeeded();
|
const pairs = getInitialPairsIfNeeded();
|
||||||
|
|
@ -261,9 +247,7 @@ export async function renderSkuLinker($app) {
|
||||||
const list = side === "L" ? pairs.map((p) => p.a) : pairs.map((p) => p.b);
|
const list = side === "L" ? pairs.map((p) => p.a) : pairs.map((p) => p.b);
|
||||||
return list.filter(
|
return list.filter(
|
||||||
(it) =>
|
(it) =>
|
||||||
it &&
|
it && it.sku !== otherSku && (!mappedSkus.has(String(it.sku)) || smwsKeyFromName(it.name || "")),
|
||||||
it.sku !== otherSku &&
|
|
||||||
(!mappedSkus.has(String(it.sku)) || smwsKeyFromName(it.name || ""))
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -271,7 +255,6 @@ export async function renderSkuLinker($app) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function attachHandlers($root, side) {
|
function attachHandlers($root, side) {
|
||||||
|
|
||||||
for (const el of Array.from($root.querySelectorAll(".thumbInternalLink"))) {
|
for (const el of Array.from($root.querySelectorAll(".thumbInternalLink"))) {
|
||||||
el.addEventListener("click", (e) => {
|
el.addEventListener("click", (e) => {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
|
@ -458,7 +441,7 @@ export async function renderSkuLinker($app) {
|
||||||
ignores: editsToSend.ignores,
|
ignores: editsToSend.ignores,
|
||||||
},
|
},
|
||||||
null,
|
null,
|
||||||
2
|
2,
|
||||||
);
|
);
|
||||||
|
|
||||||
const title = `[stviz] sku link updates (${editsToSend.links.length} link, ${editsToSend.ignores.length} ignore)`;
|
const title = `[stviz] sku link updates (${editsToSend.links.length} link, ${editsToSend.ignores.length} ignore)`;
|
||||||
|
|
@ -507,10 +490,7 @@ export async function renderSkuLinker($app) {
|
||||||
it = allAgg.find((x) => String(x?.sku || "") === canonWant);
|
it = allAgg.find((x) => String(x?.sku || "") === canonWant);
|
||||||
if (it) return it;
|
if (it) return it;
|
||||||
|
|
||||||
return (
|
return allAgg.find((x) => String(rules.canonicalSku(String(x?.sku || "")) || "") === canonWant) || null;
|
||||||
allAgg.find((x) => String(rules.canonicalSku(String(x?.sku || "")) || "") === canonWant) ||
|
|
||||||
null
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateAll() {
|
function updateAll() {
|
||||||
|
|
@ -647,7 +627,7 @@ export async function renderSkuLinker($app) {
|
||||||
for (let i = 0; i < uniq.length; i++) {
|
for (let i = 0; i < uniq.length; i++) {
|
||||||
const w = uniq[i];
|
const w = uniq[i];
|
||||||
$status.textContent = `Writing (${i + 1}/${uniq.length}): ${displaySku(
|
$status.textContent = `Writing (${i + 1}/${uniq.length}): ${displaySku(
|
||||||
w.fromSku
|
w.fromSku,
|
||||||
)} → ${displaySku(w.toSku)} …`;
|
)} → ${displaySku(w.toSku)} …`;
|
||||||
await apiWriteSkuLink(w.fromSku, w.toSku);
|
await apiWriteSkuLink(w.fromSku, w.toSku);
|
||||||
}
|
}
|
||||||
|
|
@ -764,7 +744,7 @@ export async function renderSkuLinker($app) {
|
||||||
|
|
||||||
// ✅ NEW: always include rules.links (meta can be incomplete)
|
// ✅ NEW: always include rules.links (meta can be incomplete)
|
||||||
const merged = [
|
const merged = [
|
||||||
...((rules0 && Array.isArray(rules0.links)) ? rules0.links : []),
|
...(rules0 && Array.isArray(rules0.links) ? rules0.links : []),
|
||||||
...(Array.isArray(links) ? links : []),
|
...(Array.isArray(links) ? links : []),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
@ -775,7 +755,4 @@ export async function renderSkuLinker($app) {
|
||||||
|
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -114,7 +114,7 @@ export function addPendingLink(fromSku, toSku) {
|
||||||
[
|
[
|
||||||
...pending.links.map((x) => linkKey(x.fromSku, x.toSku)),
|
...pending.links.map((x) => linkKey(x.fromSku, x.toSku)),
|
||||||
...submitted.links.map((x) => linkKey(x.fromSku, x.toSku)),
|
...submitted.links.map((x) => linkKey(x.fromSku, x.toSku)),
|
||||||
].filter(Boolean)
|
].filter(Boolean),
|
||||||
);
|
);
|
||||||
|
|
||||||
if (seen.has(k)) return false;
|
if (seen.has(k)) return false;
|
||||||
|
|
@ -137,7 +137,7 @@ export function addPendingIgnore(skuA, skuB) {
|
||||||
[
|
[
|
||||||
...pending.ignores.map((x) => pairKey(x.skuA, x.skuB)),
|
...pending.ignores.map((x) => pairKey(x.skuA, x.skuB)),
|
||||||
...submitted.ignores.map((x) => pairKey(x.skuA, x.skuB)),
|
...submitted.ignores.map((x) => pairKey(x.skuA, x.skuB)),
|
||||||
].filter(Boolean)
|
].filter(Boolean),
|
||||||
);
|
);
|
||||||
|
|
||||||
if (seen.has(k)) return false;
|
if (seen.has(k)) return false;
|
||||||
|
|
@ -164,9 +164,7 @@ export function applyPendingToMeta(meta) {
|
||||||
|
|
||||||
// merge links (dedupe by from→to)
|
// merge links (dedupe by from→to)
|
||||||
const seenL = new Set(
|
const seenL = new Set(
|
||||||
base.links
|
base.links.map((x) => linkKey(String(x?.fromSku || "").trim(), String(x?.toSku || "").trim())).filter(Boolean),
|
||||||
.map((x) => linkKey(String(x?.fromSku || "").trim(), String(x?.toSku || "").trim()))
|
|
||||||
.filter(Boolean)
|
|
||||||
);
|
);
|
||||||
for (const x of overlay.links) {
|
for (const x of overlay.links) {
|
||||||
const k = linkKey(x.fromSku, x.toSku);
|
const k = linkKey(x.fromSku, x.toSku);
|
||||||
|
|
@ -179,7 +177,7 @@ export function applyPendingToMeta(meta) {
|
||||||
const seenI = new Set(
|
const seenI = new Set(
|
||||||
base.ignores
|
base.ignores
|
||||||
.map((x) => pairKey(String(x?.skuA || x?.a || "").trim(), String(x?.skuB || x?.b || "").trim()))
|
.map((x) => pairKey(String(x?.skuA || x?.a || "").trim(), String(x?.skuB || x?.b || "").trim()))
|
||||||
.filter(Boolean)
|
.filter(Boolean),
|
||||||
);
|
);
|
||||||
for (const x of overlay.ignores) {
|
for (const x of overlay.ignores) {
|
||||||
const k = pairKey(x.skuA, x.skuB);
|
const k = pairKey(x.skuA, x.skuB);
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,9 @@
|
||||||
import { esc, renderThumbHtml, prettyTs } from "./dom.js";
|
import { esc, renderThumbHtml, prettyTs } from "./dom.js";
|
||||||
import {
|
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, parsePriceToNumber } from "./sku.js";
|
||||||
tokenizeQuery,
|
|
||||||
matchesAllTokens,
|
|
||||||
displaySku,
|
|
||||||
keySkuForRow,
|
|
||||||
parsePriceToNumber,
|
|
||||||
} from "./sku.js";
|
|
||||||
import { loadIndex, loadRecent, loadSavedQuery, saveQuery } from "./state.js";
|
import { loadIndex, loadRecent, loadSavedQuery, saveQuery } from "./state.js";
|
||||||
import { aggregateBySku } from "./catalog.js";
|
import { aggregateBySku } from "./catalog.js";
|
||||||
import { loadSkuRules } from "./mapping.js";
|
import { loadSkuRules } from "./mapping.js";
|
||||||
import {
|
import { smwsDistilleryCodesForQueryPrefix, smwsDistilleryCodeFromName } from "./smws.js";
|
||||||
smwsDistilleryCodesForQueryPrefix,
|
|
||||||
smwsDistilleryCodeFromName,
|
|
||||||
} from "./smws.js";
|
|
||||||
|
|
||||||
export function renderSearch($app) {
|
export function renderSearch($app) {
|
||||||
$app.innerHTML = `
|
$app.innerHTML = `
|
||||||
|
|
@ -126,13 +117,9 @@ export function renderSearch($app) {
|
||||||
|
|
||||||
$stores.innerHTML = stores
|
$stores.innerHTML = stores
|
||||||
.map((s, i) => {
|
.map((s, i) => {
|
||||||
const btn = `<a class="storeBtn" href="#/store/${encodeURIComponent(
|
const btn = `<a class="storeBtn" href="#/store/${encodeURIComponent(s)}">${esc(s)}</a>`;
|
||||||
s
|
|
||||||
)}">${esc(s)}</a>`;
|
|
||||||
const brk =
|
const brk =
|
||||||
i === breakAt - 1 && stores.length > 1
|
i === breakAt - 1 && stores.length > 1 ? `<span class="storeBreak" aria-hidden="true"></span>` : "";
|
||||||
? `<span class="storeBreak" aria-hidden="true"></span>`
|
|
||||||
: "";
|
|
||||||
return btn + brk;
|
return btn + brk;
|
||||||
})
|
})
|
||||||
.join("");
|
.join("");
|
||||||
|
|
@ -156,15 +143,13 @@ export function renderSearch($app) {
|
||||||
const href = urlForAgg(it, store) || String(it.sampleUrl || "").trim();
|
const href = urlForAgg(it, store) || String(it.sampleUrl || "").trim();
|
||||||
const storeBadge = href
|
const storeBadge = href
|
||||||
? `<a class="badge" href="${esc(
|
? `<a class="badge" href="${esc(
|
||||||
href
|
href,
|
||||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
store
|
store,
|
||||||
)}${esc(plus)}</a>`
|
)}${esc(plus)}</a>`
|
||||||
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
|
: `<span class="badge">${esc(store)}${esc(plus)}</span>`;
|
||||||
|
|
||||||
const skuLink = `#/link/?left=${encodeURIComponent(
|
const skuLink = `#/link/?left=${encodeURIComponent(String(it.sku || ""))}`;
|
||||||
String(it.sku || "")
|
|
||||||
)}`;
|
|
||||||
|
|
||||||
return `
|
return `
|
||||||
<div class="item" data-sku="${esc(it.sku)}">
|
<div class="item" data-sku="${esc(it.sku)}">
|
||||||
|
|
@ -176,9 +161,9 @@ export function renderSearch($app) {
|
||||||
<div class="itemTop">
|
<div class="itemTop">
|
||||||
<div class="itemName">${esc(it.name || "(no name)")}</div>
|
<div class="itemName">${esc(it.name || "(no name)")}</div>
|
||||||
<a class="badge mono skuLink" href="${esc(
|
<a class="badge mono skuLink" href="${esc(
|
||||||
skuLink
|
skuLink,
|
||||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
displaySku(it.sku)
|
displaySku(it.sku),
|
||||||
)}</a>
|
)}</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="metaRow">
|
<div class="metaRow">
|
||||||
|
|
@ -241,7 +226,10 @@ export function renderSearch($app) {
|
||||||
const storeLabelRaw = String(r?.storeLabel || r?.store || "").trim();
|
const storeLabelRaw = String(r?.storeLabel || r?.store || "").trim();
|
||||||
const bestStoreRaw = String(agg?.cheapestStoreLabel || "").trim();
|
const bestStoreRaw = String(agg?.cheapestStoreLabel || "").trim();
|
||||||
|
|
||||||
const normStore = (s) => String(s || "").trim().toLowerCase();
|
const normStore = (s) =>
|
||||||
|
String(s || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
|
||||||
// Normalize kind
|
// Normalize kind
|
||||||
let kind = String(r?.kind || "");
|
let kind = String(r?.kind || "");
|
||||||
|
|
@ -254,27 +242,16 @@ export function renderSearch($app) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const pctOff =
|
const pctOff = kind === "price_down" ? salePctOff(r?.oldPrice || "", r?.newPrice || "") : null;
|
||||||
kind === "price_down"
|
const pctUp = kind === "price_up" ? pctChange(r?.oldPrice || "", r?.newPrice || "") : null;
|
||||||
? salePctOff(r?.oldPrice || "", r?.newPrice || "")
|
|
||||||
: null;
|
|
||||||
const pctUp =
|
|
||||||
kind === "price_up"
|
|
||||||
? pctChange(r?.oldPrice || "", r?.newPrice || "")
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const isNew = kind === "new";
|
const isNew = kind === "new";
|
||||||
const storeCount = agg?.stores?.size || 0;
|
const storeCount = agg?.stores?.size || 0;
|
||||||
const isNewUnique = isNew && storeCount <= 1;
|
const isNewUnique = isNew && storeCount <= 1;
|
||||||
|
|
||||||
// Cheapest checks (use aggregate index)
|
// Cheapest checks (use aggregate index)
|
||||||
const newPriceNum =
|
const newPriceNum = kind === "price_down" || kind === "price_up" ? parsePriceToNumber(r?.newPrice || "") : null;
|
||||||
kind === "price_down" || kind === "price_up"
|
const bestPriceNum = Number.isFinite(agg?.cheapestPriceNum) ? agg.cheapestPriceNum : null;
|
||||||
? parsePriceToNumber(r?.newPrice || "")
|
|
||||||
: null;
|
|
||||||
const bestPriceNum = Number.isFinite(agg?.cheapestPriceNum)
|
|
||||||
? agg.cheapestPriceNum
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const EPS = 0.01;
|
const EPS = 0.01;
|
||||||
const priceMatchesBest =
|
const priceMatchesBest =
|
||||||
|
|
@ -283,14 +260,10 @@ export function renderSearch($app) {
|
||||||
: false;
|
: false;
|
||||||
|
|
||||||
const storeIsBest =
|
const storeIsBest =
|
||||||
normStore(storeLabelRaw) &&
|
normStore(storeLabelRaw) && normStore(bestStoreRaw) && normStore(storeLabelRaw) === normStore(bestStoreRaw);
|
||||||
normStore(bestStoreRaw) &&
|
|
||||||
normStore(storeLabelRaw) === normStore(bestStoreRaw);
|
|
||||||
|
|
||||||
const saleIsCheapestHere =
|
const saleIsCheapestHere = kind === "price_down" && storeIsBest && priceMatchesBest;
|
||||||
kind === "price_down" && storeIsBest && priceMatchesBest;
|
const saleIsTiedCheapest = kind === "price_down" && !storeIsBest && priceMatchesBest;
|
||||||
const saleIsTiedCheapest =
|
|
||||||
kind === "price_down" && !storeIsBest && priceMatchesBest;
|
|
||||||
const saleIsCheapest = saleIsCheapestHere || saleIsTiedCheapest;
|
const saleIsCheapest = saleIsCheapestHere || saleIsTiedCheapest;
|
||||||
|
|
||||||
// Bucketed scoring (higher = earlier)
|
// Bucketed scoring (higher = earlier)
|
||||||
|
|
@ -343,8 +316,7 @@ export function renderSearch($app) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const canon =
|
const canon = typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x;
|
||||||
typeof canonicalSkuFn === "function" ? canonicalSkuFn : (x) => x;
|
|
||||||
|
|
||||||
const nowMs = Date.now();
|
const nowMs = Date.now();
|
||||||
const cutoffMs = nowMs - 3 * 24 * 60 * 60 * 1000;
|
const cutoffMs = nowMs - 3 * 24 * 60 * 60 * 1000;
|
||||||
|
|
@ -400,9 +372,7 @@ export function renderSearch($app) {
|
||||||
!best ||
|
!best ||
|
||||||
meta.score > best.meta.score ||
|
meta.score > best.meta.score ||
|
||||||
(meta.score === best.meta.score && meta.tie > best.meta.tie) ||
|
(meta.score === best.meta.score && meta.tie > best.meta.tie) ||
|
||||||
(meta.score === best.meta.score &&
|
(meta.score === best.meta.score && meta.tie === best.meta.tie && ms > best.ms)
|
||||||
meta.tie === best.meta.tie &&
|
|
||||||
ms > best.ms)
|
|
||||||
) {
|
) {
|
||||||
best = { r, meta, ms };
|
best = { r, meta, ms };
|
||||||
}
|
}
|
||||||
|
|
@ -455,22 +425,18 @@ export function renderSearch($app) {
|
||||||
const href = String(r.url || "").trim();
|
const href = String(r.url || "").trim();
|
||||||
const storeBadge = href
|
const storeBadge = href
|
||||||
? `<a class="badge" href="${esc(
|
? `<a class="badge" href="${esc(
|
||||||
href
|
href,
|
||||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
(r.storeLabel || r.store || "") + plus
|
(r.storeLabel || r.store || "") + plus,
|
||||||
)}</a>`
|
)}</a>`
|
||||||
: `<span class="badge">${esc(
|
: `<span class="badge">${esc((r.storeLabel || r.store || "") + plus)}</span>`;
|
||||||
(r.storeLabel || r.store || "") + plus
|
|
||||||
)}</span>`;
|
|
||||||
|
|
||||||
const dateBadge = when
|
const dateBadge = when ? `<span class="badge mono">${esc(when)}</span>` : "";
|
||||||
? `<span class="badge mono">${esc(when)}</span>`
|
|
||||||
: "";
|
|
||||||
|
|
||||||
const offBadge =
|
const offBadge =
|
||||||
meta.kind === "price_down" && meta.pctOff !== null
|
meta.kind === "price_down" && meta.pctOff !== null
|
||||||
? `<span class="badge" style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);">[${esc(
|
? `<span class="badge" style="margin-left:6px; color:rgba(20,110,40,0.95); background:rgba(20,110,40,0.10); border:1px solid rgba(20,110,40,0.20);">[${esc(
|
||||||
meta.pctOff
|
meta.pctOff,
|
||||||
)}% Off]</span>`
|
)}% Off]</span>`
|
||||||
: "";
|
: "";
|
||||||
|
|
||||||
|
|
@ -491,9 +457,9 @@ export function renderSearch($app) {
|
||||||
<div class="itemTop">
|
<div class="itemTop">
|
||||||
<div class="itemName">${esc(r.name || "(no name)")}</div>
|
<div class="itemName">${esc(r.name || "(no name)")}</div>
|
||||||
<a class="badge mono skuLink" href="${esc(
|
<a class="badge mono skuLink" href="${esc(
|
||||||
skuLink
|
skuLink,
|
||||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
displaySku(sku)
|
displaySku(sku),
|
||||||
)}</a>
|
)}</a>
|
||||||
</div>
|
</div>
|
||||||
<div class="metaRow">
|
<div class="metaRow">
|
||||||
|
|
@ -527,9 +493,7 @@ export function renderSearch($app) {
|
||||||
const tokens = tokenizeQuery($q.value);
|
const tokens = tokenizeQuery($q.value);
|
||||||
if (!tokens.length) return;
|
if (!tokens.length) return;
|
||||||
|
|
||||||
const matches = allAgg.filter((it) =>
|
const matches = allAgg.filter((it) => matchesAllTokens(it.searchText, tokens));
|
||||||
matchesAllTokens(it.searchText, tokens)
|
|
||||||
);
|
|
||||||
|
|
||||||
const wantCodes = new Set(smwsDistilleryCodesForQueryPrefix($q.value));
|
const wantCodes = new Set(smwsDistilleryCodesForQueryPrefix($q.value));
|
||||||
if (!wantCodes.size) {
|
if (!wantCodes.size) {
|
||||||
|
|
@ -571,15 +535,11 @@ export function renderSearch($app) {
|
||||||
if (tokens.length) {
|
if (tokens.length) {
|
||||||
applySearch();
|
applySearch();
|
||||||
} else {
|
} else {
|
||||||
return loadRecent().then((recent) =>
|
return loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku));
|
||||||
renderRecent(recent, rules.canonicalSku)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch((e) => {
|
.catch((e) => {
|
||||||
$results.innerHTML = `<div class="small">Failed to load: ${esc(
|
$results.innerHTML = `<div class="small">Failed to load: ${esc(e.message)}</div>`;
|
||||||
e.message
|
|
||||||
)}</div>`;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
$clearSearch.addEventListener("click", () => {
|
$clearSearch.addEventListener("click", () => {
|
||||||
|
|
@ -588,9 +548,7 @@ export function renderSearch($app) {
|
||||||
saveQuery("");
|
saveQuery("");
|
||||||
}
|
}
|
||||||
loadSkuRules()
|
loadSkuRules()
|
||||||
.then((rules) =>
|
.then((rules) => loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku)))
|
||||||
loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku))
|
|
||||||
)
|
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
$results.innerHTML = `<div class="small">Type to search…</div>`;
|
$results.innerHTML = `<div class="small">Type to search…</div>`;
|
||||||
});
|
});
|
||||||
|
|
@ -605,9 +563,7 @@ export function renderSearch($app) {
|
||||||
const tokens = tokenizeQuery($q.value);
|
const tokens = tokenizeQuery($q.value);
|
||||||
if (!tokens.length) {
|
if (!tokens.length) {
|
||||||
loadSkuRules()
|
loadSkuRules()
|
||||||
.then((rules) =>
|
.then((rules) => loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku)))
|
||||||
loadRecent().then((recent) => renderRecent(recent, rules.canonicalSku))
|
|
||||||
)
|
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
$results.innerHTML = `<div class="small">Type to search…</div>`;
|
$results.innerHTML = `<div class="small">Type to search…</div>`;
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -57,4 +57,3 @@ export function parsePriceToNumber(v) {
|
||||||
for (const t of tokens) if (!hayNorm.includes(t)) return false;
|
for (const t of tokens) if (!hayNorm.includes(t)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1,10 +1,5 @@
|
||||||
import { esc } from "./dom.js";
|
import { esc } from "./dom.js";
|
||||||
import {
|
import { fetchJson, inferGithubOwnerRepo, githubFetchFileAtSha, githubListCommits } from "./api.js";
|
||||||
fetchJson,
|
|
||||||
inferGithubOwnerRepo,
|
|
||||||
githubFetchFileAtSha,
|
|
||||||
githubListCommits,
|
|
||||||
} from "./api.js";
|
|
||||||
import { buildStoreColorMap, storeColor, datasetStrokeWidth, lighten } from "./storeColors.js";
|
import { buildStoreColorMap, storeColor, datasetStrokeWidth, lighten } from "./storeColors.js";
|
||||||
|
|
||||||
let _chart = null;
|
let _chart = null;
|
||||||
|
|
@ -47,8 +42,7 @@ function ensureChartJs() {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const s = document.createElement("script");
|
const s = document.createElement("script");
|
||||||
// UMD build -> window.Chart
|
// UMD build -> window.Chart
|
||||||
s.src =
|
s.src = "https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js";
|
||||||
"https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js";
|
|
||||||
s.async = true;
|
s.async = true;
|
||||||
s.onload = () => resolve(window.Chart);
|
s.onload = () => resolve(window.Chart);
|
||||||
s.onerror = () => reject(new Error("Failed to load Chart.js"));
|
s.onerror = () => reject(new Error("Failed to load Chart.js"));
|
||||||
|
|
@ -117,15 +111,7 @@ function matchesAllTokens(haystack, tokens) {
|
||||||
|
|
||||||
function rowSearchText(r) {
|
function rowSearchText(r) {
|
||||||
const rep = r?.representative || {};
|
const rep = r?.representative || {};
|
||||||
return [
|
return [r?.canonSku, rep?.name, rep?.skuRaw, rep?.skuKey, rep?.categoryLabel, rep?.storeLabel, rep?.storeKey]
|
||||||
r?.canonSku,
|
|
||||||
rep?.name,
|
|
||||||
rep?.skuRaw,
|
|
||||||
rep?.skuKey,
|
|
||||||
rep?.categoryLabel,
|
|
||||||
rep?.storeLabel,
|
|
||||||
rep?.storeKey,
|
|
||||||
]
|
|
||||||
.map((x) => String(x || "").trim())
|
.map((x) => String(x || "").trim())
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join(" | ")
|
.join(" | ")
|
||||||
|
|
@ -286,9 +272,7 @@ async function loadCommitsFallback({ owner, repo, branch, relPath }) {
|
||||||
const byDate = new Map();
|
const byDate = new Map();
|
||||||
for (const c of apiCommits) {
|
for (const c of apiCommits) {
|
||||||
const sha = String(c?.sha || "");
|
const sha = String(c?.sha || "");
|
||||||
const ts = String(
|
const ts = String(c?.commit?.committer?.date || c?.commit?.author?.date || "");
|
||||||
c?.commit?.committer?.date || c?.commit?.author?.date || ""
|
|
||||||
);
|
|
||||||
const d = dateOnly(ts);
|
const d = dateOnly(ts);
|
||||||
if (!sha || !d) continue;
|
if (!sha || !d) continue;
|
||||||
if (!byDate.has(d)) byDate.set(d, { sha, date: d, ts });
|
if (!byDate.has(d)) byDate.set(d, { sha, date: d, ts });
|
||||||
|
|
@ -310,13 +294,10 @@ async function loadRawSeries({ group, size, onStatus }) {
|
||||||
|
|
||||||
const manifest = await loadCommonCommitsManifest();
|
const manifest = await loadCommonCommitsManifest();
|
||||||
|
|
||||||
let commits = Array.isArray(manifest?.files?.[rel])
|
let commits = Array.isArray(manifest?.files?.[rel]) ? manifest.files[rel] : null;
|
||||||
? manifest.files[rel]
|
|
||||||
: null;
|
|
||||||
|
|
||||||
if (!commits || !commits.length) {
|
if (!commits || !commits.length) {
|
||||||
if (typeof onStatus === "function")
|
if (typeof onStatus === "function") onStatus(`Commits manifest missing for ${rel}; using GitHub API fallback…`);
|
||||||
onStatus(`Commits manifest missing for ${rel}; using GitHub API fallback…`);
|
|
||||||
commits = await loadCommitsFallback({ owner, repo, branch, relPath: rel });
|
commits = await loadCommitsFallback({ owner, repo, branch, relPath: rel });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -328,11 +309,7 @@ async function loadRawSeries({ group, size, onStatus }) {
|
||||||
|
|
||||||
const cacheKey = `${group}:${size}`;
|
const cacheKey = `${group}:${size}`;
|
||||||
const cached = RAW_SERIES_CACHE.get(cacheKey);
|
const cached = RAW_SERIES_CACHE.get(cacheKey);
|
||||||
if (
|
if (cached && cached.latestSha === latestSha && cached.labels?.length === commits.length) {
|
||||||
cached &&
|
|
||||||
cached.latestSha === latestSha &&
|
|
||||||
cached.labels?.length === commits.length
|
|
||||||
) {
|
|
||||||
return cached;
|
return cached;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -340,21 +317,15 @@ async function loadRawSeries({ group, size, onStatus }) {
|
||||||
const limitNet = makeLimiter(NET_CONCURRENCY);
|
const limitNet = makeLimiter(NET_CONCURRENCY);
|
||||||
|
|
||||||
if (typeof onStatus === "function") onStatus(`Loading stores…`);
|
if (typeof onStatus === "function") onStatus(`Loading stores…`);
|
||||||
const newestReport = await limitNet(() =>
|
const newestReport = await limitNet(() => githubFetchFileAtSha({ owner, repo, sha: latestSha, path: rel }));
|
||||||
githubFetchFileAtSha({ owner, repo, sha: latestSha, path: rel })
|
|
||||||
);
|
|
||||||
|
|
||||||
const stores = Array.isArray(newestReport?.stores)
|
const stores = Array.isArray(newestReport?.stores) ? newestReport.stores.map(String) : [];
|
||||||
? newestReport.stores.map(String)
|
if (!stores.length) throw new Error(`No stores found in ${rel} at ${latestSha.slice(0, 7)}`);
|
||||||
: [];
|
|
||||||
if (!stores.length)
|
|
||||||
throw new Error(`No stores found in ${rel} at ${latestSha.slice(0, 7)}`);
|
|
||||||
|
|
||||||
const labels = commits.map((c) => String(c.date || "")).filter(Boolean);
|
const labels = commits.map((c) => String(c.date || "")).filter(Boolean);
|
||||||
const shaByIdx = commits.map((c) => String(c.sha || ""));
|
const shaByIdx = commits.map((c) => String(c.sha || ""));
|
||||||
|
|
||||||
if (typeof onStatus === "function")
|
if (typeof onStatus === "function") onStatus(`Loading ${labels.length} day(s)…`);
|
||||||
onStatus(`Loading ${labels.length} day(s)…`);
|
|
||||||
|
|
||||||
const reportsByIdx = new Array(shaByIdx.length).fill(null);
|
const reportsByIdx = new Array(shaByIdx.length).fill(null);
|
||||||
|
|
||||||
|
|
@ -373,15 +344,12 @@ async function loadRawSeries({ group, size, onStatus }) {
|
||||||
reportsByIdx[idx] = null;
|
reportsByIdx[idx] = null;
|
||||||
} finally {
|
} finally {
|
||||||
done++;
|
done++;
|
||||||
if (
|
if (typeof onStatus === "function" && (done % 10 === 0 || done === shaByIdx.length)) {
|
||||||
typeof onStatus === "function" &&
|
|
||||||
(done % 10 === 0 || done === shaByIdx.length)
|
|
||||||
) {
|
|
||||||
onStatus(`Loading ${labels.length} day(s)… ${done}/${labels.length}`);
|
onStatus(`Loading ${labels.length} day(s)… ${done}/${labels.length}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}),
|
||||||
)
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const out = { latestSha, labels, stores, commits, reportsByIdx };
|
const out = { latestSha, labels, stores, commits, reportsByIdx };
|
||||||
|
|
@ -440,7 +408,8 @@ function computeSeriesFromRaw(raw, filter) {
|
||||||
/* ---------------- y-axis bounds ---------------- */
|
/* ---------------- y-axis bounds ---------------- */
|
||||||
|
|
||||||
function computeYBounds(seriesByStore, minSpan = 6, pad = 1) {
|
function computeYBounds(seriesByStore, minSpan = 6, pad = 1) {
|
||||||
let mn = Infinity, mx = -Infinity;
|
let mn = Infinity,
|
||||||
|
mx = -Infinity;
|
||||||
|
|
||||||
for (const arr of Object.values(seriesByStore || {})) {
|
for (const arr of Object.values(seriesByStore || {})) {
|
||||||
if (!Array.isArray(arr)) continue;
|
if (!Array.isArray(arr)) continue;
|
||||||
|
|
@ -471,7 +440,6 @@ function computeYBounds(seriesByStore, minSpan = 6, pad = 1) {
|
||||||
return { min: mn, max: mx };
|
return { min: mn, max: mx };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* ---------------- prefs ---------------- */
|
/* ---------------- prefs ---------------- */
|
||||||
|
|
||||||
const LS_GROUP = "stviz:v1:stats:group";
|
const LS_GROUP = "stviz:v1:stats:group";
|
||||||
|
|
@ -669,9 +637,7 @@ export async function renderStats($app) {
|
||||||
|
|
||||||
function updatePriceLabel() {
|
function updatePriceLabel() {
|
||||||
if (!$priceLabel) return;
|
if (!$priceLabel) return;
|
||||||
$priceLabel.textContent = `${formatDollars(selectedMinPrice)} – ${formatDollars(
|
$priceLabel.textContent = `${formatDollars(selectedMinPrice)} – ${formatDollars(selectedMaxPrice)}`;
|
||||||
selectedMaxPrice
|
|
||||||
)}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function saveFilterPrefs(group, size) {
|
function saveFilterPrefs(group, size) {
|
||||||
|
|
@ -719,7 +685,8 @@ export async function renderStats($app) {
|
||||||
const order = stores
|
const order = stores
|
||||||
.map((s) => ({ s, v: lastFiniteFromEnd(seriesByStore[s]) }))
|
.map((s) => ({ s, v: lastFiniteFromEnd(seriesByStore[s]) }))
|
||||||
.sort((a, b) => {
|
.sort((a, b) => {
|
||||||
const av = a.v, bv = b.v;
|
const av = a.v,
|
||||||
|
bv = b.v;
|
||||||
if (av === null && bv === null) return displayStoreName(a.s).localeCompare(displayStoreName(b.s));
|
if (av === null && bv === null) return displayStoreName(a.s).localeCompare(displayStoreName(b.s));
|
||||||
if (av === null) return 1;
|
if (av === null) return 1;
|
||||||
if (bv === null) return -1;
|
if (bv === null) return -1;
|
||||||
|
|
@ -797,9 +764,7 @@ export async function renderStats($app) {
|
||||||
grid: {
|
grid: {
|
||||||
drawBorder: false,
|
drawBorder: false,
|
||||||
color: (ctx) =>
|
color: (ctx) =>
|
||||||
ctx.tick.value === 0
|
ctx.tick.value === 0 ? "rgba(154,166,178,0.35)" : "rgba(154,166,178,0.18)",
|
||||||
? "rgba(154,166,178,0.35)"
|
|
||||||
: "rgba(154,166,178,0.18)",
|
|
||||||
lineWidth: 1,
|
lineWidth: 1,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -827,8 +792,7 @@ export async function renderStats($app) {
|
||||||
|
|
||||||
// floor is ALWAYS 0 now
|
// floor is ALWAYS 0 now
|
||||||
boundMin = 0;
|
boundMin = 0;
|
||||||
boundMax =
|
boundMax = Number.isFinite(b.max) && b.max > 0 ? Math.ceil(b.max) : 1000;
|
||||||
Number.isFinite(b.max) && b.max > 0 ? Math.ceil(b.max) : 1000;
|
|
||||||
|
|
||||||
const saved = loadFilterPrefs(group, size);
|
const saved = loadFilterPrefs(group, size);
|
||||||
if ($q) $q.value = saved.q || "";
|
if ($q) $q.value = saved.q || "";
|
||||||
|
|
@ -850,8 +814,7 @@ export async function renderStats($app) {
|
||||||
selectedMinPrice = clampAndRound(wantMin);
|
selectedMinPrice = clampAndRound(wantMin);
|
||||||
selectedMaxPrice = clampAndRound(wantMax);
|
selectedMaxPrice = clampAndRound(wantMax);
|
||||||
|
|
||||||
if (selectedMinPrice > selectedMaxPrice)
|
if (selectedMinPrice > selectedMaxPrice) selectedMinPrice = selectedMaxPrice;
|
||||||
selectedMinPrice = selectedMaxPrice;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
setSliderFromPrice($minR, selectedMinPrice);
|
setSliderFromPrice($minR, selectedMinPrice);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
function normalizeId(s) {
|
function normalizeId(s) {
|
||||||
return String(s || "").toLowerCase().replace(/[^a-z0-9]+/g, "");
|
return String(s || "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Map normalized store *labels* to canonical ids used by OVERRIDES
|
// Map normalized store *labels* to canonical ids used by OVERRIDES
|
||||||
|
|
@ -45,12 +47,36 @@ const OVERRIDES = {
|
||||||
|
|
||||||
// High-contrast qualitative palette
|
// High-contrast qualitative palette
|
||||||
const PALETTE = [
|
const PALETTE = [
|
||||||
"#1F77B4", "#FF7F0E", "#2CA02C", "#D62728", "#9467BD",
|
"#1F77B4",
|
||||||
"#8C564B", "#E377C2", "#7F7F7F", "#17BECF", "#BCBD22",
|
"#FF7F0E",
|
||||||
"#AEC7E8", "#FFBB78", "#98DF8A", "#FF9896", "#C5B0D5",
|
"#2CA02C",
|
||||||
"#C49C94", "#F7B6D2", "#C7C7C7", "#9EDAE5", "#DBDB8D",
|
"#D62728",
|
||||||
"#393B79", "#637939", "#8C6D31", "#843C39", "#7B4173",
|
"#9467BD",
|
||||||
"#3182BD", "#31A354", "#756BB1", "#636363", "#E6550D",
|
"#8C564B",
|
||||||
|
"#E377C2",
|
||||||
|
"#7F7F7F",
|
||||||
|
"#17BECF",
|
||||||
|
"#BCBD22",
|
||||||
|
"#AEC7E8",
|
||||||
|
"#FFBB78",
|
||||||
|
"#98DF8A",
|
||||||
|
"#FF9896",
|
||||||
|
"#C5B0D5",
|
||||||
|
"#C49C94",
|
||||||
|
"#F7B6D2",
|
||||||
|
"#C7C7C7",
|
||||||
|
"#9EDAE5",
|
||||||
|
"#DBDB8D",
|
||||||
|
"#393B79",
|
||||||
|
"#637939",
|
||||||
|
"#8C6D31",
|
||||||
|
"#843C39",
|
||||||
|
"#7B4173",
|
||||||
|
"#3182BD",
|
||||||
|
"#31A354",
|
||||||
|
"#756BB1",
|
||||||
|
"#636363",
|
||||||
|
"#E6550D",
|
||||||
];
|
];
|
||||||
|
|
||||||
function uniq(arr) {
|
function uniq(arr) {
|
||||||
|
|
@ -89,11 +115,15 @@ const DEFAULT_UNIVERSE = buildUniverse(Object.keys(OVERRIDES), [
|
||||||
"vintage",
|
"vintage",
|
||||||
"vintagespirits",
|
"vintagespirits",
|
||||||
"willowpark",
|
"willowpark",
|
||||||
"arc"
|
"arc",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
function isWhiteHex(c) {
|
function isWhiteHex(c) {
|
||||||
return String(c || "").trim().toUpperCase() === "#FFFFFF";
|
return (
|
||||||
|
String(c || "")
|
||||||
|
.trim()
|
||||||
|
.toUpperCase() === "#FFFFFF"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildStoreColorMap(extraUniverse = []) {
|
export function buildStoreColorMap(extraUniverse = []) {
|
||||||
|
|
@ -112,9 +142,9 @@ export function buildStoreColorMap(extraUniverse = []) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter palette to avoid collisions and keep white/black reserved
|
// Filter palette to avoid collisions and keep white/black reserved
|
||||||
const palette = PALETTE
|
const palette = PALETTE.map((c) => String(c).toUpperCase()).filter(
|
||||||
.map((c) => String(c).toUpperCase())
|
(c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111",
|
||||||
.filter((c) => !used.has(c) && c !== "#FFFFFF" && c !== "#111111");
|
);
|
||||||
|
|
||||||
let pi = 0;
|
let pi = 0;
|
||||||
for (const id of universe) {
|
for (const id of universe) {
|
||||||
|
|
@ -167,8 +197,7 @@ function hexToRgb(hex) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function rgbToHex({ r, g, b }) {
|
function rgbToHex({ r, g, b }) {
|
||||||
const h = (x) =>
|
const h = (x) => clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0");
|
||||||
clamp(Math.round(x), 0, 255).toString(16).padStart(2, "0");
|
|
||||||
return `#${h(r)}${h(g)}${h(b)}`;
|
return `#${h(r)}${h(g)}${h(b)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,13 @@
|
||||||
import { esc, renderThumbHtml } from "./dom.js";
|
import { esc, renderThumbHtml } from "./dom.js";
|
||||||
import {
|
import { tokenizeQuery, matchesAllTokens, displaySku, keySkuForRow, parsePriceToNumber } from "./sku.js";
|
||||||
tokenizeQuery,
|
|
||||||
matchesAllTokens,
|
|
||||||
displaySku,
|
|
||||||
keySkuForRow,
|
|
||||||
parsePriceToNumber,
|
|
||||||
} from "./sku.js";
|
|
||||||
import { loadIndex, loadRecent } from "./state.js";
|
import { loadIndex, loadRecent } from "./state.js";
|
||||||
import { aggregateBySku } from "./catalog.js";
|
import { aggregateBySku } from "./catalog.js";
|
||||||
import { loadSkuRules } from "./mapping.js";
|
import { loadSkuRules } from "./mapping.js";
|
||||||
|
|
||||||
function normStoreLabel(s) {
|
function normStoreLabel(s) {
|
||||||
return String(s || "").trim().toLowerCase();
|
return String(s || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
function abbrevStoreLabel(s) {
|
function abbrevStoreLabel(s) {
|
||||||
|
|
@ -38,9 +34,7 @@ function readLinkHrefForSkuInStore(listingsLive, canonSku, storeLabelNorm) {
|
||||||
const store = normStoreLabel(r.storeLabel || r.store || "");
|
const store = normStoreLabel(r.storeLabel || r.store || "");
|
||||||
if (store !== storeLabelNorm) continue;
|
if (store !== storeLabelNorm) continue;
|
||||||
|
|
||||||
const skuKey = String(
|
const skuKey = String(rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r));
|
||||||
rulesCache?.canonicalSku(keySkuForRow(r)) || keySkuForRow(r)
|
|
||||||
);
|
|
||||||
if (skuKey !== canonSku) continue;
|
if (skuKey !== canonSku) continue;
|
||||||
|
|
||||||
const u = String(r.url || "").trim();
|
const u = String(r.url || "").trim();
|
||||||
|
|
@ -60,8 +54,7 @@ let rulesCache = null;
|
||||||
|
|
||||||
export async function renderStore($app, storeLabelRaw) {
|
export async function renderStore($app, storeLabelRaw) {
|
||||||
const storeLabel = String(storeLabelRaw || "").trim();
|
const storeLabel = String(storeLabelRaw || "").trim();
|
||||||
const storeLabelShort =
|
const storeLabelShort = abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
|
||||||
abbrevStoreLabel(storeLabel) || (storeLabel ? storeLabel : "Store");
|
|
||||||
|
|
||||||
$app.innerHTML = `
|
$app.innerHTML = `
|
||||||
<div class="container">
|
<div class="container">
|
||||||
|
|
@ -184,8 +177,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
// Persist max price per store (clamped later once bounds known)
|
// Persist max price per store (clamped later once bounds known)
|
||||||
const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`;
|
const LS_MAX_PRICE = `viz:storeMaxPrice:${storeNorm}`;
|
||||||
const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE);
|
const savedMaxPriceRaw = localStorage.getItem(LS_MAX_PRICE);
|
||||||
let savedMaxPrice =
|
let savedMaxPrice = savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
|
||||||
savedMaxPriceRaw !== null ? Number(savedMaxPriceRaw) : null;
|
|
||||||
if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null;
|
if (!Number.isFinite(savedMaxPrice)) savedMaxPrice = null;
|
||||||
|
|
||||||
// Persist exclusives sort per store
|
// Persist exclusives sort per store
|
||||||
|
|
@ -264,15 +256,13 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
if (!r) return null;
|
if (!r) return null;
|
||||||
|
|
||||||
const kind = normalizeKindForPrice(r);
|
const kind = normalizeKindForPrice(r);
|
||||||
if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change")
|
if (kind !== "price_down" && kind !== "price_up" && kind !== "price_change") return null;
|
||||||
return null;
|
|
||||||
|
|
||||||
const oldStr = String(r?.oldPrice || "").trim();
|
const oldStr = String(r?.oldPrice || "").trim();
|
||||||
const newStr = String(r?.newPrice || "").trim();
|
const newStr = String(r?.newPrice || "").trim();
|
||||||
const oldN = parsePriceToNumber(oldStr);
|
const oldN = parsePriceToNumber(oldStr);
|
||||||
const newN = parsePriceToNumber(newStr);
|
const newN = parsePriceToNumber(newStr);
|
||||||
if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0))
|
if (!Number.isFinite(oldN) || !Number.isFinite(newN) || !(oldN > 0)) return null;
|
||||||
return null;
|
|
||||||
|
|
||||||
const delta = newN - oldN; // negative = down
|
const delta = newN - oldN; // negative = down
|
||||||
const pct = Math.round(((newN - oldN) / oldN) * 100); // negative = down
|
const pct = Math.round(((newN - oldN) / oldN) * 100); // negative = down
|
||||||
|
|
@ -368,9 +358,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store-specific live rows only (in-stock for that store)
|
// Store-specific live rows only (in-stock for that store)
|
||||||
const rowsStoreLive = liveAll.filter(
|
const rowsStoreLive = liveAll.filter((r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm);
|
||||||
(r) => normStoreLabel(r.storeLabel || r.store || "") === storeNorm
|
|
||||||
);
|
|
||||||
|
|
||||||
// Aggregate in this store, grouped by canonical SKU (so mappings count as same bottle)
|
// Aggregate in this store, grouped by canonical SKU (so mappings count as same bottle)
|
||||||
let items = aggregateBySku(rowsStoreLive, rules.canonicalSku);
|
let items = aggregateBySku(rowsStoreLive, rules.canonicalSku);
|
||||||
|
|
@ -383,35 +371,23 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]);
|
const liveStoreSet = storesBySku.get(sku) || new Set([storeNorm]);
|
||||||
const everStoreSet = everStoresBySku.get(sku) || liveStoreSet;
|
const everStoreSet = everStoresBySku.get(sku) || liveStoreSet;
|
||||||
|
|
||||||
const soloLiveHere =
|
const soloLiveHere = liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
|
||||||
liveStoreSet.size === 1 && liveStoreSet.has(storeNorm);
|
|
||||||
const lastStock = soloLiveHere && everStoreSet.size > 1;
|
const lastStock = soloLiveHere && everStoreSet.size > 1;
|
||||||
const exclusive = soloLiveHere && !lastStock;
|
const exclusive = soloLiveHere && !lastStock;
|
||||||
|
|
||||||
const storePrice = Number.isFinite(it.cheapestPriceNum)
|
const storePrice = Number.isFinite(it.cheapestPriceNum) ? it.cheapestPriceNum : null;
|
||||||
? it.cheapestPriceNum
|
|
||||||
: null;
|
|
||||||
const bestAll = bestAllPrice(sku);
|
const bestAll = bestAllPrice(sku);
|
||||||
const other = bestOtherPrice(sku, storeNorm);
|
const other = bestOtherPrice(sku, storeNorm);
|
||||||
|
|
||||||
const isBest =
|
const isBest = storePrice !== null && bestAll !== null ? storePrice <= bestAll + EPS : false;
|
||||||
storePrice !== null && bestAll !== null
|
|
||||||
? storePrice <= bestAll + EPS
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const diffVsOtherDollar =
|
const diffVsOtherDollar = storePrice !== null && other !== null ? storePrice - other : null;
|
||||||
storePrice !== null && other !== null ? storePrice - other : null;
|
|
||||||
const diffVsOtherPct =
|
const diffVsOtherPct =
|
||||||
storePrice !== null && other !== null && other > 0
|
storePrice !== null && other !== null && other > 0 ? ((storePrice - other) / other) * 100 : null;
|
||||||
? ((storePrice - other) / other) * 100
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const diffVsBestDollar =
|
const diffVsBestDollar = storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
|
||||||
storePrice !== null && bestAll !== null ? storePrice - bestAll : null;
|
|
||||||
const diffVsBestPct =
|
const diffVsBestPct =
|
||||||
storePrice !== null && bestAll !== null && bestAll > 0
|
storePrice !== null && bestAll !== null && bestAll > 0 ? ((storePrice - bestAll) / bestAll) * 100 : null;
|
||||||
? ((storePrice - bestAll) / bestAll) * 100
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const firstSeenMs = firstSeenBySkuInStore.get(sku);
|
const firstSeenMs = firstSeenBySkuInStore.get(sku);
|
||||||
const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null;
|
const firstSeen = firstSeenMs !== undefined ? firstSeenMs : null;
|
||||||
|
|
@ -495,9 +471,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
return `$${Math.round(p)}`;
|
return `$${Math.round(p)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
let selectedMaxPrice = clampAndRound(
|
let selectedMaxPrice = clampAndRound(savedMaxPrice !== null ? savedMaxPrice : boundMax);
|
||||||
savedMaxPrice !== null ? savedMaxPrice : boundMax
|
|
||||||
);
|
|
||||||
|
|
||||||
function setSliderFromPrice(p) {
|
function setSliderFromPrice(p) {
|
||||||
const t = tFromPrice(p);
|
const t = tFromPrice(p);
|
||||||
|
|
@ -536,8 +510,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
// ---- Listing display price: keep cents (no rounding) ----
|
// ---- Listing display price: keep cents (no rounding) ----
|
||||||
function listingPriceStr(it) {
|
function listingPriceStr(it) {
|
||||||
const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null;
|
const p = it && Number.isFinite(it._storePrice) ? it._storePrice : null;
|
||||||
if (p === null)
|
if (p === null) return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
|
||||||
return it.cheapestPriceStr ? it.cheapestPriceStr : "(no price)";
|
|
||||||
return `$${p.toFixed(2)}`;
|
return `$${p.toFixed(2)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -620,9 +593,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
: "";
|
: "";
|
||||||
|
|
||||||
const bestBadge =
|
const bestBadge =
|
||||||
!it._exclusive && !it._lastStock && it._isBest
|
!it._exclusive && !it._lastStock && it._isBest ? `<span class="badge badgeBest">Best Price</span>` : "";
|
||||||
? `<span class="badge badgeBest">Best Price</span>`
|
|
||||||
: "";
|
|
||||||
|
|
||||||
const diffBadge = priceBadgeHtml(it);
|
const diffBadge = priceBadgeHtml(it);
|
||||||
const exAnnot = it._exclusive || it._lastStock ? exclusiveAnnotHtml(it) : "";
|
const exAnnot = it._exclusive || it._lastStock ? exclusiveAnnotHtml(it) : "";
|
||||||
|
|
@ -636,9 +607,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
<div class="itemTop">
|
<div class="itemTop">
|
||||||
<div class="itemName">${esc(it.name || "(no name)")}</div>
|
<div class="itemName">${esc(it.name || "(no name)")}</div>
|
||||||
<a class="badge mono skuLink" target="_blank" rel="noopener noreferrer"
|
<a class="badge mono skuLink" target="_blank" rel="noopener noreferrer"
|
||||||
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(
|
href="${esc(skuLink)}" onclick="event.stopPropagation()">${esc(displaySku(it.sku))}</a>
|
||||||
displaySku(it.sku)
|
|
||||||
)}</a>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="metaRow">
|
<div class="metaRow">
|
||||||
${specialBadge}
|
${specialBadge}
|
||||||
|
|
@ -649,9 +618,9 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
${
|
${
|
||||||
href
|
href
|
||||||
? `<a class="badge" href="${esc(
|
? `<a class="badge" href="${esc(
|
||||||
href
|
href,
|
||||||
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
)}" target="_blank" rel="noopener noreferrer" onclick="event.stopPropagation()">${esc(
|
||||||
storeLabelShort
|
storeLabelShort,
|
||||||
)}</a>`
|
)}</a>`
|
||||||
: ``
|
: ``
|
||||||
}
|
}
|
||||||
|
|
@ -686,9 +655,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (pageMax !== null) {
|
if (pageMax !== null) {
|
||||||
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(
|
$status.textContent = `In stock: ${total} item(s) (≤ ${formatDollars(selectedMaxPrice)}).`;
|
||||||
selectedMaxPrice
|
|
||||||
)}).`;
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -703,28 +670,14 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
shownCompare = 0;
|
shownCompare = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const sliceEx = filteredExclusive.slice(
|
const sliceEx = filteredExclusive.slice(shownExclusive, shownExclusive + PAGE_EACH);
|
||||||
shownExclusive,
|
const sliceCo = filteredCompare.slice(shownCompare, shownCompare + PAGE_EACH);
|
||||||
shownExclusive + PAGE_EACH
|
|
||||||
);
|
|
||||||
const sliceCo = filteredCompare.slice(
|
|
||||||
shownCompare,
|
|
||||||
shownCompare + PAGE_EACH
|
|
||||||
);
|
|
||||||
|
|
||||||
shownExclusive += sliceEx.length;
|
shownExclusive += sliceEx.length;
|
||||||
shownCompare += sliceCo.length;
|
shownCompare += sliceCo.length;
|
||||||
|
|
||||||
if (sliceEx.length)
|
if (sliceEx.length) $resultsExclusive.insertAdjacentHTML("beforeend", sliceEx.map(renderCard).join(""));
|
||||||
$resultsExclusive.insertAdjacentHTML(
|
if (sliceCo.length) $resultsCompare.insertAdjacentHTML("beforeend", sliceCo.map(renderCard).join(""));
|
||||||
"beforeend",
|
|
||||||
sliceEx.map(renderCard).join("")
|
|
||||||
);
|
|
||||||
if (sliceCo.length)
|
|
||||||
$resultsCompare.insertAdjacentHTML(
|
|
||||||
"beforeend",
|
|
||||||
sliceCo.map(renderCard).join("")
|
|
||||||
);
|
|
||||||
|
|
||||||
const total = totalFiltered();
|
const total = totalFiltered();
|
||||||
const shown = totalShown();
|
const shown = totalShown();
|
||||||
|
|
@ -778,12 +731,9 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
arr.sort((a, b) => {
|
arr.sort((a, b) => {
|
||||||
const ap = Number.isFinite(a._storePrice) ? a._storePrice : null;
|
const ap = Number.isFinite(a._storePrice) ? a._storePrice : null;
|
||||||
const bp = Number.isFinite(b._storePrice) ? b._storePrice : null;
|
const bp = Number.isFinite(b._storePrice) ? b._storePrice : null;
|
||||||
const aKey =
|
const aKey = ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
|
||||||
ap === null ? (mode === "priceAsc" ? 9e15 : -9e15) : ap;
|
const bKey = bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
|
||||||
const bKey =
|
if (aKey !== bKey) return mode === "priceAsc" ? aKey - bKey : bKey - aKey;
|
||||||
bp === null ? (mode === "priceAsc" ? 9e15 : -9e15) : bp;
|
|
||||||
if (aKey !== bKey)
|
|
||||||
return mode === "priceAsc" ? aKey - bKey : bKey - aKey;
|
|
||||||
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
|
|
@ -793,12 +743,9 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
arr.sort((a, b) => {
|
arr.sort((a, b) => {
|
||||||
const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null;
|
const ad = Number.isFinite(a._firstSeenMs) ? a._firstSeenMs : null;
|
||||||
const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null;
|
const bd = Number.isFinite(b._firstSeenMs) ? b._firstSeenMs : null;
|
||||||
const aKey =
|
const aKey = ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
|
||||||
ad === null ? (mode === "dateAsc" ? 9e15 : -9e15) : ad;
|
const bKey = bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
|
||||||
const bKey =
|
if (aKey !== bKey) return mode === "dateAsc" ? aKey - bKey : bKey - aKey;
|
||||||
bd === null ? (mode === "dateAsc" ? 9e15 : -9e15) : bd;
|
|
||||||
if (aKey !== bKey)
|
|
||||||
return mode === "dateAsc" ? aKey - bKey : bKey - aKey;
|
|
||||||
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
return (String(a.name) + a.sku).localeCompare(String(b.name) + b.sku);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -857,7 +804,7 @@ export async function renderStore($app, storeLabelRaw) {
|
||||||
if (totalShown() >= totalFiltered()) return;
|
if (totalShown() >= totalFiltered()) return;
|
||||||
renderNext(false);
|
renderNext(false);
|
||||||
},
|
},
|
||||||
{ root: null, rootMargin: "600px 0px", threshold: 0.01 }
|
{ root: null, rootMargin: "600px 0px", threshold: 0.01 },
|
||||||
);
|
);
|
||||||
io.observe($sentinel);
|
io.observe($sentinel);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue