mirror of
https://github.com/samsonjs/spirit-tracker.git
synced 2026-04-27 15:07:43 +00:00
feat: Improvements to strath
This commit is contained in:
parent
54b0c0a8b2
commit
b95bd3544a
1 changed files with 28 additions and 12 deletions
|
|
@ -1,7 +1,7 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
const { decodeHtml, stripTags, cleanText, extractFirstImgUrl } = require("../utils/html");
|
const { decodeHtml, stripTags, cleanText, extractFirstImgUrl } = require("../utils/html");
|
||||||
const { normalizeCspc } = require("../utils/sku");
|
const { normalizeCspc, pickBetterSku } = require("../utils/sku");
|
||||||
const { humanBytes } = require("../utils/bytes");
|
const { humanBytes } = require("../utils/bytes");
|
||||||
const { padLeft, padRight } = require("../utils/string");
|
const { padLeft, padRight } = require("../utils/string");
|
||||||
|
|
||||||
|
|
@ -97,6 +97,13 @@ function extractSkuFromArticle(articleHtml) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function idFromImageUrl(imgUrl) {
|
||||||
|
const s = String(imgUrl || "");
|
||||||
|
// /1487-1_... or /1487_... or /1487-... => 1487
|
||||||
|
const m = s.match(/\/(\d{1,11})(?=[-_])/);
|
||||||
|
return m && m[1] ? `id:${m[1]}` : "";
|
||||||
|
}
|
||||||
|
|
||||||
function looksInStock(articleHtml) {
|
function looksInStock(articleHtml) {
|
||||||
const a = String(articleHtml || "");
|
const a = String(articleHtml || "");
|
||||||
|
|
||||||
|
|
@ -136,21 +143,22 @@ function parseProductFromArticle(articleHtml) {
|
||||||
const price = pickPriceFromArticle(a);
|
const price = pickPriceFromArticle(a);
|
||||||
const productId = extractProductIdFromArticle(a);
|
const productId = extractProductIdFromArticle(a);
|
||||||
|
|
||||||
const skuFromHtml = extractSkuFromArticle(a);
|
|
||||||
const fallbackSku = normalizeCspc(url) || "";
|
|
||||||
|
|
||||||
const img = extractFirstImgUrl(a, "https://www.strathliquor.com/");
|
const img = extractFirstImgUrl(a, "https://www.strathliquor.com/");
|
||||||
|
|
||||||
|
const skuFromHtml = extractSkuFromArticle(a);
|
||||||
|
const skuFromImg = idFromImageUrl(img);
|
||||||
|
const fallbackSku = normalizeCspc(url) || "";
|
||||||
|
|
||||||
return {
|
return {
|
||||||
name,
|
name,
|
||||||
price,
|
price,
|
||||||
url,
|
url,
|
||||||
sku: skuFromHtml || fallbackSku,
|
sku: skuFromHtml || skuFromImg || fallbackSku,
|
||||||
productId,
|
productId,
|
||||||
img,
|
img,
|
||||||
};
|
};
|
||||||
}
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
/* ---------------- Store API paging ---------------- */
|
/* ---------------- Store API paging ---------------- */
|
||||||
|
|
||||||
|
|
@ -219,8 +227,6 @@ function normalizeProductImage(p) {
|
||||||
return direct.startsWith("//") ? `https:${direct}` : direct;
|
return direct.startsWith("//") ? `https:${direct}` : direct;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function toMoneyStringFromMinorUnits(valueStr, minorUnit) {
|
function toMoneyStringFromMinorUnits(valueStr, minorUnit) {
|
||||||
const mu = Number(minorUnit);
|
const mu = Number(minorUnit);
|
||||||
if (!Number.isFinite(mu) || mu < 0 || mu > 6) return "";
|
if (!Number.isFinite(mu) || mu < 0 || mu > 6) return "";
|
||||||
|
|
@ -393,16 +399,24 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
const sku = normalizeProductSku(p);
|
const sku = normalizeProductSku(p);
|
||||||
const productId = normalizeProductId(p);
|
const productId = normalizeProductId(p);
|
||||||
|
|
||||||
const fallbackSku = sku || normalizeCspc(url) || "";
|
|
||||||
|
|
||||||
const prev = discovered.get(url) || null;
|
const prev = discovered.get(url) || null;
|
||||||
const img = normalizeProductImage(p) || (prev && prev.img) || "";
|
|
||||||
|
const apiImg = normalizeProductImage(p) || "";
|
||||||
|
const img = apiImg || (prev && prev.img) || "";
|
||||||
|
|
||||||
|
const skuFromApiImg = idFromImageUrl(apiImg);
|
||||||
|
const fallbackSku = sku || skuFromApiImg || normalizeCspc(url) || "";
|
||||||
|
|
||||||
|
const newSku = sku || fallbackSku;
|
||||||
|
const mergedSku = pickBetterSku(newSku, prev && prev.sku);
|
||||||
|
|
||||||
|
|
||||||
discovered.set(url, {
|
discovered.set(url, {
|
||||||
name,
|
name,
|
||||||
price,
|
price,
|
||||||
url,
|
url,
|
||||||
sku: sku || fallbackSku,
|
sku: mergedSku,
|
||||||
productId,
|
productId,
|
||||||
img,
|
img,
|
||||||
});
|
});
|
||||||
|
|
@ -433,7 +447,9 @@ async function scanCategoryStrath(ctx, prevDb, report) {
|
||||||
|
|
||||||
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
ctx.logger.ok(`${ctx.catPrefixOut} | Unique products (this run): ${discovered.size}`);
|
||||||
|
|
||||||
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, { storeLabel: ctx.store.name });
|
const { merged, newItems, updatedItems, removedItems, restoredItems } = mergeDiscoveredIntoDb(prevDb, discovered, {
|
||||||
|
storeLabel: ctx.store.name,
|
||||||
|
});
|
||||||
|
|
||||||
const dbObj = buildDbObject(ctx, merged);
|
const dbObj = buildDbObject(ctx, merged);
|
||||||
writeJsonAtomic(ctx.dbFile, dbObj);
|
writeJsonAtomic(ctx.dbFile, dbObj);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue