spirit-tracker/src/main.js

#!/usr/bin/env node
"use strict";

// NOTE: store filtering is implemented here without touching utils/args.js
const fs = require("fs");
const path = require("path");

const { parseArgs, clampInt } = require("./utils/args");
const { isoTimestampFileSafe } = require("./utils/time");

const { createLogger } = require("./core/logger");
const { createHttpClient } = require("./core/http");

const { createStores, parseProductsSierra } = require("./stores");
const { runAllStores } = require("./tracker/run_all");
const { renderFinalReport } = require("./tracker/report");
const { ensureDir } = require("./tracker/db");

const DEFAULT_UA =
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0 Safari/537.36";

function resolveDir(p, fallback) {
  const v = String(p || "").trim();
  if (!v) return fallback;
  return path.isAbsolute(v) ? v : path.join(process.cwd(), v);
}

function getFlagValue(argv, flag) {
  // Supports:
  //   --stores=a,b
  //   --stores a,b
  const idx = argv.indexOf(flag);
  if (idx >= 0) return argv[idx + 1] || "";
  const pref = `${flag}=`;
  for (const a of argv) {
    if (a.startsWith(pref)) return a.slice(pref.length);
  }
  return "";
}

function normToken(s) {
  return String(s || "")
    .toLowerCase()
    .trim()
    .replace(/[^a-z0-9]+/g, "");
}

function parseStoresFilter(raw) {
  const v = String(raw || "").trim();
  if (!v) return [];
  return v
    .split(",")
    .map((x) => x.trim())
    .filter(Boolean);
}

function filterStoresOrThrow(stores, wantedListRaw) {
  const wanted = parseStoresFilter(wantedListRaw);
  if (!wanted.length) return stores;

  const wantedNorm = wanted.map(normToken).filter(Boolean);

  const matched = [];
  const missing = [];

  for (let i = 0; i < wanted.length; i++) {
    const w = wanted[i];
    const wn = wantedNorm[i];
    if (!wn) continue;

    // match against key/name/host (normalized)
    const hit = stores.find((s) => {
      const candidates = [s.key, s.name, s.host].map(normToken).filter(Boolean);
      return candidates.includes(wn);
    });

    if (hit) matched.push(hit);
    else missing.push(w);
  }

  if (missing.length) {
    const avail = stores
      .map((s) => `${s.key}${s.name ? ` (${s.name})` : ""}`)
      .join(", ");
    throw new Error(
      `Unknown store(s) in --stores: ${missing.join(", ")}\nAvailable: ${avail}`
    );
  }

  // de-dupe by key (in case name+key both matched)
  const uniq = [];
  const seen = new Set();
  for (const s of matched) {
    if (seen.has(s.key)) continue;
    seen.add(s.key);
    uniq.push(s);
  }
  return uniq;
}

async function main() {
  if (typeof fetch !== "function") {
    throw new Error(
      "Global fetch() not found. Please use Node.js 18+ (or newer). "
    );
  }

  const argv = process.argv.slice(2);
  const args = parseArgs(argv);

  const logger = createLogger({ debug: args.debug, colorize: true });

  const config = {
    debug: args.debug,
    maxPages: args.maxPages,
    concurrency: args.concurrency ?? clampInt(process.env.CONCURRENCY, 6, 1, 64),
    staggerMs:
      args.staggerMs ?? clampInt(process.env.STAGGER_MS, 150, 0, 5000),
    maxRetries: clampInt(process.env.MAX_RETRIES, 6, 0, 20),
    timeoutMs: clampInt(process.env.TIMEOUT_MS, 25000, 1000, 120000),
    discoveryGuess:
      args.guess ?? clampInt(process.env.DISCOVERY_GUESS, 20, 1, 5000),
    discoveryStep:
      args.step ?? clampInt(process.env.DISCOVERY_STEP, 5, 1, 500),
    categoryConcurrency: clampInt(process.env.CATEGORY_CONCURRENCY, 5, 1, 64),
    defaultUa: DEFAULT_UA,
    defaultParseProducts: parseProductsSierra,
    dbDir: resolveDir(
      args.dataDir ?? process.env.DATA_DIR,
      path.join(process.cwd(), "data", "db")
    ),
    reportDir: resolveDir(
      args.reportDir ?? process.env.REPORT_DIR,
      path.join(process.cwd(), "reports")
    ),
  };

  ensureDir(config.dbDir);
  ensureDir(config.reportDir);

  const http = createHttpClient({
    maxRetries: config.maxRetries,
    timeoutMs: config.timeoutMs,
    defaultUa: config.defaultUa,
    logger,
  });
  const stores = createStores({ defaultUa: config.defaultUa });

  const storesFilterRaw =
    getFlagValue(argv, "--stores") || String(process.env.STORES || "").trim();

  const storesToRun = filterStoresOrThrow(stores, storesFilterRaw);
  if (storesFilterRaw) {
    logger.info(`Stores filter: ${storesToRun.map((s) => s.key).join(", ")}`);
  }

  const report = await runAllStores(storesToRun, { config, logger, http });

  const meaningful =
    (report?.totals?.newCount || 0) +
      (report?.totals?.updatedCount || 0) +
      (report?.totals?.removedCount || 0) +
      (report?.totals?.restoredCount || 0) +
      (report?.totals?.metaChangedCount || 0) >
    0;

  const reportTextColor = renderFinalReport(report, {
    dbDir: config.dbDir,
    colorize: logger.colorize,
  });
  process.stdout.write(reportTextColor);

  if (!meaningful) {
    logger.ok("No meaningful changes; skipping report write.");
    process.exitCode = 3; // special "no-op" code
    return;
  }

  const reportTextPlain = renderFinalReport(report, {
    dbDir: config.dbDir,
    colorize: false,
  });
  const file = path.join(
    config.reportDir,
    `${isoTimestampFileSafe(new Date())}.txt`
  );
  try {
    fs.writeFileSync(file, reportTextPlain, "utf8");
    logger.ok(`Report saved: ${logger.dim(file)}`);
  } catch (e) {
    logger.warn(`Report save failed: ${e?.message || e}`);
  }
}

module.exports = { main };

if (require.main === module) {
  main().catch((e) => {
    const msg = e && e.stack ? e.stack : String(e);
    // no logger here; keep simple
    console.error(msg);
    process.exitCode = 1;
  });
}