spirit-tracker/tools/dedupe_skulinks.js

#!/usr/bin/env node
"use strict";

const fs = require("fs");
const path = require("path");

const DB_DIR = path.join(__dirname, "../data/db");
const LINKS_FILE = path.join(__dirname, "../data/sku_links.json");

function normalizeImplicitSkuKey(k) {
  const s = String(k || "").trim();
  const m = s.match(/^id:(\d{1,6})$/i);
  if (m) return String(m[1]).padStart(6, "0");
  return s;
}

// collect all valid SKUs from db files (normalized)
const validSkus = new Set();

for (const file of fs.readdirSync(DB_DIR)) {
  if (!file.endsWith(".json")) continue;
  const data = JSON.parse(fs.readFileSync(path.join(DB_DIR, file), "utf8"));
  if (!Array.isArray(data.items)) continue;
  for (const item of data.items) {
    if (!item || !item.sku) continue;
    const k = normalizeImplicitSkuKey(item.sku);
    if (k) validSkus.add(k);
  }
}

// load links
const linksData = JSON.parse(fs.readFileSync(LINKS_FILE, "utf8"));
const originalCount = Array.isArray(linksData.links) ? linksData.links.length : 0;

let prunedMissing = 0;
let prunedAuto = 0;
let prunedDup = 0;

const seen = new Set(); // dedupe after normalization
const nextLinks = [];

for (const x of Array.isArray(linksData.links) ? linksData.links : []) {
  const a = normalizeImplicitSkuKey(x?.fromSku);
  const b = normalizeImplicitSkuKey(x?.toSku);

  if (!a || !b) {
    prunedMissing++;
    continue;
  }

  // drop links that are now implicit (id:1234 <-> 001234 etc)
  if (a === b) {
    prunedAuto++;
    continue;
  }

  // keep only links where BOTH normalized skus exist in db
  if (!validSkus.has(a) || !validSkus.has(b)) {
    prunedMissing++;
    continue;
  }

  // dedupe (undirected) after normalization
  const key = a < b ? `${a}|${b}` : `${b}|${a}`;
  if (seen.has(key)) {
    prunedDup++;
    continue;
  }
  seen.add(key);

  // preserve datestamps/metadata; just normalize the SKUs
  nextLinks.push({
    ...x,
    fromSku: a,
    toSku: b,
  });
}

linksData.links = nextLinks;

// write back in place
fs.writeFileSync(LINKS_FILE, JSON.stringify(linksData, null, 2) + "\n");

const totalPruned = originalCount - linksData.links.length;

console.log(`Pruned ${totalPruned} total links`);
console.log(`- ${prunedAuto} now-implicit (id:<1-6> ↔ CSPC)`);
console.log(`- ${prunedMissing} missing/invalid vs db`);
console.log(`- ${prunedDup} duplicates after normalization`);
console.log(`Remaining ${linksData.links.length}`);