/**
 * Tag every SEC filing HTML with its generator tool.
 *
 * Usage: bun run ts/scripts/tag-generators.ts
 *
 * Reads first 20KB of each HTML file in data/raw/html/, detects the
 * generator using heuristics ported from scripts/detect_generators.py,
 * and writes a mapping JSONL to data/paragraphs/quality/generator-tags.jsonl.
 *
 * Uses Bun.spawn worker parallelism (same pattern as dapt-corpus-prep.ts).
 */

import {
  readdirSync,
  readFileSync,
  writeFileSync,
  mkdirSync,
  unlinkSync,
  createReadStream,
} from "node:fs";
import { createInterface } from "node:readline";
import { cpus } from "node:os";
import { basename } from "node:path";

const HTML_DIR = "data/raw/html";
const OUTPUT_DIR = "data/paragraphs/quality";
const OUTPUT_FILE = `${OUTPUT_DIR}/generator-tags.jsonl`;
const READ_BYTES = 20_000;

// Known SEC filing agent CIKs (accession number prefixes)
const FILING_AGENT_CIKS: Record<string, string> = {
  "0000950170": "Donnelley Financial Solutions",
  "0001193125": "Donnelley Financial Solutions",
  "0001558370": "Toppan Merrill",
  "0001654954": "Toppan Merrill",
  "0001104659": "Toppan Merrill",
};

// ─── Generator normalization ───

function normalizeGenerator(raw: string): string {
  const r = raw.trim().toLowerCase();
  if (r.includes("workiva") || r.includes("wdesk")) return "Workiva";
  if (r.includes("donnelley") || r.includes("dfin") || r.includes("rrdonnelley"))
    return "Donnelley Financial Solutions";
  if (r.includes("toppan") || (r.includes("merrill") && r.includes("bridge")))
    return "Toppan Merrill";
  if (r.includes("word") && r.includes("microsoft")) return "Microsoft Word";
  if (r.includes("excel") && r.includes("microsoft")) return "Microsoft Excel";
  if (r.includes("thunderdome")) return "ThunderDome";
  if (r.includes("goxbrl")) return "GoXBRL";
  if (r.includes("compsci")) return "CompSci Transform";
  if (r.includes("certent")) return "Certent";
  if (r.includes("iris carbon")) return "IRIS Carbon";
  if (r.includes("broadridge") || r.includes("profile")) return "Broadridge PROfile";
  if (r.includes("sec publisher")) return "SEC Publisher";
  return raw.trim();
}

// ─── Generator detection (ported from detect_generators.py) ───

function detectGenerator(filepath: string): string {
  const buf = Buffer.alloc(READ_BYTES);
  const fd = Bun.file(filepath);
  // Use sync read for worker perf
  const raw = readFileSync(filepath);
  const text = raw.subarray(0, READ_BYTES).toString("utf-8");
  const textLower = text.toLowerCase();

  // --- Explicit generator metadata ---

  // 1. <meta name="generator" content="...">
  let m: RegExpMatchArray | null;
  m =
    text.match(
      /<meta\s+name\s*=\s*["']generator["']\s+content\s*=\s*["']([^"']+)["']/i,
    ) ??
    text.match(
      /<meta\s+content\s*=\s*["']([^"']+)["']\s+name\s*=\s*["']generator["']/i,
    );
  if (m) return normalizeGenerator(m[1]!);

  // 2. <meta name="Creator" content="...">
  m = text.match(
    /<meta\s+name\s*=\s*["']Creator["']\s+content\s*=\s*["']([^"']+)["']/i,
  );
  if (m) return normalizeGenerator(m[1]!);

  // 3. <meta name="Producer" content="...">
  m = text.match(
    /<meta\s+name\s*=\s*["']Producer["']\s+content\s*=\s*["']([^"']+)["']/i,
  );
  if (m) return normalizeGenerator(m[1]!);

  // 4. ProgId meta tag
  m = text.match(
    /<meta\s+name\s*=\s*["']ProgId["']\s+content\s*=\s*["']([^"']+)["']/i,
  );
  if (m) {
    const progid = m[1]!;
    if (/word/i.test(progid)) return "Microsoft Word";
    if (/excel/i.test(progid)) return "Microsoft Excel";
    return normalizeGenerator(progid);
  }

  // --- HTML comment signatures ---

  // Workiva / Wdesk
  if (/<!--.*Created with the Workiva Platform.*-->/i.test(text)) return "Workiva";
  if (/<!--.*Copyright\s+\d{4}\s+Workiva.*-->/i.test(text)) return "Workiva";
  if (/<!--.*Document created using Wdesk.*-->/i.test(text)) return "Workiva";

  // Toppan Merrill / Bridge
  if (/<!--.*(?:Toppan\s*Merrill|iXBRL document created with.*Toppan).*-->/i.test(text))
    return "Toppan Merrill";
  if (/<!--.*Merrill\s*Bridge.*-->/i.test(text)) return "Toppan Merrill";

  // Donnelley Financial Solutions / RR Donnelley
  if (/<!--.*Donnelley Financial Solutions.*-->/i.test(text))
    return "Donnelley Financial Solutions";
  if (/<!--.*RR\s*Donnelley.*-->/i.test(text)) return "Donnelley Financial Solutions";

  // Broadridge PROfile
  if (/<!--.*Broadridge\s+PROfile.*-->/i.test(text)) return "Broadridge PROfile";
  if (textLower.includes("broadridge")) return "Broadridge PROfile";

  // SEC Publisher
  const titleMatch = text.match(/<title[^>]*>([^<]+)<\/title>/i);
  const titleText = titleMatch ? titleMatch[1]!.trim() : "";
  if (textLower.includes("sec publisher") || titleText.toLowerCase().includes("sec publisher"))
    return "SEC Publisher";

  // IRIS Carbon
  if (/<!--.*Powered by IRIS Carbon.*-->/i.test(text)) return "IRIS Carbon";

  // Certent
  if (/<!--.*Certent\s+Disclosure\s+Management.*-->/i.test(text)) return "Certent";
  if (textLower.includes("certent")) return "Certent";

  // CompSci Resources
  if (/<!--.*CompSci Resources.*-->/i.test(text)) return "CompSci Transform";

  // RDG Portal
  if (/<!--.*RDG Portal.*-->/i.test(text)) return "RDG Portal";

  // PDF to EDGAR
  if (titleText.toLowerCase() === "pdf to edgar" || textLower.slice(0, 2000).includes("pdf to edgar"))
    return "PDF to EDGAR";

  // Generic generated/created by comments
  m = text.match(/<!--\s*Generated\s+by\s+([^-]+?)-->/i);
  if (m) {
    const val = m[1]!.trim();
    if (!/^\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}/.test(val)) return normalizeGenerator(val);
  }
  m = text.match(/<!--\s*Created\s+(?:by|with)\s+([^-]+?)-->/i);
  if (m) {
    const val = m[1]!.trim();
    if (!/^\d{1,2}[/\-]\d{1,2}[/\-]\d{2,4}/.test(val)) return normalizeGenerator(val);
  }

  // --- Keyword signatures ---

  if (/\bwdesk\b/.test(textLower)) return "Workiva";
  if (/\bworkiva\b/.test(textLower)) return "Workiva";
  if (/\brrdonnelley\b/.test(textLower)) return "Donnelley Financial Solutions";
  if (/\bedgar-online\b/.test(textLower)) return "Donnelley Financial Solutions";
  if (/\btoppan\b/.test(textLower)) return "Toppan Merrill";
  if (/\bmerrill\b/.test(textLower) && /\b(?:bridge|ixbrl|xbrl)\b/.test(textLower))
    return "Toppan Merrill";
  if (/\bbowne\b/.test(textLower)) return "Toppan Merrill";
  if (/\bcompsci\b/.test(textLower)) return "CompSci Transform";
  if (/\bthunderdome\b/.test(textLower)) return "ThunderDome";
  if (/\bgoxbrl\b/.test(textLower)) return "GoXBRL";

  // CSS class naming patterns
  if (/class\s*=\s*["'][^"']*\bwk_\w+/.test(textLower)) return "Workiva";

  // --- SGML document wrapper detection ---
  const hasSgml = /<DOCUMENT>\s*\n?\s*<TYPE>/i.test(text);
  if (hasSgml) {
    const fnMatch = text.match(/<FILENAME>\s*([\w\-\.]+)/i);
    if (fnMatch) {
      const filename = fnMatch[1]!.toLowerCase();
      if (/^d\d+/.test(filename)) return "Donnelley Financial Solutions";
      if (/^tm\d+/.test(filename)) return "Toppan Merrill";
      if (/^ea\d+/.test(filename)) return "EFiling/EDGAR Agent";
    }

    if (
      textLower.includes("<!-- field: rule-page") ||
      textLower.slice(0, 5000).includes("rule-page")
    )
      return "Broadridge PROfile";

    if (textLower.includes("field: set; name: xdx")) return "EFiling XDX";

    if (textLower.slice(0, 5000).includes("<!-- field:")) return "EFiling/EDGAR Agent";

    if (/<Center><DIV STYLE="width:8\.5in"/.test(text))
      return "Donnelley Financial Solutions";

    // Check accession prefix
    const bn = basename(filepath);
    const accessionPrefix = bn.split("-")[0]!;
    if (accessionPrefix in FILING_AGENT_CIKS)
      return FILING_AGENT_CIKS[accessionPrefix]!;

    // Legacy font-based
    const fontCount = (textLower.match(/<font/g) ?? []).length;
    if (fontCount > 5) return "SGML-wrapped (legacy/font-based)";

    return "SGML-wrapped (unknown)";
  }

  // --- Inline XBRL detection ---
  const hasIxNs =
    textLower.includes("xmlns:ix=") || textLower.includes("<ix:header");

  // Structural: Donnelley uppercase P STYLE + Center DIV 8.5in
  if (
    /<P STYLE="[^"]*font-family:Times New Roman"/.test(text) &&
    /<Center><DIV STYLE="width:8\.5in"/.test(text)
  )
    return "Donnelley Financial Solutions";

  // Title tag tool names
  if (titleText) {
    const tl = titleText.toLowerCase();
    if (tl.includes("workiva") || tl.includes("wdesk")) return "Workiva";
  }

  if (hasIxNs) {
    if (textLower.includes("field: set; name: xdx")) return "EFiling XDX";
    if (textLower.includes("<!-- field: rule")) return "Broadridge PROfile";
    if (textLower.slice(0, 5000).includes("<!-- field:")) return "EFiling/EDGAR Agent";

    // Filing agent CIK-based
    const bn = basename(filepath);
    const accessionPrefix = bn.split("-")[0]!;
    if (accessionPrefix in FILING_AGENT_CIKS)
      return FILING_AGENT_CIKS[accessionPrefix]!;

    // XML declaration encoding
    if (textLower.slice(0, 200).includes('<?xml version="1.0" encoding="utf-8"'))
      return "Inline XBRL (utf-8 toolchain)";
    if (textLower.slice(0, 200).includes("<?xml version='1.0' encoding='ascii'?>")) {
      if (/<div style="display:none"><ix:header>/i.test(textLower.slice(0, 3000)))
        return "Inline XBRL (SEC/EDGAR standard)";
      return "Inline XBRL (SEC/EDGAR standard)";
    }

    return "Inline XBRL (tool unresolved)";
  }

  // --- Structural fallbacks ---
  const fontCount = (textLower.match(/<font/g) ?? []).length;
  const tdCount = (textLower.match(/<td/g) ?? []).length;
  const spanCount = (textLower.match(/<span/g) ?? []).length;

  if (fontCount > 20) return "Legacy generator (font-based)";
  if (tdCount > 50 && spanCount < 10) return "Table-based generator";

  const dataAttrCount = (textLower.match(/\bdata-\w+/g) ?? []).length;
  if (dataAttrCount > 10) return "Modern web tooling";

  return "Unknown";
}

// ─── Worker mode ───

const args = process.argv.slice(2);
if (args[0] === "--worker") {
  const startIdx = parseInt(args[1]!);
  const endIdx = parseInt(args[2]!);
  const outFile = args[3]!;

  const htmlFiles = readdirSync(HTML_DIR)
    .filter((f: string) => f.endsWith(".html"))
    .sort()
    .slice(startIdx, endIdx);

  const records: string[] = [];
  for (const file of htmlFiles) {
    const accession = file.replace(".html", "");
    const generator = detectGenerator(`${HTML_DIR}/${file}`);
    records.push(JSON.stringify({ accession, generator }));
  }

  writeFileSync(outFile, records.join("\n") + (records.length > 0 ? "\n" : ""));
  process.exit(0);
}

// ─── Main mode: orchestrate workers ───

const start = Date.now();
mkdirSync(OUTPUT_DIR, { recursive: true });

const htmlFiles = readdirSync(HTML_DIR)
  .filter((f: string) => f.endsWith(".html"))
  .sort();

const nproc = cpus().length;
const chunkSize = Math.ceil(htmlFiles.length / nproc);

process.stderr.write(
  `  Tagging generators for ${htmlFiles.length} HTML files with ${nproc} workers...\n\n`,
);

const tmpFiles: string[] = [];
const workers: ReturnType<typeof Bun.spawn>[] = [];

for (let i = 0; i < nproc; i++) {
  const startIdx = i * chunkSize;
  const endIdx = Math.min(startIdx + chunkSize, htmlFiles.length);
  if (startIdx >= htmlFiles.length) break;

  const tmpFile = `${OUTPUT_DIR}/.tmp-gen-${i}.jsonl`;
  tmpFiles.push(tmpFile);

  workers.push(
    Bun.spawn(
      [
        "bun",
        "run",
        import.meta.filename,
        "--worker",
        String(startIdx),
        String(endIdx),
        tmpFile,
      ],
      { stderr: "inherit" },
    ),
  );
}

for (const worker of workers) {
  await worker.exited;
}

process.stderr.write(`  Workers done, merging results...\n`);

// Merge and sort
type TagRecord = { accession: string; generator: string };
const allRecords: TagRecord[] = [];

for (const tmpFile of tmpFiles) {
  const rl = createInterface({ input: createReadStream(tmpFile) });
  for await (const line of rl) {
    if (line.trim()) allRecords.push(JSON.parse(line));
  }
}

allRecords.sort((a, b) => a.accession.localeCompare(b.accession));

// Write final output
writeFileSync(
  OUTPUT_FILE,
  allRecords.map((r) => JSON.stringify(r)).join("\n") + "\n",
);

// Cleanup
for (const tmpFile of tmpFiles) {
  try {
    unlinkSync(tmpFile);
  } catch {}
}

// Print summary
const counts = new Map<string, number>();
for (const r of allRecords) {
  counts.set(r.generator, (counts.get(r.generator) ?? 0) + 1);
}

const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);

const elapsed = ((Date.now() - start) / 1000).toFixed(1);
const total = allRecords.length;

console.log(`\n${"=".repeat(70)}`);
console.log(`Generator Tags Summary (${total} files, ${elapsed}s)`);
console.log(`${"=".repeat(70)}`);
console.log(`${"Generator".padEnd(45)} ${"Count".padStart(7)} ${"  %".padStart(7)}`);
console.log("-".repeat(70));

for (const [gen, count] of sorted) {
  const pct = ((count / total) * 100).toFixed(1);
  console.log(`${gen.padEnd(45)} ${String(count).padStart(7)} ${(pct + "%").padStart(7)}`);
}

console.log("-".repeat(70));
console.log(`${"TOTAL".padEnd(45)} ${String(total).padStart(7)} ${"100.0%".padStart(7)}`);
console.log(`\nOutput: ${OUTPUT_FILE}`);
