/**
 * Coordinator: splits work across N parallel bun workers, merges output.
 * Deduplicates amendment filings: keeps only the latest per (CIK, fiscalYear).
 */
import { readdirSync, readFileSync, writeFileSync, unlinkSync } from "node:fs";
import { spawn } from "node:child_process";

const HTML_DIR = "../data/raw/html";
const META_PATH = "../data/bulk/accession-meta.json";
const OUTPUT = "../data/paragraphs/paragraphs.jsonl";
const WORKERS = 16; // use 16 of 32 cores to leave headroom

const start = Date.now();

const allHtml = readdirSync(HTML_DIR).filter(f => f.endsWith(".html")).sort();
console.error(`  ${allHtml.length} HTML files total`);

// Deduplicate amendments: keep only the latest filing per (CIK, fiscalYear)
const accMeta: Record<string, { companyName: string; cik: string; ticker: string; filingDate: string; fiscalYear: number }> =
  JSON.parse(readFileSync(META_PATH, "utf-8"));

const byKey = new Map<string, { accession: string; filingDate: string }[]>();
for (const htmlFile of allHtml) {
  const acc = htmlFile.replace(".html", "");
  const meta = accMeta[acc];
  if (!meta) continue;
  const key = `${meta.cik}|${meta.fiscalYear}`;
  const group = byKey.get(key) ?? [];
  group.push({ accession: acc, filingDate: meta.filingDate });
  byKey.set(key, group);
}

const keepSet = new Set<string>();
for (const [, group] of byKey) {
  // Sort descending by filing date, keep latest
  group.sort((a, b) => b.filingDate.localeCompare(a.filingDate));
  keepSet.add(group[0]!.accession);
}

// Also keep files with no metadata (they'll be skipped by workers anyway)
const files = allHtml.filter(f => {
  const acc = f.replace(".html", "");
  return keepSet.has(acc) || !accMeta[acc];
});
const dropped = allHtml.length - files.length;
console.error(`  ${dropped} amendment duplicates dropped, ${files.length} files to process, ${WORKERS} workers`);

const chunkSize = Math.ceil(files.length / WORKERS);
// Write file list to temp file (too large for env vars)
const fileListPath = "/tmp/reparse-filelist.json";
writeFileSync(fileListPath, JSON.stringify(files));

// Spawn workers in parallel
const promises: Promise<void>[] = [];

for (let w = 0; w < WORKERS; w++) {
  const s = w * chunkSize;
  const e = Math.min(s + chunkSize, files.length);
  if (s >= files.length) break;

  const outFile = `/tmp/reparse-${w}.jsonl`;

  const p = new Promise<void>((resolve, reject) => {
    const child = spawn("bun", ["run", "src/extract/reparse-worker.ts", String(s), String(e), outFile], {
      env: { ...process.env, FILE_LIST_PATH: fileListPath },
      stdio: ["pipe", "inherit", "inherit"],
      cwd: process.cwd(),
    });
    child.on("close", (code) => {
      if (code === 0) resolve();
      else reject(new Error(`Worker ${w} exited with code ${code}`));
    });
    child.on("error", reject);
  });
  promises.push(p);
}

console.error(`  Spawned ${promises.length} workers, waiting...`);
await Promise.all(promises);

// Merge outputs
console.error("  Merging...");
const parts: string[] = [];
for (let w = 0; w < WORKERS; w++) {
  const outFile = `/tmp/reparse-${w}.jsonl`;
  try {
    const content = readFileSync(outFile, "utf-8").trim();
    if (content) parts.push(content);
    unlinkSync(outFile);
  } catch {}
}

writeFileSync(OUTPUT, parts.join("\n") + "\n");
const totalLines = parts.reduce((sum, p) => sum + p.split("\n").length, 0);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.error(`\n  ✓ Done in ${elapsed}s: ${totalLines} paragraphs`);
