/**
 * Worker: processes a slice of HTML files. Invoked as:
 *   bun run reparse-worker.ts <start> <end> <outfile>
 * Reads file list from stdin (one filename per line).
 */
import { readFileSync, writeFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { v4 as uuidv4 } from "uuid";

const HTML_DIR = "../data/raw/html";
const META_PATH = "../data/bulk/accession-meta.json";

// ─── Inline everything to avoid import overhead ───

function stripHtml(html: string): string {
  return html
    .replace(/<script[\s\S]*?<\/script>/gi, "")
    .replace(/<style[\s\S]*?<\/style>/gi, "")
    .replace(/<noscript[\s\S]*?<\/noscript>/gi, "")
    // Collapse adjacent inline element boundaries to prevent word splitting
    // (e.g., <span>It</span><span>em 2</span> → Item 2)
    // Preserve existing whitespace between tags as a single space; if no whitespace, join directly
    .replace(/<\/(span|a|b|i|u|em|strong|font)>(\s*)<(?:span|a|b|i|u|em|strong|font)[^>]*>/gi, (_m, _tag, ws) => ws.length > 0 ? " " : "")
    .replace(/<\/ix:[a-z]+>(\s*)<ix:[a-z]+[^>]*>/gi, (_m, ws) => ws.length > 0 ? " " : "")
    .replace(/<\/(p|div|tr|li|h[1-6]|td|th)>/gi, "\n")
    .replace(/<(br|hr)\s*\/?>/gi, "\n")
    .replace(/<[^>]+>/g, " ")
    .replace(/&nbsp;|&#160;|&#xa0;/gi, " ")
    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
    .replace(/&quot;|&ldquo;|&rdquo;|&#8220;|&#8221;|&#147;|&#148;/g, '"')
    .replace(/&#39;|&apos;|&rsquo;|&lsquo;|&#8216;|&#8217;|&#146;/g, "'")
    .replace(/&mdash;|&#8212;|&#151;/g, "—").replace(/&ndash;|&#8211;|&#150;/g, "–")
    .replace(/&bull;|&#8226;|&#149;/g, "•")
    .replace(/&minus;|&#8722;/g, "-")
    .replace(/&sect;|&#167;/g, "§")
    .replace(/&#153;/g, "™")
    .replace(/&#x([0-9a-fA-F]+);/gi, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
    .replace(/&#\d+;/g, " ").replace(/&\w+;/g, " ")
    // Collapse spaces within lines but PRESERVE newlines as section boundaries
    .replace(/[^\S\n]+/g, " ")
    // Fix missing spaces: sentence.Next → sentence. Next
    .replace(/([a-z])\.([A-Z])/g, "$1. $2")
    // Fix missing spaces: comma,Next → comma, Next (but not thousands like 1,000)
    .replace(/([a-z]),([A-Z])/g, "$1, $2")
    // Fix missing spaces: semicolon;Next → semicolon; Next
    .replace(/([a-z]);([A-Z])/g, "$1; $2")
    // Fix bullet spacing: •Text → • Text
    .replace(/•([A-Za-z])/g, "• $1")
    // Fix sub-item markers: a.Text → a. Text (single lowercase letter followed by period and uppercase)
    .replace(/\b([a-z])\.([A-Z])/g, "$1. $2")
    // Greek question mark (U+037E) → semicolon
    .replace(/\u037e/g, ";")
    // Fix inline element joins that created camelCase with common English words
    .replace(/([a-z])(The|Our|We|This|These|That|Its|His|Her|In|As|For|And|Or|If|An|It|To|By|On|At|No|Of|All|Any|Has|Was|Is|Are|Not|May|Can|Will|Such|Also|But|Each|New|So|Up|With|From)\b/g, "$1 $2")
    // Fix colon-joins: word:Word → word: Word (exclude URLs)
    .replace(/([a-z]):([A-Z])/g, "$1: $2")
    // Fix ISO standard joins: ISO/IEC27001 → ISO/IEC 27001, ISO27001 → ISO 27001
    .replace(/\b(ISO(?:\/IEC)?)(\d)/g, "$1 $2")
    .replace(/(Standardization)(\d)/g, "$1 $2")
    // Fix PDF extraction artifact: space before punctuation ("Director ," → "Director,")
    .replace(/ ([,;:.!?)])/g, "$1");
}

// Heading pattern: "Item 1C" must appear near the START of the line (not a mid-sentence cross-reference)
const ITEM_1C_HEADING = /^\s*(\u2022\s*)?item\s*1c[\.\s\u00a0—–:-]/i;
const NEXT_ITEM = /^\s*items?\s*(\d+[a-z]?\.?\d*)/i; // \s* allows "Item2.", items? allows plural typos
const SECTION_END = /^\s*(signatures?|part\s*(ii|2|iii|3|iv|4)|exhibit\s*index|financial\s+statements|management['']?s?\s+financial\s+discussion)/i;
// Combined-filing subsidiary headers (e.g., "ENTERGY ARKANSAS, LLC AND SUBSIDIARIES MANAGEMENT'S FINANCIAL DISCUSSION")
// Must be ALL-CAPS and short (< 120 chars) to avoid matching body text
const SUBSIDIARY_HEADER = /^[A-Z][A-Z\s,.'&-]{5,}(?:LLC|INC|CORP|COMPANY|L\.?P\.?)\b.*\bAND\s+SUBSIDIARIES\b/;
// Additional section headings that always appear AFTER Item 1C
const POST_1C_SECTION = /^\s*(properties|legal\s+proceedings|mine\s+safety|market\s+for\s+registrant|selected\s+financial|management.s\s+discussion|equity\s+compensation\s+plan|stock\s+performance)/i;
// Standalone section headings (must be short to avoid matching cross-references in body text)
const SHORT_SECTION_HEADING = /^\s*(risk\s+factors|controls\s+and\s+procedures|unresolved\s+staff\s+comments|glossary[\s\w]*|employees?\s+and\s+(consultants|human)|subsidiaries|executive\s+officers)\s*$/i;

function extractItem1C(text: string): string | null {
  // Rejoin broken headings: "ITEM\n2. PROPERTIES" → "ITEM 2. PROPERTIES"
  const rawLines = text.split("\n").map(l => l.replace(/[^\S\n]+/g, " ").trim()).filter(l => l.length > 0);
  const lines: string[] = [];
  for (let i = 0; i < rawLines.length; i++) {
    const line = rawLines[i]!;
    // If this line is just "ITEM" or "Item" alone, join with next line
    if (/^\s*(item|ITEM|Item)\s*$/i.test(line) && i + 1 < rawLines.length && /^\s*\d/.test(rawLines[i + 1]!)) {
      lines.push(line + " " + rawLines[i + 1]!);
      i++; // skip next line
    } else {
      lines.push(line);
    }
  }

  let startIdx = -1;
  for (let i = 0; i < lines.length; i++) {
    if (ITEM_1C_HEADING.test(lines[i]!) && lines[i]!.length < 300) startIdx = i + 1;
  }
  if (startIdx === -1) return null;

  let endIdx = lines.length;
  for (let i = startIdx; i < lines.length; i++) {
    const line = lines[i]!;
    if (line.length > 300) continue;
    if (SECTION_END.test(line)) { endIdx = i; break; }
    if (POST_1C_SECTION.test(line) && line.length < 150) { endIdx = i; break; }
    if (SHORT_SECTION_HEADING.test(line)) { endIdx = i; break; }
    const m = NEXT_ITEM.exec(line);
    // Only treat as section boundary if line is short (real headings) — not mid-sentence cross-references
    if (m && m[1]!.toLowerCase() !== "1c" && line.length < 120) { endIdx = i; break; }
  }

  // First pass: strip page artifacts (headers, footers, page numbers, running titles)
  const contentLines: string[] = [];
  for (let i = startIdx; i < endIdx; i++) {
    const line = lines[i]!;
    if (line.length < 3) continue;
    // Page numbers: bare numbers "28", "- 12 -", "F-3", "P58", "A-1", "Page 19"
    if (/^[-–—\s]*[A-Za-z]?[-–—]?\s*\d+[-–—\s]*$/.test(line)) continue;
    if (/^page\s+\d+$/i.test(line)) continue;
    // Running headers/footers
    if (/^table of conten\s*t?s?[\s\/]*$/i.test(line)) continue;
    if (/^part\s+[iv]+$/i.test(line) && line.length < 15) continue;
    if (/^\(?\s*back\s+to\s+(index|top|toc)\s*\)?$/i.test(line)) continue;
    if (/^index$/i.test(line)) continue;
    // Company name + form type headers (e.g., "Constellation Brands, Inc. FY 2025 Form 10-K")
    if (/form\s+10-[kq]/i.test(line) && line.length < 120) continue;
    // Standalone year headers (e.g., "2024 Form 10-K", "FY 2025")
    if (/^\d{4}\s+(form|annual)/i.test(line) && line.length < 40) continue;
    // Generic page decorations: short non-sentence lines that don't contain cybersecurity content
    // (e.g., "#WORTHREACHINGFOR I 28", "OTHER KEY INFORMATION", company mottos)
    if (line.length < 50 && /^[A-Z#@\s\d]+$/.test(line) && !/CYBER|SECURITY|RISK|BOARD|INCIDENT/i.test(line)) continue;
    // Combined-filing subsidiary running headers (e.g., "GENERAL MOTORS COMPANY AND SUBSIDIARIES")
    if (line.length < 120 && SUBSIDIARY_HEADER.test(line)) continue;
    // Short lines with pipe separators (page section headers like "Risk Factors | General Risk")
    if (line.length < 80 && /\|/.test(line) && !/cyber|security|incident|threat/i.test(line)) continue;
    // "Table of Contents" prefix stuck to content — strip the prefix
    if (/^\d*\s*table\s+of\s+contents\s+/i.test(line)) {
      const stripped = line.replace(/^\d*\s*table\s+of\s+contents\s+/i, "").trim();
      if (stripped.length >= 5 && /[.!?]/.test(stripped)) { contentLines.push(stripped); }
      continue;
    }
    contentLines.push(line);
  }

  // Second pass: merge continuation lines.
  // Also skip short non-content lines that sit between a broken sentence.
  const merged: string[] = [];
  for (let ci = 0; ci < contentLines.length; ci++) {
    const line = contentLines[ci]!;
    const prev = merged.length > 0 ? merged[merged.length - 1]! : "";
    const prevIsBroken = merged.length > 0 && !/[.!?:;\")\u201d]\s*$/.test(prev) && prev.length > 15;

    // If this is a short line sitting between a broken prev and a lowercase continuation,
    // it's likely a page artifact we didn't catch — skip it
    if (prevIsBroken && line.length < 80 && ci + 1 < contentLines.length && /^[a-z]/.test(contentLines[ci + 1]!)) {
      const hasContentWords = /\b(we|our|the|is|are|has|have|its|this|each|all|any|not|may|can|will|such|including|cybersecurity|security|risk|board|management|incident|threat|assess|monitor|oversee|protect|comply)\b/i.test(line);
      if (!hasContentWords) {
        continue;
      }
    }

    // Merge if: current starts lowercase AND prev doesn't end with sentence punctuation
    const lcContinuation = /^[a-z]/.test(line) && prevIsBroken;
    // Merge continuation phrases even when prev has terminal punctuation
    const phraseContinuation = merged.length > 0 && CONTINUATION_RE.test(line) && prev.length > 15;
    // When prev is mid-sentence, aggressively merge non-heading content lines
    // (catches uppercase continuations like "Manager as well as..." after page breaks)
    const brokenMerge = prevIsBroken && line.length > 40 &&
      !/^\s*(item|part|signatures?|exhibit|properties|legal\s+proceedings)/i.test(line);
    if (lcContinuation || phraseContinuation || brokenMerge) {
      merged[merged.length - 1] = prev + " " + line;
    } else {
      merged.push(line);
    }
  }

  const blocks: string[] = [];
  let words = 0;
  for (const line of merged) {
    if (line.length < 5) continue;
    blocks.push(line);
    words += line.split(/\s+/).length;
    if (blocks.length >= 50 || words >= 15000) break;
  }
  return blocks.length > 0 ? blocks.join("\n\n") : null;
}

// Inline segmentation to avoid import chain
const MIN_WORDS = 20;
const MAX_SEG_WORDS = 500;

const BULLET_RE = /^[•\-·▪►■◦‣⁃–—]\s/;
const CONTAINS_BULLET = /[•·▪►■◦‣⁃–—]\s/;
const ENDS_WITH_COLON = /:\s*$/;
const TERMINAL_PUNCT = /[.!?;")\u201d]\s*$/;
const STARTS_LOWERCASE = /^[a-z]/;
const LEADING_PUNCT = /^[":.,;]\s*/;
const CONTINUATION_RE = /^(and |or |including |such as |along with |that |which |where |whether |as well as |as described |for example|for more |pursuant to |in addition )/i;

function wordCount(t: string): number { return t.split(/\s+/).filter(w => w.length > 0).length; }

function normalizeForHash(text: string): string {
  return text.toLowerCase().replace(/\s+/g, " ").replace(/['']/g, "'").replace(/[""]/g, '"').replace(/[—–]/g, "-").trim();
}
function textHash(text: string): string {
  return createHash("sha256").update(normalizeForHash(text)).digest("hex").slice(0, 16);
}

function splitSentences(text: string): string[] {
  const sentences = text.match(/[^.!?]+[.!?]+[\s"]*/g) ?? [text];
  const chunks: string[] = [];
  let cur = "";
  for (const s of sentences) {
    const combined = cur + s;
    if (wordCount(combined) > MAX_SEG_WORDS && cur.length > 0) { chunks.push(cur.trim()); cur = s; }
    else cur = combined;
  }
  if (cur.trim().length > 0) chunks.push(cur.trim());
  return chunks;
}

function mergeBlocks(rawBlocks: string[]): string[] {
  const cleaned: string[] = [];
  for (const raw of rawBlocks) {
    const text = raw.replace(/\s+/g, " ").trim();
    if (text.length > 0) cleaned.push(text);
  }
  const merged: string[] = [];
  for (const block of cleaned) {
    if (merged.length === 0) { merged.push(block); continue; }
    const prev = merged[merged.length - 1]!;
    const prevWc = wordCount(prev);
    const combinedWc = prevWc + wordCount(block);
    if (combinedWc > MAX_SEG_WORDS) { merged.push(block); continue; }
    const isBullet = BULLET_RE.test(block);
    const prevContainsBullet = CONTAINS_BULLET.test(prev);
    const prevEndsColon = ENDS_WITH_COLON.test(prev);
    const prevNoTerminal = !TERMINAL_PUNCT.test(prev);
    if (isBullet && (prevEndsColon || prevContainsBullet)) { merged[merged.length - 1] = prev + " " + block; continue; }
    if (STARTS_LOWERCASE.test(block) && prevNoTerminal && !isBullet && prevWc > 5) { merged[merged.length - 1] = prev + " " + block; continue; }
    if (CONTINUATION_RE.test(block) && !isBullet) { merged[merged.length - 1] = prev + " " + block; continue; }
    merged.push(block);
  }
  return merged;
}

interface Para {
  id: string; text: string; textHash: string; wordCount: number;
  paragraphIndex: number; filing: Record<string, unknown>;
}

function segment(sectionText: string, filing: Record<string, unknown>): Para[] {
  const rawBlocks = sectionText.split(/\n\n+/);
  const blocks = mergeBlocks(rawBlocks);
  const paras: Para[] = [];
  const seen = new Set<string>();
  let idx = 0;

  function addPara(text: string) {
    const stripped = text.replace(LEADING_PUNCT, "");
    if (!stripped) return;
    const hash = textHash(stripped);
    if (seen.has(hash)) return;
    seen.add(hash);
    const wc = wordCount(stripped);
    if (wc < MIN_WORDS) return;
    if (wc > MAX_SEG_WORDS) {
      for (const chunk of splitSentences(stripped)) {
        const cwc = wordCount(chunk);
        if (cwc < MIN_WORDS) continue;
        const chunkHash = textHash(chunk);
        if (seen.has(chunkHash)) continue;
        seen.add(chunkHash);
        paras.push({ id: uuidv4(), text: chunk, textHash: chunkHash, wordCount: cwc, paragraphIndex: idx++, filing });
      }
    } else {
      paras.push({ id: uuidv4(), text: stripped, textHash: hash, wordCount: wc, paragraphIndex: idx++, filing });
    }
  }

  for (const block of blocks) {
    const stripped = block.replace(LEADING_PUNCT, "");
    if (!stripped) continue;
    const wc = wordCount(stripped);
    // Short blocks: append to previous paragraph instead of dropping,
    // but only if it completes a sentence or previous was already broken
    if (wc < MIN_WORDS && paras.length > 0) {
      const prev = paras[paras.length - 1]!;
      const fragmentCompletes = TERMINAL_PUNCT.test(stripped);
      const prevAlreadyBroken = !TERMINAL_PUNCT.test(prev.text);
      if (fragmentCompletes || prevAlreadyBroken) {
        const combined = prev.text + " " + stripped;
        const combinedWc = wordCount(combined);
        if (combinedWc <= MAX_SEG_WORDS) {
          prev.text = combined;
          prev.textHash = textHash(combined);
          prev.wordCount = combinedWc;
          continue;
        }
      }
    }
    addPara(stripped);
  }
  return paras;
}

// ─── Main ───

const [startStr, endStr, outFile] = process.argv.slice(2);
const startI = parseInt(startStr!);
const endI = parseInt(endStr!);

// Read file list from temp file
const allFiles: string[] = JSON.parse(readFileSync(process.env.FILE_LIST_PATH!, "utf-8"));
const files = allFiles.slice(startI, endI);

// Load metadata
const accMeta: Record<string, { companyName: string; cik: string; ticker: string; filingDate: string; fiscalYear: number }> =
  JSON.parse(readFileSync(META_PATH, "utf-8"));

const output: string[] = [];
let parsed = 0;

for (const file of files) {
  const acc = file.replace(".html", "");
  const meta = accMeta[acc];
  if (!meta) continue;

  const html = readFileSync(`${HTML_DIR}/${file}`, "utf-8");
  const section = extractItem1C(stripHtml(html));
  if (!section) continue;

  const filing = {
    companyName: meta.companyName, cik: meta.cik, ticker: meta.ticker,
    filingType: "10-K", filingDate: meta.filingDate, fiscalYear: meta.fiscalYear,
    accessionNumber: acc, secItem: "Item 1C",
  };

  const paras = segment(section, filing);
  for (const p of paras) output.push(JSON.stringify(p));
  parsed++;
}

writeFileSync(outFile!, output.join("\n") + (output.length > 0 ? "\n" : ""));
process.stderr.write(`  Worker [${startI}-${endI}]: ${parsed} filings → ${output.length} paragraphs\n`);
