/**
 * Diagnostic: Download sample 10-K filings and check why Item 1C extraction fails.
 * Tests the parser against real filings to determine skip rate causes.
 */
import {
  enumerateFilings,
  getCompanySubmissions,
  filingDocUrl,
  edgarHtml,
} from "../extract/edgar-client.ts";
import { extractItem1C } from "../extract/parse-item1c.ts";

const START_DATE = "2024-02-01"; // Firmly post-rule
const END_DATE = "2024-04-30";   // Peak 10-K filing season
const SAMPLE_SIZE = 100;

async function main() {
  process.stderr.write("  Enumerating 10-K filings...\n");
  const entries = await enumerateFilings({
    formTypes: ["10-K"],
    startDate: START_DATE,
    endDate: END_DATE,
  });

  // Sample evenly across the list (not just first N which may cluster by CIK)
  const step = Math.max(1, Math.floor(entries.length / SAMPLE_SIZE));
  const sample = entries.filter((_, i) => i % step === 0).slice(0, SAMPLE_SIZE);

  process.stderr.write(`  Testing ${sample.length} filings from ${entries.length} total\n\n`);

  let found = 0;
  let notFound = 0;
  let notInSubmissions = 0;
  let errors = 0;
  const skipReasons: Array<{ company: string; cik: string; reason: string; snippets?: string[] }> = [];

  for (let i = 0; i < sample.length; i++) {
    const entry = sample[i]!;
    try {
      const { meta, filings } = await getCompanySubmissions(entry.cik);
      const filing = filings.find((f) => f.accessionNumber === entry.accessionNumber);

      if (!filing) {
        notInSubmissions++;
        skipReasons.push({
          company: entry.companyName,
          cik: entry.cik,
          reason: "not-in-submissions",
        });
        continue;
      }

      const url = filingDocUrl(entry.cik, entry.accessionNumber, filing.primaryDocument);
      const html = await edgarHtml(url);
      const result = extractItem1C(html);

      if (result) {
        found++;
      } else {
        notFound++;
        // Check if HTML contains Item 1C at all
        const lowerHtml = html.toLowerCase();
        const hasItem1C = /item\s*1c/i.test(lowerHtml);
        const hasCybersecurity = /cybersecurity/i.test(lowerHtml);
        const isHtml = html.includes("<html") || html.includes("<HTML");
        const isPdf = filing.primaryDocument.endsWith(".pdf");
        const fileType = filing.primaryDocument.split(".").pop() ?? "unknown";

        // Get what's around "Item 1C" if it exists
        const snippets: string[] = [];
        const matches = lowerHtml.matchAll(/item\s*1c/gi);
        for (const m of matches) {
          const start = Math.max(0, m.index! - 50);
          const end = Math.min(lowerHtml.length, m.index! + 60);
          snippets.push(lowerHtml.slice(start, end).replace(/\s+/g, " "));
        }

        let reason = "no-item1c-text";
        if (isPdf) reason = "pdf-file";
        else if (!isHtml) reason = `non-html-${fileType}`;
        else if (hasItem1C) reason = "item1c-in-html-but-parser-missed";
        else if (!hasCybersecurity) reason = "no-cybersecurity-mention";
        else reason = "has-cybersecurity-no-item1c-heading";

        skipReasons.push({
          company: entry.companyName,
          cik: entry.cik,
          reason,
          snippets: snippets.length > 0 ? snippets.slice(0, 3) : undefined,
        });
      }

      if ((i + 1) % 10 === 0) {
        process.stderr.write(`  ${i + 1}/${sample.length}: ${found} found, ${notFound} not found, ${notInSubmissions} not in submissions\n`);
      }
    } catch (error) {
      errors++;
      skipReasons.push({
        company: entry.companyName,
        cik: entry.cik,
        reason: `error: ${error instanceof Error ? error.message : String(error)}`,
      });
    }
  }

  console.log("\n═══ Diagnostic Results ═══\n");
  console.log(`Total tested:       ${sample.length}`);
  console.log(`Item 1C found:      ${found} (${((found / sample.length) * 100).toFixed(1)}%)`);
  console.log(`Item 1C not found:  ${notFound} (${((notFound / sample.length) * 100).toFixed(1)}%)`);
  console.log(`Not in submissions: ${notInSubmissions}`);
  console.log(`Errors:             ${errors}`);

  // Breakdown of skip reasons
  const reasonCounts = new Map<string, number>();
  for (const s of skipReasons) {
    reasonCounts.set(s.reason, (reasonCounts.get(s.reason) ?? 0) + 1);
  }

  console.log("\n─── Skip Reason Breakdown ───\n");
  for (const [reason, count] of [...reasonCounts.entries()].sort((a, b) => b[1] - a[1])) {
    console.log(`  ${reason.padEnd(45)} ${count}`);
  }

  // Show parser-missed cases (the ones we can fix)
  const parserMissed = skipReasons.filter((s) => s.reason === "item1c-in-html-but-parser-missed");
  if (parserMissed.length > 0) {
    console.log(`\n─── Parser Missed (${parserMissed.length} — fixable) ───\n`);
    for (const s of parserMissed.slice(0, 10)) {
      console.log(`  ${s.company} (CIK ${s.cik})`);
      if (s.snippets) {
        for (const snip of s.snippets) {
          console.log(`    "${snip}"`);
        }
      }
    }
  }

  // Show some examples of each type
  console.log("\n─── Sample Skipped Filings ───\n");
  for (const [reason] of [...reasonCounts.entries()].sort((a, b) => b[1] - a[1])) {
    const examples = skipReasons.filter((s) => s.reason === reason).slice(0, 3);
    console.log(`  ${reason}:`);
    for (const e of examples) {
      console.log(`    - ${e.company} (CIK ${e.cik})`);
    }
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});
