import {
  enumerateFilings,
  getCompanySubmissions,
  filingDocUrl,
  edgarHtml,
  type CompanyMeta,
} from "./edgar-client.ts";
import { hasBulkSubmissions, readBulkSubmissions } from "./bulk-submissions.ts";
import { extractItem1C, extractItem105 } from "./parse-item1c.ts";
import { segmentParagraphs } from "./segment.ts";
import { appendJsonl } from "../lib/jsonl.ts";
import { loadCompletedIds } from "../lib/checkpoint.ts";
import { writeFile, mkdir } from "node:fs/promises";
import { existsSync, readFileSync } from "node:fs";
import type { FilingMeta } from "@sec-cybert/schemas/paragraph.ts";

interface ExtractOpts {
  outputPath: string;
  errorsPath: string;
  filingType: "10-K" | "8-K";
  startDate: string;
  endDate: string;
  limit?: number;
}

interface ExtractStats {
  filingsProcessed: number;
  filingsSkipped: number;
  filingsErrored: number;
  paragraphsExtracted: number;
  startTime: number;
}

/** Guess fiscal year from filing date. 10-Ks filed Jan-Jun are typically for prior fiscal year. */
function guessFiscalYear(filingDate: string): number {
  const year = parseInt(filingDate.slice(0, 4), 10);
  const month = parseInt(filingDate.slice(5, 7), 10);
  return month <= 6 ? year - 1 : year;
}

const HTML_CACHE_DIR = "../data/raw/html";

/** Cache filing HTML locally so we can re-parse without re-downloading. */
async function cacheHtml(accessionNumber: string, html: string): Promise<void> {
  const dir = HTML_CACHE_DIR;
  if (!existsSync(dir)) await mkdir(dir, { recursive: true });
  const path = `${dir}/${accessionNumber}.html`;
  await writeFile(path, html);
}

/** Read cached HTML if available. */
function readCachedHtml(accessionNumber: string): string | null {
  const path = `${HTML_CACHE_DIR}/${accessionNumber}.html`;
  if (!existsSync(path)) return null;
  return readFileSync(path, "utf-8");
}

function renderProgress(stats: ExtractStats, total: number) {
  const elapsed = (Date.now() - stats.startTime) / 1000;
  const pct = total > 0 ? ((stats.filingsProcessed / total) * 100).toFixed(1) : "0.0";
  const rate = elapsed > 0 ? (stats.filingsProcessed / elapsed) * 60 : 0;

  process.stderr.write(
    `\x1b[2K\r  Extract: ${stats.filingsProcessed}/${total} filings (${pct}%) │ ` +
      `${stats.paragraphsExtracted} paragraphs │ ` +
      `${stats.filingsErrored} errors │ ` +
      `${rate.toFixed(1)} filings/min`,
  );
}

/**
 * Run the full 10-K extraction pipeline:
 * 1. Enumerate ALL 10-K filings in date range via master.idx (complete, deterministic)
 * 2. For each filing, get company metadata + primary document URL
 * 3. Download HTML, extract Item 1C
 * 4. Segment into paragraphs
 * 5. Append to JSONL (crash-safe)
 */
export async function extract10K(opts: ExtractOpts): Promise<void> {
  const { outputPath, errorsPath, startDate, endDate, limit } = opts;

  process.stderr.write(`  Enumerating 10-K filings via master.idx (${startDate} to ${endDate})...\n`);

  const allEntries = await enumerateFilings({
    formTypes: ["10-K", "10-K/A"],
    startDate,
    endDate,
  });

  process.stderr.write(`  Found ${allEntries.length} 10-K filings total\n`);

  // Apply limit if specified
  const entries = limit ? allEntries.slice(0, limit) : allEntries;
  if (limit) {
    process.stderr.write(`  Limited to ${entries.length} filings\n`);
  }

  // Resume: check what accession numbers we've already processed
  const processedFilingsPath = outputPath.replace(".jsonl", "-processed.jsonl");
  const { completedIds: processedFilings } = await loadCompletedIds(processedFilingsPath);

  const remaining = entries.filter((r) => !processedFilings.has(r.accessionNumber));
  process.stderr.write(
    `  ${remaining.length} filings remaining (${processedFilings.size} already processed)\n\n`,
  );

  const stats: ExtractStats = {
    filingsProcessed: 0,
    filingsSkipped: 0,
    filingsErrored: 0,
    paragraphsExtracted: 0,
    startTime: Date.now(),
  };

  const useBulk = hasBulkSubmissions();
  if (useBulk) {
    process.stderr.write("  Using bulk submissions.zip for metadata (fast mode)\n");
  }

  // Caches: company metadata + filing lists (avoid re-fetching for same CIK)
  const metaCache = new Map<string, CompanyMeta>();
  const filingsCache = new Map<string, Awaited<ReturnType<typeof getCompanySubmissions>>["filings"]>();

  for (const ref of remaining) {
    try {
      // Get company metadata + filing list (cached per CIK, bulk or API)
      let meta = metaCache.get(ref.cik);
      let companyFilings = filingsCache.get(ref.cik);
      if (!meta || !companyFilings) {
        if (useBulk) {
          const bulk = readBulkSubmissions(ref.cik);
          if (bulk) {
            meta = bulk.meta;
            companyFilings = bulk.filings;
          }
        }
        if (!meta || !companyFilings) {
          const submissions = await getCompanySubmissions(ref.cik);
          meta = submissions.meta;
          companyFilings = submissions.filings;
        }
        metaCache.set(ref.cik, meta);
        filingsCache.set(ref.cik, companyFilings);
      }

      // Find the primary document for this specific filing
      const filing = companyFilings.find(
        (f) => f.accessionNumber === ref.accessionNumber,
      );

      if (!filing) {
        stats.filingsSkipped++;
        await appendJsonl(processedFilingsPath, {
          paragraphId: ref.accessionNumber,
          status: "skipped-not-found",
        });
        continue;
      }

      // Download the filing HTML (cache locally for parser iteration)
      let html = readCachedHtml(ref.accessionNumber);
      if (!html) {
        const url = filingDocUrl(ref.cik, ref.accessionNumber, filing.primaryDocument);
        html = await edgarHtml(url);
        await cacheHtml(ref.accessionNumber, html);
      }

      // Extract Item 1C
      const sectionText = extractItem1C(html);
      if (!sectionText) {
        stats.filingsSkipped++;
        await appendJsonl(processedFilingsPath, {
          paragraphId: ref.accessionNumber,
          status: "no-item1c",
        });
        continue;
      }

      // Build filing metadata
      const ticker = meta.tickers[0] ?? "";
      const filingMeta: FilingMeta = {
        companyName: meta.name,
        cik: meta.cik,
        ticker,
        filingType: "10-K",
        filingDate: ref.dateFiled,
        fiscalYear: guessFiscalYear(ref.dateFiled),
        accessionNumber: ref.accessionNumber,
        secItem: "Item 1C",
      };

      // Segment into paragraphs
      const paragraphs = segmentParagraphs(sectionText, filingMeta);

      // Append each paragraph
      for (const p of paragraphs) {
        await appendJsonl(outputPath, p);
      }

      stats.paragraphsExtracted += paragraphs.length;
      stats.filingsProcessed++;
      await appendJsonl(processedFilingsPath, {
        paragraphId: ref.accessionNumber,
        status: "complete",
        paragraphs: paragraphs.length,
      });

      if (stats.filingsProcessed % 25 === 0) {
        renderProgress(stats, remaining.length);
      }
    } catch (error) {
      stats.filingsErrored++;
      await appendJsonl(errorsPath, {
        accessionNumber: ref.accessionNumber,
        cik: ref.cik,
        error: error instanceof Error ? error.message : String(error),
        timestamp: new Date().toISOString(),
      });

      if (stats.filingsErrored > 500) {
        process.stderr.write("\n  ✖ Too many errors (>500). Stopping.\n");
        break;
      }
    }
  }

  process.stderr.write(
    `\n\n  ✓ Extraction complete:\n` +
      `    ${stats.filingsProcessed} filings processed\n` +
      `    ${stats.filingsSkipped} skipped (no Item 1C or not found)\n` +
      `    ${stats.filingsErrored} errors\n` +
      `    ${stats.paragraphsExtracted} paragraphs extracted\n`,
  );
}

/**
 * Re-parse all cached HTML files with the current parser.
 * Uses pre-extracted meta-lookup.json + master.idx for metadata (no per-CIK ZIP reads).
 * Rewrites paragraphs.jsonl from scratch.
 */
export async function reparse10K(opts: { outputPath: string }): Promise<void> {
  const { outputPath } = opts;
  const { readdirSync } = await import("node:fs");

  const htmlFiles = readdirSync(HTML_CACHE_DIR)
    .filter((f) => f.endsWith(".html"))
    .sort();

  process.stderr.write(`  Found ${htmlFiles.length} cached HTML files\n`);

  // Load pre-extracted metadata lookup (built from bulk submissions.zip)
  const metaLookupPath = "../data/bulk/meta-lookup.json";
  if (!existsSync(metaLookupPath)) {
    process.stderr.write("  ✖ meta-lookup.json not found. Run the metadata extraction first.\n");
    process.exit(1);
  }
  process.stderr.write("  Loading meta-lookup.json...\n");
  const metaLookup: Record<string, { name: string; tickers: string[]; sic: string; sicDescription: string; stateOfIncorporation: string }> =
    JSON.parse(readFileSync(metaLookupPath, "utf-8"));
  process.stderr.write(`  Loaded metadata for ${Object.keys(metaLookup).length} companies\n`);

  // Load master index for accession → CIK mapping
  const entries = await enumerateFilings({
    formTypes: ["10-K", "10-K/A"],
    startDate: "2023-12-15",
    endDate: "2025-12-31",
  });
  const entryByAccession = new Map(entries.map((e) => [e.accessionNumber, e]));

  let processed = 0;
  let skipped = 0;
  let totalParagraphs = 0;

  const { writeJsonl } = await import("../lib/jsonl.ts");
  const allParagraphs: import("@sec-cybert/schemas/paragraph.ts").Paragraph[] = [];

  for (const file of htmlFiles) {
    const accession = file.replace(".html", "");
    const entry = entryByAccession.get(accession);
    if (!entry) {
      skipped++;
      continue;
    }

    const html = readFileSync(`${HTML_CACHE_DIR}/${file}`, "utf-8");
    const sectionText = extractItem1C(html);
    if (!sectionText) {
      skipped++;
      continue;
    }

    // Get metadata from pre-extracted lookup
    const meta = metaLookup[entry.cik];
    const ticker = meta?.tickers[0] ?? "";

    const filingMeta: FilingMeta = {
      companyName: meta?.name ?? entry.companyName,
      cik: entry.cik,
      ticker,
      filingType: "10-K",
      filingDate: entry.dateFiled,
      fiscalYear: guessFiscalYear(entry.dateFiled),
      accessionNumber: accession,
      secItem: "Item 1C",
    };

    const paragraphs = segmentParagraphs(sectionText, filingMeta);
    allParagraphs.push(...paragraphs);
    totalParagraphs += paragraphs.length;
    processed++;

    if (processed % 500 === 0) {
      process.stderr.write(`\x1b[2K\r  Reparse: ${processed}/${htmlFiles.length} files, ${totalParagraphs} paragraphs, ${skipped} skipped`);
    }
  }

  await writeJsonl(outputPath, allParagraphs);

  process.stderr.write(
    `\n\n  ✓ Reparse complete:\n` +
      `    ${processed} filings parsed\n` +
      `    ${skipped} skipped (no Item 1C or no metadata)\n` +
      `    ${totalParagraphs} paragraphs written to ${outputPath}\n`,
  );
}

/**
 * Re-parse cached 8-K HTML files with the current parser.
 * Uses accession-meta-8k.json for metadata (no network, no EFTS).
 */
export async function reparse8K(opts: { outputPath: string }): Promise<void> {
  const { outputPath } = opts;

  if (!existsSync(ACCESSION_META_8K_PATH)) {
    process.stderr.write("  ✖ accession-meta-8k.json not found. Run extract:8k first.\n");
    process.exit(1);
  }

  const accMeta: Record<string, {
    companyName: string;
    cik: string;
    ticker: string;
    filingDate: string;
    form: string;
    secItem: string;
  }> = JSON.parse(readFileSync(ACCESSION_META_8K_PATH, "utf-8"));

  const accessions = Object.keys(accMeta);
  process.stderr.write(`  ${accessions.length} 8-K entries in accession-meta-8k.json\n`);

  // Check which HTML files exist
  const htmlFiles = accessions
    .filter((acc) => existsSync(`${HTML_CACHE_DIR}/${acc}.html`))
    .sort();
  process.stderr.write(`  ${htmlFiles.length} cached HTML files found\n`);

  let processed = 0;
  let skipped = 0;
  let totalParagraphs = 0;

  const { writeJsonl } = await import("../lib/jsonl.ts");
  const allParagraphs: import("@sec-cybert/schemas/paragraph.ts").Paragraph[] = [];

  for (const accession of htmlFiles) {
    const meta = accMeta[accession]!;
    const html = readFileSync(`${HTML_CACHE_DIR}/${accession}.html`, "utf-8");
    const extracted = extractItem105(html);

    if (!extracted) {
      skipped++;
      continue;
    }

    const filingMeta: FilingMeta = {
      companyName: meta.companyName,
      cik: meta.cik,
      ticker: meta.ticker,
      filingType: "8-K",
      filingDate: meta.filingDate,
      fiscalYear: parseInt(meta.filingDate.slice(0, 4), 10),
      accessionNumber: accession,
      secItem: extracted.item,
    };

    const paragraphs = segmentParagraphs(extracted.text, filingMeta);
    allParagraphs.push(...paragraphs);
    totalParagraphs += paragraphs.length;
    processed++;

    if (processed % 50 === 0) {
      process.stderr.write(`\x1b[2K\r  Reparse 8-K: ${processed}/${htmlFiles.length} files, ${totalParagraphs} paragraphs, ${skipped} skipped`);
    }
  }

  await writeJsonl(outputPath, allParagraphs);

  process.stderr.write(
    `\n\n  ✓ 8-K reparse complete:\n` +
      `    ${processed} filings parsed\n` +
      `    ${skipped} skipped (no cybersecurity item found)\n` +
      `    ${totalParagraphs} paragraphs written to ${outputPath}\n`,
  );
}

/**
 * Merge 10-K and 8-K paragraphs into a clean training corpus.
 *
 * Cleanup steps:
 * 1. Remove paragraphs from truncated filings (last paragraph lacks terminal punctuation)
 * 2. Deduplicate by textHash (keep first occurrence — 10-K before 8-K)
 *
 * Writes the merged corpus to outputPath.
 */
export async function mergeTrainingData(opts: {
  tenKPath: string;
  eightKPath: string;
  outputPath: string;
}): Promise<void> {
  const { writeJsonl } = await import("../lib/jsonl.ts");
  const { readJsonl } = await import("../lib/jsonl.ts");
  const { Paragraph } = await import("@sec-cybert/schemas/paragraph.ts");

  // Load both corpora
  const { records: tenK, skipped: s1 } = await readJsonl(opts.tenKPath, Paragraph);
  const { records: eightK, skipped: s2 } = await readJsonl(opts.eightKPath, Paragraph);
  if (s1 > 0) process.stderr.write(`  ⚠ Skipped ${s1} invalid 10-K lines\n`);
  if (s2 > 0) process.stderr.write(`  ⚠ Skipped ${s2} invalid 8-K lines\n`);

  const all = [...tenK, ...eightK];
  process.stderr.write(`  Loaded ${tenK.length} 10-K + ${eightK.length} 8-K = ${all.length} total paragraphs\n`);

  // Step 1: Identify truncated filings (last paragraph lacks terminal punctuation)
  const byFiling = new Map<string, typeof all>();
  for (const p of all) {
    const acc = p.filing.accessionNumber;
    const existing = byFiling.get(acc);
    if (existing) existing.push(p);
    else byFiling.set(acc, [p]);
  }

  const truncatedFilings = new Set<string>();
  for (const [acc, pars] of byFiling) {
    const last = pars.reduce((a, b) => (a.paragraphIndex > b.paragraphIndex ? a : b));
    if (!/[.!?;"\u201d)\]]\s*$/.test(last.text)) {
      truncatedFilings.add(acc);
    }
  }
  process.stderr.write(`  Removing ${truncatedFilings.size} truncated filings\n`);

  const afterTruncFilter = all.filter((p) => !truncatedFilings.has(p.filing.accessionNumber));
  process.stderr.write(`  After truncation filter: ${afterTruncFilter.length} paragraphs\n`);

  // Step 2: Deduplicate by textHash (keep first occurrence)
  const seenHashes = new Set<string>();
  const deduped: typeof all = [];
  for (const p of afterTruncFilter) {
    if (!seenHashes.has(p.textHash)) {
      seenHashes.add(p.textHash);
      deduped.push(p);
    }
  }

  process.stderr.write(`  After dedup: ${deduped.length} unique paragraphs\n`);

  // Stats
  const filingTypes = new Map<string, number>();
  for (const p of deduped) {
    const t = p.filing.filingType;
    filingTypes.set(t, (filingTypes.get(t) ?? 0) + 1);
  }

  const uniqueFilings = new Set(deduped.map((p) => p.filing.accessionNumber)).size;
  const uniqueCompanies = new Set(deduped.map((p) => p.filing.cik)).size;

  // Write output
  await writeJsonl(opts.outputPath, deduped);

  process.stderr.write(
    `\n  ✓ Training corpus written to ${opts.outputPath}\n` +
      `    ${deduped.length} paragraphs from ${uniqueFilings} filings (${uniqueCompanies} companies)\n` +
      `    By type: ${[...filingTypes.entries()].map(([t, c]) => `${t}: ${c}`).join(", ")}\n` +
      `    Removed: ${truncatedFilings.size} truncated filings, ${afterTruncFilter.length - deduped.length} duplicate paragraphs\n`,
  );
}

const ACCESSION_META_8K_PATH = "../data/bulk/accession-meta-8k.json";

interface AccessionMeta8K {
  companyName: string;
  cik: string;
  ticker: string;
  filingDate: string;
  form: string;
  items: string;
  primaryDocument: string;
}

/**
 * Run the 8-K extraction pipeline for cybersecurity incident disclosures.
 *
 * Discovery uses accession-meta-8k.json built by scan-8k-items.py
 * (deterministic scan of bulk submissions.zip for all 8-K filings with Item 1.05).
 * This is complete — no EFTS keyword search gaps.
 *
 * For each filing: download HTML → extract Item 1.05 → segment → append JSONL.
 */
export async function extract8K(opts: ExtractOpts): Promise<void> {
  const { outputPath, errorsPath, limit } = opts;

  if (!existsSync(ACCESSION_META_8K_PATH)) {
    process.stderr.write("  ✖ accession-meta-8k.json not found.\n");
    process.stderr.write("  Run: uv run ts/src/extract/scan-8k-items.py\n");
    process.exit(1);
  }

  const accMeta: Record<string, AccessionMeta8K> =
    JSON.parse(readFileSync(ACCESSION_META_8K_PATH, "utf-8"));

  let entries = Object.entries(accMeta);
  process.stderr.write(`  ${entries.length} Item 1.05 8-K filings in accession-meta-8k.json\n`);

  if (limit) {
    entries = entries.slice(0, limit);
    process.stderr.write(`  Limited to ${entries.length}\n`);
  }

  const processedFilingsPath = outputPath.replace(".jsonl", "-processed.jsonl");
  const { completedIds: processedFilings } = await loadCompletedIds(processedFilingsPath);
  const remaining = entries.filter(([acc]) => !processedFilings.has(acc));

  process.stderr.write(`  ${remaining.length} remaining (${processedFilings.size} already processed)\n\n`);

  const stats: ExtractStats = {
    filingsProcessed: 0,
    filingsSkipped: 0,
    filingsErrored: 0,
    paragraphsExtracted: 0,
    startTime: Date.now(),
  };

  for (const [accession, meta] of remaining) {
    try {
      // Download HTML (primaryDocument comes from bulk metadata — no API calls needed)
      let html = readCachedHtml(accession);
      if (!html) {
        const url = filingDocUrl(meta.cik, accession, meta.primaryDocument);
        html = await edgarHtml(url);
        await cacheHtml(accession, html);
      }

      const extracted = extractItem105(html);

      if (!extracted) {
        stats.filingsSkipped++;
        await appendJsonl(processedFilingsPath, {
          paragraphId: accession,
          status: "no-item105",
        });
        continue;
      }

      const filingMeta: FilingMeta = {
        companyName: meta.companyName,
        cik: meta.cik,
        ticker: meta.ticker,
        filingType: "8-K",
        filingDate: meta.filingDate,
        fiscalYear: parseInt(meta.filingDate.slice(0, 4), 10),
        accessionNumber: accession,
        secItem: extracted.item,
      };

      const paragraphs = segmentParagraphs(extracted.text, filingMeta);
      for (const p of paragraphs) {
        await appendJsonl(outputPath, p);
      }

      stats.paragraphsExtracted += paragraphs.length;
      stats.filingsProcessed++;
      await appendJsonl(processedFilingsPath, {
        paragraphId: accession,
        status: "complete",
        paragraphs: paragraphs.length,
      });

      if (stats.filingsProcessed % 5 === 0) {
        renderProgress(stats, remaining.length);
      }
    } catch (error) {
      stats.filingsErrored++;
      await appendJsonl(errorsPath, {
        accessionNumber: accession,
        cik: meta.cik,
        error: error instanceof Error ? error.message : String(error),
        timestamp: new Date().toISOString(),
      });
    }
  }

  process.stderr.write(
    `\n\n  ✓ 8-K extraction complete:\n` +
      `    ${stats.filingsProcessed} filings processed\n` +
      `    ${stats.filingsSkipped} skipped (no cybersecurity item extracted)\n` +
      `    ${stats.filingsErrored} errors\n` +
      `    ${stats.paragraphsExtracted} paragraphs extracted\n`,
  );
}
