/**
 * Corpus statistical analysis for extraction QA and representative sampling.
 * Produces a comprehensive breakdown saved as JSON + human-readable report.
 */
import { readJsonl } from "../lib/jsonl.ts";
import { Paragraph } from "@sec-cybert/schemas/paragraph.ts";
import type { Paragraph as ParagraphType } from "@sec-cybert/schemas/paragraph.ts";
import { writeFile, mkdir } from "node:fs/promises";
import { dirname } from "node:path";

const DATA = "../data";
const PARAGRAPHS_PATH = `${DATA}/paragraphs/paragraphs.jsonl`;

// ─── Statistical helpers ───

function median(sorted: number[]): number {
  if (sorted.length === 0) return 0;
  const mid = Math.floor(sorted.length / 2);
  return sorted.length % 2 === 0 ? (sorted[mid - 1]! + sorted[mid]!) / 2 : sorted[mid]!;
}

function percentile(sorted: number[], p: number): number {
  if (sorted.length === 0) return 0;
  const idx = (p / 100) * (sorted.length - 1);
  const lower = Math.floor(idx);
  const upper = Math.ceil(idx);
  if (lower === upper) return sorted[lower]!;
  return sorted[lower]! + (sorted[upper]! - sorted[lower]!) * (idx - lower);
}

function mean(arr: number[]): number {
  return arr.length === 0 ? 0 : arr.reduce((a, b) => a + b, 0) / arr.length;
}

function stddev(arr: number[]): number {
  if (arr.length < 2) return 0;
  const m = mean(arr);
  return Math.sqrt(arr.reduce((sum, v) => sum + (v - m) ** 2, 0) / (arr.length - 1));
}

function skewness(arr: number[]): number {
  if (arr.length < 3) return 0;
  const m = mean(arr);
  const s = stddev(arr);
  if (s === 0) return 0;
  const n = arr.length;
  return (n / ((n - 1) * (n - 2))) * arr.reduce((sum, v) => sum + ((v - m) / s) ** 3, 0);
}

function kurtosis(arr: number[]): number {
  if (arr.length < 4) return 0;
  const m = mean(arr);
  const s = stddev(arr);
  if (s === 0) return 0;
  const n = arr.length;
  const k4 = arr.reduce((sum, v) => sum + ((v - m) / s) ** 4, 0) / n;
  return k4 - 3; // excess kurtosis
}

interface DistributionStats {
  count: number;
  mean: number;
  stddev: number;
  min: number;
  p5: number;
  p25: number;
  median: number;
  p75: number;
  p95: number;
  max: number;
  skewness: number;
  kurtosis: number;
}

function distributionStats(values: number[]): DistributionStats {
  const sorted = [...values].sort((a, b) => a - b);
  return {
    count: values.length,
    mean: round(mean(values)),
    stddev: round(stddev(values)),
    min: sorted[0] ?? 0,
    p5: round(percentile(sorted, 5)),
    p25: round(percentile(sorted, 25)),
    median: round(median(sorted)),
    p75: round(percentile(sorted, 75)),
    p95: round(percentile(sorted, 95)),
    max: sorted[sorted.length - 1] ?? 0,
    skewness: round(skewness(values)),
    kurtosis: round(kurtosis(values)),
  };
}

function round(n: number, decimals = 2): number {
  return Math.round(n * 10 ** decimals) / 10 ** decimals;
}

// ─── Histogram builder ───

function histogram(values: number[], binCount: number): Array<{ binStart: number; binEnd: number; count: number; pct: number }> {
  if (values.length === 0) return [];
  const sorted = [...values].sort((a, b) => a - b);
  const min = sorted[0]!;
  const max = sorted[sorted.length - 1]!;
  const binWidth = Math.ceil((max - min) / binCount) || 1;

  const bins: Array<{ binStart: number; binEnd: number; count: number; pct: number }> = [];
  for (let i = 0; i < binCount; i++) {
    const binStart = min + i * binWidth;
    const binEnd = binStart + binWidth;
    const count = sorted.filter((v) => v >= binStart && (i === binCount - 1 ? v <= binEnd : v < binEnd)).length;
    bins.push({ binStart, binEnd, count, pct: round((count / values.length) * 100) });
  }
  return bins;
}

// ─── Analysis functions ───

interface CompanyStats {
  cik: string;
  name: string;
  ticker: string;
  filingType: string;
  filingDate: string;
  paragraphCount: number;
  totalWords: number;
  avgWordCount: number;
  medianWordCount: number;
}

function analyzeByCompany(paragraphs: ParagraphType[]): CompanyStats[] {
  const byAccession = new Map<string, ParagraphType[]>();
  for (const p of paragraphs) {
    const key = p.filing.accessionNumber;
    const existing = byAccession.get(key) ?? [];
    existing.push(p);
    byAccession.set(key, existing);
  }

  const stats: CompanyStats[] = [];
  for (const [, paras] of byAccession) {
    const first = paras[0]!;
    const wcs = paras.map((p) => p.wordCount);
    const sorted = [...wcs].sort((a, b) => a - b);
    stats.push({
      cik: first.filing.cik,
      name: first.filing.companyName,
      ticker: first.filing.ticker,
      filingType: first.filing.filingType,
      filingDate: first.filing.filingDate,
      paragraphCount: paras.length,
      totalWords: wcs.reduce((a, b) => a + b, 0),
      avgWordCount: round(mean(wcs)),
      medianWordCount: median(sorted),
    });
  }

  return stats.sort((a, b) => b.paragraphCount - a.paragraphCount);
}

function analyzeWordCountDistribution(paragraphs: ParagraphType[]) {
  const wcs = paragraphs.map((p) => p.wordCount);
  return {
    overall: distributionStats(wcs),
    histogram: histogram(wcs, 20),
    buckets: {
      "20-50": wcs.filter((w) => w >= 20 && w < 50).length,
      "50-100": wcs.filter((w) => w >= 50 && w < 100).length,
      "100-200": wcs.filter((w) => w >= 100 && w < 200).length,
      "200-300": wcs.filter((w) => w >= 200 && w < 300).length,
      "300-400": wcs.filter((w) => w >= 300 && w < 400).length,
      "400-500": wcs.filter((w) => w >= 400 && w <= 500).length,
      "500+": wcs.filter((w) => w > 500).length,
    },
  };
}

function analyzeFilingTypeBreakdown(paragraphs: ParagraphType[]) {
  const byType = new Map<string, ParagraphType[]>();
  for (const p of paragraphs) {
    const key = p.filing.filingType;
    const existing = byType.get(key) ?? [];
    existing.push(p);
    byType.set(key, existing);
  }

  const result: Record<string, { paragraphs: number; companies: number; wordCountDist: DistributionStats }> = {};
  for (const [type, paras] of byType) {
    const companies = new Set(paras.map((p) => p.filing.cik));
    result[type] = {
      paragraphs: paras.length,
      companies: companies.size,
      wordCountDist: distributionStats(paras.map((p) => p.wordCount)),
    };
  }
  return result;
}

function analyzeTemporalDistribution(paragraphs: ParagraphType[]) {
  // By filing date (month)
  const byMonth = new Map<string, number>();
  const byYear = new Map<number, number>();
  for (const p of paragraphs) {
    const month = p.filing.filingDate.slice(0, 7); // YYYY-MM
    byMonth.set(month, (byMonth.get(month) ?? 0) + 1);
    byYear.set(p.filing.fiscalYear, (byYear.get(p.filing.fiscalYear) ?? 0) + 1);
  }

  return {
    byFilingMonth: Object.fromEntries([...byMonth.entries()].sort()),
    byFiscalYear: Object.fromEntries([...byYear.entries()].sort()),
  };
}

function analyzeSecItemDistribution(paragraphs: ParagraphType[]) {
  const byItem = new Map<string, number>();
  for (const p of paragraphs) {
    const item = p.filing.secItem;
    byItem.set(item, (byItem.get(item) ?? 0) + 1);
  }
  return Object.fromEntries([...byItem.entries()].sort());
}

function analyzeParagraphPosition(paragraphs: ParagraphType[]) {
  // How paragraph index relates to content — early vs late in section
  const indices = paragraphs.map((p) => p.paragraphIndex);
  return {
    indexDistribution: distributionStats(indices),
    // Paragraphs per position bucket
    positionBuckets: {
      "0-2 (opening)": indices.filter((i) => i <= 2).length,
      "3-5 (early)": indices.filter((i) => i >= 3 && i <= 5).length,
      "6-10 (middle)": indices.filter((i) => i >= 6 && i <= 10).length,
      "11-20 (late)": indices.filter((i) => i >= 11 && i <= 20).length,
      "21+ (deep)": indices.filter((i) => i >= 21).length,
    },
  };
}

function analyzeTextLengthByPosition(paragraphs: ParagraphType[]) {
  // Do paragraphs get shorter/longer deeper into the section?
  const byBucket: Record<string, number[]> = {
    "0-2": [],
    "3-5": [],
    "6-10": [],
    "11-20": [],
    "21+": [],
  };
  for (const p of paragraphs) {
    const idx = p.paragraphIndex;
    if (idx <= 2) byBucket["0-2"]!.push(p.wordCount);
    else if (idx <= 5) byBucket["3-5"]!.push(p.wordCount);
    else if (idx <= 10) byBucket["6-10"]!.push(p.wordCount);
    else if (idx <= 20) byBucket["11-20"]!.push(p.wordCount);
    else byBucket["21+"]!.push(p.wordCount);
  }

  const result: Record<string, DistributionStats> = {};
  for (const [bucket, wcs] of Object.entries(byBucket)) {
    if (wcs.length > 0) result[bucket] = distributionStats(wcs);
  }
  return result;
}

function analyzeCompanySizeDistribution(companyStats: CompanyStats[]) {
  const paraCounts = companyStats.map((c) => c.paragraphCount);
  const totalWords = companyStats.map((c) => c.totalWords);

  return {
    paragraphsPerFiling: distributionStats(paraCounts),
    wordsPerFiling: distributionStats(totalWords),
    paragraphHistogram: histogram(paraCounts, 15),
    sizeCategories: {
      "tiny (1-3 paras)": paraCounts.filter((c) => c >= 1 && c <= 3).length,
      "small (4-8 paras)": paraCounts.filter((c) => c >= 4 && c <= 8).length,
      "medium (9-15 paras)": paraCounts.filter((c) => c >= 9 && c <= 15).length,
      "large (16-30 paras)": paraCounts.filter((c) => c >= 16 && c <= 30).length,
      "very large (31+ paras)": paraCounts.filter((c) => c >= 31).length,
    },
  };
}

function analyzeTickerCoverage(paragraphs: ParagraphType[]) {
  const withTicker = new Set<string>();
  const withoutTicker = new Set<string>();
  for (const p of paragraphs) {
    if (p.filing.ticker) withTicker.add(p.filing.cik);
    else withoutTicker.add(p.filing.cik);
  }
  return {
    companiesWithTicker: withTicker.size,
    companiesWithoutTicker: withoutTicker.size,
    tickerCoveragePct: round((withTicker.size / (withTicker.size + withoutTicker.size)) * 100),
  };
}

function detectOutliers(companyStats: CompanyStats[]) {
  const paraCounts = companyStats.map((c) => c.paragraphCount);
  const sorted = [...paraCounts].sort((a, b) => a - b);
  const q1 = percentile(sorted, 25);
  const q3 = percentile(sorted, 75);
  const iqr = q3 - q1;
  const upperFence = q3 + 1.5 * iqr;
  const lowerFence = q1 - 1.5 * iqr;

  return {
    q1,
    q3,
    iqr: round(iqr),
    upperFence: round(upperFence),
    lowerFence: round(lowerFence),
    outlierFilings: companyStats
      .filter((c) => c.paragraphCount > upperFence || c.paragraphCount < lowerFence)
      .map((c) => ({
        name: c.name,
        ticker: c.ticker,
        paragraphCount: c.paragraphCount,
        totalWords: c.totalWords,
        direction: c.paragraphCount > upperFence ? "high" : "low",
      })),
  };
}

// ─── Text content heuristics ───

function analyzeContentSignals(paragraphs: ParagraphType[]) {
  // Simple keyword presence analysis to estimate content distribution
  const signals = {
    boardGovernance: 0,
    managementRole: 0,
    riskProcess: 0,
    thirdParty: 0,
    incidentDisclosure: 0,
    strategyIntegration: 0,
    boilerplate: 0,
  };

  const patterns = {
    boardGovernance: /\b(board\s+of\s+directors|board\s+oversight|audit\s+committee|board\s+member|director|board\s+level|governance\s+committee)\b/i,
    managementRole: /\b(chief\s+information\s+security\s+officer|CISO|chief\s+technology\s+officer|CTO|chief\s+information\s+officer|CIO|management\s+team|security\s+team|reports?\s+to\s+the)\b/i,
    riskProcess: /\b(risk\s+assessment|risk\s+management|vulnerability\s+(assessment|scanning)|penetration\s+test|threat\s+intelligence|risk\s+framework|NIST|ISO\s*27001|identify.*assess.*mitigate)\b/i,
    thirdParty: /\b(third[- ]party|vendor|supplier|service\s+provider|outsourc|contractor|supply\s+chain|due\s+diligence)\b/i,
    incidentDisclosure: /\b(incident|breach|unauthorized\s+access|data\s+loss|ransomware|cyber\s*attack|compromise[d]?|disruption|material\s+impact)\b/i,
    strategyIntegration: /\b(strateg(y|ic)|business\s+objective|integrated|enterprise[- ]wide|competitive\s+advantage|digital\s+transformation|investment\s+in\s+cyber)\b/i,
    boilerplate: /\b(no\s+assurance|cannot\s+guarantee|may\s+not\s+be\s+successful|there\s+can\s+be\s+no|subject\s+to\s+risks|could\s+have\s+a\s+material|may\s+adversely\s+affect)\b/i,
  };

  for (const p of paragraphs) {
    for (const [key, pattern] of Object.entries(patterns)) {
      if (pattern.test(p.text)) {
        signals[key as keyof typeof signals]++;
      }
    }
  }

  // Also compute overlap — paragraphs matching multiple signals
  let multiSignal = 0;
  let noSignal = 0;
  for (const p of paragraphs) {
    let matches = 0;
    for (const pattern of Object.values(patterns)) {
      if (pattern.test(p.text)) matches++;
    }
    if (matches > 1) multiSignal++;
    if (matches === 0) noSignal++;
  }

  return {
    signalCounts: signals,
    signalPcts: Object.fromEntries(
      Object.entries(signals).map(([k, v]) => [k, round((v / paragraphs.length) * 100)]),
    ),
    multiSignalParagraphs: multiSignal,
    noSignalParagraphs: noSignal,
    multiSignalPct: round((multiSignal / paragraphs.length) * 100),
    noSignalPct: round((noSignal / paragraphs.length) * 100),
  };
}

// ─── Report generation ───

function generateReport(analysis: Record<string, unknown>): string {
  const lines: string[] = [];
  const hr = "═".repeat(72);
  const sr = "─".repeat(72);

  lines.push(hr);
  lines.push("  SEC-cyBERT Corpus Statistical Analysis");
  lines.push(`  Generated: ${new Date().toISOString()}`);
  lines.push(`  Data: ${PARAGRAPHS_PATH}`);
  lines.push(hr);
  lines.push("");

  // Overview
  const overview = analysis["overview"] as Record<string, number>;
  lines.push("1. OVERVIEW");
  lines.push(sr);
  lines.push(`  Total paragraphs:    ${overview["totalParagraphs"]}`);
  lines.push(`  Total filings:       ${overview["totalFilings"]}`);
  lines.push(`  Unique companies:    ${overview["uniqueCompanies"]}`);
  lines.push(`  Total words:         ${overview["totalWords"]?.toLocaleString()}`);
  lines.push(`  Avg words/paragraph: ${overview["avgWordsPerParagraph"]}`);
  lines.push(`  Avg paras/filing:    ${overview["avgParagraphsPerFiling"]}`);
  lines.push("");

  // Filing type breakdown
  const filingTypes = analysis["filingTypeBreakdown"] as Record<string, { paragraphs: number; companies: number; wordCountDist: DistributionStats }>;
  lines.push("2. FILING TYPE BREAKDOWN");
  lines.push(sr);
  for (const [type, data] of Object.entries(filingTypes)) {
    lines.push(`  ${type}: ${data.paragraphs} paragraphs from ${data.companies} companies`);
    lines.push(`    Word count: mean=${data.wordCountDist.mean}, median=${data.wordCountDist.median}, std=${data.wordCountDist.stddev}`);
  }
  lines.push("");

  // Word count distribution
  const wcDist = analysis["wordCountDistribution"] as { overall: DistributionStats; buckets: Record<string, number> };
  lines.push("3. WORD COUNT DISTRIBUTION");
  lines.push(sr);
  const ov = wcDist.overall;
  lines.push(`  Mean:     ${ov.mean} (std: ${ov.stddev})`);
  lines.push(`  Median:   ${ov.median}`);
  lines.push(`  Range:    [${ov.min}, ${ov.max}]`);
  lines.push(`  IQR:      [${ov.p25}, ${ov.p75}]`);
  lines.push(`  Skewness: ${ov.skewness} (${ov.skewness > 0.5 ? "right-skewed" : ov.skewness < -0.5 ? "left-skewed" : "roughly symmetric"})`);
  lines.push(`  Kurtosis: ${ov.kurtosis} (excess; ${ov.kurtosis > 1 ? "heavy-tailed" : ov.kurtosis < -1 ? "light-tailed" : "near-normal"})`);
  lines.push("");
  lines.push("  Buckets:");
  for (const [bucket, count] of Object.entries(wcDist.buckets)) {
    const pct = round((count / overview["totalParagraphs"]!) * 100);
    const bar = "█".repeat(Math.round(pct));
    lines.push(`    ${bucket.padEnd(12)} ${String(count).padStart(5)} (${String(pct).padStart(5)}%) ${bar}`);
  }
  lines.push("");

  // Company size distribution
  const compSize = analysis["companySizeDistribution"] as { paragraphsPerFiling: DistributionStats; wordsPerFiling: DistributionStats; sizeCategories: Record<string, number> };
  lines.push("4. FILING SIZE DISTRIBUTION (paragraphs per filing)");
  lines.push(sr);
  const pf = compSize.paragraphsPerFiling;
  lines.push(`  Mean:   ${pf.mean} (std: ${pf.stddev})`);
  lines.push(`  Median: ${pf.median}`);
  lines.push(`  Range:  [${pf.min}, ${pf.max}]`);
  lines.push("");
  lines.push("  Size categories:");
  const totalFilings = Object.values(compSize.sizeCategories).reduce((a, b) => a + b, 0);
  for (const [cat, count] of Object.entries(compSize.sizeCategories)) {
    const pct = round((count / totalFilings) * 100);
    const bar = "█".repeat(Math.round(pct / 2));
    lines.push(`    ${cat.padEnd(25)} ${String(count).padStart(4)} (${String(pct).padStart(5)}%) ${bar}`);
  }
  lines.push("");

  // Temporal
  const temporal = analysis["temporalDistribution"] as { byFilingMonth: Record<string, number>; byFiscalYear: Record<string, number> };
  lines.push("5. TEMPORAL DISTRIBUTION");
  lines.push(sr);
  lines.push("  By filing month:");
  for (const [month, count] of Object.entries(temporal.byFilingMonth)) {
    const bar = "█".repeat(Math.round(count / 20));
    lines.push(`    ${month}  ${String(count).padStart(5)} ${bar}`);
  }
  lines.push("");
  lines.push("  By fiscal year:");
  for (const [year, count] of Object.entries(temporal.byFiscalYear)) {
    lines.push(`    ${year}  ${count}`);
  }
  lines.push("");

  // Paragraph position
  const position = analysis["paragraphPosition"] as { positionBuckets: Record<string, number> };
  lines.push("6. PARAGRAPH POSITION IN SECTION");
  lines.push(sr);
  for (const [pos, count] of Object.entries(position.positionBuckets)) {
    const pct = round((count / overview["totalParagraphs"]!) * 100);
    lines.push(`    ${pos.padEnd(20)} ${String(count).padStart(5)} (${String(pct).padStart(5)}%)`);
  }
  lines.push("");

  // Content signals
  const content = analysis["contentSignals"] as { signalCounts: Record<string, number>; signalPcts: Record<string, number>; multiSignalParagraphs: number; noSignalParagraphs: number; multiSignalPct: number; noSignalPct: number };
  lines.push("7. HEURISTIC CONTENT SIGNAL ANALYSIS");
  lines.push(sr);
  lines.push("  (Keyword-based estimates — NOT ground truth labels)");
  lines.push("");
  for (const [signal, count] of Object.entries(content.signalCounts)) {
    const pct = content.signalPcts[signal]!;
    const bar = "█".repeat(Math.round(pct / 2));
    lines.push(`    ${signal.padEnd(22)} ${String(count).padStart(5)} (${String(pct).padStart(5)}%) ${bar}`);
  }
  lines.push("");
  lines.push(`  Multi-signal paragraphs: ${content.multiSignalParagraphs} (${content.multiSignalPct}%)`);
  lines.push(`  No-signal paragraphs:    ${content.noSignalParagraphs} (${content.noSignalPct}%)`);
  lines.push("");

  // Outliers
  const outliers = analysis["outliers"] as { upperFence: number; lowerFence: number; iqr: number; outlierFilings: Array<{ name: string; ticker: string; paragraphCount: number; totalWords: number; direction: string }> };
  lines.push("8. OUTLIER FILINGS (IQR method)");
  lines.push(sr);
  lines.push(`  IQR: ${outliers.iqr}, Upper fence: ${outliers.upperFence}, Lower fence: ${outliers.lowerFence}`);
  lines.push(`  ${outliers.outlierFilings.length} outlier filings:`);
  for (const o of outliers.outlierFilings.slice(0, 20)) {
    lines.push(`    ${o.direction.toUpperCase().padEnd(5)} ${(o.ticker || o.name).padEnd(30)} ${o.paragraphCount} paras, ${o.totalWords} words`);
  }
  lines.push("");

  // Ticker coverage
  const ticker = analysis["tickerCoverage"] as { companiesWithTicker: number; companiesWithoutTicker: number; tickerCoveragePct: number };
  lines.push("9. TICKER COVERAGE");
  lines.push(sr);
  lines.push(`  With ticker:    ${ticker.companiesWithTicker} companies (${ticker.tickerCoveragePct}%)`);
  lines.push(`  Without ticker: ${ticker.companiesWithoutTicker} companies`);
  lines.push("");

  // Top 20 companies by paragraph count
  const companies = analysis["companyStats"] as CompanyStats[];
  lines.push("10. TOP 20 COMPANIES BY PARAGRAPH COUNT");
  lines.push(sr);
  lines.push(`    ${"Company".padEnd(40)} ${"Ticker".padEnd(8)} ${"Type".padEnd(5)} ${"Paras".padStart(6)} ${"Words".padStart(7)} ${"Avg WC".padStart(7)}`);
  for (const c of companies.slice(0, 20)) {
    lines.push(
      `    ${c.name.slice(0, 39).padEnd(40)} ${(c.ticker || "-").padEnd(8)} ${c.filingType.padEnd(5)} ${String(c.paragraphCount).padStart(6)} ${String(c.totalWords).padStart(7)} ${String(c.avgWordCount).padStart(7)}`,
    );
  }
  lines.push("");

  // Bottom 10
  lines.push("11. BOTTOM 10 COMPANIES BY PARAGRAPH COUNT");
  lines.push(sr);
  for (const c of companies.slice(-10).reverse()) {
    lines.push(
      `    ${c.name.slice(0, 39).padEnd(40)} ${(c.ticker || "-").padEnd(8)} ${c.filingType.padEnd(5)} ${String(c.paragraphCount).padStart(6)} ${String(c.totalWords).padStart(7)}`,
    );
  }
  lines.push("");

  // Sampling recommendations
  lines.push("12. SAMPLING RECOMMENDATIONS");
  lines.push(sr);
  const totalParas = overview["totalParagraphs"]!;
  lines.push(`  For a representative sample of ~200 paragraphs (${round((200 / totalParas) * 100)}% of corpus):`);
  lines.push("  Stratify by:");
  lines.push("    - Filing type (10-K vs 8-K, proportional)");
  lines.push("    - Paragraph position (opening/middle/late)");
  lines.push("    - Word count quartile (short/medium/long/very long)");
  lines.push("    - Content signal diversity (ensure all categories represented)");
  lines.push("    - Company size (small/medium/large filings)");
  lines.push("");
  lines.push("  Ensure minimum 5 paragraphs per stratum for statistical validity.");
  lines.push("");

  lines.push(hr);
  lines.push("  END OF ANALYSIS");
  lines.push(hr);

  return lines.join("\n");
}

// ─── Main ───

async function main() {
  process.stderr.write("  Loading paragraphs...\n");
  const { records: paragraphs, skipped } = await readJsonl(PARAGRAPHS_PATH, Paragraph);
  if (skipped > 0) process.stderr.write(`  Warning: ${skipped} invalid lines skipped\n`);
  process.stderr.write(`  Loaded ${paragraphs.length} paragraphs\n\n`);

  process.stderr.write("  Computing statistics...\n");

  const companyStats = analyzeByCompany(paragraphs);
  const uniqueCompanies = new Set(paragraphs.map((p) => p.filing.cik)).size;
  const uniqueFilings = new Set(paragraphs.map((p) => p.filing.accessionNumber)).size;
  const totalWords = paragraphs.reduce((sum, p) => sum + p.wordCount, 0);

  const analysis = {
    generatedAt: new Date().toISOString(),
    dataPath: PARAGRAPHS_PATH,
    overview: {
      totalParagraphs: paragraphs.length,
      totalFilings: uniqueFilings,
      uniqueCompanies,
      totalWords,
      avgWordsPerParagraph: round(totalWords / paragraphs.length),
      avgParagraphsPerFiling: round(paragraphs.length / uniqueFilings),
    },
    filingTypeBreakdown: analyzeFilingTypeBreakdown(paragraphs),
    wordCountDistribution: analyzeWordCountDistribution(paragraphs),
    companySizeDistribution: analyzeCompanySizeDistribution(companyStats),
    temporalDistribution: analyzeTemporalDistribution(paragraphs),
    paragraphPosition: analyzeParagraphPosition(paragraphs),
    textLengthByPosition: analyzeTextLengthByPosition(paragraphs),
    contentSignals: analyzeContentSignals(paragraphs),
    outliers: detectOutliers(companyStats),
    tickerCoverage: analyzeTickerCoverage(paragraphs),
    secItemDistribution: analyzeSecItemDistribution(paragraphs),
    companyStats,
  };

  // Save JSON
  const jsonPath = `${DATA}/analysis/corpus-stats.json`;
  const jsonDir = dirname(jsonPath);
  await mkdir(jsonDir, { recursive: true });
  await writeFile(jsonPath, JSON.stringify(analysis, null, 2));
  process.stderr.write(`  Saved JSON: ${jsonPath}\n`);

  // Save report
  const report = generateReport(analysis);
  const reportPath = `${DATA}/analysis/corpus-stats.txt`;
  await writeFile(reportPath, report);
  process.stderr.write(`  Saved report: ${reportPath}\n`);

  // Print report to stdout
  console.log(report);
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});
