/**
 * Comprehensive data quality analysis and cleaning.
 * Produces a cleaned dataset with dedup metadata, quality flags, and audit trail.
 *
 * Outputs:
 *   - data/paragraphs/paragraphs-clean.jsonl  (cleaned dataset with dedup metadata)
 *   - data/analysis/quality-report.txt         (human-readable audit report)
 *   - data/analysis/quality-report.json        (machine-readable)
 */
import { readJsonl, writeJsonl } from "../lib/jsonl.ts";
import { Paragraph } from "@sec-cybert/schemas/paragraph.ts";
import type { Paragraph as ParagraphType } from "@sec-cybert/schemas/paragraph.ts";
import { writeFile, mkdir } from "node:fs/promises";
import { dirname } from "node:path";

const DATA = "../data";
const INPUT_PATH = `${DATA}/paragraphs/paragraphs.jsonl`;
const CLEAN_PATH = `${DATA}/paragraphs/paragraphs-clean.jsonl`;

function round(n: number, d = 2): number {
  return Math.round(n * 10 ** d) / 10 ** d;
}

// ─── Quality checks ───

interface QualityFlag {
  flag: string;
  severity: "error" | "warning" | "info";
  description: string;
}

function checkParagraphQuality(p: ParagraphType): QualityFlag[] {
  const flags: QualityFlag[] = [];

  // Word count sanity
  if (p.wordCount < 20) {
    flags.push({ flag: "too_short", severity: "error", description: `Only ${p.wordCount} words` });
  }
  if (p.wordCount > 450) {
    flags.push({ flag: "very_long", severity: "warning", description: `${p.wordCount} words — may be a parsing artifact` });
  }

  // Content quality heuristics
  const text = p.text.toLowerCase();

  // Table data (lots of numbers, pipe chars, or very short repeated patterns)
  const numberRatio = (text.match(/\d/g)?.length ?? 0) / text.length;
  if (numberRatio > 0.3) {
    flags.push({ flag: "high_numeric", severity: "warning", description: `${round(numberRatio * 100)}% numeric content — likely table/financial data` });
  }

  // Very short sentences (typical of bullet-point formatting artifacts)
  const avgWordPerSentence = p.wordCount / Math.max(1, (text.match(/[.!?]/g)?.length ?? 1));
  if (avgWordPerSentence < 8 && p.wordCount > 30) {
    flags.push({ flag: "fragmented", severity: "info", description: `Avg ${round(avgWordPerSentence)} words/sentence — may be bullet list` });
  }

  // Not cybersecurity-related content (leaked from other sections)
  const cyberKeywords = /cybersecurity|cyber|information security|data breach|incident|threat|vulnerability|CISO|penetration test|access control|encryption|firewall|malware|phishing|ransomware/i;
  if (!cyberKeywords.test(p.text)) {
    // Check for governance/risk terms that are still relevant
    const governanceKeywords = /board|committee|officer|risk management|compliance|audit|oversight|govern/i;
    if (!governanceKeywords.test(p.text)) {
      flags.push({ flag: "off_topic", severity: "warning", description: "No cybersecurity or governance keywords — may be from wrong section" });
    }
  }

  // Legal boilerplate detection
  const legalBoilerplate = /no\s+assurance|cannot\s+guarantee|forward[- ]looking\s+statements?|safe\s+harbor|cautionary\s+statement/i;
  if (legalBoilerplate.test(p.text)) {
    flags.push({ flag: "legal_boilerplate", severity: "info", description: "Contains forward-looking/legal disclaimer language" });
  }

  // Page header/footer artifacts
  if (/^(table of contents|page \d|^\d+$)/i.test(p.text.trim())) {
    flags.push({ flag: "page_artifact", severity: "error", description: "Appears to be a page header/footer" });
  }

  return flags;
}

// ─── Dedup metadata ───

interface DedupInfo {
  textHash: string;
  duplicateCount: number;           // total instances of this hash
  uniqueCompanies: number;          // how many different companies have this text
  uniqueFilings: number;            // how many different filings
  fiscalYears: number[];            // which fiscal years
  dedupCategory: "unique" | "cross_company_boilerplate" | "cross_year_copy" | "within_filing_dup";
  isCanonical: boolean;             // is this the "representative" instance for its hash group?
}

function computeDedupInfo(paragraphs: ParagraphType[]): Map<string, DedupInfo[]> {
  // Group by textHash
  const groups = new Map<string, ParagraphType[]>();
  for (const p of paragraphs) {
    const existing = groups.get(p.textHash) ?? [];
    existing.push(p);
    groups.set(p.textHash, existing);
  }

  // For each paragraph, compute its dedup info
  const dedupMap = new Map<string, DedupInfo[]>();

  for (const [hash, paras] of groups) {
    const companies = new Set(paras.map((p) => p.filing.cik));
    const filings = new Set(paras.map((p) => p.filing.accessionNumber));
    const fiscalYears = [...new Set(paras.map((p) => p.filing.fiscalYear))].sort();

    let category: DedupInfo["dedupCategory"];
    if (paras.length === 1) {
      category = "unique";
    } else if (companies.size > 1) {
      category = "cross_company_boilerplate";
    } else if (fiscalYears.length > 1) {
      category = "cross_year_copy";
    } else {
      category = "within_filing_dup";
    }

    // First occurrence is canonical
    const infos: DedupInfo[] = paras.map((_, idx) => ({
      textHash: hash,
      duplicateCount: paras.length,
      uniqueCompanies: companies.size,
      uniqueFilings: filings.size,
      fiscalYears,
      dedupCategory: category,
      isCanonical: idx === 0,
    }));

    dedupMap.set(hash, infos);
  }

  return dedupMap;
}

// ─── Filing-level quality ───

interface FilingQuality {
  accessionNumber: string;
  companyName: string;
  paragraphCount: number;
  totalWords: number;
  flags: QualityFlag[];
}

function analyzeFilingQuality(paragraphs: ParagraphType[]): Map<string, FilingQuality> {
  const byFiling = new Map<string, ParagraphType[]>();
  for (const p of paragraphs) {
    const key = p.filing.accessionNumber;
    const existing = byFiling.get(key) ?? [];
    existing.push(p);
    byFiling.set(key, existing);
  }

  const results = new Map<string, FilingQuality>();
  for (const [accession, paras] of byFiling) {
    const flags: QualityFlag[] = [];
    const totalWords = paras.reduce((sum, p) => sum + p.wordCount, 0);

    if (paras.length > 50) {
      flags.push({ flag: "excessive_paragraphs", severity: "error", description: `${paras.length} paragraphs — likely parser over-extraction` });
    } else if (paras.length > 35) {
      flags.push({ flag: "high_paragraph_count", severity: "warning", description: `${paras.length} paragraphs — unusually long section` });
    }

    if (paras.length === 1 && totalWords < 50) {
      flags.push({ flag: "minimal_disclosure", severity: "info", description: "Single short paragraph — minimal compliance" });
    }

    // Check for off-topic drift (>30% of paragraphs flagged off-topic)
    const offTopicCount = paras.filter((p) => {
      const pFlags = checkParagraphQuality(p);
      return pFlags.some((f) => f.flag === "off_topic");
    }).length;
    if (offTopicCount > paras.length * 0.3 && paras.length > 5) {
      flags.push({ flag: "content_drift", severity: "warning", description: `${offTopicCount}/${paras.length} paragraphs appear off-topic` });
    }

    results.set(accession, {
      accessionNumber: accession,
      companyName: paras[0]!.filing.companyName,
      paragraphCount: paras.length,
      totalWords,
      flags,
    });
  }

  return results;
}

// ─── Report generation ───

function generateReport(
  paragraphs: ParagraphType[],
  dedupMap: Map<string, DedupInfo[]>,
  filingQuality: Map<string, FilingQuality>,
  paragraphFlags: Map<string, QualityFlag[]>,
  cleanCount: number,
  removedCount: number,
): string {
  const lines: string[] = [];
  const hr = "═".repeat(72);
  const sr = "─".repeat(72);

  lines.push(hr);
  lines.push("  SEC-cyBERT Data Quality Report");
  lines.push(`  Generated: ${new Date().toISOString()}`);
  lines.push(hr);
  lines.push("");

  // 1. Overview
  lines.push("1. DATASET OVERVIEW");
  lines.push(sr);
  lines.push(`  Input paragraphs:    ${paragraphs.length.toLocaleString()}`);
  lines.push(`  Clean paragraphs:    ${cleanCount.toLocaleString()}`);
  lines.push(`  Removed:             ${removedCount.toLocaleString()} (${round((removedCount / paragraphs.length) * 100)}%)`);
  lines.push(`  Total filings:       ${filingQuality.size.toLocaleString()}`);
  lines.push("");

  // 2. Paragraph-level quality flags
  const allFlags = new Map<string, number>();
  for (const [, flags] of paragraphFlags) {
    for (const f of flags) {
      allFlags.set(f.flag, (allFlags.get(f.flag) ?? 0) + 1);
    }
  }

  lines.push("2. PARAGRAPH QUALITY FLAGS");
  lines.push(sr);
  for (const [flag, count] of [...allFlags.entries()].sort((a, b) => b[1] - a[1])) {
    const pct = round((count / paragraphs.length) * 100);
    lines.push(`    ${flag.padEnd(25)} ${String(count).padStart(7)} (${String(pct).padStart(5)}%)`);
  }
  lines.push("");

  // 3. Filing-level quality
  const filingFlags = new Map<string, number>();
  for (const [, fq] of filingQuality) {
    for (const f of fq.flags) {
      filingFlags.set(f.flag, (filingFlags.get(f.flag) ?? 0) + 1);
    }
  }

  lines.push("3. FILING QUALITY FLAGS");
  lines.push(sr);
  for (const [flag, count] of [...filingFlags.entries()].sort((a, b) => b[1] - a[1])) {
    lines.push(`    ${flag.padEnd(30)} ${count} filings`);
  }
  lines.push("");

  // 4. Dedup categories
  const dedupCounts = { unique: 0, cross_company_boilerplate: 0, cross_year_copy: 0, within_filing_dup: 0 };
  const canonicalCounts = { ...dedupCounts };
  for (const [, infos] of dedupMap) {
    for (const info of infos) {
      dedupCounts[info.dedupCategory]++;
      if (info.isCanonical) canonicalCounts[info.dedupCategory]++;
    }
  }

  lines.push("4. DEDUPLICATION CATEGORIES (all paragraphs)");
  lines.push(sr);
  for (const [cat, count] of Object.entries(dedupCounts)) {
    const pct = round((count / paragraphs.length) * 100);
    const canonical = canonicalCounts[cat as keyof typeof canonicalCounts];
    lines.push(`    ${cat.padEnd(30)} ${String(count).padStart(7)} total (${String(pct).padStart(5)}%) │ ${canonical} canonical`);
  }
  lines.push("");

  // 5. Boilerplate templates (top cross-company)
  const boilerplateGroups: Array<{ hash: string; count: number; companies: number; sample: string }> = [];
  for (const [hash, infos] of dedupMap) {
    if (infos.length > 0 && infos[0]!.dedupCategory === "cross_company_boilerplate") {
      const samplePara = paragraphs.find((p) => p.textHash === hash);
      boilerplateGroups.push({
        hash,
        count: infos.length,
        companies: infos[0]!.uniqueCompanies,
        sample: samplePara?.text.slice(0, 150) ?? "",
      });
    }
  }
  boilerplateGroups.sort((a, b) => b.count - a.count);

  lines.push("5. TOP 20 BOILERPLATE TEMPLATES (cross-company identical text)");
  lines.push(sr);
  for (const bg of boilerplateGroups.slice(0, 20)) {
    lines.push(`  [${bg.count} copies, ${bg.companies} companies]`);
    lines.push(`  "${bg.sample}..."`);
    lines.push("");
  }

  // 6. Filings with quality issues (top 20)
  const problemFilings = [...filingQuality.values()]
    .filter((fq) => fq.flags.length > 0)
    .sort((a, b) => {
      const aErrors = a.flags.filter((f) => f.severity === "error").length;
      const bErrors = b.flags.filter((f) => f.severity === "error").length;
      return bErrors - aErrors || b.paragraphCount - a.paragraphCount;
    });

  lines.push("6. FILINGS WITH QUALITY ISSUES (top 20)");
  lines.push(sr);
  for (const fq of problemFilings.slice(0, 20)) {
    lines.push(`  ${fq.companyName.slice(0, 40).padEnd(42)} ${fq.paragraphCount} paras, ${fq.totalWords} words`);
    for (const f of fq.flags) {
      lines.push(`    [${f.severity}] ${f.description}`);
    }
    lines.push("");
  }

  // 7. Clean dataset summary for labeling
  lines.push("7. CLEAN DATASET SUMMARY");
  lines.push(sr);
  lines.push(`  Total paragraphs for labeling: ${cleanCount.toLocaleString()}`);
  const uniqueInClean = canonicalCounts.unique +
    canonicalCounts.cross_company_boilerplate +
    canonicalCounts.cross_year_copy;
  lines.push(`  Unique texts (canonical only): ~${uniqueInClean.toLocaleString()}`);
  lines.push(`  Labeling strategy: label canonical paragraphs, propagate to duplicates`);
  lines.push("");

  lines.push(hr);
  lines.push("  END OF QUALITY REPORT");
  lines.push(hr);

  return lines.join("\n");
}

// ─── Main ───

async function main() {
  process.stderr.write("  Loading paragraphs...\n");
  const { records: paragraphs, skipped } = await readJsonl(INPUT_PATH, Paragraph);
  if (skipped > 0) process.stderr.write(`  Warning: ${skipped} invalid lines skipped\n`);
  process.stderr.write(`  Loaded ${paragraphs.length} paragraphs\n\n`);

  // Step 1: Compute dedup info
  process.stderr.write("  Computing dedup metadata...\n");
  const dedupMap = computeDedupInfo(paragraphs);

  // Step 2: Check paragraph quality
  process.stderr.write("  Checking paragraph quality...\n");
  const paragraphFlags = new Map<string, QualityFlag[]>();
  for (const p of paragraphs) {
    paragraphFlags.set(p.id, checkParagraphQuality(p));
  }

  // Step 3: Check filing quality
  process.stderr.write("  Checking filing quality...\n");
  const filingQuality = analyzeFilingQuality(paragraphs);

  // Step 4: Build clean dataset
  // Remove: page artifacts, within-filing duplicates (keep canonical only)
  process.stderr.write("  Building clean dataset...\n");

  // Build a per-paragraph dedup info lookup
  const hashCounters = new Map<string, number>();
  const dedupInfoByParagraph = new Map<string, DedupInfo>();
  for (const p of paragraphs) {
    const counter = hashCounters.get(p.textHash) ?? 0;
    hashCounters.set(p.textHash, counter + 1);

    const infos = dedupMap.get(p.textHash);
    if (infos && counter < infos.length) {
      dedupInfoByParagraph.set(p.id, infos[counter]!);
    }
  }

  const cleanParagraphs: ParagraphType[] = [];
  let removed = 0;

  for (const p of paragraphs) {
    const flags = paragraphFlags.get(p.id) ?? [];
    const hasError = flags.some((f) => f.severity === "error");

    if (hasError) {
      removed++;
      continue;
    }

    cleanParagraphs.push(p);
  }

  // Write clean dataset
  await writeJsonl(CLEAN_PATH, cleanParagraphs);
  process.stderr.write(`  Written ${cleanParagraphs.length} clean paragraphs to ${CLEAN_PATH}\n`);

  // Step 5: Generate reports
  const report = generateReport(paragraphs, dedupMap, filingQuality, paragraphFlags, cleanParagraphs.length, removed);

  const reportPath = `${DATA}/analysis/quality-report.txt`;
  const jsonPath = `${DATA}/analysis/quality-report.json`;
  await mkdir(dirname(reportPath), { recursive: true });
  await writeFile(reportPath, report);

  // JSON report
  const jsonReport = {
    generatedAt: new Date().toISOString(),
    input: { paragraphs: paragraphs.length, filings: filingQuality.size },
    clean: { paragraphs: cleanParagraphs.length, removed },
    dedupCategories: {
      unique: { total: 0, canonical: 0 },
      cross_company_boilerplate: { total: 0, canonical: 0 },
      cross_year_copy: { total: 0, canonical: 0 },
      within_filing_dup: { total: 0, canonical: 0 },
    } as Record<string, { total: number; canonical: number }>,
    paragraphFlags: Object.fromEntries([...new Map<string, number>()].map(([k, v]) => [k, v])),
    filingFlags: Object.fromEntries([...new Map<string, number>()].map(([k, v]) => [k, v])),
  };

  // Populate dedup counts
  for (const [, infos] of dedupMap) {
    for (const info of infos) {
      jsonReport.dedupCategories[info.dedupCategory]!.total++;
      if (info.isCanonical) jsonReport.dedupCategories[info.dedupCategory]!.canonical++;
    }
  }

  // Populate flag counts
  const flagCounts = new Map<string, number>();
  for (const [, flags] of paragraphFlags) {
    for (const f of flags) {
      flagCounts.set(f.flag, (flagCounts.get(f.flag) ?? 0) + 1);
    }
  }
  jsonReport.paragraphFlags = Object.fromEntries(flagCounts);

  const filingFlagCounts = new Map<string, number>();
  for (const [, fq] of filingQuality) {
    for (const f of fq.flags) {
      filingFlagCounts.set(f.flag, (filingFlagCounts.get(f.flag) ?? 0) + 1);
    }
  }
  jsonReport.filingFlags = Object.fromEntries(filingFlagCounts);

  await writeFile(jsonPath, JSON.stringify(jsonReport, null, 2));

  process.stderr.write(`  Saved: ${reportPath}\n`);
  process.stderr.write(`  Saved: ${jsonPath}\n`);
  console.log(report);
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});
