const USER_AGENT = "sec-cyBERT research@sec-cybert.dev";
const BASE_DELAY_MS = 110; // ~9 req/s to stay safely under 10 req/s limit

let lastRequestTime = 0;

/** Rate-limited fetch for EDGAR APIs. Enforces 10 req/s with sequential throttle. */
async function edgarFetch(url: string): Promise<Response> {
  const now = Date.now();
  const elapsed = now - lastRequestTime;
  if (elapsed < BASE_DELAY_MS) {
    await new Promise((r) => setTimeout(r, BASE_DELAY_MS - elapsed));
  }
  lastRequestTime = Date.now();

  const res = await fetch(url, {
    headers: {
      "User-Agent": USER_AGENT,
      Accept: "application/json, text/html, */*",
    },
  });

  if (res.status === 403) {
    throw new Error(`EDGAR 403 Forbidden — check User-Agent header. URL: ${url}`);
  }
  if (res.status === 429) {
    // Back off and retry once
    process.stderr.write("  ⟳ EDGAR 429 — backing off 5s\n");
    await new Promise((r) => setTimeout(r, 5000));
    lastRequestTime = Date.now();
    return fetch(url, {
      headers: { "User-Agent": USER_AGENT, Accept: "application/json, text/html, */*" },
    });
  }
  if (!res.ok) {
    throw new Error(`EDGAR HTTP ${res.status} for ${url}`);
  }

  return res;
}

/** Fetch JSON from EDGAR. */
export async function edgarJson<T>(url: string): Promise<T> {
  const res = await edgarFetch(url);
  return res.json() as Promise<T>;
}

/** Fetch HTML from EDGAR. */
export async function edgarHtml(url: string): Promise<string> {
  const res = await edgarFetch(url);
  return res.text();
}

// ─── EDGAR Full-Text Search API ───

interface EftsHit {
  _id: string;
  _score: number;
  _source: {
    ciks: string[];
    display_names: string[];
    root_forms: string[];
    form: string;
    file_date: string;
    adsh: string; // accession number with dashes
    period_ending: string;
    sics: string[];
    items: string[];
  };
}

interface EftsResponse {
  hits: {
    total: { value: number; relation: string };
    hits: EftsHit[];
  };
}

export interface FilingRef {
  cik: string;
  displayName: string;
  accessionNumber: string;
  form: string;
  filingDate: string;
  periodEnding: string;
  sic: string;
}

/**
 * Search EDGAR full-text search for filings.
 * Paginates automatically up to maxResults.
 */
export async function searchFilings(opts: {
  query: string;
  forms: string;
  startDate: string;
  endDate: string;
  maxResults?: number;
}): Promise<FilingRef[]> {
  const { query, forms, startDate, endDate, maxResults = 10000 } = opts;
  const results: FilingRef[] = [];
  const pageSize = 100;

  for (let from = 0; from < maxResults; from += pageSize) {
    const params = new URLSearchParams({
      q: query,
      forms,
      dateRange: "custom",
      startdt: startDate,
      enddt: endDate,
      _source: "true",
      from: String(from),
      size: String(pageSize),
    });

    const url = `https://efts.sec.gov/LATEST/search-index?${params}`;
    const data = await edgarJson<EftsResponse>(url);

    for (const hit of data.hits.hits) {
      const s = hit._source;
      results.push({
        cik: s.ciks[0] ?? "",
        displayName: s.display_names[0] ?? "",
        accessionNumber: s.adsh,
        form: s.form,
        filingDate: s.file_date,
        periodEnding: s.period_ending,
        sic: s.sics[0] ?? "",
      });
    }

    // Trim to maxResults
    if (results.length >= maxResults) {
      results.length = maxResults;
      process.stderr.write(`  Fetched ${results.length}/${data.hits.total.value} filing refs (capped at ${maxResults})\n`);
      break;
    }

    process.stderr.write(
      `  Fetched ${results.length}/${data.hits.total.value} filing refs\n`,
    );

    if (results.length >= data.hits.total.value) break;
  }

  return results;
}

// ─── Master Index (bulk filing discovery) ───

export interface MasterIndexEntry {
  cik: string;
  companyName: string;
  formType: string;
  dateFiled: string;
  filename: string; // edgar/data/{cik}/{accession}.txt
  accessionNumber: string;
}

/**
 * Download and parse a quarterly master.idx file from EDGAR.
 * These files list every filing made in that quarter.
 * Format: CIK|Company Name|Form Type|Date Filed|Filename
 */
async function fetchMasterIndex(year: number, quarter: number): Promise<MasterIndexEntry[]> {
  const url = `https://www.sec.gov/Archives/edgar/full-index/${year}/QTR${quarter}/master.idx`;
  const res = await edgarFetch(url);
  const text = await res.text();

  const entries: MasterIndexEntry[] = [];
  const lines = text.split("\n");

  // Skip header lines (description, blank lines, separator)
  let dataStarted = false;
  for (const line of lines) {
    if (line.startsWith("---")) {
      dataStarted = true;
      continue;
    }
    if (!dataStarted) continue;

    const parts = line.split("|");
    if (parts.length < 5) continue;

    const [cik, companyName, formType, dateFiled, filename] = parts;

    // Extract accession number from filename: edgar/data/{cik}/{accession}.txt
    const accessionMatch = filename!.match(/(\d{10}-\d{2}-\d{6})/);
    if (!accessionMatch) continue;

    entries.push({
      cik: cik!.trim(),
      companyName: companyName!.trim(),
      formType: formType!.trim(),
      dateFiled: dateFiled!.trim(),
      filename: filename!.trim(),
      accessionNumber: accessionMatch[1]!,
    });
  }

  return entries;
}

/**
 * Enumerate all filings of given form types across a date range using master.idx files.
 * This is complete and deterministic — no full-text search limitations.
 */
export async function enumerateFilings(opts: {
  formTypes: string[];
  startDate: string;
  endDate: string;
}): Promise<MasterIndexEntry[]> {
  const { formTypes, startDate, endDate } = opts;
  const formSet = new Set(formTypes);

  // Determine which quarters to fetch
  const startYear = parseInt(startDate.slice(0, 4));
  const startMonth = parseInt(startDate.slice(5, 7));
  const endYear = parseInt(endDate.slice(0, 4));
  const endMonth = parseInt(endDate.slice(5, 7));

  const quarters: Array<{ year: number; quarter: number }> = [];
  for (let y = startYear; y <= endYear; y++) {
    for (let q = 1; q <= 4; q++) {
      const qStartMonth = (q - 1) * 3 + 1;
      const qEndMonth = q * 3;
      // Check if this quarter overlaps with our date range
      if (y === startYear && qEndMonth < startMonth) continue;
      if (y === endYear && qStartMonth > endMonth) continue;
      quarters.push({ year: y, quarter: q });
    }
  }

  const allEntries: MasterIndexEntry[] = [];

  for (const { year, quarter } of quarters) {
    process.stderr.write(`  Fetching master.idx for ${year}/QTR${quarter}...\n`);
    try {
      const entries = await fetchMasterIndex(year, quarter);
      const filtered = entries.filter(
        (e) => formSet.has(e.formType) && e.dateFiled >= startDate && e.dateFiled <= endDate,
      );
      allEntries.push(...filtered);
      process.stderr.write(`    ${filtered.length} ${formTypes.join("/")} filings\n`);
    } catch (error) {
      // Future quarters may not exist yet
      process.stderr.write(`    (not available)\n`);
    }
  }

  return allEntries;
}

// ─── Company Submissions API ───

interface SubmissionsResponse {
  cik: string;
  entityType: string;
  sic: string;
  sicDescription: string;
  name: string;
  tickers: string[];
  exchanges: string[];
  stateOfIncorporation: string;
  filings: {
    recent: {
      accessionNumber: string[];
      filingDate: string[];
      reportDate: string[];
      form: string[];
      primaryDocument: string[];
      primaryDocDescription: string[];
      items: string[];
    };
    files: Array<{ name: string; filingCount: number }>;
  };
}

export interface CompanyMeta {
  cik: string;
  name: string;
  tickers: string[];
  sic: string;
  sicDescription: string;
  stateOfIncorporation: string;
}

export interface CompanyFiling {
  accessionNumber: string;
  filingDate: string;
  reportDate: string;
  form: string;
  primaryDocument: string;
  items: string;
}

/** Get company metadata and recent filings from the submissions API. */
export async function getCompanySubmissions(cik: string): Promise<{
  meta: CompanyMeta;
  filings: CompanyFiling[];
}> {
  const paddedCik = cik.replace(/^0+/, "").padStart(10, "0");
  const url = `https://data.sec.gov/submissions/CIK${paddedCik}.json`;
  const data = await edgarJson<SubmissionsResponse>(url);

  const recent = data.filings.recent;
  const filings: CompanyFiling[] = [];
  for (let i = 0; i < recent.accessionNumber.length; i++) {
    filings.push({
      accessionNumber: recent.accessionNumber[i]!,
      filingDate: recent.filingDate[i]!,
      reportDate: recent.reportDate[i]!,
      form: recent.form[i]!,
      primaryDocument: recent.primaryDocument[i]!,
      items: recent.items[i]!,
    });
  }

  return {
    meta: {
      cik: data.cik,
      name: data.name,
      tickers: data.tickers,
      sic: data.sic,
      sicDescription: data.sicDescription,
      stateOfIncorporation: data.stateOfIncorporation,
    },
    filings,
  };
}

/** Build the URL to download a filing's primary document. */
export function filingDocUrl(cik: string, accessionNumber: string, primaryDocument: string): string {
  const cleanCik = cik.replace(/^0+/, "");
  const cleanAccession = accessionNumber.replace(/-/g, "");
  return `https://www.sec.gov/Archives/edgar/data/${cleanCik}/${cleanAccession}/${primaryDocument}`;
}
