SEC-cyBERT/labelapp/app/api/metrics/__test__/metrics.test.ts

import { describe, test, expect, beforeAll, afterAll, mock } from "bun:test";
import { db } from "@/db";
import {
  annotators,
  paragraphs,
  assignments,
  humanLabels,
  adjudications,
} from "@/db/schema";
import { eq, inArray } from "drizzle-orm";

const cookieJar = new Map<string, string>();
const mockCookieStore = {
  get(name: string) {
    const value = cookieJar.get(name);
    return value ? { value } : undefined;
  },
  set(name: string, value: string, _opts?: unknown) {
    cookieJar.set(name, value);
  },
  delete(name: string) {
    cookieJar.delete(name);
  },
};

mock.module("next/headers", () => ({
  cookies: async () => mockCookieStore,
}));

const { createSession } = await import("@/lib/auth");
const { GET } = await import("../route");

const ADMIN_USER = {
  id: "joey",
  displayName: "Joey",
  password: "adminpass",
};

const TEST_ANNOTATORS = [
  { id: "met-ann-1", displayName: "Metric Annotator 1", password: "pass1" },
  { id: "met-ann-2", displayName: "Metric Annotator 2", password: "pass2" },
  { id: "met-ann-3", displayName: "Metric Annotator 3", password: "pass3" },
];

const TEST_PARAGRAPHS = [
  {
    id: "met-para-1",
    text: "Board oversight of cybersecurity risk.",
    wordCount: 6,
    paragraphIndex: 0,
    companyName: "Metrics Corp",
    cik: "0008888888",
    ticker: "METT",
    filingType: "10-K",
    filingDate: "2024-03-15",
    fiscalYear: 2024,
    accessionNumber: "0008888888-24-000001",
    secItem: "Item 1C",
  },
  {
    id: "met-para-2",
    text: "CISO manages program.",
    wordCount: 3,
    paragraphIndex: 1,
    companyName: "Metrics Corp",
    cik: "0008888888",
    ticker: "METT",
    filingType: "10-K",
    filingDate: "2024-03-15",
    fiscalYear: 2024,
    accessionNumber: "0008888888-24-000001",
    secItem: "Item 1C",
  },
  {
    id: "met-para-3",
    text: "Third party vendor management policies.",
    wordCount: 5,
    paragraphIndex: 2,
    companyName: "Metrics Corp",
    cik: "0008888888",
    ticker: "METT",
    filingType: "10-K",
    filingDate: "2024-03-15",
    fiscalYear: 2024,
    accessionNumber: "0008888888-24-000001",
    secItem: "Item 1C",
  },
];

const PARAGRAPH_IDS = TEST_PARAGRAPHS.map((p) => p.id);
const ALL_ANNOTATOR_IDS = [
  ADMIN_USER.id,
  ...TEST_ANNOTATORS.map((a) => a.id),
];

async function cleanup() {
  await db
    .delete(adjudications)
    .where(inArray(adjudications.paragraphId, PARAGRAPH_IDS));
  await db
    .delete(humanLabels)
    .where(inArray(humanLabels.paragraphId, PARAGRAPH_IDS));
  await db
    .delete(assignments)
    .where(inArray(assignments.paragraphId, PARAGRAPH_IDS));
  await db.delete(paragraphs).where(inArray(paragraphs.id, PARAGRAPH_IDS));
  for (const a of TEST_ANNOTATORS) {
    await db.delete(annotators).where(eq(annotators.id, a.id));
  }
  await db.delete(annotators).where(eq(annotators.id, ADMIN_USER.id));
}

beforeAll(async () => {
  await cleanup();

  await db
    .insert(annotators)
    .values([ADMIN_USER, ...TEST_ANNOTATORS])
    .onConflictDoNothing();
  await db.insert(paragraphs).values(TEST_PARAGRAPHS);

  // Assign all paragraphs to all 3 annotators
  const assignmentRows = TEST_PARAGRAPHS.flatMap((p) =>
    TEST_ANNOTATORS.map((a) => ({
      paragraphId: p.id,
      annotatorId: a.id,
    })),
  );
  await db.insert(assignments).values(assignmentRows);

  // Create labels with known patterns:
  // Para 1: all 3 agree → Board Governance, specificity 3
  // Para 2: 2 agree, 1 differs → Management Role (2/3), specificity 2 (2/3)
  // Para 3: all 3 agree → Third-Party Risk, specificity 2
  await db.insert(humanLabels).values([
    // Para 1 - full agreement
    { paragraphId: "met-para-1", annotatorId: "met-ann-1", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" },
    { paragraphId: "met-para-1", annotatorId: "met-ann-2", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" },
    { paragraphId: "met-para-1", annotatorId: "met-ann-3", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" },
    // Para 2 - 2/3 agreement
    { paragraphId: "met-para-2", annotatorId: "met-ann-1", contentCategory: "Management Role", specificityLevel: 2, sessionId: "met-session" },
    { paragraphId: "met-para-2", annotatorId: "met-ann-2", contentCategory: "Management Role", specificityLevel: 2, sessionId: "met-session" },
    { paragraphId: "met-para-2", annotatorId: "met-ann-3", contentCategory: "Board Governance", specificityLevel: 1, sessionId: "met-session" },
    // Para 3 - full agreement
    { paragraphId: "met-para-3", annotatorId: "met-ann-1", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" },
    { paragraphId: "met-para-3", annotatorId: "met-ann-2", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" },
    { paragraphId: "met-para-3", annotatorId: "met-ann-3", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" },
  ]);

  // Create one adjudication
  await db.insert(adjudications).values({
    paragraphId: "met-para-1",
    finalCategory: "Board Governance",
    finalSpecificity: 3,
    method: "consensus",
  });

  cookieJar.clear();
  await createSession(ADMIN_USER.id);
});

afterAll(async () => {
  await cleanup();
});

describe("GET /api/metrics", () => {
  test("returns expected progress numbers", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);

    // 3 test paragraphs all have 3 labels → all fully labeled
    expect(data.progress.fullyLabeled).toBe(3);
    expect(data.progress.adjudicated).toBe(1);

    // Per-annotator: each should have 3 completed (one label per paragraph)
    for (const ann of data.progress.perAnnotator) {
      if (TEST_ANNOTATORS.some((a) => a.id === ann.id)) {
        expect(ann.completed).toBe(3);
        expect(ann.total).toBe(3);
      }
    }
  });

  test("returns valid kappa value", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);

    // With high agreement (2/3 paragraphs fully agree, 1 has 2/3 majority), kappa should be high
    expect(data.agreement.avgKappa).toBeGreaterThan(0);
    expect(data.agreement.avgKappa).toBeLessThanOrEqual(1);

    // Kappa matrix should have our 3 annotators
    expect(data.agreement.kappaMatrix.annotators.length).toBeGreaterThanOrEqual(3);
    expect(data.agreement.kappaMatrix.values.length).toBe(
      data.agreement.kappaMatrix.annotators.length,
    );
  });

  test("returns valid alpha value", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);

    // Krippendorff's alpha should be defined and within valid range
    expect(typeof data.agreement.krippendorffsAlpha).toBe("number");
    expect(data.agreement.krippendorffsAlpha).toBeLessThanOrEqual(1);
  });

  test("returns consensus rate", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);

    // 2 out of 3 paragraphs have full category agreement
    // Consensus rate should be ~0.667
    expect(data.agreement.consensusRate).toBeGreaterThan(0.5);
    expect(data.agreement.consensusRate).toBeLessThanOrEqual(1);
  });

  test("returns confusion matrix with correct dimensions", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);
    expect(data.confusionMatrix.labels).toHaveLength(7);
    expect(data.confusionMatrix.matrix).toHaveLength(7);
    for (const row of data.confusionMatrix.matrix) {
      expect(row).toHaveLength(7);
    }
  });

  test("returns per-category agreement", async () => {
    const res = await GET();
    const data = await res.json();

    expect(res.status).toBe(200);
    expect(data.agreement.perCategory).toBeDefined();

    // Board Governance appears in all 3 paragraphs' labels but only fully agreed in para 1
    expect(typeof data.agreement.perCategory["Board Governance"]).toBe(
      "number",
    );
  });
});

describe("GET /api/metrics - unauthorized", () => {
  test("returns 401 without session", async () => {
    cookieJar.clear();
    const res = await GET();
    expect(res.status).toBe(401);
  });

  test("returns 403 for non-admin user", async () => {
    cookieJar.clear();
    await createSession(TEST_ANNOTATORS[0].id);
    const res = await GET();
    expect(res.status).toBe(403);

    // Restore admin session
    cookieJar.clear();
    await createSession(ADMIN_USER.id);
  });
});