import { describe, test, expect, beforeAll, afterAll, mock } from "bun:test"; import { db } from "@/db"; import { annotators, paragraphs, assignments, humanLabels, adjudications, } from "@/db/schema"; import { eq, inArray } from "drizzle-orm"; const cookieJar = new Map(); const mockCookieStore = { get(name: string) { const value = cookieJar.get(name); return value ? { value } : undefined; }, set(name: string, value: string, _opts?: unknown) { cookieJar.set(name, value); }, delete(name: string) { cookieJar.delete(name); }, }; mock.module("next/headers", () => ({ cookies: async () => mockCookieStore, })); const { createSession } = await import("@/lib/auth"); const { GET } = await import("../route"); const ADMIN_USER = { id: "admin", displayName: "Admin", password: "adminpass", }; const TEST_ANNOTATORS = [ { id: "met-ann-1", displayName: "Metric Annotator 1", password: "pass1" }, { id: "met-ann-2", displayName: "Metric Annotator 2", password: "pass2" }, { id: "met-ann-3", displayName: "Metric Annotator 3", password: "pass3" }, ]; const TEST_PARAGRAPHS = [ { id: "met-para-1", text: "Board oversight of cybersecurity risk.", wordCount: 6, paragraphIndex: 0, companyName: "Metrics Corp", cik: "0008888888", ticker: "METT", filingType: "10-K", filingDate: "2024-03-15", fiscalYear: 2024, accessionNumber: "0008888888-24-000001", secItem: "Item 1C", }, { id: "met-para-2", text: "CISO manages program.", wordCount: 3, paragraphIndex: 1, companyName: "Metrics Corp", cik: "0008888888", ticker: "METT", filingType: "10-K", filingDate: "2024-03-15", fiscalYear: 2024, accessionNumber: "0008888888-24-000001", secItem: "Item 1C", }, { id: "met-para-3", text: "Third party vendor management policies.", wordCount: 5, paragraphIndex: 2, companyName: "Metrics Corp", cik: "0008888888", ticker: "METT", filingType: "10-K", filingDate: "2024-03-15", fiscalYear: 2024, accessionNumber: "0008888888-24-000001", secItem: "Item 1C", }, ]; const PARAGRAPH_IDS = TEST_PARAGRAPHS.map((p) => p.id); const ALL_ANNOTATOR_IDS = [ ADMIN_USER.id, ...TEST_ANNOTATORS.map((a) => a.id), ]; async function cleanup() { await db .delete(adjudications) .where(inArray(adjudications.paragraphId, PARAGRAPH_IDS)); await db .delete(humanLabels) .where(inArray(humanLabels.paragraphId, PARAGRAPH_IDS)); await db .delete(assignments) .where(inArray(assignments.paragraphId, PARAGRAPH_IDS)); await db.delete(paragraphs).where(inArray(paragraphs.id, PARAGRAPH_IDS)); for (const a of TEST_ANNOTATORS) { await db.delete(annotators).where(eq(annotators.id, a.id)); } await db.delete(annotators).where(eq(annotators.id, ADMIN_USER.id)); } beforeAll(async () => { await cleanup(); await db .insert(annotators) .values([ADMIN_USER, ...TEST_ANNOTATORS]) .onConflictDoNothing(); await db.insert(paragraphs).values(TEST_PARAGRAPHS); // Assign all paragraphs to all 3 annotators const assignmentRows = TEST_PARAGRAPHS.flatMap((p) => TEST_ANNOTATORS.map((a) => ({ paragraphId: p.id, annotatorId: a.id, })), ); await db.insert(assignments).values(assignmentRows); // Create labels with known patterns: // Para 1: all 3 agree → Board Governance, specificity 3 // Para 2: 2 agree, 1 differs → Management Role (2/3), specificity 2 (2/3) // Para 3: all 3 agree → Third-Party Risk, specificity 2 await db.insert(humanLabels).values([ // Para 1 - full agreement { paragraphId: "met-para-1", annotatorId: "met-ann-1", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" }, { paragraphId: "met-para-1", annotatorId: "met-ann-2", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" }, { paragraphId: "met-para-1", annotatorId: "met-ann-3", contentCategory: "Board Governance", specificityLevel: 3, sessionId: "met-session" }, // Para 2 - 2/3 agreement { paragraphId: "met-para-2", annotatorId: "met-ann-1", contentCategory: "Management Role", specificityLevel: 2, sessionId: "met-session" }, { paragraphId: "met-para-2", annotatorId: "met-ann-2", contentCategory: "Management Role", specificityLevel: 2, sessionId: "met-session" }, { paragraphId: "met-para-2", annotatorId: "met-ann-3", contentCategory: "Board Governance", specificityLevel: 1, sessionId: "met-session" }, // Para 3 - full agreement { paragraphId: "met-para-3", annotatorId: "met-ann-1", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" }, { paragraphId: "met-para-3", annotatorId: "met-ann-2", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" }, { paragraphId: "met-para-3", annotatorId: "met-ann-3", contentCategory: "Third-Party Risk", specificityLevel: 2, sessionId: "met-session" }, ]); // Create one adjudication await db.insert(adjudications).values({ paragraphId: "met-para-1", finalCategory: "Board Governance", finalSpecificity: 3, method: "consensus", }); cookieJar.clear(); await createSession(ADMIN_USER.id); }); afterAll(async () => { await cleanup(); }); describe("GET /api/metrics", () => { test("returns expected progress numbers", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); // 3 test paragraphs all have 3 labels → all fully labeled expect(data.progress.fullyLabeled).toBe(3); expect(data.progress.adjudicated).toBe(1); // Per-annotator: each should have 3 completed (one label per paragraph) for (const ann of data.progress.perAnnotator) { if (TEST_ANNOTATORS.some((a) => a.id === ann.id)) { expect(ann.completed).toBe(3); expect(ann.total).toBe(3); } } }); test("returns valid kappa value", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); // With high agreement (2/3 paragraphs fully agree, 1 has 2/3 majority), kappa should be high expect(data.agreement.avgKappa).toBeGreaterThan(0); expect(data.agreement.avgKappa).toBeLessThanOrEqual(1); // Kappa matrix should have our 3 annotators expect(data.agreement.kappaMatrix.annotators.length).toBeGreaterThanOrEqual(3); expect(data.agreement.kappaMatrix.values.length).toBe( data.agreement.kappaMatrix.annotators.length, ); }); test("returns valid alpha value", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); // Krippendorff's alpha should be defined and within valid range expect(typeof data.agreement.krippendorffsAlpha).toBe("number"); expect(data.agreement.krippendorffsAlpha).toBeLessThanOrEqual(1); }); test("returns consensus rate", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); // 2 out of 3 paragraphs have full category agreement // Consensus rate should be ~0.667 expect(data.agreement.consensusRate).toBeGreaterThan(0.5); expect(data.agreement.consensusRate).toBeLessThanOrEqual(1); }); test("returns confusion matrix with correct dimensions", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); expect(data.confusionMatrix.labels).toHaveLength(7); expect(data.confusionMatrix.matrix).toHaveLength(7); for (const row of data.confusionMatrix.matrix) { expect(row).toHaveLength(7); } }); test("returns per-category agreement", async () => { const res = await GET(); const data = await res.json(); expect(res.status).toBe(200); expect(data.agreement.perCategory).toBeDefined(); // Board Governance appears in all 3 paragraphs' labels but only fully agreed in para 1 expect(typeof data.agreement.perCategory["Board Governance"]).toBe( "number", ); }); }); describe("GET /api/metrics - unauthorized", () => { test("returns 401 without session", async () => { cookieJar.clear(); const res = await GET(); expect(res.status).toBe(401); }); test("returns 403 for non-admin user", async () => { cookieJar.clear(); await createSession(TEST_ANNOTATORS[0].id); const res = await GET(); expect(res.status).toBe(403); // Restore admin session cookieJar.clear(); await createSession(ADMIN_USER.id); }); });