2026-04-05 00:55:53 -04:00

751 lines
24 KiB
TypeScript

"use client";
import { useEffect, useState, useCallback } from "react";
import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs";
import {
Card,
CardHeader,
CardTitle,
CardDescription,
CardContent,
} from "@/components/ui/card";
import {
Table,
TableHeader,
TableBody,
TableHead,
TableRow,
TableCell,
} from "@/components/ui/table";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogDescription,
DialogFooter,
} from "@/components/ui/dialog";
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
import { Textarea } from "@/components/ui/textarea";
import { Progress, ProgressLabel, ProgressValue } from "@/components/ui/progress";
import { Separator } from "@/components/ui/separator";
const CATEGORIES = [
"Board Governance",
"Management Role",
"Risk Management Process",
"Third-Party Risk",
"Incident Disclosure",
"Strategy Integration",
"None/Other",
] as const;
const SPECIFICITIES = [
{ value: 1, label: "1 - Generic/Boilerplate" },
{ value: 2, label: "2 - Domain-Adapted" },
{ value: 3, label: "3 - Firm-Specific" },
{ value: 4, label: "4 - Quantified-Verifiable" },
] as const;
interface QueueLabel {
annotatorId: string;
displayName: string;
category: string;
specificity: number;
notes: string | null;
}
interface QueueItem {
paragraphId: string;
paragraphText: string;
labels: QueueLabel[];
stage1Category: string | null;
stage1Specificity: number | null;
splitSeverity: number;
}
interface AnnotatorProgress {
id: string;
displayName: string;
completed: number;
total: number;
}
interface MetricsData {
progress: {
totalParagraphs: number;
fullyLabeled: number;
adjudicated: number;
perAnnotator: AnnotatorProgress[];
};
agreement: {
consensusRate: number;
avgKappa: number;
kappaMatrix: {
annotators: string[];
values: number[][];
};
krippendorffsAlpha: number;
perCategory: Record<string, number>;
};
confusionMatrix: {
labels: string[];
matrix: number[][];
};
}
function kappaColor(value: number): string {
if (value >= 0.75) return "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/30 dark:text-emerald-300";
if (value >= 0.5) return "bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-300";
return "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300";
}
function heatmapColor(value: number, max: number): string {
if (max === 0) return "";
const intensity = value / max;
if (intensity > 0.7) return "bg-blue-200 dark:bg-blue-900/50";
if (intensity > 0.4) return "bg-blue-100 dark:bg-blue-900/30";
if (intensity > 0.1) return "bg-blue-50 dark:bg-blue-900/15";
return "";
}
function AdjudicationQueue() {
const [queue, setQueue] = useState<QueueItem[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [selectedItem, setSelectedItem] = useState<QueueItem | null>(null);
const [dialogOpen, setDialogOpen] = useState(false);
const [resolveCategory, setResolveCategory] = useState("");
const [resolveSpecificity, setResolveSpecificity] = useState("");
const [resolveNotes, setResolveNotes] = useState("");
const [submitting, setSubmitting] = useState(false);
const [expandedParagraphs, setExpandedParagraphs] = useState<Set<string>>(
new Set(),
);
const fetchQueue = useCallback(async () => {
setLoading(true);
setError(null);
try {
const res = await fetch("/api/adjudicate");
if (!res.ok) {
const data = await res.json();
throw new Error(data.error ?? "Failed to load queue");
}
const data = await res.json();
setQueue(data.queue);
} catch (e) {
setError(e instanceof Error ? e.message : "Unknown error");
} finally {
setLoading(false);
}
}, []);
useEffect(() => {
fetchQueue();
}, [fetchQueue]);
function openResolveDialog(item: QueueItem) {
setSelectedItem(item);
setResolveCategory("");
setResolveSpecificity("");
setResolveNotes("");
setDialogOpen(true);
}
async function handleSubmitResolution() {
if (!selectedItem || !resolveCategory || !resolveSpecificity) return;
setSubmitting(true);
try {
const res = await fetch("/api/adjudicate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
paragraphId: selectedItem.paragraphId,
finalCategory: resolveCategory,
finalSpecificity: Number(resolveSpecificity),
notes: resolveNotes || undefined,
}),
});
if (!res.ok) {
const data = await res.json();
throw new Error(data.error ?? "Failed to resolve");
}
setQueue((prev) =>
prev.filter((item) => item.paragraphId !== selectedItem.paragraphId),
);
setDialogOpen(false);
} catch (e) {
setError(e instanceof Error ? e.message : "Unknown error");
} finally {
setSubmitting(false);
}
}
function toggleExpand(paragraphId: string) {
setExpandedParagraphs((prev) => {
const next = new Set(prev);
if (next.has(paragraphId)) {
next.delete(paragraphId);
} else {
next.add(paragraphId);
}
return next;
});
}
if (loading) {
return (
<Card>
<CardContent className="py-8 text-center text-muted-foreground">
Loading adjudication queue...
</CardContent>
</Card>
);
}
if (error) {
return (
<Card>
<CardContent className="py-8 text-center text-red-600">{error}</CardContent>
</Card>
);
}
return (
<>
<Card>
<CardHeader>
<CardTitle className="text-lg">Adjudication Queue</CardTitle>
<CardDescription>
{queue.length === 0
? "All disagreements have been resolved."
: `${queue.length} paragraph${queue.length === 1 ? "" : "s"} need${queue.length === 1 ? "s" : ""} adjudication`}
</CardDescription>
</CardHeader>
<CardContent>
{queue.length === 0 ? (
<p className="text-sm text-muted-foreground">
No paragraphs pending manual review. Consensus and majority
agreements are auto-resolved.
</p>
) : (
<div className="space-y-4">
{queue.map((item) => {
const isExpanded = expandedParagraphs.has(item.paragraphId);
const truncatedText =
item.paragraphText.length > 200 && !isExpanded
? item.paragraphText.slice(0, 200) + "..."
: item.paragraphText;
return (
<Card key={item.paragraphId} className="border">
<CardContent className="pt-4 space-y-3">
{/* Severity badge */}
<div className="flex items-center justify-between">
<Badge
variant={
item.splitSeverity >= 3
? "destructive"
: "secondary"
}
>
{item.splitSeverity >= 3
? "3-way split"
: "2-way split"}
</Badge>
<span className="text-xs text-muted-foreground font-mono">
{item.paragraphId}
</span>
</div>
{/* Paragraph text */}
<div>
<p className="text-sm leading-relaxed">
{truncatedText}
</p>
{item.paragraphText.length > 200 && (
<button
onClick={() => toggleExpand(item.paragraphId)}
className="text-xs text-primary hover:underline mt-1"
>
{isExpanded ? "Show less" : "Show more"}
</button>
)}
</div>
{/* Stage 1 reference */}
{item.stage1Category && (
<p className="text-xs text-muted-foreground">
Stage 1: {item.stage1Category}
{item.stage1Specificity !== null &&
` (specificity ${item.stage1Specificity})`}
</p>
)}
<Separator />
{/* Annotator labels side by side */}
<Table>
<TableHeader>
<TableRow>
<TableHead>Annotator</TableHead>
<TableHead>Category</TableHead>
<TableHead>Specificity</TableHead>
<TableHead>Notes</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{item.labels.map((label) => (
<TableRow key={label.annotatorId}>
<TableCell className="font-medium">
{label.displayName}
</TableCell>
<TableCell>{label.category}</TableCell>
<TableCell>{label.specificity}</TableCell>
<TableCell className="text-muted-foreground max-w-[200px] truncate">
{label.notes ?? "-"}
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
<div className="flex justify-end">
<Button onClick={() => openResolveDialog(item)}>
Resolve
</Button>
</div>
</CardContent>
</Card>
);
})}
</div>
)}
</CardContent>
</Card>
{/* Resolution Dialog */}
<Dialog open={dialogOpen} onOpenChange={setDialogOpen}>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>Resolve Adjudication</DialogTitle>
<DialogDescription>
Select the final category and specificity for this paragraph.
</DialogDescription>
</DialogHeader>
<div className="space-y-5 py-2">
{/* Category selection */}
<div className="space-y-2">
<label className="text-sm font-medium">Content Category</label>
<RadioGroup
value={resolveCategory}
onValueChange={setResolveCategory}
>
{CATEGORIES.map((cat) => (
<div key={cat} className="flex items-center gap-2">
<RadioGroupItem value={cat} />
<label className="text-sm cursor-pointer">{cat}</label>
</div>
))}
</RadioGroup>
</div>
{/* Specificity selection */}
<div className="space-y-2">
<label className="text-sm font-medium">Specificity Level</label>
<RadioGroup
value={resolveSpecificity}
onValueChange={setResolveSpecificity}
>
{SPECIFICITIES.map((spec) => (
<div key={spec.value} className="flex items-center gap-2">
<RadioGroupItem value={String(spec.value)} />
<label className="text-sm cursor-pointer">
{spec.label}
</label>
</div>
))}
</RadioGroup>
</div>
{/* Notes */}
<div className="space-y-2">
<label className="text-sm font-medium">
Notes{" "}
<span className="text-muted-foreground font-normal">
(optional)
</span>
</label>
<Textarea
value={resolveNotes}
onChange={(e) => setResolveNotes(e.target.value)}
placeholder="Reasoning for this decision..."
rows={3}
/>
</div>
</div>
<DialogFooter>
<Button
onClick={handleSubmitResolution}
disabled={
!resolveCategory || !resolveSpecificity || submitting
}
>
{submitting ? "Submitting..." : "Submit Resolution"}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</>
);
}
function MetricsDashboard() {
const [metrics, setMetrics] = useState<MetricsData | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
async function fetchMetrics() {
try {
const res = await fetch("/api/metrics");
if (!res.ok) {
const data = await res.json();
throw new Error(data.error ?? "Failed to load metrics");
}
const data = await res.json();
setMetrics(data);
} catch (e) {
setError(e instanceof Error ? e.message : "Unknown error");
} finally {
setLoading(false);
}
}
fetchMetrics();
}, []);
if (loading) {
return (
<Card>
<CardContent className="py-8 text-center text-muted-foreground">
Computing metrics...
</CardContent>
</Card>
);
}
if (error || !metrics) {
return (
<Card>
<CardContent className="py-8 text-center text-red-600">
{error ?? "Failed to load metrics"}
</CardContent>
</Card>
);
}
const { progress, agreement, confusionMatrix: cm } = metrics;
const labeledPct =
progress.totalParagraphs > 0
? Math.round((progress.fullyLabeled / progress.totalParagraphs) * 100)
: 0;
const confMax = Math.max(...cm.matrix.flat(), 1);
return (
<div className="space-y-6">
{/* Progress Section */}
<Card>
<CardHeader>
<CardTitle className="text-lg">Progress</CardTitle>
<CardDescription>
{progress.fullyLabeled} / {progress.totalParagraphs} fully labeled,{" "}
{progress.adjudicated} adjudicated
</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
<Progress value={labeledPct}>
<ProgressLabel>Fully Labeled</ProgressLabel>
<ProgressValue />
</Progress>
<Separator />
<h4 className="text-sm font-medium">Per-Annotator Progress</h4>
<Table>
<TableHeader>
<TableRow>
<TableHead>Annotator</TableHead>
<TableHead className="text-right">Completed</TableHead>
<TableHead className="text-right">Total</TableHead>
<TableHead className="text-right">%</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{progress.perAnnotator.map((a) => {
const pct =
a.total > 0 ? Math.round((a.completed / a.total) * 100) : 0;
return (
<TableRow key={a.id}>
<TableCell className="font-medium">
{a.displayName}
</TableCell>
<TableCell className="text-right tabular-nums">
{a.completed}
</TableCell>
<TableCell className="text-right tabular-nums">
{a.total}
</TableCell>
<TableCell className="text-right tabular-nums">
{pct}%
</TableCell>
</TableRow>
);
})}
</TableBody>
</Table>
</CardContent>
</Card>
{/* Agreement Section */}
<Card>
<CardHeader>
<CardTitle className="text-lg">Inter-Rater Agreement</CardTitle>
</CardHeader>
<CardContent className="space-y-6">
{/* Summary stats */}
<div className="grid grid-cols-1 sm:grid-cols-3 gap-4">
<Card className="border">
<CardContent className="pt-4 pb-3 text-center">
<p className="text-2xl font-bold tabular-nums">
{(agreement.consensusRate * 100).toFixed(1)}%
</p>
<p className="text-xs text-muted-foreground mt-1">
Consensus Rate (all 3 agree)
</p>
</CardContent>
</Card>
<Card className="border">
<CardContent className="pt-4 pb-3 text-center">
<p className="text-2xl font-bold tabular-nums">
{agreement.avgKappa.toFixed(3)}
</p>
<p className="text-xs text-muted-foreground mt-1">
Avg Cohen&apos;s Kappa
</p>
</CardContent>
</Card>
<Card className="border">
<CardContent className="pt-4 pb-3 text-center">
<p className="text-2xl font-bold tabular-nums">
{agreement.krippendorffsAlpha.toFixed(3)}
</p>
<p className="text-xs text-muted-foreground mt-1">
Krippendorff&apos;s Alpha (specificity)
</p>
</CardContent>
</Card>
</div>
<Separator />
{/* Kappa Matrix */}
{agreement.kappaMatrix.annotators.length > 0 && (
<div>
<h4 className="text-sm font-medium mb-3">
Pairwise Cohen&apos;s Kappa
</h4>
<div className="overflow-x-auto">
<Table>
<TableHeader>
<TableRow>
<TableHead />
{agreement.kappaMatrix.annotators.map((name) => (
<TableHead
key={name}
className="text-center text-xs px-1"
>
{name}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{agreement.kappaMatrix.annotators.map((name, i) => (
<TableRow key={name}>
<TableCell className="font-medium text-xs">
{name}
</TableCell>
{agreement.kappaMatrix.values[i].map((val, j) => (
<TableCell
key={j}
className={`text-center text-xs tabular-nums px-1 ${
i === j ? "bg-muted" : kappaColor(val)
}`}
>
{i === j ? "-" : val.toFixed(2)}
</TableCell>
))}
</TableRow>
))}
</TableBody>
</Table>
</div>
<div className="flex items-center gap-4 mt-2 text-xs text-muted-foreground">
<span className="flex items-center gap-1">
<span className="inline-block w-3 h-3 rounded bg-emerald-100 dark:bg-emerald-900/30" />
Good (&ge;0.75)
</span>
<span className="flex items-center gap-1">
<span className="inline-block w-3 h-3 rounded bg-amber-100 dark:bg-amber-900/30" />
Moderate (0.5-0.75)
</span>
<span className="flex items-center gap-1">
<span className="inline-block w-3 h-3 rounded bg-red-100 dark:bg-red-900/30" />
Poor (&lt;0.5)
</span>
</div>
</div>
)}
<Separator />
{/* Per-Category Agreement */}
<div>
<h4 className="text-sm font-medium mb-3">
Per-Category Agreement Rates
</h4>
<Table>
<TableHeader>
<TableRow>
<TableHead>Category</TableHead>
<TableHead className="text-right">Agreement</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{CATEGORIES.map((cat) => {
const rate = agreement.perCategory[cat] ?? 0;
return (
<TableRow key={cat}>
<TableCell>{cat}</TableCell>
<TableCell className="text-right tabular-nums">
<Badge
variant={
rate >= 0.75
? "default"
: rate >= 0.5
? "secondary"
: "destructive"
}
>
{(rate * 100).toFixed(1)}%
</Badge>
</TableCell>
</TableRow>
);
})}
</TableBody>
</Table>
</div>
</CardContent>
</Card>
{/* Confusion Matrix */}
<Card>
<CardHeader>
<CardTitle className="text-lg">Confusion Matrix</CardTitle>
<CardDescription>
Aggregated across all annotator pairs (row = annotator A, col =
annotator B)
</CardDescription>
</CardHeader>
<CardContent>
<div className="overflow-x-auto">
<Table>
<TableHeader>
<TableRow>
<TableHead className="text-xs" />
{cm.labels.map((label) => (
<TableHead
key={label}
className="text-center text-xs px-1 max-w-[80px]"
title={label}
>
{label.length > 12
? label.slice(0, 10) + ".."
: label}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{cm.labels.map((rowLabel, i) => (
<TableRow key={rowLabel}>
<TableCell
className="font-medium text-xs max-w-[100px] truncate"
title={rowLabel}
>
{rowLabel.length > 14
? rowLabel.slice(0, 12) + ".."
: rowLabel}
</TableCell>
{cm.matrix[i].map((val, j) => (
<TableCell
key={j}
className={`text-center text-xs tabular-nums px-1 ${
i === j
? "font-bold"
: ""
} ${heatmapColor(val, confMax)}`}
>
{val}
</TableCell>
))}
</TableRow>
))}
</TableBody>
</Table>
</div>
</CardContent>
</Card>
</div>
);
}
export default function AdminPage() {
return (
<div className="min-h-screen bg-background p-4 sm:p-8">
<div className="mx-auto max-w-5xl space-y-6">
<div>
<h1 className="text-2xl font-bold">Admin Dashboard</h1>
<p className="text-sm text-muted-foreground mt-1">
SEC cyBERT Labeling Project
</p>
</div>
<Tabs defaultValue="adjudication">
<TabsList>
<TabsTrigger value="adjudication">Adjudication Queue</TabsTrigger>
<TabsTrigger value="metrics">Metrics Dashboard</TabsTrigger>
</TabsList>
<TabsContent value="adjudication">
<AdjudicationQueue />
</TabsContent>
<TabsContent value="metrics">
<MetricsDashboard />
</TabsContent>
</Tabs>
</div>
</div>
);
}