626 lines
29 KiB
Python
626 lines
29 KiB
Python
"""
|
|
Gold Set Adjudication — Experimental Harness
|
|
=============================================
|
|
|
|
Runs the adjudication pipeline with toggleable interventions, one variable
|
|
at a time, and produces comparable metrics for each configuration.
|
|
|
|
Experiments:
|
|
baseline — Current production adjudication (92 T5 cases)
|
|
exp1_gemini — Exclude Gemini from MR↔RMP axis when Gemini voted MR
|
|
exp2_board — Board-removal test overrides for BG↔RMP T5 cases
|
|
exp3_committee — Committee-level test overrides for BG↔MR T5 cases
|
|
exp4_idsi — ID↔SI volume-dominant tiebreaker
|
|
exp5_spec — Specificity hybrid (human unanimous → human, split → model)
|
|
combined — All validated interventions stacked
|
|
|
|
Usage:
|
|
uv run scripts/adjudicate-gold-experiment.py [experiment_name|all]
|
|
"""
|
|
import json
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
# ── IMPORTS FROM PRODUCTION SCRIPT ──────────────────────────────────────
|
|
# These are the existing overrides from adjudicate-gold.py, kept identical
|
|
# so the baseline matches production exactly.
|
|
|
|
SI_NO_OVERRIDES: dict[str, tuple[str, str]] = {
|
|
"026c8eca": ("None/Other", "Speculation: 'could potentially result in' -- no materiality assessment"),
|
|
"160fec46": ("None/Other", "Resource lament: 'do not have manpower' -- no materiality assessment"),
|
|
"1f29ea8c": ("None/Other", "Speculation: 'could have material adverse effect' boilerplate"),
|
|
"20c70335": ("None/Other", "Risk list: 'A breach could lead to...' -- enumeration, not assessment"),
|
|
"303685cf": ("None/Other", "Speculation: 'could materially adversely affect'"),
|
|
"7d021fcc": ("None/Other", "Speculation: 'could...have a material adverse effect'"),
|
|
"7ef53cab": ("None/Other", "Risk enumeration: 'could lead to... could disrupt... could steal...'"),
|
|
"a0d01951": ("None/Other", "Speculation: 'could adversely affect our business'"),
|
|
"aaa8974b": ("None/Other", "Speculation: 'could potentially have a material impact' -- Case 9 fix"),
|
|
"b058dca1": ("None/Other", "Speculation: 'could disrupt our operations'"),
|
|
"b1b216b6": ("None/Other", "Speculation: 'could materially adversely affect'"),
|
|
"dc8a2798": ("None/Other", "Speculation: 'If compromised, we could be subject to...'"),
|
|
"e4bd0e2f": ("None/Other", "Speculation: 'could have material adverse impact'"),
|
|
"f4656a7e": ("None/Other", "Threat enumeration under SI-sounding header -- no assessment"),
|
|
"2e8cbdbf": ("None/Other", "Cross-ref: 'We describe whether and how... under the headings [risk factors]'"),
|
|
"75de7441": ("None/Other", "Cross-ref: 'We describe whether and how... under the heading [risk factor]'"),
|
|
"78cad2a1": ("None/Other", "Cross-ref: 'In our Risk Factors, we describe whether and how...'"),
|
|
"3879887f": ("None/Other", "Brief incident mention + 'See Item 1A' cross-reference"),
|
|
"f026f2be": ("None/Other", "Risk factor heading/cross-reference -- not an assessment"),
|
|
"5df3a6c9": ("None/Other", "IT importance statement -- no assessment. H=1/3 SI"),
|
|
"d5dc17c2": ("None/Other", "Risk enumeration -- no assessment. H=1/3 SI"),
|
|
"c10f2a54": ("None/Other", "Early-stage/SPAC + weak negative assertion. SPAC rule dominates"),
|
|
"45961c99": ("None/Other", "Past disruption but no materiality language. Primarily speculation"),
|
|
"1673f332": ("None/Other", "SPAC with assessment at end -- SPAC rule dominates per Case 8"),
|
|
"f75ac78a": ("Risk Management Process", "Resource expenditure on cybersecurity -- RMP per person-removal test"),
|
|
"367108c2": ("Strategy Integration", "Negative assertion: 'not aware of having experienced any prior material data breaches'"),
|
|
"837e31d5": ("Strategy Integration", "Negative assertion: 'did not experience any cybersecurity incident during 2024'"),
|
|
}
|
|
|
|
T5_CODEBOOK_OVERRIDES: dict[str, tuple[str, str]] = {
|
|
"15e7cf99": ("Strategy Integration", "SI/ID tiebreaker: 'have not encountered any risks' -- materiality assessment, no specific incident described"),
|
|
"6dc6bb4a": ("Incident Disclosure", "SI/ID tiebreaker: 'ransomware attack in October 2021' -- describes specific incident with date"),
|
|
"c71739a9": ("Risk Management Process", "TP/RMP: Fund relies on CCO and adviser's risk management expertise -- third parties supporting internal process"),
|
|
}
|
|
|
|
# ── EXPERIMENT-SPECIFIC OVERRIDES ───────────────────────────────────────
|
|
|
|
# Exp 2/3: Board-removal + committee-level test overrides (with-board paragraphs)
|
|
# These 5 paragraphs mention "board" so the automated no-board test can't catch them.
|
|
# Each read manually; board-removal test applied to determine if board mention is
|
|
# incidental or substantive.
|
|
MANUAL_BOARD_OVERRIDES: dict[str, tuple[str, str]] = {
|
|
# Board = 1/5 sentences + final notification clause. CISO/ISIRT/incident
|
|
# response plan dominate the content. Board oversight is incidental attribution.
|
|
"22da6695": ("Risk Management Process",
|
|
"Board-removal: 'Board is also responsible for approval' (1 sentence) + "
|
|
"'notifying the Board' (final clause). Remove → CISO + IS Program + incident "
|
|
"response plan. Process dominates."),
|
|
# Titled 'Management's Role.' Compliance Committee = management-level (CIO,
|
|
# executives). Board mentioned 2x as information destination only.
|
|
"a2ff7e1e": ("Management Role",
|
|
"Committee-level: Compliance Committee is management-level (O'Reilly executives). "
|
|
"Board is incidental destination (2 clauses). Titled 'Management's Role.'"),
|
|
# Very brief (3 sentences). Management oversees + board notification + 'Public
|
|
# Offering' (registration statement). Board is incident notification only.
|
|
"cb518f47": ("Management Role",
|
|
"Board-removal: remove notification sentence → 'management oversees cybersecurity.' "
|
|
"Board is incident notification destination only. Brief paragraph."),
|
|
}
|
|
|
|
# Exp 4: Codebook tiebreaker overrides (beyond existing T5_CODEBOOK_OVERRIDES)
|
|
# Each paragraph read in full and classified by codebook rules.
|
|
CODEBOOK_OVERRIDES: dict[str, tuple[str, str]] = {
|
|
# ── ID↔SI: negative assertion = materiality assessment → SI ──────────
|
|
"0ceeb618": ("Strategy Integration",
|
|
"ID/SI: Opens with negative assertion ('no material incidents'), Feb 2025 "
|
|
"incident is brief context + 'has not had material impact' conclusion. "
|
|
"Materiality assessment frame dominates → SI"),
|
|
"cc82eb9f": ("Strategy Integration",
|
|
"ID/SI: June 2018 incident is example within broader negative materiality "
|
|
"assertion ('have not materially affected us'). Assessment frame dominates → SI"),
|
|
# ── SPAC rule (Case 8): pre-revenue company → N/O ────────────────────
|
|
"203ccd43": ("None/Other",
|
|
"SPAC: 'once the Company commences operations' — pre-revenue company. "
|
|
"Case 8: SPAC → N/O regardless of management role language"),
|
|
# ── ID→RMP: post-incident improvements, no incident described ────────
|
|
"f549fd64": ("Risk Management Process",
|
|
"ID/RMP: 'Following this cybersecurity event' — refers to incident without "
|
|
"describing it. 100% of content is hardening, training, MFA, EDR — pure RMP"),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class ExperimentConfig:
|
|
name: str
|
|
description: str
|
|
exclude_gemini_mr_rmp: bool = False
|
|
apply_board_removal: bool = False
|
|
apply_committee_level: bool = False
|
|
apply_idsi_tiebreaker: bool = False
|
|
apply_specificity_hybrid: bool = False
|
|
# Text-based: remove BG model votes when "board" absent from paragraph text
|
|
apply_no_board_bg_removal: bool = False
|
|
|
|
|
|
@dataclass
|
|
class ExperimentResult:
|
|
config: ExperimentConfig
|
|
total: int = 0
|
|
tier_counts: dict[str, int] = field(default_factory=dict)
|
|
category_dist: dict[str, int] = field(default_factory=dict)
|
|
human_maj_dist: dict[str, int] = field(default_factory=dict)
|
|
flipped_from_human: int = 0
|
|
source_accuracy: dict[str, float] = field(default_factory=dict)
|
|
t5_by_axis: dict[str, int] = field(default_factory=dict)
|
|
t5_weak_plurality: int = 0 # 4-5/9
|
|
results: list[dict] = field(default_factory=list)
|
|
spec_changes: int = 0
|
|
|
|
|
|
def load_jsonl(path: Path) -> list[dict]:
|
|
with open(path) as f:
|
|
return [json.loads(line) for line in f]
|
|
|
|
|
|
def majority_vote(votes: list[str]) -> str | None:
|
|
if not votes:
|
|
return None
|
|
return Counter(votes).most_common(1)[0][0]
|
|
|
|
|
|
def get_confusion_axis(human_votes: dict, model_votes: dict) -> str:
|
|
"""Identify the confusion axis from vote distributions."""
|
|
all_cats = sorted(set(list(human_votes.keys()) + list(model_votes.keys())))
|
|
if len(all_cats) == 2:
|
|
return f"{all_cats[0]}↔{all_cats[1]}"
|
|
return "↔".join(all_cats)
|
|
|
|
|
|
def run_experiment(config: ExperimentConfig) -> ExperimentResult:
|
|
"""Run adjudication with a specific experimental configuration."""
|
|
|
|
# ── Load data ─────────────────────────────────────────────────────
|
|
human_labels: dict[str, list[dict]] = defaultdict(list)
|
|
for r in load_jsonl(ROOT / "data/gold/human-labels-raw.jsonl"):
|
|
human_labels[r["paragraphId"]].append({
|
|
"cat": r["contentCategory"],
|
|
"spec": r["specificityLevel"],
|
|
"annotator": r["annotatorName"],
|
|
})
|
|
|
|
confusion_pids = {r["paragraphId"] for r in load_jsonl(ROOT / "data/gold/holdout-rerun-v35.jsonl")}
|
|
|
|
TOP6 = ["Opus", "GPT-5.4", "Gemini", "GLM-5", "Kimi", "MIMO"]
|
|
|
|
def load_model_cats(files: dict[str, Path]) -> dict[str, dict[str, str]]:
|
|
result: dict[str, dict[str, str]] = {}
|
|
for name, path in files.items():
|
|
result[name] = {}
|
|
if path.exists():
|
|
for r in load_jsonl(path):
|
|
cat = r.get("label", {}).get("content_category") or r.get("content_category")
|
|
if cat:
|
|
result[name][r["paragraphId"]] = cat
|
|
# Also load specificity for exp5
|
|
result[f"{name}_spec"] = {}
|
|
if path.exists():
|
|
for r in load_jsonl(path):
|
|
spec = r.get("label", {}).get("specificity_level") or r.get("specificity_level")
|
|
if spec is not None:
|
|
result[f"{name}_spec"][r["paragraphId"]] = spec
|
|
return result
|
|
|
|
v30_cats = load_model_cats({
|
|
"Opus": ROOT / "data/annotations/golden/opus.jsonl",
|
|
"GPT-5.4": ROOT / "data/annotations/bench-holdout/gpt-5.4.jsonl",
|
|
"Gemini": ROOT / "data/annotations/bench-holdout/gemini-3.1-pro-preview.jsonl",
|
|
"GLM-5": ROOT / "data/annotations/bench-holdout/glm-5:exacto.jsonl",
|
|
"Kimi": ROOT / "data/annotations/bench-holdout/kimi-k2.5.jsonl",
|
|
"MIMO": ROOT / "data/annotations/bench-holdout/mimo-v2-pro:exacto.jsonl",
|
|
})
|
|
|
|
v35_cats = load_model_cats({
|
|
"Opus": ROOT / "data/annotations/golden-v35/opus.jsonl",
|
|
"GPT-5.4": ROOT / "data/annotations/bench-holdout-v35/gpt-5.4.jsonl",
|
|
"Gemini": ROOT / "data/annotations/bench-holdout-v35/gemini-3.1-pro-preview.jsonl",
|
|
"GLM-5": ROOT / "data/annotations/bench-holdout-v35/glm-5:exacto.jsonl",
|
|
"Kimi": ROOT / "data/annotations/bench-holdout-v35/kimi-k2.5.jsonl",
|
|
"MIMO": ROOT / "data/annotations/bench-holdout-v35/mimo-v2-pro:exacto.jsonl",
|
|
})
|
|
|
|
# Merge v3.0 + v3.5 (v3.5 for confusion PIDs)
|
|
model_cats: dict[str, dict[str, str]] = {}
|
|
model_specs: dict[str, dict[str, int]] = {}
|
|
for m in TOP6:
|
|
model_cats[m] = {}
|
|
model_specs[m] = {}
|
|
for pid in human_labels:
|
|
if pid in confusion_pids and pid in v35_cats.get(m, {}):
|
|
model_cats[m][pid] = v35_cats[m][pid]
|
|
elif pid in v30_cats.get(m, {}):
|
|
model_cats[m][pid] = v30_cats[m][pid]
|
|
# Specificity (always v3.0 for full coverage)
|
|
if pid in v30_cats.get(f"{m}_spec", {}):
|
|
model_specs[m][pid] = v30_cats[f"{m}_spec"][pid]
|
|
|
|
# ── Adjudicate ────────────────────────────────────────────────────
|
|
result = ExperimentResult(config=config)
|
|
tier_counts: Counter[str] = Counter()
|
|
|
|
for pid in sorted(human_labels.keys()):
|
|
h_cats = [l["cat"] for l in human_labels[pid]]
|
|
h_specs = [l["spec"] for l in human_labels[pid]]
|
|
h_cat_maj = majority_vote(h_cats)
|
|
h_spec_maj = majority_vote(h_specs)
|
|
h_spec_unanimous = len(set(h_specs)) == 1
|
|
|
|
# Use full model panel for tier calculation (T1-T4 stability)
|
|
active_models = list(TOP6)
|
|
|
|
m_cats_list = [model_cats[m][pid] for m in active_models if pid in model_cats[m]]
|
|
m_cat_maj = majority_vote(m_cats_list)
|
|
m_cat_unanimous = len(set(m_cats_list)) == 1 and len(m_cats_list) == len(active_models)
|
|
|
|
all_signals = h_cats + m_cats_list
|
|
signal_counter = Counter(all_signals)
|
|
total_signals = len(all_signals)
|
|
top_signal, top_count = signal_counter.most_common(1)[0]
|
|
|
|
short_pid = pid[:8]
|
|
si_override = SI_NO_OVERRIDES.get(short_pid)
|
|
|
|
gold_cat: str | None = None
|
|
tier: str = ""
|
|
reason: str = ""
|
|
|
|
if si_override:
|
|
gold_cat = si_override[0]
|
|
tier = "T3-rule"
|
|
reason = f"SI/NO override: {si_override[1]}"
|
|
elif top_count >= 8 and total_signals >= 8:
|
|
gold_cat = top_signal
|
|
tier = "T1-super"
|
|
reason = f"{top_count}/{total_signals} signals agree"
|
|
elif h_cat_maj == m_cat_maj:
|
|
gold_cat = h_cat_maj
|
|
tier = "T2-cross"
|
|
reason = "Human + model majority agree"
|
|
elif m_cat_unanimous:
|
|
gold_cat = m_cat_maj
|
|
tier = "T4-model"
|
|
h_count = Counter(h_cats).most_common(1)[0][1]
|
|
reason = f"{len(m_cats_list)}/{len(m_cats_list)} models unanimous ({m_cat_maj}) vs human {h_count}/3 ({h_cat_maj})"
|
|
else:
|
|
# Check rule-based overrides
|
|
t5_override = T5_CODEBOOK_OVERRIDES.get(short_pid)
|
|
|
|
# Exp 2/3: Manual board-removal + committee-level test (with-board paragraphs)
|
|
board_override = MANUAL_BOARD_OVERRIDES.get(short_pid) if (config.apply_board_removal or config.apply_committee_level) else None
|
|
|
|
# Exp 4: Codebook tiebreaker overrides
|
|
codebook_override = CODEBOOK_OVERRIDES.get(short_pid) if config.apply_idsi_tiebreaker else None
|
|
|
|
if t5_override:
|
|
gold_cat = t5_override[0]
|
|
tier = "T3-rule"
|
|
reason = f"T5 codebook override: {t5_override[1]}"
|
|
elif board_override:
|
|
gold_cat = board_override[0]
|
|
tier = "T3-rule"
|
|
reason = f"Board/committee test: {board_override[1]}"
|
|
elif codebook_override:
|
|
gold_cat = codebook_override[0]
|
|
tier = "T3-rule"
|
|
reason = f"Codebook tiebreaker: {codebook_override[1]}"
|
|
else:
|
|
t5_signals = list(all_signals)
|
|
t5_total = total_signals
|
|
suffix = ""
|
|
|
|
# ── Exp 1: Gemini exclusion at T5 resolution only ─────
|
|
if config.exclude_gemini_mr_rmp:
|
|
gemini_cat = model_cats.get("Gemini", {}).get(pid)
|
|
if gemini_cat == "Management Role":
|
|
other_m_cats = [model_cats[m][pid] for m in TOP6 if m != "Gemini" and pid in model_cats[m]]
|
|
other_m_maj = majority_vote(other_m_cats) if other_m_cats else None
|
|
if other_m_maj != "Management Role":
|
|
t5_signals = h_cats + other_m_cats
|
|
t5_total = len(t5_signals)
|
|
suffix += " [Gemini MR excluded]"
|
|
|
|
# ── Exp 2b: No-board BG vote removal ─────────────────
|
|
# If "board" (case-insensitive) doesn't appear in the paragraph
|
|
# text, BG model votes are provably unsupported — the paragraph
|
|
# can't be about board governance if it never mentions the board.
|
|
# Remove those BG signals and recalculate plurality.
|
|
if config.apply_no_board_bg_removal:
|
|
para_texts = load_paragraph_texts()
|
|
para_text = para_texts.get(pid, "")
|
|
if "board" not in para_text.lower():
|
|
bg_count = sum(1 for s in t5_signals if s == "Board Governance")
|
|
if bg_count > 0:
|
|
t5_signals = [s for s in t5_signals if s != "Board Governance"]
|
|
t5_total = len(t5_signals)
|
|
if t5_signals:
|
|
suffix += f" [BG removed: no 'board' in text, {bg_count} votes dropped]"
|
|
|
|
if t5_signals:
|
|
t5_counter = Counter(t5_signals)
|
|
t5_top, t5_top_count = t5_counter.most_common(1)[0]
|
|
else:
|
|
t5_top, t5_top_count = top_signal, top_count
|
|
|
|
gold_cat = t5_top
|
|
tier = "T5-plurality"
|
|
reason = f"Mixed: human={h_cat_maj}, model={m_cat_maj}, plurality={t5_top} ({t5_top_count}/{t5_total}){suffix}"
|
|
|
|
# ── Specificity ───────────────────────────────────────────────
|
|
if config.apply_specificity_hybrid and not h_spec_unanimous:
|
|
# Human split → use model majority
|
|
m_specs = [model_specs[m][pid] for m in TOP6 if pid in model_specs[m]]
|
|
if m_specs:
|
|
gold_spec = majority_vote([str(s) for s in m_specs])
|
|
gold_spec = int(gold_spec) if gold_spec else h_spec_maj
|
|
if gold_spec != h_spec_maj:
|
|
result.spec_changes += 1
|
|
else:
|
|
gold_spec = h_spec_maj
|
|
else:
|
|
gold_spec = h_spec_maj
|
|
|
|
tier_counts[tier] += 1
|
|
|
|
row = {
|
|
"paragraphId": pid,
|
|
"gold_category": gold_cat,
|
|
"gold_specificity": gold_spec,
|
|
"tier": tier,
|
|
"reason": reason,
|
|
"human_majority": h_cat_maj,
|
|
"model_majority": m_cat_maj,
|
|
"human_votes": dict(Counter(h_cats)),
|
|
"model_votes": dict(Counter(m_cats_list)),
|
|
}
|
|
result.results.append(row)
|
|
|
|
if tier == "T5-plurality":
|
|
axis = get_confusion_axis(dict(Counter(h_cats)), dict(Counter(m_cats_list)))
|
|
result.t5_by_axis[axis] = result.t5_by_axis.get(axis, 0) + 1
|
|
if top_count <= 5:
|
|
result.t5_weak_plurality += 1
|
|
|
|
result.total = len(result.results)
|
|
result.tier_counts = dict(sorted(tier_counts.items()))
|
|
result.flipped_from_human = sum(1 for r in result.results if r["gold_category"] != r["human_majority"])
|
|
result.category_dist = dict(Counter(r["gold_category"] for r in result.results))
|
|
result.human_maj_dist = dict(Counter(r["human_majority"] for r in result.results))
|
|
|
|
# Source accuracy vs gold
|
|
gold_by_pid = {r["paragraphId"]: r["gold_category"] for r in result.results}
|
|
|
|
# Human annotators
|
|
annotator_names = sorted(set(l["annotator"] for labels in human_labels.values() for l in labels))
|
|
for ann in annotator_names:
|
|
agree = total = 0
|
|
for pid, labels in human_labels.items():
|
|
for l in labels:
|
|
if l["annotator"] == ann and pid in gold_by_pid:
|
|
total += 1
|
|
if l["cat"] == gold_by_pid[pid]:
|
|
agree += 1
|
|
if total > 0:
|
|
result.source_accuracy[f"H:{ann}"] = agree / total
|
|
|
|
# Models (v3.0 on full 1200)
|
|
for m in TOP6:
|
|
agree = total = 0
|
|
for pid in gold_by_pid:
|
|
if pid in v30_cats.get(m, {}):
|
|
total += 1
|
|
if v30_cats[m][pid] == gold_by_pid[pid]:
|
|
agree += 1
|
|
if total > 0:
|
|
result.source_accuracy[f"M:{m}"] = agree / total
|
|
|
|
return result
|
|
|
|
|
|
def print_result(r: ExperimentResult, baseline: ExperimentResult | None = None) -> None:
|
|
"""Print experiment results with optional delta from baseline."""
|
|
print(f"\n{'=' * 90}")
|
|
print(f"EXPERIMENT: {r.config.name}")
|
|
print(f" {r.config.description}")
|
|
print(f"{'=' * 90}")
|
|
|
|
print(f"\nTier distribution:")
|
|
for tier in ["T1-super", "T2-cross", "T3-rule", "T4-model", "T5-plurality"]:
|
|
count = r.tier_counts.get(tier, 0)
|
|
pct = count / r.total * 100
|
|
delta = ""
|
|
if baseline:
|
|
bc = baseline.tier_counts.get(tier, 0)
|
|
if count != bc:
|
|
delta = f" (Δ {count - bc:+d})"
|
|
print(f" {tier:<16} {count:>5} ({pct:.1f}%){delta}")
|
|
|
|
print(f"\nGold ≠ human majority: {r.flipped_from_human} ({r.flipped_from_human / r.total:.1%})")
|
|
if baseline and r.flipped_from_human != baseline.flipped_from_human:
|
|
print(f" (Δ {r.flipped_from_human - baseline.flipped_from_human:+d})")
|
|
|
|
if r.t5_by_axis:
|
|
t5_total = sum(r.t5_by_axis.values())
|
|
print(f"\nT5 remaining ({t5_total} cases):")
|
|
for axis, count in sorted(r.t5_by_axis.items(), key=lambda x: -x[1])[:10]:
|
|
print(f" {axis:<60} {count:>3}")
|
|
print(f" Weak plurality (4-5/9): {r.t5_weak_plurality}")
|
|
|
|
print(f"\nCategory distribution (gold):")
|
|
all_cats = sorted(set(list(r.category_dist.keys()) + list(r.human_maj_dist.keys())))
|
|
print(f" {'Category':<25} {'Gold':>6} {'H-Maj':>6} {'Δ':>5}", end="")
|
|
if baseline:
|
|
print(f" {'Prev':>6} {'ΔExp':>5}", end="")
|
|
print()
|
|
for cat in all_cats:
|
|
g = r.category_dist.get(cat, 0)
|
|
h = r.human_maj_dist.get(cat, 0)
|
|
line = f" {cat:<25} {g:>6} {h:>6} {g - h:>+5}"
|
|
if baseline:
|
|
bg = baseline.category_dist.get(cat, 0)
|
|
line += f" {bg:>6} {g - bg:>+5}"
|
|
print(line)
|
|
|
|
print(f"\nSource accuracy vs gold:")
|
|
# Sort by accuracy descending
|
|
for source, acc in sorted(r.source_accuracy.items(), key=lambda x: -x[1]):
|
|
delta = ""
|
|
if baseline and source in baseline.source_accuracy:
|
|
ba = baseline.source_accuracy[source]
|
|
diff = acc - ba
|
|
if abs(diff) >= 0.0005:
|
|
delta = f" (Δ {diff:+.1%})"
|
|
print(f" {source:<16} {acc:.1%}{delta}")
|
|
|
|
if r.config.apply_specificity_hybrid:
|
|
print(f"\nSpecificity: {r.spec_changes} labels changed from human majority to model majority")
|
|
|
|
|
|
def diff_results(a: ExperimentResult, b: ExperimentResult) -> list[dict]:
|
|
"""Find paragraphs where gold_category differs between two experiments."""
|
|
a_map = {r["paragraphId"]: r for r in a.results}
|
|
b_map = {r["paragraphId"]: r for r in b.results}
|
|
diffs = []
|
|
for pid in sorted(a_map.keys()):
|
|
if a_map[pid]["gold_category"] != b_map[pid]["gold_category"]:
|
|
diffs.append({
|
|
"paragraphId": pid,
|
|
"before": a_map[pid]["gold_category"],
|
|
"after": b_map[pid]["gold_category"],
|
|
"before_tier": a_map[pid]["tier"],
|
|
"after_tier": b_map[pid]["tier"],
|
|
"human_majority": a_map[pid]["human_majority"],
|
|
"reason_after": b_map[pid]["reason"],
|
|
})
|
|
return diffs
|
|
|
|
|
|
# ── PARAGRAPH TEXT LOADER (for text-based tests) ───────────────────────
|
|
_paragraph_texts: dict[str, str] | None = None
|
|
|
|
def load_paragraph_texts() -> dict[str, str]:
|
|
global _paragraph_texts
|
|
if _paragraph_texts is None:
|
|
_paragraph_texts = {}
|
|
for r in load_jsonl(ROOT / "data/gold/paragraphs-holdout.jsonl"):
|
|
_paragraph_texts[r["id"]] = r["text"]
|
|
return _paragraph_texts
|
|
|
|
|
|
EXPERIMENTS = {
|
|
"baseline": ExperimentConfig(
|
|
name="baseline",
|
|
description="Current production adjudication (no changes)",
|
|
),
|
|
"exp1_gemini": ExperimentConfig(
|
|
name="exp1_gemini",
|
|
description="Exclude Gemini from MR↔RMP axis when Gemini voted MR",
|
|
exclude_gemini_mr_rmp=True,
|
|
),
|
|
"exp2_board": ExperimentConfig(
|
|
name="exp2_board",
|
|
description="Board-removal test overrides for BG↔RMP T5 cases",
|
|
apply_board_removal=True,
|
|
),
|
|
"exp2b_noboard": ExperimentConfig(
|
|
name="exp2b_noboard",
|
|
description="Remove BG model votes when 'board' absent from paragraph text (automated, verifiable)",
|
|
apply_no_board_bg_removal=True,
|
|
),
|
|
"exp3_committee": ExperimentConfig(
|
|
name="exp3_committee",
|
|
description="Committee-level test overrides for BG↔MR T5 cases",
|
|
apply_committee_level=True,
|
|
),
|
|
"exp4_idsi": ExperimentConfig(
|
|
name="exp4_idsi",
|
|
description="ID↔SI volume-dominant tiebreaker",
|
|
apply_idsi_tiebreaker=True,
|
|
),
|
|
"exp5_spec": ExperimentConfig(
|
|
name="exp5_spec",
|
|
description="Specificity hybrid: human unanimous → human, split → model majority",
|
|
apply_specificity_hybrid=True,
|
|
),
|
|
"combined": ExperimentConfig(
|
|
name="combined",
|
|
description="All validated interventions: no-board BG removal + manual board overrides + codebook tiebreakers + specificity hybrid",
|
|
apply_no_board_bg_removal=True,
|
|
apply_board_removal=True,
|
|
apply_idsi_tiebreaker=True,
|
|
apply_specificity_hybrid=True,
|
|
),
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
experiments_to_run = sys.argv[1:] if len(sys.argv) > 1 else ["all"]
|
|
|
|
if "all" in experiments_to_run:
|
|
experiments_to_run = list(EXPERIMENTS.keys())
|
|
|
|
# Always run baseline first
|
|
if "baseline" not in experiments_to_run:
|
|
experiments_to_run.insert(0, "baseline")
|
|
|
|
results: dict[str, ExperimentResult] = {}
|
|
baseline: ExperimentResult | None = None
|
|
|
|
for exp_name in experiments_to_run:
|
|
if exp_name not in EXPERIMENTS:
|
|
print(f"Unknown experiment: {exp_name}")
|
|
continue
|
|
|
|
r = run_experiment(EXPERIMENTS[exp_name])
|
|
results[exp_name] = r
|
|
|
|
if exp_name == "baseline":
|
|
baseline = r
|
|
print_result(r)
|
|
else:
|
|
print_result(r, baseline)
|
|
|
|
# Show specific label changes
|
|
if baseline:
|
|
diffs = diff_results(baseline, r)
|
|
if diffs:
|
|
print(f"\n Label changes ({len(diffs)}):")
|
|
for d in diffs:
|
|
print(f" {d['paragraphId'][:8]}: {d['before']:<25} → {d['after']:<25} (H={d['human_majority']}) [{d['after_tier']}]")
|
|
|
|
# ── Acceptance criteria check ─────────────────────────────────────
|
|
if baseline and len(results) > 1:
|
|
print(f"\n{'=' * 90}")
|
|
print("ACCEPTANCE CRITERIA SUMMARY")
|
|
print(f"{'=' * 90}")
|
|
print(f"\nCriteria:")
|
|
print(f" 1. T5 count decreases (fewer arbitrary resolutions)")
|
|
print(f" 2. Source accuracy: no model/human drops >1% (intervention isn't distorting)")
|
|
print(f" 3. Category distribution: no category shifts >±5% of its baseline count")
|
|
print(f" 4. Changes are principled (each has documented codebook justification)")
|
|
print()
|
|
|
|
for exp_name, r in results.items():
|
|
if exp_name == "baseline":
|
|
continue
|
|
t5_base = baseline.tier_counts.get("T5-plurality", 0)
|
|
t5_exp = r.tier_counts.get("T5-plurality", 0)
|
|
t5_pass = t5_exp <= t5_base
|
|
|
|
max_acc_drop = 0.0
|
|
for source in baseline.source_accuracy:
|
|
if source in r.source_accuracy:
|
|
drop = baseline.source_accuracy[source] - r.source_accuracy[source]
|
|
max_acc_drop = max(max_acc_drop, drop)
|
|
acc_pass = max_acc_drop < 0.01
|
|
|
|
max_cat_shift_pct = 0.0
|
|
for cat in baseline.category_dist:
|
|
base_n = baseline.category_dist.get(cat, 0)
|
|
exp_n = r.category_dist.get(cat, 0)
|
|
if base_n > 0:
|
|
shift = abs(exp_n - base_n) / base_n
|
|
max_cat_shift_pct = max(max_cat_shift_pct, shift)
|
|
cat_pass = max_cat_shift_pct < 0.05
|
|
|
|
status = "✓ PASS" if (t5_pass and acc_pass and cat_pass) else "✗ FAIL"
|
|
print(f" {exp_name:<20} {status}")
|
|
print(f" T5: {t5_base} → {t5_exp} (Δ {t5_exp - t5_base:+d}) {'✓' if t5_pass else '✗'}")
|
|
print(f" Max accuracy drop: {max_acc_drop:.2%} {'✓' if acc_pass else '✗'}")
|
|
print(f" Max category shift: {max_cat_shift_pct:.1%} {'✓' if cat_pass else '✗'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|