@@ -1071,8 +1094,9 @@ export default function CodebookPage() {
framework alignment, ERM integration. Remove “Our CISO
oversees” and it still makes complete sense as a process
description. →{" "}
- Risk Management Process, Specificity 2 (NIST CSF
- framework, no firm-specific facts beyond that).
+ Risk Management Process, Specificity 3 (CISO is
+ on the IS list — specificity rates the whole paragraph, not just
+ the category-relevant parts).
diff --git a/labelapp/lib/warmup-paragraphs.ts b/labelapp/lib/warmup-paragraphs.ts
index 511c455..efaca78 100644
--- a/labelapp/lib/warmup-paragraphs.ts
+++ b/labelapp/lib/warmup-paragraphs.ts
@@ -11,9 +11,9 @@ export const WARMUP_PARAGRAPHS: WarmupParagraph[] = [
id: "warmup-1",
text: "The Board of Directors oversees the Company's management of cybersecurity risks. The Board has delegated oversight of cybersecurity and data privacy matters to the Audit Committee, which receives quarterly reports from management on the Company's cybersecurity risk management program, recent threats, and any incidents.",
goldCategory: "Board Governance",
- goldSpecificity: 3,
+ goldSpecificity: 1,
explanation:
- "Board Governance because the Board of Directors and Audit Committee are the grammatical subjects performing the primary actions (overseeing, delegating, receiving reports). Specificity 3 (Firm-Specific) because the paragraph describes a specific delegation structure (to the Audit Committee) with a defined briefing cadence. While 'Audit Committee' alone is generic (NOT list), the delegation of cybersecurity oversight to it is a firm-specific organizational choice. No QV-eligible facts present (no specific numbers, dates, named entities, or tools).",
+ "Board Governance because the paragraph answers 'How does the board oversee cybersecurity?' — the Board and Audit Committee are overseeing, delegating, and receiving reports. Specificity 1 (Generic Boilerplate) because every term is on the NOT list: 'Board of Directors,' 'Audit Committee' (every public company has one), 'management,' and 'quarterly' (generic cadence). No cybersecurity domain terminology, no firm-specific facts (IS list), no QV-eligible facts.",
},
{
id: "warmup-2",
diff --git a/python/src/data/__init__.py b/python/src/data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python/src/data/corpus.py b/python/src/data/corpus.py
new file mode 100644
index 0000000..7e4ca0d
--- /dev/null
+++ b/python/src/data/corpus.py
@@ -0,0 +1,141 @@
+"""DAPT/TAPT corpus loading and tokenization.
+
+Loads clean text from JSONL (sharded directory or single file),
+tokenizes, and concatenates + chunks into fixed-length sequences
+for masked language modeling.
+"""
+
+from os import cpu_count
+from pathlib import Path
+
+from datasets import Dataset, load_dataset
+from transformers import PreTrainedTokenizerBase
+
+
+def load_corpus(corpus_path: str, text_field: str = "text") -> Dataset:
+ """Load JSONL file(s) into a HuggingFace Dataset.
+
+ Args:
+ corpus_path: Path to a single JSONL file or a directory of JSONL shards.
+ text_field: Name of the JSON field containing the text.
+
+ Returns:
+ A Dataset with at least a `text_field` column.
+ """
+ p = Path(corpus_path)
+ if p.is_file():
+ ds = load_dataset("json", data_files=str(p), split="train")
+ elif p.is_dir():
+ ds = load_dataset("json", data_dir=str(p), split="train")
+ else:
+ raise FileNotFoundError(f"Corpus path not found: {corpus_path}")
+
+ if text_field not in ds.column_names:
+ raise ValueError(
+ f"Text field '{text_field}' not found. "
+ f"Available columns: {ds.column_names}"
+ )
+ return ds
+
+
+def tokenize_and_chunk(
+ dataset: Dataset,
+ tokenizer: PreTrainedTokenizerBase,
+ text_field: str = "text",
+ max_seq_length: int = 2048,
+ whole_word_mask: bool = False,
+) -> Dataset:
+ """Tokenize texts for MLM pre-training.
+
+ Two modes:
+ - **DAPT** (whole_word_mask=False): Concatenate all tokens and chunk into
+ fixed-length sequences. Standard for pre-training on long documents —
+ maximizes GPU utilization by avoiding padding.
+ - **TAPT** (whole_word_mask=True): Tokenize each text individually with
+ truncation, preserving word_ids for whole-word masking. The data
+ collator handles dynamic padding per batch.
+
+ Args:
+ dataset: Dataset with a text column.
+ tokenizer: Pretrained tokenizer.
+ text_field: Column name containing raw text.
+ max_seq_length: Length of each output sequence.
+ whole_word_mask: If True, return per-sample word_ids instead
+ of concatenate-and-chunk. Required for whole-word masking.
+
+ Returns:
+ Dataset with `input_ids` and `attention_mask` columns.
+ If whole_word_mask=True, also includes `word_ids`.
+ """
+ nproc = cpu_count() or 1
+
+ if whole_word_mask:
+ # Tokenize each paragraph individually, extracting word_ids for
+ # whole-word masking. We use word_ids() (from the fast tokenizer)
+ # instead of offset_mapping because BPE tokenizers absorb leading
+ # spaces into tokens, making all offsets contiguous — the upstream
+ # offset_mapping-based boundary detection treats entire sequences
+ # as one "word".
+ def tokenize_fn(examples: dict) -> dict:
+ encodings = tokenizer(
+ examples[text_field],
+ truncation=True,
+ max_length=max_seq_length,
+ )
+ batch_word_ids = []
+ for i in range(len(examples[text_field])):
+ wids = encodings.word_ids(i)
+ # None (special tokens) → -1 for Arrow integer storage
+ batch_word_ids.append([w if w is not None else -1 for w in wids])
+ return {
+ "input_ids": encodings["input_ids"],
+ "attention_mask": encodings["attention_mask"],
+ "word_ids": batch_word_ids,
+ }
+
+ return dataset.map(
+ tokenize_fn,
+ batched=True,
+ batch_size=100,
+ num_proc=nproc,
+ remove_columns=dataset.column_names,
+ desc="Tokenizing (whole-word mask)",
+ )
+
+ # DAPT mode: concatenate-then-chunk
+ def tokenize_fn(examples: dict) -> dict:
+ return tokenizer(
+ examples[text_field],
+ truncation=False,
+ add_special_tokens=False,
+ )
+
+ tokenized = dataset.map(
+ tokenize_fn,
+ batched=True,
+ batch_size=100,
+ num_proc=nproc,
+ remove_columns=dataset.column_names,
+ desc="Tokenizing",
+ )
+
+ # Concatenate-then-chunk in parallel batches. Each batch of documents
+ # has its tokens concatenated and sliced into fixed-length sequences.
+ # Only the tail of each batch is dropped (not each document), so token
+ # waste is negligible (~nproc * max_seq_length tokens total).
+ def chunk_fn(examples: dict) -> dict:
+ all_ids: list[int] = []
+ all_mask: list[int] = []
+ for ids, mask in zip(examples["input_ids"], examples["attention_mask"]):
+ all_ids.extend(ids)
+ all_mask.extend(mask)
+ n = (len(all_ids) // max_seq_length) * max_seq_length
+ return {
+ "input_ids": [all_ids[i : i + max_seq_length] for i in range(0, n, max_seq_length)],
+ "attention_mask": [all_mask[i : i + max_seq_length] for i in range(0, n, max_seq_length)],
+ }
+
+ return tokenized.map(
+ chunk_fn, batched=True, batch_size=500, num_proc=nproc,
+ remove_columns=tokenized.column_names, desc="Chunking",
+ )
diff --git a/ts/data/pilot/pilot-report.txt b/ts/data/pilot/pilot-report.txt
new file mode 100644
index 0000000..d9d2aed
--- /dev/null
+++ b/ts/data/pilot/pilot-report.txt
@@ -0,0 +1,66 @@
+SEC-cyBERT Prompt Pilot Report — 2026-03-28T05:14:43.318Z
+Prompt version: v1.0
+Sample: 0 paragraphs, seed=42
+Models: google/gemini-3.1-flash-lite-preview, xiaomi/mimo-v2-flash, x-ai/grok-4.1-fast
+Total cost: $0.0000
+
+═══ PER-MODEL STATS ═══
+
+google/gemini-3.1-flash-lite-preview:
+ Cost: $0.0000 ($NaN/para)
+ Tokens: 0 in, 0 out, 0 reasoning
+ Avg latency: 0ms
+ Categories:
+ Specificity:
+ Category confidence:
+ Specificity confidence:
+
+xiaomi/mimo-v2-flash:
+ Cost: $0.0000 ($NaN/para)
+ Tokens: 0 in, 0 out, 0 reasoning
+ Avg latency: 0ms
+ Categories:
+ Specificity:
+ Category confidence:
+ Specificity confidence:
+
+x-ai/grok-4.1-fast:
+ Cost: $0.0000 ($NaN/para)
+ Tokens: 0 in, 0 out, 0 reasoning
+ Avg latency: 0ms
+ Categories:
+ Specificity:
+ Category confidence:
+ Specificity confidence:
+
+
+═══ AGREEMENT ANALYSIS ═══
+Paragraphs with all 3 models: 0
+
+Content Category Agreement:
+ 3/3 unanimous: 0/0 (NaN%)
+ 2/3 majority: 0/0 (NaN%)
+ All disagree: 0/0 (NaN%)
+
+Specificity Level Agreement:
+ 3/3 unanimous: 0/0 (NaN%)
+ 2/3 majority: 0/0 (NaN%)
+ All disagree: 0/0 (NaN%)
+
+Both dimensions 3/3: 0/0 (NaN%)
+
+Pairwise category agreement:
+ gemini-3.1-flash-lite-preview × mimo-v2-flash: 0/0 (NaN%)
+ gemini-3.1-flash-lite-preview × grok-4.1-fast: 0/0 (NaN%)
+ mimo-v2-flash × grok-4.1-fast: 0/0 (NaN%)
+
+
+═══ DISAGREEMENT DETAILS ═══
+
+
+═══ COST PROJECTIONS (50K paragraphs) ═══
+ TOTAL Stage 1 (all 3 models): ~$NaN
+
+ Observed disagreement rate: NaN%
+ Estimated Stage 2 judge calls: ~NaN
+ (Judge cost depends on Sonnet 4.6 pricing — see OpenRouter)
\ No newline at end of file
diff --git a/ts/data/pilot/pilot-sample.jsonl b/ts/data/pilot/pilot-sample.jsonl
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/ts/data/pilot/pilot-sample.jsonl
@@ -0,0 +1 @@
+