labelapp timing fixes and migration

This commit is contained in:
Joey Eamigh 2026-03-29 16:37:51 -04:00
parent a9a7d59603
commit ca4bc288c9
No known key found for this signature in database
GPG Key ID: CE8C05DFFC53C9CB
17 changed files with 1010 additions and 22 deletions

View File

@ -18,6 +18,49 @@ Bun workspace monorepo. Three packages:
| Docker compose (Postgres) | `docker-compose.yaml` (root) | | Docker compose (Postgres) | `docker-compose.yaml` (root) |
| DB credentials | `sec_cybert` / `sec_cybert` / `sec_cybert` on localhost:5432 | | DB credentials | `sec_cybert` / `sec_cybert` / `sec_cybert` on localhost:5432 |
## Root scripts
All commands run from repo root via `bun run <script>`. No need to cd into subpackages.
### Labelapp (`la:*`)
| Script | What it does |
|--------|-------------|
| `la:dev` | Start Next.js dev server (Turbopack) |
| `la:build` | Production build |
| `la:typecheck` | TypeScript type-check |
| `la:lint` | ESLint |
| `la:test` | API tests + Playwright E2E |
| `la:test:api` | API tests only (`bun test`) |
| `la:test:e2e` | Playwright E2E only |
| `la:db:generate` | Generate Drizzle migration |
| `la:db:migrate` | Apply Drizzle migrations |
| `la:db:studio` | Drizzle Studio (DB browser) |
| `la:seed` | Seed paragraphs + annotations |
| `la:sample` | Run paragraph sampling |
| `la:assign` | Generate annotator assignments |
| `la:export` | Export labels |
| `la:docker` | Build + push Docker image |
### GenAI pipeline (`ts:*`)
| Script | What it does |
|--------|-------------|
| `ts:sec` | CLI entrypoint (`bun run ts/src/cli.ts`) |
| `ts:typecheck` | TypeScript type-check |
### Python training (`py:*`)
| Script | What it does |
|--------|-------------|
| `py:train` | CLI entrypoint (`uv run main.py` — pass subcommand as arg, e.g. `bun run py:train dapt --config ...`) |
### Cross-package
| Script | What it does |
|--------|-------------|
| `typecheck` | Type-check all TS packages in parallel |
## Rules ## Rules
- `bun` for all JS/TS. `uv` for Python. - `bun` for all JS/TS. `uv` for Python.
@ -26,4 +69,5 @@ Bun workspace monorepo. Three packages:
- No parallel codepaths. Find and extend existing code before writing new. - No parallel codepaths. Find and extend existing code before writing new.
- Schemas live in `packages/schemas/` — do not duplicate type definitions elsewhere. - Schemas live in `packages/schemas/` — do not duplicate type definitions elsewhere.
- `labelapp/` uses flat layout (no `src/` dir): `app/`, `db/`, `lib/`, `components/` at root. - `labelapp/` uses flat layout (no `src/` dir): `app/`, `db/`, `lib/`, `components/` at root.
- `labelapp/` uses file-based Drizzle migrations (`drizzle-kit generate` + `drizzle-kit migrate`), not `push`.
- Tests: `bun test` for backend route integration (`__test__/` dirs adjacent to routes), Playwright for E2E (`tests/`). - Tests: `bun test` for backend route integration (`__test__/` dirs adjacent to routes), Playwright for E2E (`tests/`).

View File

@ -4,6 +4,10 @@
"workspaces": { "workspaces": {
"": { "": {
"name": "sec-cybert-monorepo", "name": "sec-cybert-monorepo",
"devDependencies": {
"@types/bun": "^1.3.11",
"@types/node": "^25.5.0",
},
}, },
"labelapp": { "labelapp": {
"name": "labelapp", "name": "labelapp",
@ -56,7 +60,8 @@
"zod": "^4.3.6", "zod": "^4.3.6",
}, },
"devDependencies": { "devDependencies": {
"@types/bun": "latest", "@types/bun": "^1.3.11",
"@types/node": "^25.5.0",
"@types/uuid": "^11.0.0", "@types/uuid": "^11.0.0",
}, },
"peerDependencies": { "peerDependencies": {
@ -419,7 +424,7 @@
"@types/json5": ["@types/json5@0.0.29", "", {}, "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ=="], "@types/json5": ["@types/json5@0.0.29", "", {}, "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ=="],
"@types/node": ["@types/node@20.19.37", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw=="], "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="], "@types/react": ["@types/react@19.2.14", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w=="],
@ -1403,7 +1408,7 @@
"undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="], "undici": ["undici@7.24.6", "", {}, "sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA=="],
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"unicorn-magic": ["unicorn-magic@0.3.0", "", {}, "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA=="], "unicorn-magic": ["unicorn-magic@0.3.0", "", {}, "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA=="],
@ -1515,8 +1520,6 @@
"body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
"bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"cliui/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "cliui/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
"cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "cliui/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
@ -1541,6 +1544,8 @@
"is-bun-module/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], "is-bun-module/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="],
"labelapp/@types/node": ["@types/node@20.19.37", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw=="],
"log-symbols/chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], "log-symbols/chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
"log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], "log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="],
@ -1649,14 +1654,14 @@
"ajv-formats/ajv/json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], "ajv-formats/ajv/json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
"bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "cliui/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "cliui/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
"eslint-plugin-import/tsconfig-paths/json5": ["json5@1.0.2", "", { "dependencies": { "minimist": "^1.2.0" }, "bin": { "json5": "lib/cli.js" } }, "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="], "eslint-plugin-import/tsconfig-paths/json5": ["json5@1.0.2", "", { "dependencies": { "minimist": "^1.2.0" }, "bin": { "json5": "lib/cli.js" } }, "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="],
"labelapp/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
"p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
"tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.4", "", { "os": "aix", "cpu": "ppc64" }, "sha512-cQPwL2mp2nSmHHJlCyoXgHGhbEPMrEEU5xhkcy3Hs/O7nGZqEpZ2sUtLaL9MORLtDfRvVl2/3PAuEkYZH0Ty8Q=="], "tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.27.4", "", { "os": "aix", "cpu": "ppc64" }, "sha512-cQPwL2mp2nSmHHJlCyoXgHGhbEPMrEEU5xhkcy3Hs/O7nGZqEpZ2sUtLaL9MORLtDfRvVl2/3PAuEkYZH0Ty8Q=="],

View File

@ -358,7 +358,7 @@ This enables confidence-stratified training data: high-confidence judge labels g
--- ---
## Phase 7: Revised Data Quality Strategy (Current) ## Phase 7: Revised Data Quality Strategy
The post-Stage 1 analysis and judge benchmarking led to a fundamental reassessment of our approach. The post-Stage 1 analysis and judge benchmarking led to a fundamental reassessment of our approach.
@ -394,7 +394,123 @@ Expected total: ~46,000-48,000 paragraphs at ~93-95% label accuracy.
--- ---
## Phase 8: Pre-Training Strategy — DAPT + TAPT ## Phase 8: Human Labeling Webapp (Labelapp)
### Why Build a Webapp?
The project requires 1,200 human-labeled paragraphs as a gold holdout set — the calibration metric for everything downstream. Six student annotators, three per paragraph, 600 per person. The labels need to be reliable enough to benchmark the GenAI pipeline and validate the final classifier.
The alternative was everyone tagging in a shared JSON file or spreadsheet. That would almost certainly produce poor data quality. The failure modes are well-documented in annotation literature and we'd hit all of them:
- **Inconsistent category names.** Free-text entry in a spreadsheet means "Risk Management Process" vs "Risk Mgmt" vs "RMP" vs "3" — all referring to the same class but requiring manual reconciliation.
- **Skipped or double-labeled paragraphs.** No enforced assignment tracking means annotators can accidentally skip paragraphs or label the same one twice without anyone noticing until export.
- **No codebook enforcement.** The labeling codebook has 7 categories, 4 specificity levels, 5 decision rules, and 3 codebook rulings (v3.0). Without quiz gating, annotators can start labeling without understanding the materiality disclaimer ruling, the person-vs-function test, or the QV counting threshold — exactly the boundaries where annotation quality lives or dies.
- **No feedback loop.** In a spreadsheet, an annotator who misunderstands the SPAC ruling labels 600 paragraphs before anyone catches it. A webapp with warmup feedback catches misunderstanding in the first 5 paragraphs.
- **No timing data.** For the writeup, we need per-paragraph labeling times to report annotator effort and identify paragraphs that are disproportionately hard. A spreadsheet gives you nothing; even a basic timer gives you wall-clock time corrupted by idle periods.
A purpose-built labeling tool turns all of these failure modes into solved problems. Constrained radio buttons eliminate typos. Server-side assignment tracking prevents skips and duplicates. Quiz gating enforces codebook knowledge. Warmup paragraphs with gold feedback catch misunderstandings early. Active timing with idle detection gives clean data for the writeup.
### The Onboarding Funnel
Every annotation session follows the same enforced path:
1. **Login** → annotator selects their name, enters password. Session cookie (HMAC-SHA256 signed, 8-hour expiry).
2. **Dashboard** → shows progress, links to training materials or labeling.
3. **Quiz** → 8 questions (2 per type), random draw from a bank of ~30. Four question types target the exact codebook boundaries that cause the most disagreement in the GenAI pipeline:
- **Person-vs-function** (Management Role vs RMP) — the #1 disagreement axis (2,290 disputes in Stage 1)
- **Materiality disclaimers** (Strategy Integration vs None/Other) — resolved ~1,094 disputes via codebook ruling
- **QV fact counting** (Specificity 3 vs 4) — the hardest specificity boundary
- **SPAC exception** (None/Other for shell companies)
- Pass threshold: 7/8 correct. Immediate feedback with codebook explanation after each answer. Failed → review mistakes → retry.
4. **Warmup** → 5 pre-selected paragraphs with known gold labels. Identical UI to real labeling, but after submit, the annotator sees the gold answer + explanation. This catches systematic misunderstandings before they contaminate 600 labels.
5. **Labeling** → the real thing. 600 assigned paragraphs per annotator.
The quiz questions are not random trivia — they're targeted at the exact confusion axes that the GenAI pipeline struggles with. If an annotator can't reliably distinguish Management Role from RMP, their labels on that axis are noise. Better to catch that before they start than after.
### Labeling Interface Design
The labeling UI prioritizes speed and consistency:
- **Paragraph display:** Full text with filing metadata badges (company, ticker, filing type, date, SEC item) in the header bar.
- **Constrained input:** Radio buttons for both category (7 options) and specificity (4 options). No free-text entry for classifications.
- **Keyboard shortcuts:** 1-7 for category, Q/W/E/R for specificity, N to focus notes, Enter to submit. An experienced annotator never touches the mouse.
- **Codebook sidebar:** Floating button opens a slide-out panel with all category definitions, IS/NOT lists, specificity levels, and decision rules. Always one click away — annotators don't need to switch to a separate document.
- **Progress bar:** Shows completed/total in the header. Annotators know where they stand.
- **Notes field:** Optional free-text for edge cases or uncertainty. Useful for adjudication — if an annotator flags "this could be either Management Role or RMP, went with RMP because the person-vs-function test says..." that reasoning helps the adjudicator.
### Sampling Strategy
The 1,200 paragraphs are not randomly sampled. Random sampling from 50K paragraphs would over-represent the easy cases (Board Governance at Specificity 1 is unambiguous) and under-represent the hard cases that actually test annotation quality.
Instead, the sampling is stratified by the disagreement patterns discovered in the Stage 1 analysis (Phase 5):
| Stratum | Count | Why |
|---------|-------|-----|
| Management ↔ RMP split votes | 120 | #1 disagreement axis — validates the person-vs-function ruling |
| None/Other ↔ Strategy splits | 80 | Materiality disclaimer boundary |
| Specificity [3,4] splits | 80 | QV counting — the hardest specificity boundary |
| Board ↔ Management splits | 80 | Board/management interface |
| Rare category guarantee | 120 | ≥15 per category, extra for Incident Disclosure (sparse) |
| Proportional stratified random | 720 | Fill remaining from category × specificity cells |
This ensures the gold set is informative where it matters most: at the decision boundaries where both humans and models are most likely to disagree.
### Assignment: Balanced Incomplete Block Design (BIBD)
Each paragraph gets exactly 3 of 6 annotators. The assignment uses a balanced incomplete block design:
- C(6,3) = 20 unique triples. Assign 60 paragraphs to each triple.
- Each annotator appears in C(5,2) = 10 triples → 10 × 60 = 600 paragraphs per person.
- Every annotator pair shares equal paragraph overlap → pairwise Cohen's Kappa is statistically valid across all 15 pairs.
This is important for the writeup: we can report inter-rater reliability as a full pairwise matrix, not just an average that hides weak pairs.
### Active Timer and Idle Detection
The initial implementation tracked raw wall-clock `duration_ms` per label — `Date.now()` when the paragraph loaded, minus `Date.now()` at submit. This is corrupted by any idle time (annotator walks away, checks email, gets coffee).
We added `useActiveTimer`, a React hook that tracks active vs idle time using mouse/keyboard/scroll/focus events with a 30-second idle threshold. When no activity is detected for 30 seconds, the timer pauses and the header shows an amber "idle" indicator. Both `duration_ms` (wall-clock) and `active_ms` (idle-excluded) are submitted with every label.
For the writeup, `active_ms` is the metric to report — it reflects actual cognitive effort per paragraph. `duration_ms` is retained for completeness. Pre-existing labels (before the timer change) have `active_ms = NULL` and are excluded from timing analysis.
### Infrastructure Decisions
**Stack:** Next.js (App Router) + Drizzle ORM + Postgres + Tailwind + shadcn/ui. Deployed via Docker with a Postgres sidecar.
**Migrations:** Switched from `drizzle-kit push --force` (schema diffing at startup) to file-based Drizzle migrations (`drizzle-kit generate` + `drizzle-kit migrate`). A `scripts/ensure-migration-baseline.ts` script handles the transition for existing databases by seeding the migration journal with the baseline hash.
**Monorepo:** The labelapp triggered converting the repo to a Bun workspace monorepo with shared Zod schemas (`packages/schemas/`). This ensures the labelapp's category/specificity enums are identical to the GenAI pipeline's — no possibility of a mismatch between what the models label and what the humans label.
### Adjudication
After all 3 annotators label a paragraph:
- **3/3 agree** on both dimensions → consensus (no intervention needed)
- **2/3 agree** on both dimensions → majority rules
- **Otherwise** → flagged for admin adjudication
The admin page shows disputed paragraphs with all 3 labels side-by-side, annotator notes, and Stage 1 consensus for reference. The adjudicator picks a label, enters a custom one, or marks it for team discussion. Adjudications are stored separately from labels for audit trail.
### Key Technical Artifacts
| Artifact | Location |
|----------|----------|
| Implementation plan | `docs/labelapp-plan.md` |
| Agent guide | `labelapp/AGENTS.md` |
| Database schema | `labelapp/db/schema.ts` |
| Active timer hook | `labelapp/hooks/use-active-timer.ts` |
| Labeling UI | `labelapp/app/label/page.tsx` |
| Quiz questions | `labelapp/lib/quiz-questions.ts` |
| Warmup paragraphs | `labelapp/lib/warmup-paragraphs.ts` |
| BIBD assignment generator | `labelapp/lib/assignment.ts` |
| IRR metrics (Kappa, Alpha) | `labelapp/lib/metrics.ts` |
| Stratified sampling | `labelapp/lib/sampling.ts` |
| Baseline migration | `labelapp/drizzle/0000_baseline.sql` |
| Migration transition script | `labelapp/scripts/ensure-migration-baseline.ts` |
| Docker entrypoint | `labelapp/entrypoint.sh` |
---
## Phase 9: Pre-Training Strategy — DAPT + TAPT
### The Decision: Own Filings Over PleIAs/SEC ### The Decision: Own Filings Over PleIAs/SEC
@ -463,13 +579,14 @@ Only nano's portion ($21.24) of the first run was wasted — the gemini and grok
| Prompt iteration + model benchmarking | ~4h | 12+ prompt versions, 6 model candidates, pilot analysis | | Prompt iteration + model benchmarking | ~4h | 12+ prompt versions, 6 model candidates, pilot analysis |
| Post-Stage 1 analysis + Stage 2 planning | ~5h | Distributional analysis, model bias discovery, codebook v3.0 rulings, judge benchmarking, strategy revision | | Post-Stage 1 analysis + Stage 2 planning | ~5h | Distributional analysis, model bias discovery, codebook v3.0 rulings, judge benchmarking, strategy revision |
| Documentation + narrative | ~2h | Codebook updates, narrative writing, technical guide updates | | Documentation + narrative | ~2h | Codebook updates, narrative writing, technical guide updates |
| **Total to date** | **~23h** | | | Labelapp build + infrastructure | ~8h | Monorepo restructure, Next.js app, quiz/warmup/labeling flows, BIBD assignment, sampling, Docker deployment, timer + migration infrastructure |
| **Total to date** | **~31h** | |
### Remaining Work (estimated) ### Remaining Work (estimated)
| Phase | Est. Hours | Est. Cost | | Phase | Est. Hours | Est. Cost |
|-------|-----------|-----------| |-------|-----------|-----------|
| Human labeling site + 1,200 labels | ~8-10h | $0 (team labor) | | Human labeling (1,200 paragraphs, 6 annotators) | ~6-8h | $0 (team labor) |
| Stage 2 judge production run (~3-5K paragraphs) | ~1h | ~$20-40 | | Stage 2 judge production run (~3-5K paragraphs) | ~1h | ~$20-40 |
| Training data assembly | ~2h | $0 | | Training data assembly | ~2h | $0 |
| DAPT pre-training | ~48-72h GPU | $0 (own 3090) | | DAPT pre-training | ~48-72h GPU | $0 (own 3090) |

View File

@ -174,6 +174,7 @@ export const humanLabels = pgTable("human_labels", {
labeledAt: timestamp("labeled_at").notNull().defaultNow(), labeledAt: timestamp("labeled_at").notNull().defaultNow(),
sessionId: text("session_id").notNull(), sessionId: text("session_id").notNull(),
durationMs: integer("duration_ms"), durationMs: integer("duration_ms"),
activeMs: integer("active_ms"), // idle-excluded active time
}, (t) => [unique().on(t.paragraphId, t.annotatorId)]); }, (t) => [unique().on(t.paragraphId, t.annotatorId)]);
export const quizSessions = pgTable("quiz_sessions", { export const quizSessions = pgTable("quiz_sessions", {
@ -310,7 +311,7 @@ labelapp/
- **Keyboard shortcuts:** 1-7 category, Shift+1-4 specificity, Enter submit - **Keyboard shortcuts:** 1-7 category, Shift+1-4 specificity, Enter submit
- **Sidebar (collapsible):** Codebook quick-reference (category defs, IS/NOT lists, decision rules) - **Sidebar (collapsible):** Codebook quick-reference (category defs, IS/NOT lists, decision rules)
- **Progress bar:** "47 / 600 completed" - **Progress bar:** "47 / 600 completed"
- **Hidden timer:** tracks `duration_ms` per paragraph - **Active timer:** visible timer in header bar tracks active labeling time with idle detection (30s threshold). Submits both `duration_ms` (wall-clock) and `active_ms` (idle-excluded) per paragraph. Timer pauses and shows amber "idle" indicator when no mouse/keyboard/scroll activity detected.
### Sampling (1,200 from ~50K) ### Sampling (1,200 from ~50K)
Script reads Stage 1 consensus from JSONL, stratifies: Script reads Stage 1 consensus from JSONL, stratifies:

View File

@ -33,10 +33,11 @@ COPY --from=builder /app/labelapp/.next/standalone ./
COPY --from=builder /app/labelapp/.next/static ./labelapp/.next/static COPY --from=builder /app/labelapp/.next/static ./labelapp/.next/static
COPY --from=builder /app/labelapp/public ./labelapp/public COPY --from=builder /app/labelapp/public ./labelapp/public
# Drizzle migration tooling (drizzle-kit push needs these) # Drizzle migration tooling
COPY --from=deps /app/node_modules ./node_modules COPY --from=deps /app/node_modules ./node_modules
COPY --from=deps /app/labelapp/node_modules ./labelapp/node_modules COPY --from=deps /app/labelapp/node_modules ./labelapp/node_modules
COPY --from=builder /app/labelapp/drizzle.config.ts ./labelapp/ COPY --from=builder /app/labelapp/drizzle.config.ts ./labelapp/
COPY --from=builder /app/labelapp/drizzle/ ./labelapp/drizzle/
COPY --from=builder /app/labelapp/db/ ./labelapp/db/ COPY --from=builder /app/labelapp/db/ ./labelapp/db/
COPY --from=builder /app/packages/schemas/ ./packages/schemas/ COPY --from=builder /app/packages/schemas/ ./packages/schemas/
COPY --from=builder /app/package.json ./ COPY --from=builder /app/package.json ./

View File

@ -122,7 +122,7 @@ export async function POST(request: Request) {
} }
const body = await request.json(); const body = await request.json();
const { paragraphId, contentCategory, specificityLevel, notes, sessionId, durationMs } = const { paragraphId, contentCategory, specificityLevel, notes, sessionId, durationMs, activeMs } =
body as { body as {
paragraphId?: string; paragraphId?: string;
contentCategory?: string; contentCategory?: string;
@ -130,6 +130,7 @@ export async function POST(request: Request) {
notes?: string; notes?: string;
sessionId?: string; sessionId?: string;
durationMs?: number; durationMs?: number;
activeMs?: number;
}; };
if (!paragraphId || !contentCategory || !specificityLevel || !sessionId) { if (!paragraphId || !contentCategory || !specificityLevel || !sessionId) {
@ -199,6 +200,7 @@ export async function POST(request: Request) {
notes: notes || null, notes: notes || null,
sessionId, sessionId,
durationMs: durationMs ?? null, durationMs: durationMs ?? null,
activeMs: activeMs ?? null,
}); });
// Get updated progress // Get updated progress

View File

@ -25,7 +25,15 @@ import {
import { ScrollArea } from "@/components/ui/scroll-area"; import { ScrollArea } from "@/components/ui/scroll-area";
import { Separator } from "@/components/ui/separator"; import { Separator } from "@/components/ui/separator";
import { Badge } from "@/components/ui/badge"; import { Badge } from "@/components/ui/badge";
import { BookOpen, CheckCircle2 } from "lucide-react"; import { BookOpen, CheckCircle2, Clock, Pause } from "lucide-react";
import { useActiveTimer } from "@/hooks/use-active-timer";
function formatTime(ms: number): string {
const totalSeconds = Math.floor(ms / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
const CATEGORIES = [ const CATEGORIES = [
"Board Governance", "Board Governance",
@ -72,7 +80,7 @@ export default function LabelPage() {
const [submitting, setSubmitting] = useState(false); const [submitting, setSubmitting] = useState(false);
const [done, setDone] = useState(false); const [done, setDone] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const timerStart = useRef<number>(Date.now()); const timer = useActiveTimer();
const sessionId = useRef<string>(crypto.randomUUID()); const sessionId = useRef<string>(crypto.randomUUID());
const notesRef = useRef<HTMLTextAreaElement>(null); const notesRef = useRef<HTMLTextAreaElement>(null);
@ -104,13 +112,13 @@ export default function LabelPage() {
setCategory(""); setCategory("");
setSpecificity(0); setSpecificity(0);
setNotes(""); setNotes("");
timerStart.current = Date.now(); timer.reset();
} catch { } catch {
setError("Network error"); setError("Network error");
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, [router]); }, [router, timer.reset]);
useEffect(() => { useEffect(() => {
fetchNext(); fetchNext();
@ -119,7 +127,8 @@ export default function LabelPage() {
const handleSubmit = useCallback(async () => { const handleSubmit = useCallback(async () => {
if (!category || !specificity || !paragraph || submitting) return; if (!category || !specificity || !paragraph || submitting) return;
const durationMs = Date.now() - timerStart.current; const durationMs = timer.totalMs;
const activeMs = timer.activeMs;
setSubmitting(true); setSubmitting(true);
try { try {
@ -133,6 +142,7 @@ export default function LabelPage() {
notes: notes.trim() || undefined, notes: notes.trim() || undefined,
sessionId: sessionId.current, sessionId: sessionId.current,
durationMs, durationMs,
activeMs,
}), }),
}); });
@ -239,6 +249,19 @@ export default function LabelPage() {
<Badge variant="outline">{paragraph.secItem}</Badge> <Badge variant="outline">{paragraph.secItem}</Badge>
</div> </div>
<div className="flex items-center gap-4"> <div className="flex items-center gap-4">
<div className="flex items-center gap-1.5 text-sm tabular-nums">
{timer.isIdle ? (
<Pause className="size-3.5 text-amber-500" />
) : (
<Clock className="size-3.5 text-muted-foreground" />
)}
<span className={timer.isIdle ? "text-amber-500" : "text-muted-foreground"}>
{formatTime(timer.activeMs)}
</span>
{timer.isIdle && (
<span className="text-xs text-amber-500/70">idle</span>
)}
</div>
<Progress <Progress
value={progress.completed} value={progress.completed}
max={progress.total} max={progress.total}

View File

@ -66,6 +66,7 @@ export const humanLabels = pgTable(
labeledAt: timestamp("labeled_at").notNull().defaultNow(), labeledAt: timestamp("labeled_at").notNull().defaultNow(),
sessionId: text("session_id").notNull(), sessionId: text("session_id").notNull(),
durationMs: integer("duration_ms"), durationMs: integer("duration_ms"),
activeMs: integer("active_ms"),
}, },
(t) => [unique().on(t.paragraphId, t.annotatorId)], (t) => [unique().on(t.paragraphId, t.annotatorId)],
); );

View File

@ -2,6 +2,7 @@ import { defineConfig } from "drizzle-kit";
export default defineConfig({ export default defineConfig({
schema: "./db/schema.ts", schema: "./db/schema.ts",
out: "./drizzle",
dialect: "postgresql", dialect: "postgresql",
dbCredentials: { dbCredentials: {
url: process.env.DATABASE_URL!, url: process.env.DATABASE_URL!,

View File

@ -0,0 +1,75 @@
CREATE TABLE "adjudications" (
"paragraph_id" text PRIMARY KEY NOT NULL,
"final_category" text NOT NULL,
"final_specificity" integer NOT NULL,
"method" text NOT NULL,
"adjudicator_id" text,
"notes" text,
"resolved_at" timestamp DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE TABLE "annotators" (
"id" text PRIMARY KEY NOT NULL,
"display_name" text NOT NULL,
"password" text NOT NULL,
"onboarded_at" timestamp
);
--> statement-breakpoint
CREATE TABLE "assignments" (
"paragraph_id" text NOT NULL,
"annotator_id" text NOT NULL,
"assigned_at" timestamp DEFAULT now() NOT NULL,
"is_warmup" boolean DEFAULT false NOT NULL,
CONSTRAINT "assignments_paragraph_id_annotator_id_unique" UNIQUE("paragraph_id","annotator_id")
);
--> statement-breakpoint
CREATE TABLE "human_labels" (
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "human_labels_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
"paragraph_id" text NOT NULL,
"annotator_id" text NOT NULL,
"content_category" text NOT NULL,
"specificity_level" integer NOT NULL,
"notes" text,
"labeled_at" timestamp DEFAULT now() NOT NULL,
"session_id" text NOT NULL,
"duration_ms" integer,
"active_ms" integer,
CONSTRAINT "human_labels_paragraph_id_annotator_id_unique" UNIQUE("paragraph_id","annotator_id")
);
--> statement-breakpoint
CREATE TABLE "paragraphs" (
"id" text PRIMARY KEY NOT NULL,
"text" text NOT NULL,
"word_count" integer NOT NULL,
"paragraph_index" integer NOT NULL,
"company_name" text NOT NULL,
"cik" text NOT NULL,
"ticker" text,
"filing_type" text NOT NULL,
"filing_date" text NOT NULL,
"fiscal_year" integer NOT NULL,
"accession_number" text NOT NULL,
"sec_item" text NOT NULL,
"stage1_category" text,
"stage1_specificity" integer,
"stage1_method" text,
"stage1_confidence" real
);
--> statement-breakpoint
CREATE TABLE "quiz_sessions" (
"id" text PRIMARY KEY NOT NULL,
"annotator_id" text NOT NULL,
"started_at" timestamp DEFAULT now() NOT NULL,
"completed_at" timestamp,
"passed" boolean DEFAULT false NOT NULL,
"score" integer DEFAULT 0 NOT NULL,
"total_questions" integer NOT NULL,
"answers" text DEFAULT '[]' NOT NULL
);
--> statement-breakpoint
ALTER TABLE "adjudications" ADD CONSTRAINT "adjudications_paragraph_id_paragraphs_id_fk" FOREIGN KEY ("paragraph_id") REFERENCES "public"."paragraphs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "assignments" ADD CONSTRAINT "assignments_paragraph_id_paragraphs_id_fk" FOREIGN KEY ("paragraph_id") REFERENCES "public"."paragraphs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "assignments" ADD CONSTRAINT "assignments_annotator_id_annotators_id_fk" FOREIGN KEY ("annotator_id") REFERENCES "public"."annotators"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "human_labels" ADD CONSTRAINT "human_labels_paragraph_id_paragraphs_id_fk" FOREIGN KEY ("paragraph_id") REFERENCES "public"."paragraphs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "human_labels" ADD CONSTRAINT "human_labels_annotator_id_annotators_id_fk" FOREIGN KEY ("annotator_id") REFERENCES "public"."annotators"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "quiz_sessions" ADD CONSTRAINT "quiz_sessions_annotator_id_annotators_id_fk" FOREIGN KEY ("annotator_id") REFERENCES "public"."annotators"("id") ON DELETE no action ON UPDATE no action;

View File

@ -0,0 +1,510 @@
{
"id": "392c9bd4-1b68-4e32-86b0-be7abc632b44",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.adjudications": {
"name": "adjudications",
"schema": "",
"columns": {
"paragraph_id": {
"name": "paragraph_id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"final_category": {
"name": "final_category",
"type": "text",
"primaryKey": false,
"notNull": true
},
"final_specificity": {
"name": "final_specificity",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"method": {
"name": "method",
"type": "text",
"primaryKey": false,
"notNull": true
},
"adjudicator_id": {
"name": "adjudicator_id",
"type": "text",
"primaryKey": false,
"notNull": false
},
"notes": {
"name": "notes",
"type": "text",
"primaryKey": false,
"notNull": false
},
"resolved_at": {
"name": "resolved_at",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {
"adjudications_paragraph_id_paragraphs_id_fk": {
"name": "adjudications_paragraph_id_paragraphs_id_fk",
"tableFrom": "adjudications",
"tableTo": "paragraphs",
"columnsFrom": [
"paragraph_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.annotators": {
"name": "annotators",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"display_name": {
"name": "display_name",
"type": "text",
"primaryKey": false,
"notNull": true
},
"password": {
"name": "password",
"type": "text",
"primaryKey": false,
"notNull": true
},
"onboarded_at": {
"name": "onboarded_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.assignments": {
"name": "assignments",
"schema": "",
"columns": {
"paragraph_id": {
"name": "paragraph_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"annotator_id": {
"name": "annotator_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"assigned_at": {
"name": "assigned_at",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"is_warmup": {
"name": "is_warmup",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"default": false
}
},
"indexes": {},
"foreignKeys": {
"assignments_paragraph_id_paragraphs_id_fk": {
"name": "assignments_paragraph_id_paragraphs_id_fk",
"tableFrom": "assignments",
"tableTo": "paragraphs",
"columnsFrom": [
"paragraph_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"assignments_annotator_id_annotators_id_fk": {
"name": "assignments_annotator_id_annotators_id_fk",
"tableFrom": "assignments",
"tableTo": "annotators",
"columnsFrom": [
"annotator_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {
"assignments_paragraph_id_annotator_id_unique": {
"name": "assignments_paragraph_id_annotator_id_unique",
"nullsNotDistinct": false,
"columns": [
"paragraph_id",
"annotator_id"
]
}
},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.human_labels": {
"name": "human_labels",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"identity": {
"type": "always",
"name": "human_labels_id_seq",
"schema": "public",
"increment": "1",
"startWith": "1",
"minValue": "1",
"maxValue": "2147483647",
"cache": "1",
"cycle": false
}
},
"paragraph_id": {
"name": "paragraph_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"annotator_id": {
"name": "annotator_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"content_category": {
"name": "content_category",
"type": "text",
"primaryKey": false,
"notNull": true
},
"specificity_level": {
"name": "specificity_level",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"notes": {
"name": "notes",
"type": "text",
"primaryKey": false,
"notNull": false
},
"labeled_at": {
"name": "labeled_at",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"session_id": {
"name": "session_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"duration_ms": {
"name": "duration_ms",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"active_ms": {
"name": "active_ms",
"type": "integer",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {
"human_labels_paragraph_id_paragraphs_id_fk": {
"name": "human_labels_paragraph_id_paragraphs_id_fk",
"tableFrom": "human_labels",
"tableTo": "paragraphs",
"columnsFrom": [
"paragraph_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
},
"human_labels_annotator_id_annotators_id_fk": {
"name": "human_labels_annotator_id_annotators_id_fk",
"tableFrom": "human_labels",
"tableTo": "annotators",
"columnsFrom": [
"annotator_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {
"human_labels_paragraph_id_annotator_id_unique": {
"name": "human_labels_paragraph_id_annotator_id_unique",
"nullsNotDistinct": false,
"columns": [
"paragraph_id",
"annotator_id"
]
}
},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.paragraphs": {
"name": "paragraphs",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"text": {
"name": "text",
"type": "text",
"primaryKey": false,
"notNull": true
},
"word_count": {
"name": "word_count",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"paragraph_index": {
"name": "paragraph_index",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"company_name": {
"name": "company_name",
"type": "text",
"primaryKey": false,
"notNull": true
},
"cik": {
"name": "cik",
"type": "text",
"primaryKey": false,
"notNull": true
},
"ticker": {
"name": "ticker",
"type": "text",
"primaryKey": false,
"notNull": false
},
"filing_type": {
"name": "filing_type",
"type": "text",
"primaryKey": false,
"notNull": true
},
"filing_date": {
"name": "filing_date",
"type": "text",
"primaryKey": false,
"notNull": true
},
"fiscal_year": {
"name": "fiscal_year",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"accession_number": {
"name": "accession_number",
"type": "text",
"primaryKey": false,
"notNull": true
},
"sec_item": {
"name": "sec_item",
"type": "text",
"primaryKey": false,
"notNull": true
},
"stage1_category": {
"name": "stage1_category",
"type": "text",
"primaryKey": false,
"notNull": false
},
"stage1_specificity": {
"name": "stage1_specificity",
"type": "integer",
"primaryKey": false,
"notNull": false
},
"stage1_method": {
"name": "stage1_method",
"type": "text",
"primaryKey": false,
"notNull": false
},
"stage1_confidence": {
"name": "stage1_confidence",
"type": "real",
"primaryKey": false,
"notNull": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
},
"public.quiz_sessions": {
"name": "quiz_sessions",
"schema": "",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true
},
"annotator_id": {
"name": "annotator_id",
"type": "text",
"primaryKey": false,
"notNull": true
},
"started_at": {
"name": "started_at",
"type": "timestamp",
"primaryKey": false,
"notNull": true,
"default": "now()"
},
"completed_at": {
"name": "completed_at",
"type": "timestamp",
"primaryKey": false,
"notNull": false
},
"passed": {
"name": "passed",
"type": "boolean",
"primaryKey": false,
"notNull": true,
"default": false
},
"score": {
"name": "score",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 0
},
"total_questions": {
"name": "total_questions",
"type": "integer",
"primaryKey": false,
"notNull": true
},
"answers": {
"name": "answers",
"type": "text",
"primaryKey": false,
"notNull": true,
"default": "'[]'"
}
},
"indexes": {},
"foreignKeys": {
"quiz_sessions_annotator_id_annotators_id_fk": {
"name": "quiz_sessions_annotator_id_annotators_id_fk",
"tableFrom": "quiz_sessions",
"tableTo": "annotators",
"columnsFrom": [
"annotator_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"policies": {},
"checkConstraints": {},
"isRLSEnabled": false
}
},
"enums": {},
"schemas": {},
"sequences": {},
"roles": {},
"policies": {},
"views": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "postgresql",
"entries": [
{
"idx": 0,
"version": "7",
"when": 1774815564792,
"tag": "0000_baseline",
"breakpoints": true
}
]
}

View File

@ -3,8 +3,11 @@ set -euo pipefail
cd /app/labelapp cd /app/labelapp
echo "==> Ensuring migration baseline..."
bun run scripts/ensure-migration-baseline.ts
echo "==> Running Drizzle migrations..." echo "==> Running Drizzle migrations..."
bunx drizzle-kit push --force bunx drizzle-kit migrate
echo "==> Checking if database needs seeding..." echo "==> Checking if database needs seeding..."
ROW_COUNT=$(bun --eval " ROW_COUNT=$(bun --eval "

View File

@ -0,0 +1,105 @@
import { useCallback, useEffect, useRef, useState } from "react";
const IDLE_THRESHOLD_MS = 30_000; // 30 seconds of inactivity = idle
const TICK_INTERVAL_MS = 1_000; // Update display every second
const ACTIVITY_EVENTS = [
"mousemove",
"mousedown",
"keydown",
"scroll",
"touchstart",
] as const;
export function useActiveTimer() {
const [activeMs, setActiveMs] = useState(0);
const [totalMs, setTotalMs] = useState(0);
const [isIdle, setIsIdle] = useState(false);
// Refs for values that change frequently but shouldn't trigger re-renders
const activeMsRef = useRef(0);
const totalMsRef = useRef(0);
const lastActivityRef = useRef(Date.now());
const lastTickRef = useRef(Date.now());
const idleRef = useRef(false);
const startedRef = useRef(Date.now());
const reset = useCallback(() => {
const now = Date.now();
activeMsRef.current = 0;
totalMsRef.current = 0;
lastActivityRef.current = now;
lastTickRef.current = now;
startedRef.current = now;
idleRef.current = false;
setActiveMs(0);
setTotalMs(0);
setIsIdle(false);
}, []);
// Mark activity on user interaction
useEffect(() => {
function onActivity() {
lastActivityRef.current = Date.now();
if (idleRef.current) {
idleRef.current = false;
setIsIdle(false);
// Reset the tick baseline so we don't count the idle gap as active
lastTickRef.current = Date.now();
}
}
for (const event of ACTIVITY_EVENTS) {
window.addEventListener(event, onActivity, { passive: true });
}
// Also track when the tab regains focus
window.addEventListener("focus", onActivity);
return () => {
for (const event of ACTIVITY_EVENTS) {
window.removeEventListener(event, onActivity);
}
window.removeEventListener("focus", onActivity);
};
}, []);
// Tick loop: accumulate active time and detect idle transitions
useEffect(() => {
const interval = setInterval(() => {
const now = Date.now();
const elapsed = now - lastTickRef.current;
lastTickRef.current = now;
// Always accumulate wall-clock time
totalMsRef.current = now - startedRef.current;
setTotalMs(totalMsRef.current);
const timeSinceActivity = now - lastActivityRef.current;
if (timeSinceActivity >= IDLE_THRESHOLD_MS) {
// User is idle — don't accumulate active time
if (!idleRef.current) {
idleRef.current = true;
setIsIdle(true);
}
} else {
// User is active — accumulate
activeMsRef.current += elapsed;
setActiveMs(activeMsRef.current);
}
}, TICK_INTERVAL_MS);
return () => clearInterval(interval);
}, []);
return {
/** Milliseconds the user was actively engaged (idle time excluded) */
activeMs,
/** Total wall-clock milliseconds since last reset */
totalMs,
/** Whether the user is currently idle */
isIdle,
/** Reset both counters (call when loading a new paragraph) */
reset,
};
}

View File

@ -6,7 +6,10 @@
"dev": "next dev --turbopack", "dev": "next dev --turbopack",
"build": "next build", "build": "next build",
"start": "next start", "start": "next start",
"typecheck": "tsc --noEmit",
"lint": "eslint", "lint": "eslint",
"db:generate": "drizzle-kit generate",
"db:migrate": "drizzle-kit migrate",
"db:push": "drizzle-kit push", "db:push": "drizzle-kit push",
"db:studio": "drizzle-kit studio", "db:studio": "drizzle-kit studio",
"seed": "bun run scripts/seed.ts", "seed": "bun run scripts/seed.ts",

View File

@ -0,0 +1,62 @@
/**
* Ensures the baseline migration is marked as applied for databases
* that were created with `drizzle-kit push` before we switched to
* file-based migrations. Safe to run on fresh databases (no-op).
*/
import postgres from "postgres";
import { readFileSync } from "fs";
import { createHash } from "crypto";
import { resolve } from "path";
const sql = postgres(process.env.DATABASE_URL!);
try {
// Check if any application tables exist (indicator of a push-created DB)
const [{ exists: tablesExist }] = await sql`
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = 'paragraphs'
) as exists
`;
if (!tablesExist) {
console.log("Fresh database — baseline seeding not needed.");
await sql.end();
process.exit(0);
}
// Ensure the drizzle schema + migrations table exist
await sql`CREATE SCHEMA IF NOT EXISTS drizzle`;
await sql`
CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
id serial PRIMARY KEY,
hash text NOT NULL,
created_at bigint
)
`;
// Check if any migrations are already recorded
const [{ count }] = await sql`
SELECT count(*)::int as count FROM drizzle.__drizzle_migrations
`;
if (Number(count) > 0) {
console.log("Migration history already exists — baseline seeding not needed.");
await sql.end();
process.exit(0);
}
// Compute hash of the baseline migration and insert it
const baselinePath = resolve(import.meta.dirname!, "../drizzle/0000_baseline.sql");
const content = readFileSync(baselinePath, "utf-8");
const hash = createHash("sha256").update(content).digest("hex");
await sql`
INSERT INTO drizzle.__drizzle_migrations (hash, created_at)
VALUES (${hash}, ${Date.now()})
`;
console.log("Baseline migration marked as applied (push → migrate transition).");
} finally {
await sql.end();
}

View File

@ -2,11 +2,33 @@
"name": "sec-cybert-monorepo", "name": "sec-cybert-monorepo",
"private": true, "private": true,
"scripts": { "scripts": {
"labelapp:docker": "docker build -f labelapp/Dockerfile -t registry.claiborne.soy/labelapp:latest . --push" "la:dev": "bun run --filter labelapp dev",
"la:build": "bun run --filter labelapp build",
"la:typecheck": "bun run --filter labelapp typecheck",
"la:lint": "bun run --filter labelapp lint",
"la:test": "bun run --filter labelapp test",
"la:test:api": "bun run --filter labelapp test:api",
"la:test:e2e": "bun run --filter labelapp test:e2e",
"la:db:generate": "bun run --filter labelapp db:generate",
"la:db:migrate": "bun run --filter labelapp db:migrate",
"la:db:studio": "bun run --filter labelapp db:studio",
"la:seed": "bun run --filter labelapp seed",
"la:sample": "bun run --filter labelapp sample",
"la:assign": "bun run --filter labelapp assign",
"la:export": "bun run --filter labelapp export",
"la:docker": "docker build -f labelapp/Dockerfile -t registry.claiborne.soy/labelapp:latest . --push",
"ts:sec": "bun run --filter sec-cybert sec",
"ts:typecheck": "bun run --filter sec-cybert typecheck",
"py:train": "cd python && uv run main.py",
"typecheck": "bun run --filter '*' typecheck"
}, },
"workspaces": [ "workspaces": [
"packages/*", "packages/*",
"ts", "ts",
"labelapp" "labelapp"
] ],
"devDependencies": {
"@types/bun": "^1.3.11",
"@types/node": "^25.5.0"
}
} }