diff --git a/labelapp/app/api/onboarding/route.ts b/labelapp/app/api/onboarding/route.ts new file mode 100644 index 0000000..ce8457a --- /dev/null +++ b/labelapp/app/api/onboarding/route.ts @@ -0,0 +1,43 @@ +import { NextResponse } from "next/server"; +import { db } from "@/db"; +import { annotators } from "@/db/schema"; +import { eq } from "drizzle-orm"; +import { getSession } from "@/lib/auth"; + +export async function GET() { + const session = await getSession(); + if (!session) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const [annotator] = await db + .select({ onboardedAt: annotators.onboardedAt }) + .from(annotators) + .where(eq(annotators.id, session.annotatorId)) + .limit(1); + + return NextResponse.json({ + onboarded: !!annotator?.onboardedAt, + }); +} + +export async function POST(request: Request) { + const session = await getSession(); + if (!session) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const body = await request.json(); + const { action } = body as { action?: string }; + + if (action === "complete") { + await db + .update(annotators) + .set({ onboardedAt: new Date() }) + .where(eq(annotators.id, session.annotatorId)); + + return NextResponse.json({ ok: true }); + } + + return NextResponse.json({ error: "Invalid action" }, { status: 400 }); +} diff --git a/labelapp/app/codebook/page.tsx b/labelapp/app/codebook/page.tsx new file mode 100644 index 0000000..3f3c0e2 --- /dev/null +++ b/labelapp/app/codebook/page.tsx @@ -0,0 +1,1133 @@ +import Link from "next/link"; +import { Badge } from "@/components/ui/badge"; +import { Separator } from "@/components/ui/separator"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +function ExampleBlock({ + text, + category, + specificity, + explanation, +}: { + text: string; + category: string; + specificity: number; + explanation: string; +}) { + return ( +
+

{text}

+
+ {category} + Specificity {specificity} + {explanation} +
+
+ ); +} + +function ISItem({ children }: { children: React.ReactNode }) { + return ( +
  • {children}
    • + ); +} + +function NOTItem({ children }: { children: React.ReactNode }) { + return ( +
  • {children}
    • + ); +} + +function SectionHeading({ + id, + level, + children, +}: { + id: string; + level: 2 | 3; + children: React.ReactNode; +}) { + const Tag = level === 2 ? "h2" : "h3"; + return ( + + {children} + + ); +} + +const tocSections = [ + { id: "overview", label: "Overview" }, + { id: "content-categories", label: "Content Categories" }, + { id: "decision-rules", label: "Category Decision Rules" }, + { id: "specificity-levels", label: "Specificity Levels" }, + { id: "borderline-cases", label: "Borderline Cases" }, +]; + +export default function CodebookPage() { + return ( +
    + {/* Sticky header */} +
    +
    +

    Labeling Codebook

    + + ← Back to Dashboard + +
    +
    + +
    + {/* Table of Contents */} + + + {/* ================================================================ + SECTION 1: OVERVIEW + ================================================================ */} +
    + + 1. Overview + + +

    + Unit of analysis: One paragraph from an SEC filing + (Item 1C of 10-K, or Item 1.05/8.01/7.01 of 8-K). +

    +

    + Classification type: Multi-class (single-label), + NOT multi-label. Each paragraph receives exactly one content + category. +

    +

    + Each paragraph receives two labels: +

    +
      +
    1. + Content Category — single-label, one of 7 + mutually exclusive classes +
      2. +
    2. + Specificity Level — ordinal integer 1–4 +
      3. +
    +

    + None/Other policy: Required. Since this is + multi-class (not multi-label), we need a catch-all for paragraphs + that don’t fit the 6 substantive categories. A paragraph + receives None/Other when it contains no cybersecurity-specific + disclosure content (e.g., forward-looking statement disclaimers, + section headers, general business language). +

    +
    + + + + {/* ================================================================ + SECTION 2: CONTENT CATEGORIES + ================================================================ */} +
    + + 2. Content Categories + +

    + Each paragraph is assigned exactly one content + category. If a paragraph spans multiple categories, assign the{" "} + dominant category — the one that best describes the + paragraph’s primary communicative purpose. +

    + + {/* ---------- Board Governance ---------- */} +
    + + Board Governance + +
      +
    • SEC basis: Item 106(c)(1)
      • +
    • + Covers: Board or committee oversight of + cybersecurity risks, briefing frequency, board member + cybersecurity expertise +
      • +
    • + Key markers: “Audit Committee,” + “Board of Directors oversees,” “quarterly + briefings,” “board-level expertise,” + “board committee” +
      • +
    • + Assign when: The grammatical subject performing + the primary action is the board or a board committee +
      • +
    + + + + +
    + + {/* ---------- Management Role ---------- */} +
    + + Management Role + +
      +
    • SEC basis: Item 106(c)(2)
      • +
    • + Covers: The specific person filling a + cybersecurity leadership position: their name, qualifications, + career history, credentials, tenure, reporting lines, management + committees responsible for cybersecurity +
      • +
    • + Key markers: “Chief Information Security + Officer,” “reports to,” “years of + experience,” “management committee,” + “CISSP,” “CISM,” named individuals, + career background +
      • +
    • + Assign when: The paragraph tells you something + about who the person is — their background, + credentials, experience, or reporting structure. A paragraph that + names a CISO/CIO/CTO and then describes what the cybersecurity{" "} + program does is NOT Management Role — it is Risk + Management Process with an incidental role attribution. The test + is whether the paragraph is about the person or + about the function. +
      • +
    + +
    +

    The person-vs-function test:

    +

    + If you removed the role holder’s name, title, + qualifications, and background from the paragraph and the + remaining content still describes substantive cybersecurity + activities, processes, or oversight → the paragraph is about + the function (Risk Management Process), not the person + (Management Role). Management Role requires the person’s + identity or credentials to be the primary content, not just a + brief attribution of who runs the program. +

    +
    + + + + + +
    + + {/* ---------- Risk Management Process ---------- */} +
    + + Risk Management Process + +
      +
    • SEC basis: Item 106(b)
      • +
    • + Covers: Risk assessment methodology, framework + adoption (NIST, ISO, etc.), vulnerability management, monitoring, + incident response planning, tabletop exercises, ERM integration +
      • +
    • + Key markers: “NIST CSF,” “ISO + 27001,” “risk assessment,” “vulnerability + management,” “tabletop exercises,” + “incident response plan,” “SOC,” + “SIEM” +
      • +
    • + Assign when: The paragraph primarily describes + the company’s internal cybersecurity processes, tools, or + methodologies +
      • +
    + + + + +
    + + {/* ---------- Third-Party Risk ---------- */} +
    + + Third-Party Risk + +
      +
    • SEC basis: Item 106(b)
      • +
    • + Covers: Vendor/supplier risk oversight, external + assessor engagement, contractual security requirements, supply + chain risk management +
      • +
    • + Key markers: “third-party,” + “service providers,” “vendor risk,” + “external auditors,” “supply chain,” + “SOC 2 report,” “contractual + requirements” +
      • +
    • + Assign when: The central topic is oversight of + external parties’ cybersecurity, not the company’s + own internal processes +
      • +
    + + + + +
    + + {/* ---------- Incident Disclosure ---------- */} +
    + + Incident Disclosure + +
      +
    • + SEC basis: 8-K Item 1.05 (and 8.01/7.01 + post-May 2024) +
      • +
    • + Covers: Description of cybersecurity incidents — + nature, scope, timing, impact assessment, remediation actions, + ongoing investigation +
      • +
    • + Key markers: “unauthorized access,” + “detected,” “incident,” + “remediation,” “impacted,” + “forensic investigation,” “breach,” + “compromised” +
      • +
    • + Assign when: The paragraph primarily describes + what happened in a cybersecurity incident +
      • +
    + + + + +
    + + {/* ---------- Strategy Integration ---------- */} +
    + + Strategy Integration + +
      +
    • SEC basis: Item 106(b)(2)
      • +
    • + Covers: Material impact (or lack thereof) on + business strategy or financials, cybersecurity insurance, + investment/resource allocation, cost of incidents +
      • +
    • + Key markers: “business strategy,” + “insurance,” “investment,” + “material,” “financial condition,” + “budget,” “not materially affected,” + “results of operations” +
      • +
    • + Assign when: The paragraph primarily discusses + business/financial consequences or strategic response to cyber + risk, not the risk management activities themselves +
      • +
    • + Includes materiality disclaimers: Any paragraph + that explicitly assesses whether cybersecurity risks have or + could “materially affect” the company’s + business, strategy, financial condition, or results of operations + is Strategy Integration — even if the assessment is boilerplate. + The company is making a strategic judgment about cyber risk + impact, which is the essence of this category. A cross-reference + to Risk Factors appended to a materiality assessment does not + change the classification. +
      • +
    + + + + + +
    + + {/* ---------- None/Other ---------- */} +
    + + None/Other + +
      +
    • + Covers: Forward-looking statement disclaimers, + section headers, cross-references to other filing sections, + general business language that mentions cybersecurity + incidentally, text erroneously extracted from outside Item + 1C/1.05 +
      • +
    • + No specificity scoring needed: Always assign + Specificity 1 for None/Other paragraphs (since there is no + cybersecurity disclosure to rate) +
      • +
    • + SPACs and shell companies: Companies that + explicitly state they have no operations, no cybersecurity + program, or no formal processes receive None/Other regardless of + incidental mentions of board oversight or risk acknowledgment. + The absence of a program is not a description of a program. + Paragraphs like “We have not adopted any cybersecurity risk + management program. Our board is generally responsible for + oversight” are None/Other — the board mention is + perfunctory, not substantive governance disclosure. +
      • +
    • + Distinguishing from Strategy Integration: A pure + cross-reference (“See Item 1A, Risk Factors”) with + no materiality assessment is None/Other. But if the paragraph + includes an explicit materiality conclusion (“have not + materially affected our business strategy”), it becomes + Strategy Integration even if a cross-reference is also present. + The test: does the paragraph make a substantive claim about + cybersecurity’s impact on the business? If yes → + Strategy Integration. If it only points elsewhere → + None/Other. +
      • +
    + + + + + + +
    +
    + + + + {/* ================================================================ + SECTION 3: CATEGORY DECISION RULES + ================================================================ */} +
    + + 3. Category Decision Rules + + + {/* Rule 1 */} +
    + + Rule 1: Dominant Category + +

    + If a paragraph spans multiple categories, assign the one whose + topic occupies the most text or is the paragraph’s primary + communicative purpose. +

    +
    + + {/* Rule 2 */} +
    + + Rule 2: Board vs. Management + + + + + Signal + Category + + + + + Board/committee is the grammatical subject + Board Governance + + + Board delegates responsibility to management + Board Governance + + + Management role reports TO the board + Management Role + + + Management role’s qualifications are described + Management Role + + + “Board oversees... CISO reports to Board quarterly” + Board Governance (board is primary actor) + + + “CISO reports quarterly to the Board on...” + Management Role (CISO is primary actor) + + +
    +
    + + {/* Rule 2b */} +
    + + Rule 2b: Person-vs-Function Test (Management Role vs. Risk Management Process) + +

    + This is the single most common source of annotator disagreement. + The line is: is the paragraph about the person or about the function? +

    + + + + Signal + Category + + + + + The person’s background, credentials, tenure, experience, education, career history + Management Role + + + The person’s name is given + Management Role (strong signal) + + + Reporting lines as primary content (who reports to whom, management committee structure) + Management Role + + + Role title mentioned as attribution (“Our CISO oversees...”) followed by process description + Risk Management Process + + + Activities, tools, methodologies, frameworks as the primary content + Risk Management Process + + + The paragraph would still make sense if you removed the role title and replaced it with “the Company” + Risk Management Process + + +
    +
    +

    Key principle:

    +

    + Naming a cybersecurity leadership title (CISO, CIO, CTO, VP of + Security) does not make a paragraph Management Role. The title is + often an incidental attribution — the paragraph names who is + responsible then describes what the program does. If the + paragraph’s substantive content is about processes, + activities, or tools, it is Risk Management Process regardless of + how many times a role title appears. Management Role requires the + paragraph’s content to be about the person — who + they are, what makes them qualified, how long they’ve + served, what their background is. +

    +
    +
    + + {/* Rule 3 */} +
    + + Rule 3: Risk Management vs. Third-Party + + + + + Signal + Category + + + + + Company’s own internal processes, tools, teams + Risk Management Process + + + Third parties mentioned as ONE component of internal program + Risk Management Process + + + Vendor oversight is the CENTRAL topic + Third-Party Risk + + + External assessor hired to test the company + Risk Management Process (they serve the company) + + + Requirements imposed ON vendors + Third-Party Risk + + +
    +
    + + {/* Rule 4 */} +
    + + Rule 4: Incident vs. Strategy + + + + + Signal + Category + + + + + Describes what happened (timeline, scope, response) + Incident Disclosure + + + Describes business impact of an incident (costs, revenue, insurance claim) + Strategy Integration + + + Mixed: “We detected X... at a cost of $Y” + Assign based on which is dominant — if cost is one sentence in a paragraph about the incident → Incident Disclosure + + +
    +
    + + {/* Rule 5 */} +
    + + Rule 5: None/Other Threshold + +

    + Assign None/Other ONLY when the paragraph contains no substantive + cybersecurity disclosure content. If a paragraph mentions + cybersecurity even briefly in service of a disclosure obligation, + assign the relevant content category. +

    +
    +

    Exception — SPACs and no-operations companies:

    +

    + A paragraph that explicitly states the company has no + cybersecurity program, no operations, or no formal processes is + None/Other even if it perfunctorily mentions board oversight or + risk acknowledgment. The absence of a program is not substantive + disclosure. +

    +
    +
    + + {/* Rule 6 */} +
    + + Rule 6: Materiality Disclaimers → Strategy Integration + +

    + Any paragraph that explicitly assesses whether cybersecurity risks + or incidents have “materially affected” (or are + “reasonably likely to materially affect”) the + company’s business strategy, results of operations, or + financial condition is Strategy Integration — even + when the assessment is boilerplate. The materiality assessment is + the substantive content. A cross-reference to Risk Factors appended + to a materiality assessment does not change the classification to + None/Other. Only a pure cross-reference with no + materiality conclusion is None/Other. +

    +
    +
    + + + + {/* ================================================================ + SECTION 4: SPECIFICITY LEVELS + ================================================================ */} +
    + + 4. Specificity Levels + +

    + Each paragraph receives a specificity level (1–4) indicating + how company-specific the disclosure is. Apply the decision test in + order — stop at the first “yes.” +

    + + {/* Decision Test */} +
    + + Decision Test (Waterfall) + +
      +
    1. + Count hard verifiable facts ONLY (specific + dates, dollar amounts, headcounts/percentages, named third-party + firms, named products/tools, named certifications). TWO or more? + → Quantified-Verifiable (4) +
      2. +
    2. + Does it contain at least one fact from the IS list + below? → Firm-Specific (3) +
      3. +
    3. + Does it name a recognized standard (NIST, ISO + 27001, SOC 2, CIS, GDPR, PCI DSS, HIPAA)? →{" "} + Sector-Adapted (2) +
      4. +
    4. + None of the above? →{" "} + Generic Boilerplate (1) +
      5. +
    +

    + None/Other paragraphs always receive Specificity 1. +

    +
    + + {/* Level Definitions */} +
    + + Level Definitions + + + + + Level + Name + Description + + + + + 1 + Generic Boilerplate + + Could paste into any company’s filing unchanged. No + named entities, frameworks, roles, dates, or specific details. + + + + 2 + Sector-Adapted + + Names a specific recognized standard (NIST, ISO 27001, SOC 2, + etc.) but contains nothing unique to THIS company. General + practices (pen testing, vulnerability scanning, tabletop + exercises) do NOT qualify — only named standards. + + + + 3 + Firm-Specific + + Contains at least one fact from the IS list that identifies + something unique to THIS company’s disclosure. + + + + 4 + Quantified-Verifiable + + Contains TWO or more hard verifiable facts (see QV-eligible + list). One fact = Firm-Specific, not QV. + + + +
    +
    + + {/* IS list */} +
    + + IS a Specific Fact (any ONE → at least Firm-Specific) + +
      + Cybersecurity-specific titles: CISO, CTO, CIO, VP of IT/Security, Information Security Officer, Director of IT Security, HSE Director overseeing cybersecurity, Chief Digital Officer (when overseeing cyber), Cybersecurity Director + Named non-generic committees: Technology Committee, Cybersecurity Committee, Risk Committee, ERM Committee (NOT “Audit Committee” — that exists at every public company) + Specific team/department compositions: “Legal, Compliance, and Finance” (but NOT just “a cross-functional team”) + Specific dates: “In December 2023”, “On May 6, 2024”, “fiscal 2025” + Named internal programs with unique identifiers: “Cyber Incident Response Plan (CIRP)” (must have a distinguishing name/abbreviation — generic “incident response plan” does not qualify) + Named products, systems, tools: Splunk, CrowdStrike Falcon, Azure Sentinel, ServiceNow + Named third-party firms: Mandiant, Deloitte, CrowdStrike, PwC + Specific numbers: headcounts, dollar amounts, percentages, exact durations (“17 years”, “12 professionals”) + Certification claims: “We maintain ISO 27001 certification” (holding a certification is more than naming a standard) + Named universities in credential context: “Ph.D. from Princeton University” (independently verifiable) +
    +
    + + {/* IS NOT list */} +
    + + IS NOT a Specific Fact (do NOT use to justify Firm-Specific) + +
      + Generic governance: “the Board”, “Board of Directors”, “management”, “Audit Committee”, “the Committee” + Generic C-suite: CEO, CFO, COO, President, General Counsel — these exist at every company and are not cybersecurity-specific + Generic IT leadership (NOT cybersecurity-specific): “Head of IT”, “IT Manager”, “Director of IT”, “Chief Compliance Officer”, “Associate Vice President of IT” — these are general corporate/IT titles, not cybersecurity roles per the IS list + Unnamed entities: “third-party experts”, “external consultants”, “cybersecurity firms”, “managed service provider” + Generic cadences: “quarterly”, “annual”, “periodic”, “regular” — without exact dates + Boilerplate phrases: “cybersecurity risks”, “material adverse effect”, “business operations”, “financial condition” + Standard incident language: “forensic investigation”, “law enforcement”, “regulatory obligations”, “incident response protocols” + Vague quantifiers: “certain systems”, “some employees”, “a number of”, “a portion of” + Common practices: “penetration testing”, “vulnerability scanning”, “tabletop exercises”, “phishing simulations”, “security awareness training” + Generic program names: “incident response plan”, “business continuity plan”, “cybersecurity program”, “Third-Party Risk Management Program”, “Company-wide training” — no unique identifier or distinguishing abbreviation + Company self-references: the company’s own name, “the Company”, “the Bank”, subsidiary names, filing form types + Company milestones: “since our IPO”, “since inception” — not cybersecurity facts +
    +
    + + {/* QV-Eligible Facts */} +
    + + QV-Eligible Facts (count toward the 2-fact threshold for Quantified-Verifiable) + +
      +
    • Specific dates (month+year or exact date)
      • +
    • Dollar amounts, headcounts, percentages
      • +
    • Named third-party firms (Mandiant, CrowdStrike, Deloitte)
      • +
    • Named products/tools (Splunk, Azure Sentinel)
      • +
    • Named certifications held by individuals (CISSP, CISM, CEH)
      • +
    • Years of experience as a specific number (“17 years”, “over 20 years”)
      • +
    • Named universities in credential context
      • +
    +
    + + {/* Do NOT count toward QV */} +
    + + Do NOT Count Toward QV (these trigger Firm-Specific but not QV) + +
      +
    • Named roles (CISO, CIO)
      • +
    • Named committees
      • +
    • Named frameworks (NIST, ISO 27001) — these trigger Sector-Adapted
      • +
    • Team compositions, reporting structures
      • +
    • Named internal programs
      • +
    • Generic degrees without named university (“BS in Management”)
      • +
    +
    + + {/* Validation Step */} +
    + + Validation Step + +

    + Before finalizing specificity, review the extracted facts. Remove + any that appear on the NOT list. If no facts remain after filtering + → Generic Boilerplate (or Sector-Adapted if a named standard + is present). Do not let NOT-list items inflate the specificity + rating. +

    +
    +
    + + + + {/* ================================================================ + SECTION 5: BORDERLINE CASES + ================================================================ */} +
    + + 5. Borderline Cases + + + {/* Case 1 */} +
    +

    Case 1: Framework mention + firm-specific fact

    +
    +

    “We follow NIST CSF and our CISO oversees the program.”

    +
    +

    + The NIST mention → Level 2 anchor. The CISO reference → + firm-specific. Apply boundary rule 2→3:{" "} + “Does it mention anything unique to THIS company?” Yes + (CISO role exists at this company) →{" "} + Level 3. +

    +
    + + {/* Case 2 */} +
    +

    Case 2: Named role but generic description

    +
    +

    “Our Chief Information Security Officer is responsible for managing cybersecurity risks.”

    +
    +

    + Names a role (CISO) → potentially Level 3. But the description + is completely generic. Apply judgment: the mere + existence of a CISO title is firm-specific (not all companies have + one). → Level 3. If the paragraph said + “a senior executive is responsible” without naming the + role → Level 1. +

    +
    + + {/* Case 3 */} +
    +

    Case 3: Specificity-rich None/Other

    +
    +

    “On March 15, 2025, we filed a Current Report on Form 8-K disclosing a cybersecurity incident. For details, see our Form 8-K filed March 15, 2025, accession number 0001193125-25-012345.”

    +
    +

    + Contains specific dates and filing numbers, but the paragraph + itself contains no disclosure content — it’s a + cross-reference. → None/Other, Specificity 1.{" "} + Specificity only applies to disclosure substance, not to metadata. +

    +
    + + {/* Case 4 */} +
    +

    Case 4: Hypothetical incident language in 10-K

    +
    +

    “We may experience cybersecurity incidents that could disrupt our operations.”

    +
    +

    + This appears in Item 1C, not an 8-K. It describes no actual + incident. →{" "} + + Risk Management Process or Strategy Integration (depending on + context), NOT Incident Disclosure. + {" "} + Incident Disclosure is reserved for descriptions of events that + actually occurred. +

    +
    + + {/* Case 5 */} +
    +

    Case 5: Dual-category paragraph

    +
    +

    “The Audit Committee oversees our cybersecurity program, which is led by our CISO who holds CISSP certification and reports quarterly to the Committee.”

    +
    +

    + Board (Audit Committee oversees) + Management (CISO qualifications, + reporting). The opening clause sets the frame: this is about the + Audit Committee’s oversight, and the CISO detail is + subordinate. → Board Governance, Specificity 3. +

    +
    + + {/* Case 6 */} +
    +

    Case 6: Management Role vs. Risk Management Process — the person-vs-function test

    + +
    +

    “Our CISO oversees the Company’s cybersecurity program, which includes risk assessments, vulnerability scanning, and incident response planning. The program is aligned with the NIST CSF framework and integrated into our enterprise risk management process.”

    +
    +

    + The CISO is named as attribution, but the paragraph is about what + the program does — assessments, scanning, response planning, + framework alignment, ERM integration. Remove “Our CISO + oversees” and it still makes complete sense as a process + description. →{" "} + Risk Management Process, Specificity 2 (NIST CSF + framework, no firm-specific facts beyond that). +

    + +
    +

    “Our CISO has over 20 years of experience in cybersecurity and holds CISSP and CISM certifications. She reports directly to the CIO and oversees a team of 12 security professionals. Prior to joining the Company in 2019, she served as VP of Security at a Fortune 500 technology firm.”

    +
    +

    + The entire paragraph is about the person: experience, + certifications, reporting line, team size, tenure, prior role. + → Management Role, Specificity 4 (years of + experience + team headcount + named certifications = multiple + QV-eligible facts). +

    +
    + + {/* Case 7 */} +
    +

    Case 7: Materiality disclaimer — Strategy Integration vs. None/Other

    + +
    +

    “We have not identified any cybersecurity incidents or threats that have materially affected our business strategy, results of operations, or financial condition. However, like other companies, we have experienced threats from time to time. For more information, see Item 1A, Risk Factors.”

    +
    +

    + Contains an explicit materiality assessment (“materially + affected... business strategy, results of operations, or financial + condition”). The cross-reference and generic threat mention + are noise. →{" "} + Strategy Integration, Specificity 1. +

    + +
    +

    “For additional information about risks related to our information technology systems, see Part I, Item 1A, ‘Risk Factors.’”

    +
    +

    + No materiality assessment. Pure cross-reference. →{" "} + None/Other, Specificity 1. +

    +
    + + {/* Case 8 */} +
    +

    Case 8: SPAC / no-operations company

    +
    +

    “We are a special purpose acquisition company with no business operations. We have not adopted any cybersecurity risk management program or formal processes. Our Board of Directors is generally responsible for oversight of cybersecurity risks, if any. We have not encountered any cybersecurity incidents since our IPO.”

    +
    +

    + Despite touching RMP (no program), Board Governance (board is + responsible), and Strategy Integration (no incidents), the paragraph + contains no substantive disclosure. The company explicitly has no + program, and the board mention is perfunctory (“generally + responsible... if any”). The absence of a program is not a + program description. →{" "} + None/Other, Specificity 1. +

    +
    +
    + + {/* Bottom spacer */} +
    +
    +
    + ); +} diff --git a/labelapp/app/dashboard/page.tsx b/labelapp/app/dashboard/page.tsx index 0ea2c79..ec8bab1 100644 --- a/labelapp/app/dashboard/page.tsx +++ b/labelapp/app/dashboard/page.tsx @@ -19,13 +19,18 @@ export default async function DashboardPage() { if (!session) redirect("/"); const [annotator] = await db - .select({ displayName: annotators.displayName }) + .select({ + displayName: annotators.displayName, + onboardedAt: annotators.onboardedAt, + }) .from(annotators) .where(eq(annotators.id, session.annotatorId)) .limit(1); if (!annotator) redirect("/"); + const isOnboarded = !!annotator.onboardedAt; + return (
    @@ -38,8 +43,19 @@ export default async function DashboardPage() { - - + {isOnboarded ? ( + + + + ) : ( + + + + )} + + {session.annotatorId === "admin" && ( diff --git a/labelapp/app/label/page.tsx b/labelapp/app/label/page.tsx index 7768d4c..b0b9585 100644 --- a/labelapp/app/label/page.tsx +++ b/labelapp/app/label/page.tsx @@ -158,9 +158,10 @@ export default function LabelPage() { if (e.target instanceof HTMLTextAreaElement) return; if (done || loading) return; - if (e.shiftKey && e.key >= "1" && e.key <= "4") { + const specKeys: Record = { q: 1, w: 2, e: 3, r: 4 }; + if (specKeys[e.key.toLowerCase()]) { e.preventDefault(); - setSpecificity(parseInt(e.key)); + setSpecificity(specKeys[e.key.toLowerCase()]); } else if (!e.shiftKey && e.key >= "1" && e.key <= "7") { e.preventDefault(); setCategory(CATEGORIES[parseInt(e.key) - 1]); @@ -248,11 +249,13 @@ export default function LabelPage() { {() => `${progress.completed} / ${progress.total}`} -
    + {/* Floating codebook button */} + + {/* Main content */}
    {/* Paragraph display */} @@ -314,9 +317,9 @@ export default function LabelPage() { htmlFor={`spec-${i}`} className="cursor-pointer text-sm" > - {i + 1}. {label}{" "} + {label}{" "} - Shift+{i + 1} + {["Q", "W", "E", "R"][i]} @@ -326,6 +329,19 @@ export default function LabelPage() { + {/* Submit button */} + + {/* Notes */} @@ -346,19 +362,6 @@ export default function LabelPage() { /> - - {/* Submit button */} -
    ); @@ -366,20 +369,18 @@ export default function LabelPage() { function CodebookSidebar() { return ( - - - - - - - Labeling Codebook - Quick reference for categories and specificity levels - - -
    +
    + + + + + + + Labeling Codebook + Quick reference for categories and specificity levels + + +
    {/* Categories */}

    @@ -488,9 +489,9 @@ function CodebookSidebar() { named firms, headcounts) for Specificity 4. - Boilerplate "no material impact" language = typically - None/Other or Specificity 1 unless containing specific - incident details. + Any "materially affected" assessment = Strategy Integration, + even if boilerplate. Pure cross-reference with no materiality + conclusion = None/Other. Choose the category whose content occupies the majority of the @@ -498,10 +499,20 @@ function CodebookSidebar() {

    -
    - - - + + + Open full codebook reference → + +
    +     +     +     + ); } diff --git a/labelapp/app/label/warmup/page.tsx b/labelapp/app/label/warmup/page.tsx index 57770f5..c712840 100644 --- a/labelapp/app/label/warmup/page.tsx +++ b/labelapp/app/label/warmup/page.tsx @@ -142,9 +142,10 @@ export default function WarmupPage() { return; } - if (e.shiftKey && e.key >= "1" && e.key <= "4") { + const specKeys: Record = { q: 1, w: 2, e: 3, r: 4 }; + if (specKeys[e.key.toLowerCase()]) { e.preventDefault(); - setSpecificity(parseInt(e.key)); + setSpecificity(specKeys[e.key.toLowerCase()]); } else if (!e.shiftKey && e.key >= "1" && e.key <= "7") { e.preventDefault(); setCategory(CATEGORIES[parseInt(e.key) - 1]); @@ -276,9 +277,9 @@ export default function WarmupPage() { htmlFor={`spec-${i}`} className="cursor-pointer text-sm" > - {i + 1}. {label}{" "} + {label}{" "} - Shift+{i + 1} + {["Q", "W", "E", "R"][i]} diff --git a/labelapp/app/onboarding/page.tsx b/labelapp/app/onboarding/page.tsx new file mode 100644 index 0000000..247c7e2 --- /dev/null +++ b/labelapp/app/onboarding/page.tsx @@ -0,0 +1,247 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Progress } from "@/components/ui/progress"; +import { Separator } from "@/components/ui/separator"; +import { ONBOARDING_STEPS } from "@/lib/onboarding-content"; +import { ChevronLeft, ChevronRight, GraduationCap } from "lucide-react"; + +export default function OnboardingPage() { + const router = useRouter(); + const [currentStep, setCurrentStep] = useState(0); + const [maxVisited, setMaxVisited] = useState(0); + const [completing, setCompleting] = useState(false); + + const step = ONBOARDING_STEPS[currentStep]; + const isLastStep = currentStep === ONBOARDING_STEPS.length - 1; + const progress = ((currentStep + 1) / ONBOARDING_STEPS.length) * 100; + + // Check if already onboarded + useEffect(() => { + fetch("/api/onboarding") + .then((res) => res.json()) + .then((data) => { + if (data.onboarded) router.push("/dashboard"); + }); + }, [router]); + + const goNext = useCallback(() => { + if (isLastStep) return; + const next = currentStep + 1; + setCurrentStep(next); + setMaxVisited((prev) => Math.max(prev, next)); + window.scrollTo(0, 0); + }, [currentStep, isLastStep]); + + const goBack = useCallback(() => { + if (currentStep === 0) return; + setCurrentStep(currentStep - 1); + window.scrollTo(0, 0); + }, [currentStep]); + + const completeOnboarding = useCallback(async () => { + setCompleting(true); + try { + const res = await fetch("/api/onboarding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action: "complete" }), + }); + if (res.ok) { + router.push("/dashboard"); + } + } finally { + setCompleting(false); + } + }, [router]); + + // Keyboard navigation + useEffect(() => { + function handleKeyDown(e: KeyboardEvent) { + if (e.key === "ArrowRight" && !isLastStep) { + e.preventDefault(); + goNext(); + } else if (e.key === "ArrowLeft" && currentStep > 0) { + e.preventDefault(); + goBack(); + } + } + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [currentStep, isLastStep, goNext, goBack]); + + return ( +
    + {/* Progress header */} +
    +
    + + Step {currentStep + 1} of {ONBOARDING_STEPS.length} + + + + Full Codebook + +
    +
    + + {/* Step nav dots */} +
    + {ONBOARDING_STEPS.map((s, i) => ( +
    + + {/* Main content */} +
    + + +
    + + {currentStep + 1} + +
    + {step.title} + + {step.subtitle} + +
    +
    +     + + {/* Main text content */} +
    + {step.content.map((paragraph, i) => ( +

    + {paragraph} +

    + ))} +
    + + {/* Examples */} + {step.examples && step.examples.length > 0 && ( + <> + +
    +

    + Examples +

    + {step.examples.map((example, i) => ( +
    +

    + “{example.text}” +

    +
    + {example.category && ( + {example.category} + )} + {example.specificity && ( + {example.specificity} + )} +
    +

    + {example.explanation} +

    +
    + ))} +
    + + )} + + {/* Key points */} + {step.keyPoints && step.keyPoints.length > 0 && ( + <> + +
    +

    + Key Points +

    +
      + {step.keyPoints.map((point, i) => ( +
    • + + {point} +
      • + ))} +
    +
    + + )} + + {/* Tip */} + {step.tip && ( +
    +

    + Tip: + {step.tip} +

    +
    + )} +     + + + + {isLastStep ? ( + + ) : ( + + )} + +     +
    +
    + ); +} diff --git a/labelapp/db/schema.ts b/labelapp/db/schema.ts index 597a667..69a5e16 100644 --- a/labelapp/db/schema.ts +++ b/labelapp/db/schema.ts @@ -32,6 +32,7 @@ export const annotators = pgTable("annotators", { id: text("id").primaryKey(), displayName: text("display_name").notNull(), password: text("password").notNull(), + onboardedAt: timestamp("onboarded_at"), }); export const assignments = pgTable( diff --git a/labelapp/lib/onboarding-content.ts b/labelapp/lib/onboarding-content.ts new file mode 100644 index 0000000..e59504d --- /dev/null +++ b/labelapp/lib/onboarding-content.ts @@ -0,0 +1,566 @@ +export interface OnboardingExample { + text: string; + category?: string; + specificity?: string; + explanation: string; +} + +export interface OnboardingStep { + id: number; + title: string; + subtitle: string; + content: string[]; + examples?: OnboardingExample[]; + keyPoints?: string[]; + tip?: string; +} + +export const ONBOARDING_STEPS: OnboardingStep[] = [ + // ── Step 1: What You'll Be Doing ────────────────────────────────────── + { + id: 1, + title: "What You'll Be Doing", + subtitle: "A quick overview of the labeling task", + content: [ + "You're going to read short paragraphs from SEC filings about cybersecurity and label them. No prior knowledge of SEC filings or cybersecurity is needed — we'll teach you everything right here.", + "Since 2023, the SEC requires every public company to disclose how they handle cybersecurity risk (in their annual 10-K filings, Item 1C) and any cybersecurity incidents (in 8-K filings). These disclosures are what you'll be reading.", + "Your job is simple: read each paragraph and answer two questions about it. That's it.", + "This tool is building a gold-standard dataset for training an AI classifier. There are 6 annotators total, with 3 annotators labeling each paragraph. You'll label roughly 600 paragraphs out of 1,200 total.", + ], + keyPoints: [ + "Each paragraph gets two labels: a Content Category and a Specificity Level.", + "You don't need any background in finance, law, or cybersecurity.", + "Your labels are the ground truth that an AI model will learn from — accuracy matters.", + ], + }, + + // ── Step 2: The Two Questions ───────────────────────────────────────── + { + id: 2, + title: "The Two Questions", + subtitle: "Every paragraph gets exactly two labels", + content: [ + 'Question 1: "What is this paragraph about?" — this is the Content Category. You pick one of 7 options.', + 'Question 2: "How specific is this paragraph?" — this is the Specificity Level. You pick one of 4 levels.', + "Every paragraph gets exactly one answer for each question. This is single-label classification — pick the BEST fit, not multiple labels.", + ], + keyPoints: [ + "Content Category: one of 7 mutually exclusive options.", + "Specificity Level: one of 4 levels from vague to very specific.", + "Always pick the single best answer for each dimension.", + ], + }, + + // ── Step 3: Content Categories Overview ─────────────────────────────── + { + id: 3, + title: "Content Categories Overview", + subtitle: "The 7 categories at a glance", + content: [ + "There are 7 mutually exclusive content categories. Here's a plain-English way to think about each one:", + "Board Governance — Who's in charge at the board level?", + "Management Role — Who's the person running cybersecurity?", + "Risk Management Process — What does the company's cyber program actually do?", + "Third-Party Risk — How do they handle vendor/supplier risk?", + "Incident Disclosure — Did something bad actually happen?", + "Strategy Integration — What does cyber risk mean for the business/money?", + "None/Other — None of the above.", + "If a paragraph touches multiple categories, pick the dominant one — the category that takes up most of the paragraph's text.", + ], + keyPoints: [ + "7 categories, mutually exclusive — always pick exactly one.", + "When in doubt, pick the category that dominates the paragraph.", + ], + }, + + // ── Step 4: Board Governance ────────────────────────────────────────── + { + id: 4, + title: "Board Governance", + subtitle: "Board or committee oversight of cybersecurity risk", + content: [ + "Definition: Board or committee oversight of cybersecurity risk. The board or a board committee is the grammatical subject performing the primary action.", + 'Look for language like: "Board of Directors oversees," "Audit Committee," "quarterly briefings," "board-level expertise."', + "IS Board Governance: Board receives reports, Audit Committee oversees cyber risk, directors review cybersecurity matters.", + "NOT Board Governance: CISO reports TO the board (that's Management Role — the CISO is the subject), board mentioned only in passing.", + ], + examples: [ + { + text: "The Board of Directors oversees the Company's management of cybersecurity risks.", + category: "Board Governance", + explanation: + "The board is the subject doing the overseeing. Classic Board Governance.", + }, + { + text: "The Audit Committee receives quarterly reports from the CISO and conducts an annual deep-dive review of the Company's cybersecurity program, threat landscape, and incident response readiness.", + category: "Board Governance", + explanation: + "Even though the CISO is mentioned, the Audit Committee is the one performing the actions (receiving, conducting). The committee is the grammatical subject.", + }, + { + text: "Our Board of Directors recognizes the critical importance of maintaining the trust and confidence of our customers and stakeholders, and cybersecurity risk is an area of increasing focus for our Board.", + category: "Board Governance", + explanation: + "Generic statement about board awareness — still Board Governance because the board is the subject performing the action (recognizing).", + }, + ], + tip: "The key test is always: who is the grammatical subject? If the board or a board committee is doing the action, it's Board Governance.", + }, + + // ── Step 5: Management Role ─────────────────────────────────────────── + { + id: 5, + title: "Management Role", + subtitle: "Named officers or management teams responsible for cybersecurity", + content: [ + "Definition: Named officers or management teams responsible for cybersecurity. A specific person or management function is the grammatical subject.", + "This category is about WHO THE PERSON IS — their background, credentials, experience, reporting structure.", + "IS Management Role: CISO's qualifications described, VP of Security's background, management committee structure.", + "NOT Management Role: CISO mentioned once and then the paragraph describes the program (that's Risk Management Process).", + ], + examples: [ + { + text: "Our Vice President of Information Security, who holds CISSP and CISM certifications and has over 20 years of experience in cybersecurity, reports directly to our Chief Information Officer.", + category: "Management Role", + explanation: + "It's about the person — their certifications, experience, and reporting line.", + }, + { + text: "Our CISO, Sarah Chen, leads a dedicated cybersecurity team of 35 professionals. Ms. Chen joined the Company in 2019 after serving as Deputy CISO at a Fortune 100 financial services firm.", + category: "Management Role", + explanation: + "The paragraph tells you about Sarah Chen as a person — name, team, tenure, prior role.", + }, + { + text: "Management is responsible for assessing and managing cybersecurity risks within the organization.", + category: "Management Role", + explanation: + "Generic, but still about who is responsible (management as the subject), not what the program does.", + }, + ], + tip: "Ask yourself: is this paragraph telling me about a person (or role), or about what the cybersecurity program does? If it's about the person, it's Management Role.", + }, + + // ── Step 6: Board vs Management — The Key Test ──────────────────────── + { + id: 6, + title: "Board vs Management — The Key Test", + subtitle: "The #1 source of confusion between annotators", + content: [ + "This is the single most common mistake. The test is simple: WHO is the grammatical subject performing the action?", + "Board or committee is the subject → Board Governance.", + "Named officer or management team is the subject → Management Role.", + 'The Person-vs-Function test: If you removed the person\'s name, title, and credentials, does the paragraph still describe cybersecurity activities? If YES → it\'s about the function (Risk Management Process), not the person (Management Role). Naming a cybersecurity title like "CISO" or "CIO" does NOT automatically make it Management Role. The title is often just attribution before describing the program.', + ], + examples: [ + { + text: "The Board has delegated oversight of cybersecurity matters to the Audit Committee, which meets quarterly with the CISO.", + category: "Board Governance", + explanation: + "Board is the subject. The CISO is mentioned incidentally.", + }, + { + text: "Our CISO reports quarterly to the Board on cybersecurity threats and program performance.", + category: "Management Role", + explanation: + "The CISO is the subject performing the action (reporting).", + }, + { + text: "Our CISO oversees the Company's cybersecurity program, which includes risk assessments, vulnerability scanning, penetration testing, and incident response planning aligned with the NIST CSF framework.", + category: "Risk Management Process", + explanation: + 'NOT Management Role! The CISO is mentioned once as attribution, but the paragraph is about what the program does. Remove "Our CISO oversees" and it still makes complete sense as a description of the program.', + }, + ], + keyPoints: [ + "Who is the grammatical subject? Board → Board Governance. Officer → Management Role.", + "Person-vs-Function test: remove the name/title — does the paragraph still describe activities? If yes, it's Risk Management Process.", + "A CISO mention does NOT automatically mean Management Role.", + ], + }, + + // ── Step 7: Risk Management Process ─────────────────────────────────── + { + id: 7, + title: "Risk Management Process", + subtitle: "Internal cybersecurity program mechanics", + content: [ + "Definition: Internal cybersecurity program mechanics — frameworks, assessments, controls, training, monitoring.", + 'This is the "what do they actually do" category.', + "IS Risk Management Process: NIST framework adoption, penetration testing, employee training, SOC operations, vulnerability scanning.", + "NOT Risk Management Process: Vendor assessments (that's Third-Party Risk), incident response actions during a real incident (that's Incident Disclosure).", + ], + examples: [ + { + text: "We maintain a cybersecurity risk management program that is integrated into our overall enterprise risk management framework.", + category: "Risk Management Process", + explanation: + "Describing the program's existence and its integration into enterprise risk management.", + }, + { + text: "Our cybersecurity program is aligned with the NIST Cybersecurity Framework and incorporates elements of ISO 27001. We conduct regular risk assessments and penetration testing.", + category: "Risk Management Process", + explanation: + "Framework adoption and specific program activities. All about the internal program.", + }, + { + text: "We operate a 24/7 Security Operations Center that uses Splunk SIEM and CrowdStrike Falcon endpoint detection. Our incident response team conducts quarterly tabletop exercises simulating ransomware and supply chain compromise scenarios.", + category: "Risk Management Process", + explanation: + "Specific tools, team operations, and exercises. Very detailed, but still about the internal program — no real incident happened.", + }, + ], + tip: "If the paragraph describes what the cybersecurity program does day-to-day, it's almost always Risk Management Process.", + }, + + // ── Step 8: Third-Party Risk ────────────────────────────────────────── + { + id: 8, + title: "Third-Party Risk", + subtitle: "Oversight of external parties' cybersecurity", + content: [ + "Definition: Oversight of external parties' cybersecurity — vendors, suppliers, service providers.", + "The key question: is the CENTRAL topic about overseeing someone outside the company?", + "IS Third-Party Risk: Vendor assessments, third-party audits, supply chain monitoring, SOC 2 requirements for vendors.", + "NOT Third-Party Risk: Internal program that happens to use third-party tools (Risk Management Process), hiring a firm to test YOUR systems (Risk Management Process).", + ], + examples: [ + { + text: "We face cybersecurity risks associated with our use of third-party service providers who may have access to our systems and data.", + category: "Third-Party Risk", + explanation: "About vendor risk exposure — the central topic is external parties.", + }, + { + text: "Our vendor risk management program requires all third-party service providers with access to sensitive data to meet minimum security standards, including SOC 2 Type II certification.", + category: "Third-Party Risk", + explanation: "Requirements imposed on vendors. The focus is on what the company demands from external parties.", + }, + { + text: "We assessed 312 vendors in fiscal 2024. All Tier 1 vendors are required to provide annual SOC 2 Type II reports. 14 vendors were placed on remediation plans and 3 vendor relationships were terminated.", + category: "Third-Party Risk", + explanation: + "Very specific vendor oversight with hard numbers. Still Third-Party Risk because the central topic is vendor management.", + }, + ], + tip: 'Watch out for the "hired a firm" trap. If a company says "we engaged Mandiant to conduct penetration testing," that\'s Risk Management Process — Mandiant is testing the company\'s own systems, not being overseen as a vendor.', + }, + + // ── Step 9: Incident Disclosure ─────────────────────────────────────── + { + id: 9, + title: "Incident Disclosure", + subtitle: "Description of an actual cybersecurity incident", + content: [ + "Definition: Description of an actual cybersecurity incident — what happened, the timeline, the impact, the response.", + "Key word: ACTUAL. Something really happened. Not hypothetical.", + 'IS Incident Disclosure: "We detected unauthorized access," specific breach details, forensic investigation results.', + 'NOT Incident Disclosure: Generic "we may experience incidents" (that\'s hypothetical risk language), incident response PLANS (that\'s Risk Management Process).', + ], + examples: [ + { + text: "On January 15, 2024, we detected unauthorized access to our customer support portal. The threat actor exploited a known vulnerability in a third-party software component.", + category: "Incident Disclosure", + explanation: + "A real event with a real date and real details. Something actually happened.", + }, + { + text: "In December 2023, the Company experienced a cybersecurity incident involving unauthorized access to certain internal systems. The Company promptly took steps to contain and remediate the incident.", + category: "Incident Disclosure", + explanation: + "Real event, though vaguer than the previous example. Still describes something that actually occurred.", + }, + { + text: "We have experienced, and may in the future experience, cybersecurity incidents that could have a material adverse effect on our business.", + category: "None/Other", + explanation: + "NOT Incident Disclosure. This is hypothetical risk language — no actual incident is described. Depending on context, this would be Strategy Integration or None/Other.", + }, + ], + keyPoints: [ + "The incident must be REAL — something that actually happened.", + "Hypothetical risk language (\"we may experience\") is never Incident Disclosure.", + "Incident response plans and tabletop exercises are Risk Management Process, not Incident Disclosure.", + ], + }, + + // ── Step 10: Strategy Integration ───────────────────────────────────── + { + id: 10, + title: "Strategy Integration", + subtitle: "Business and financial consequences of cyber risk", + content: [ + "Definition: Business/financial consequences of cyber risk — budget, insurance, M&A impact, competitive impact, materiality assessments.", + 'This is the "what does it mean for the business" category.', + 'IMPORTANT RULE: Any paragraph that says cybersecurity risks have or could "materially affect" the business → Strategy Integration, even if it\'s boilerplate language.', + 'IS Strategy Integration: Cyber budget amounts, insurance coverage, "not materially affected" statements, cost of incidents.', + "NOT Strategy Integration: Technical program costs mentioned in passing (that's Risk Management Process).", + ], + examples: [ + { + text: "We increased our cybersecurity budget by 32% to $45M in fiscal 2024. We maintain cyber liability insurance with $100M in aggregate coverage through AIG and Chubb.", + category: "Strategy Integration", + explanation: + "Dollar amounts and business decisions about cyber spending. This is about what cyber risk means for the business financially.", + }, + { + text: "Cybersecurity risks have not materially affected, and are not reasonably likely to materially affect, our business strategy, results of operations, or financial condition.", + category: "Strategy Integration", + explanation: + "This is boilerplate that appears in thousands of filings, but it IS a materiality assessment — the company is making a strategic judgment about impact. Always Strategy Integration.", + }, + { + text: "We have not identified any cybersecurity incidents that have materially affected us. For more information, see Item 1A, Risk Factors.", + category: "Strategy Integration", + explanation: + "The materiality assessment is the key content. The cross-reference at the end is just noise.", + }, + ], + tip: 'The "materially affect" rule is one of the most important. Whenever you see the word "materially" in the context of business impact, think Strategy Integration.', + }, + + // ── Step 11: None/Other ─────────────────────────────────────────────── + { + id: 11, + title: "None/Other", + subtitle: "Doesn't fit any of the above categories", + content: [ + "Definition: Doesn't fit any of the other 6 categories. Generic corporate language, section headers, cross-references, non-cyber content.", + "Specificity is always 1 (Generic Boilerplate) for None/Other paragraphs — there's no cyber content to rate.", + 'SPAC rule: Companies that say "we have no operations" or "we have not adopted any cybersecurity program" → None/Other, even if they mention the board.', + 'Cross-reference vs materiality: "See Item 1A, Risk Factors" alone = None/Other. But if it also includes "have not materially affected our business" = Strategy Integration.', + ], + examples: [ + { + text: "Item 1C. Cybersecurity", + category: "None/Other", + explanation: "Just a section header. No disclosure content.", + }, + { + text: "For additional information about risks related to our information technology systems, see Part I, Item 1A, 'Risk Factors.'", + category: "None/Other", + explanation: + "Pure cross-reference with no disclosure content of its own.", + }, + { + text: "We are a special purpose acquisition company with no business operations. We have not adopted any cybersecurity risk management program. Our Board of Directors is generally responsible for oversight of cybersecurity risks, if any.", + category: "None/Other", + explanation: + "Despite mentioning the Board, this company has no program — the board mention is perfunctory. SPACs with no operations get None/Other.", + }, + ], + keyPoints: [ + "None/Other always gets Specificity 1.", + "SPACs with no operations → None/Other, even if they mention the board.", + "Pure cross-references → None/Other. Cross-references with materiality language → Strategy Integration.", + ], + }, + + // ── Step 12: Decision Rules Recap ───────────────────────────────────── + { + id: 12, + title: "Decision Rules Recap", + subtitle: "Quick-reference rules for tricky cases", + content: [ + "Here are the 6 decision rules that handle the most common edge cases:", + "Rule 1 — Dominant Category: If a paragraph spans multiple categories, assign the one that takes up the most text.", + "Rule 2 — Board vs Management: Who is the grammatical subject? Board/committee → Board Governance. Named officer/team → Management Role.", + "Rule 2b — Person vs Function: Is the paragraph about the person or what the program does? Remove the name/title — if the paragraph still describes activities, it's Risk Management Process.", + "Rule 3 — Risk Management vs Third-Party: Is the central topic internal processes or vendor oversight?", + "Rule 4 — Incident vs Strategy: What happened (Incident Disclosure) vs what it means for the business (Strategy Integration).", + "Rule 5 — None/Other Threshold: Only assign None/Other when there's no substantive cyber disclosure.", + "Rule 6 — Materiality Disclaimers: Any paragraph with a materiality assessment always goes to Strategy Integration.", + ], + keyPoints: [ + "Rule 1: Dominant category wins when a paragraph spans multiple topics.", + "Rule 2: Grammatical subject determines Board Governance vs Management Role.", + "Rule 2b: Person-vs-Function test — remove the name and see if the paragraph still works.", + "Rule 3: Internal processes → Risk Management Process. Vendor oversight → Third-Party Risk.", + "Rule 4: Real event → Incident Disclosure. Business impact → Strategy Integration.", + "Rule 5: None/Other only when there's no substantive disclosure.", + "Rule 6: Materiality disclaimers → always Strategy Integration.", + ], + }, + + // ── Step 13: Specificity — What It Measures ─────────────────────────── + { + id: 13, + title: "Specificity — What It Measures", + subtitle: "The second dimension: how company-specific is the disclosure?", + content: [ + "Now for the second question you'll answer for every paragraph: Specificity Level.", + 'Think of it this way: "Could you paste this paragraph into any company\'s filing and it would still make sense?"', + "If yes → low specificity (it's generic boilerplate).", + "If no, because it mentions this specific company's people, tools, numbers, or dates → high specificity.", + "There are 4 levels, from vague to very specific:", + "Level 1 — Generic Boilerplate: Could appear in any filing unchanged.", + "Level 2 — Sector-Adapted: Names a recognized standard but nothing unique to this company.", + "Level 3 — Firm-Specific: Contains at least one fact unique to this company.", + "Level 4 — Quantified-Verifiable: Contains two or more hard verifiable facts.", + "None/Other paragraphs always get Specificity 1.", + ], + keyPoints: [ + "Specificity measures how unique the disclosure is to this specific company.", + "4 levels: Generic Boilerplate (1) → Sector-Adapted (2) → Firm-Specific (3) → Quantified-Verifiable (4).", + "None/Other paragraphs are always Specificity 1.", + ], + }, + + // ── Step 14: Generic Boilerplate & Sector-Adapted (Levels 1-2) ─────── + { + id: 14, + title: "Generic Boilerplate & Sector-Adapted (Levels 1-2)", + subtitle: "The lower specificity levels", + content: [ + "Level 1 — Generic Boilerplate: Could appear in any company's filing unchanged. No named frameworks, tools, people, dates, or quantities.", + "Level 2 — Sector-Adapted: Names a recognized standard (NIST, ISO 27001, SOC 2, PCI DSS, HIPAA, etc.) but nothing unique to THIS company.", + "The jump from Level 1 to Level 2: does the paragraph name a specific standard or framework? If yes, and there are no other company-specific facts, it's Level 2.", + ], + examples: [ + { + text: "We maintain a cybersecurity risk management program designed to identify, assess, and manage material cybersecurity risks.", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "Could be any company. No named frameworks, tools, people, or details of any kind.", + }, + { + text: "Management is responsible for assessing and managing cybersecurity risks within the organization.", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "No named roles, frameworks, or details. Completely interchangeable between companies.", + }, + { + text: "Our cybersecurity program is aligned with the NIST Cybersecurity Framework and incorporates elements of ISO 27001.", + specificity: "Level 2 — Sector-Adapted", + explanation: + "Names NIST and ISO 27001, but nothing unique to this company — many companies say the exact same thing.", + }, + { + text: "We conduct regular risk assessments, vulnerability scanning, and penetration testing as part of our continuous monitoring approach.", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "NOT Sector-Adapted. These are common practices, but they don't name a specific standard. Activities alone don't bump you to Level 2.", + }, + ], + tip: "Common practices like penetration testing and vulnerability scanning do NOT trigger Level 2. You need a named standard or framework (NIST, ISO, SOC 2, etc.) to reach Level 2.", + }, + + // ── Step 15: Firm-Specific & Quantified-Verifiable (Levels 3-4) ────── + { + id: 15, + title: "Firm-Specific & Quantified-Verifiable (Levels 3-4)", + subtitle: "The higher specificity levels", + content: [ + "Level 3 — Firm-Specific: Contains at least one fact unique to THIS company.", + "Level 4 — Quantified-Verifiable: Contains TWO or more hard verifiable facts.", + "What counts as a specific fact (triggers Level 3): cybersecurity-specific titles (CISO, CTO, CIO, VP of Security), named non-generic committees (Technology Committee, Cybersecurity Committee — NOT Audit Committee since every company has one), specific dates, named tools (Splunk, CrowdStrike, Azure Sentinel), named third-party firms (Mandiant, Deloitte), specific numbers (headcounts, dollar amounts, percentages).", + "What does NOT count as a specific fact: generic governance terms (the Board, Audit Committee, management), generic C-suite titles (CEO, CFO, COO — not cybersecurity-specific), unnamed entities (third-party experts, external consultants), generic cadences (quarterly, annual without exact dates), common practices (penetration testing, vulnerability scanning).", + ], + examples: [ + { + text: "Our CISO oversees a team of 12 security professionals.", + specificity: "Level 3 — Firm-Specific", + explanation: + "CISO (cybersecurity-specific title) is one specific fact. But just one fact, so Level 3, not Level 4.", + }, + { + text: "Our CISO, Sarah Chen, holds CISSP and CISM certifications and has over 20 years of experience. She joined the Company in 2019 after serving as Deputy CISO at a Fortune 100 firm.", + specificity: "Level 4 — Quantified-Verifiable", + explanation: + "Named person + named certifications + years of experience + specific year = 4+ verifiable facts. Easily clears the 2-fact threshold for Level 4.", + }, + { + text: "The Audit Committee oversees cybersecurity risk.", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "\"Audit Committee\" is NOT a specific fact — every public company has one. No other specifics present.", + }, + ], + keyPoints: [ + "Cybersecurity-specific titles (CISO, CIO, CTO) count as specific facts. Generic titles (CEO, CFO) do not.", + "Audit Committee is NOT a specific fact — it's generic governance.", + "Level 4 requires two or more HARD verifiable facts: dates, dollars, headcounts, named firms, named tools, named certifications, years of experience.", + "Named roles and committees trigger Level 3 but do NOT count toward the Level 4 threshold.", + ], + }, + + // ── Step 16: The Specificity Decision Test ──────────────────────────── + { + id: 16, + title: "The Specificity Decision Test", + subtitle: "A step-by-step waterfall to determine specificity", + content: [ + "Apply this waterfall — stop at the first yes:", + "Step A: Count ONLY hard verifiable facts: specific dates (month+year or exact date), dollar amounts, headcounts, percentages, named third-party firms, named products/tools, named certifications held by individuals, years of experience as a specific number. Two or more? → Level 4 (Quantified-Verifiable).", + "Step B: At least one fact from the specific-facts list (cybersecurity titles, named committees, named tools, named firms, specific dates, specific numbers)? → Level 3 (Firm-Specific).", + "Step C: Names a recognized standard (NIST, ISO, SOC 2, PCI DSS, HIPAA, etc.)? → Level 2 (Sector-Adapted).", + "Step D: None of the above? → Level 1 (Generic Boilerplate).", + "Important: named roles (CISO, CIO) and named committees trigger Firm-Specific (Level 3) but do NOT count toward the 2-fact threshold for Quantified-Verifiable (Level 4). Named frameworks (NIST, ISO) also do not count toward Level 4.", + ], + examples: [ + { + text: "We operate a 24/7 Security Operations Center that uses Splunk SIEM and CrowdStrike Falcon endpoint detection. Our incident response team conducts quarterly tabletop exercises.", + specificity: "Level 4 — Quantified-Verifiable", + explanation: + "Count QV facts: Splunk (named tool) + CrowdStrike Falcon (named tool) = 2 hard verifiable facts. Meets the threshold → Level 4.", + }, + { + text: "Our CISO oversees the cybersecurity program aligned with NIST CSF.", + specificity: "Level 3 — Firm-Specific", + explanation: + "Count QV facts: none (CISO is a role, NIST is a framework — neither counts toward QV). Specific-facts list: CISO (yes, cybersecurity-specific title). → Level 3.", + }, + { + text: "Our cybersecurity program incorporates elements of ISO 27001.", + specificity: "Level 2 — Sector-Adapted", + explanation: + "Count QV facts: none. Specific-facts list: none (ISO is a standard, not a firm-specific fact). Named standard: ISO 27001 (yes). → Level 2.", + }, + ], + tip: "When in doubt, count the verifiable facts on your fingers. If you can point to two things that an outside observer could independently confirm, it's Level 4.", + }, + + // ── Step 17: Putting It All Together ────────────────────────────────── + { + id: 17, + title: "Putting It All Together", + subtitle: "Borderline cases that exercise both dimensions", + content: [ + "Let's work through some tricky examples that require you to assign BOTH a content category and a specificity level. These are the kinds of paragraphs that trip people up.", + ], + examples: [ + { + text: "The Audit Committee, which includes two members with significant technology expertise, receives quarterly reports from the CISO and conducts an annual deep-dive review of the cybersecurity program.", + category: "Board Governance", + specificity: "Level 3 — Firm-Specific", + explanation: + "Board Governance because the Audit Committee is the grammatical subject doing the actions (receiving reports, conducting reviews). Specificity: CISO is a cybersecurity-specific title — that's one specific fact, so Level 3. The Audit Committee itself doesn't count as a specific fact (every company has one).", + }, + { + text: "Our CISO oversees the Company's cybersecurity program, which includes risk assessments, vulnerability scanning, penetration testing, and incident response planning aligned with the NIST CSF framework.", + category: "Risk Management Process", + specificity: "Level 3 — Firm-Specific", + explanation: + "Risk Management Process, NOT Management Role — the paragraph is about what the program does, and the CISO is just attribution. Apply the Person-vs-Function test: remove \"Our CISO oversees\" and the paragraph still describes the program perfectly. For specificity, the CISO mention still counts as a firm-specific fact even when it's just attribution, so Level 3.", + }, + { + text: "Cybersecurity risks have not materially affected our business strategy, results of operations, or financial condition. For more information, see Item 1A, Risk Factors.", + category: "Strategy Integration", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "Strategy Integration because the materiality assessment is the key content — the cross-reference is just noise. Generic Boilerplate because there are no specific facts, no named frameworks — this is pure boilerplate language that appears in thousands of filings.", + }, + { + text: "We are a blank check company formed for the purpose of effecting a merger. We have not adopted any cybersecurity risk management program or formal processes for assessing cybersecurity risk.", + category: "None/Other", + specificity: "Level 1 — Generic Boilerplate", + explanation: + "None/Other because there's no substantive disclosure — this is a SPAC with no cybersecurity program. Specificity is always Level 1 for None/Other.", + }, + ], + keyPoints: [ + "Always assign both a content category AND a specificity level.", + "The Person-vs-Function test and the Specificity Decision Test work together — use both.", + "You're ready for the quiz! You'll answer 8 questions testing these concepts. You need 7/8 to pass.", + ], + }, +]; diff --git a/labelapp/playwright-report/index.html b/labelapp/playwright-report/index.html new file mode 100644 index 0000000..577ba01 --- /dev/null +++ b/labelapp/playwright-report/index.html @@ -0,0 +1,85 @@ + + + + + + + + + Playwright Test Report + + + + +
    + + + \ No newline at end of file diff --git a/labelapp/proxy.ts b/labelapp/proxy.ts index bb7ddbd..6e6be3b 100644 --- a/labelapp/proxy.ts +++ b/labelapp/proxy.ts @@ -2,7 +2,7 @@ import { NextResponse } from "next/server"; import type { NextRequest } from "next/server"; import { verifyAndDecode } from "@/lib/auth"; -const protectedPrefixes = ["/dashboard", "/quiz", "/label", "/admin"]; +const protectedPrefixes = ["/dashboard", "/quiz", "/label", "/admin", "/onboarding", "/codebook"]; export function proxy(request: NextRequest) { const path = request.nextUrl.pathname; diff --git a/labelapp/tests/02-onboarding.spec.ts b/labelapp/tests/02-onboarding.spec.ts new file mode 100644 index 0000000..14c0446 --- /dev/null +++ b/labelapp/tests/02-onboarding.spec.ts @@ -0,0 +1,71 @@ +import { test, expect } from "@playwright/test"; +import { loginAs } from "./helpers/login"; +import { clearOnboarding } from "./helpers/reset-db"; + +test.describe("Onboarding", () => { + test.beforeAll(async () => { + // Clear onboarding state so joey sees the training flow + await clearOnboarding("joey"); + }); + + test("dashboard shows 'Complete Training' for new annotator", async ({ + page, + }) => { + await loginAs(page, "joey"); + await expect(page.getByText("Complete Training")).toBeVisible(); + }); + + test("clicking 'Complete Training' navigates to /onboarding", async ({ + page, + }) => { + await loginAs(page, "joey"); + await page.getByText("Complete Training").click(); + await page.waitForURL("/onboarding"); + }); + + test("onboarding shows step 1 and progress", async ({ page }) => { + await loginAs(page, "joey"); + await page.goto("/onboarding"); + await expect(page.getByText("Step 1 of 17")).toBeVisible(); + await expect(page.getByText("What You'll Be Doing")).toBeVisible(); + }); + + test("can navigate through steps", async ({ page }) => { + await loginAs(page, "joey"); + await page.goto("/onboarding"); + + // Step 1 + await expect(page.getByText("Step 1 of 17")).toBeVisible(); + await page.getByRole("button", { name: /continue/i }).click(); + + // Step 2 + await expect(page.getByText("Step 2 of 17")).toBeVisible(); + await expect(page.getByText("The Two Questions")).toBeVisible(); + + // Can go back + await page.getByRole("button", { name: /back/i }).click(); + await expect(page.getByText("Step 1 of 17")).toBeVisible(); + }); + + test("completing onboarding redirects to dashboard with 'Start Labeling Session'", async ({ + page, + }) => { + await loginAs(page, "joey"); + await page.goto("/onboarding"); + + // Navigate through all 17 steps + for (let i = 0; i < 16; i++) { + await page.getByRole("button", { name: /continue/i }).click(); + } + + // Last step should show completion button + await expect(page.getByText("Step 17 of 17")).toBeVisible(); + await page.getByRole("button", { name: /ready/i }).click(); + + // Should redirect to dashboard + await page.waitForURL("/dashboard"); + + // Dashboard should now show "Start Labeling Session" instead of "Complete Training" + await expect(page.getByText("Start Labeling Session")).toBeVisible(); + }); +}); diff --git a/labelapp/tests/02-quiz.spec.ts b/labelapp/tests/03-quiz.spec.ts similarity index 78% rename from labelapp/tests/02-quiz.spec.ts rename to labelapp/tests/03-quiz.spec.ts index 029b934..6126786 100644 --- a/labelapp/tests/02-quiz.spec.ts +++ b/labelapp/tests/03-quiz.spec.ts @@ -1,7 +1,15 @@ import { test, expect } from "@playwright/test"; import { loginAs } from "./helpers/login"; +import { markOnboarded, clearQuizSessions } from "./helpers/reset-db"; test.describe("Quiz System", () => { + test.beforeAll(async () => { + // Ensure joey is onboarded so "Start Labeling Session" appears + await markOnboarded("joey"); + // Clear any existing quiz sessions so the quiz page shows "Start Quiz" + await clearQuizSessions("joey"); + }); + test.beforeEach(async ({ page }) => { await loginAs(page, "joey"); }); diff --git a/labelapp/tests/helpers/reset-db.ts b/labelapp/tests/helpers/reset-db.ts index 542c072..9973d40 100644 --- a/labelapp/tests/helpers/reset-db.ts +++ b/labelapp/tests/helpers/reset-db.ts @@ -35,6 +35,24 @@ export async function seedE2EData() { `); } +export async function clearQuizSessions(annotatorId: string) { + await db.execute( + sql`DELETE FROM quiz_sessions WHERE annotator_id = ${annotatorId}`, + ); +} + +export async function markOnboarded(annotatorId: string) { + await db.execute( + sql`UPDATE annotators SET onboarded_at = NOW() WHERE id = ${annotatorId}`, + ); +} + +export async function clearOnboarding(annotatorId: string) { + await db.execute( + sql`UPDATE annotators SET onboarded_at = NULL WHERE id = ${annotatorId}`, + ); +} + export async function cleanup() { await client.end(); }