80 lines
2.6 KiB
TypeScript
80 lines
2.6 KiB
TypeScript
/**
|
|
* Quick probe: test which OpenRouter models support structured output.
|
|
* Sends one paragraph to each candidate model, reports success/failure.
|
|
*/
|
|
import { generateText, Output } from "ai";
|
|
import { openrouter } from "../src/lib/openrouter.ts";
|
|
import { LabelOutput } from "@sec-cybert/schemas/label.ts";
|
|
import { SYSTEM_PROMPT, buildUserPrompt } from "../src/label/prompts.ts";
|
|
import type { Paragraph } from "@sec-cybert/schemas/paragraph.ts";
|
|
|
|
const TEST_PARAGRAPH: Paragraph = {
|
|
id: "00000000-0000-0000-0000-000000000001",
|
|
text: "The Board of Directors oversees the Company's management of cybersecurity risks. The Board has delegated oversight of cybersecurity and data privacy matters to the Audit Committee, which receives quarterly reports from the CISO.",
|
|
textHash: "test",
|
|
wordCount: 38,
|
|
paragraphIndex: 0,
|
|
filing: {
|
|
companyName: "Test Corp",
|
|
cik: "0000000001",
|
|
ticker: "TEST",
|
|
filingType: "10-K",
|
|
filingDate: "2024-03-15",
|
|
fiscalYear: 2023,
|
|
accessionNumber: "0000000001-24-000001",
|
|
secItem: "Item 1C",
|
|
},
|
|
};
|
|
|
|
const CANDIDATES = [
|
|
// Cheap/fast tier - good for Stage 1
|
|
"google/gemini-3.1-flash-lite-preview",
|
|
"x-ai/grok-4.1-fast",
|
|
"openai/gpt-4.1-mini",
|
|
"openai/gpt-4.1-nano",
|
|
"anthropic/claude-haiku-4.5",
|
|
"google/gemini-3.1-flash-preview",
|
|
"deepseek/deepseek-chat-v3-0324:free",
|
|
"meta-llama/llama-4-maverick",
|
|
"qwen/qwen3-235b-a22b",
|
|
];
|
|
|
|
async function testModel(modelId: string): Promise<void> {
|
|
const start = Date.now();
|
|
try {
|
|
const result = await generateText({
|
|
model: openrouter(modelId),
|
|
output: Output.object({ schema: LabelOutput }),
|
|
system: SYSTEM_PROMPT,
|
|
prompt: buildUserPrompt(TEST_PARAGRAPH),
|
|
temperature: 0,
|
|
providerOptions: {
|
|
openrouter: {
|
|
reasoning: { effort: "low" },
|
|
usage: { include: true },
|
|
},
|
|
},
|
|
});
|
|
|
|
const latency = Date.now() - start;
|
|
const output = result.output;
|
|
const raw = result.usage as { raw?: { cost?: number } };
|
|
const cost = raw.raw?.cost ?? 0;
|
|
|
|
if (output) {
|
|
console.log(` ✓ ${modelId.padEnd(45)} ${latency}ms $${cost.toFixed(6)} → ${output.content_category}, spec=${output.specificity_level}`);
|
|
} else {
|
|
console.log(` ✖ ${modelId.padEnd(45)} ${latency}ms No output`);
|
|
}
|
|
} catch (error) {
|
|
const latency = Date.now() - start;
|
|
const msg = error instanceof Error ? error.message.slice(0, 80) : String(error);
|
|
console.log(` ✖ ${modelId.padEnd(45)} ${latency}ms ${msg}`);
|
|
}
|
|
}
|
|
|
|
console.log("Testing structured output support across OpenRouter models...\n");
|
|
for (const model of CANDIDATES) {
|
|
await testModel(model);
|
|
}
|