[
  {
    "variant": "onnx-fp32",
    "model_mb": 1583.256294,
    "ms_per_sample": 12.703279327494482,
    "throughput_per_s": 78.71983085781946,
    "peak_vram_mb": 8228.0,
    "load_vram_mb": 2074.0,
    "GPT-5.4_cat_f1": 0.9336741161693523,
    "GPT-5.4_spec_f1": 0.8951731906425856,
    "GPT-5.4_cat_mcc": 0.9226990724708704,
    "GPT-5.4_spec_qwk": 0.9324447137231142,
    "GPT-5.4_spec_mae": 0.1175,
    "GPT-5.4_cat_ece": 0.05386760701735813,
    "GPT-5.4_spec_ece": 0.07004604930679002,
    "Opus-4.6_cat_f1": 0.922684387023173,
    "Opus-4.6_spec_f1": 0.8833694419146193,
    "Opus-4.6_cat_mcc": 0.909266938399113,
    "Opus-4.6_spec_qwk": 0.9227008860372746,
    "Opus-4.6_spec_mae": 0.13583333333333333,
    "Opus-4.6_cat_ece": 0.06540583113829297,
    "Opus-4.6_spec_ece": 0.08156729981303217
  },
  {
    "variant": "onnx-fp16",
    "error": "Fail: [ONNXRuntimeError] : 1 : FAIL : Load model from /home/joey/Documents/sec-cyBERT/results/eval/onnx/models/model_fp16.onnx failed:Type Error: Type parameter (T) of Optype (Mul) bound to different types (tensor(float) and tensor(float16) in node (/model/backbone/rotary_emb_1/Mul_2)."
  },
  {
    "variant": "onnx-int8-dyn",
    "model_mb": 553.381903,
    "ms_per_sample": 95.90791940659983,
    "throughput_per_s": 10.42666764316426,
    "peak_vram_mb": 7188.0,
    "load_vram_mb": 266.0,
    "GPT-5.4_cat_f1": 0.3971686880679718,
    "GPT-5.4_spec_f1": 0.3364003775746365,
    "GPT-5.4_cat_mcc": 0.3459776856134484,
    "GPT-5.4_spec_qwk": 0.4412945592628398,
    "GPT-5.4_spec_mae": 0.7225,
    "GPT-5.4_cat_ece": 0.23434762333830195,
    "GPT-5.4_spec_ece": 0.35458642202119034,
    "Opus-4.6_cat_f1": 0.4150626036637055,
    "Opus-4.6_spec_f1": 0.3204015536108683,
    "Opus-4.6_cat_mcc": 0.3663561834842673,
    "Opus-4.6_spec_qwk": 0.43979676755288855,
    "Opus-4.6_spec_mae": 0.7375,
    "Opus-4.6_cat_ece": 0.2126809566716353,
    "Opus-4.6_spec_ece": 0.37541975535452365
  }
]