{
  "benchmark": "Atlan Decision Bench bench-v4",
  "dataset": "atlanai/decision-bench",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "checkpoint_step": 4153,
  "calibrated": true,
  "extended": {
    "n": 949,
    "scored": 949,
    "errors": 0,
    "accuracy": 0.7249736564805058,
    "wilson_95": [
      0.6957000006593718,
      0.7524333126501542
    ],
    "brier": 0.35628704320295485,
    "nll": 0.6233650821561088,
    "ece_10bin": 0.03043775777042981,
    "mean_latency_ms": 89.61731881559167,
    "median_latency_ms": 55.447272999572306,
    "mean_input_tokens": 732.9515279241307,
    "max_input_tokens": 4696,
    "correct": 688,
    "context_limit": 8192
  },
  "strict": {
    "n": 949,
    "scored": 912,
    "errors": 37,
    "accuracy": 0.7060063224446786,
    "wilson_95": [
      0.6762363604064593,
      0.7341152215701795
    ],
    "brier": 0.34924103534634166,
    "nll": 0.6173254965496132,
    "ece_10bin": 0.03315468122692484,
    "mean_latency_ms": 70.71173217104244,
    "median_latency_ms": 55.31234199997925,
    "mean_input_tokens": 632.046052631579,
    "correct": 670,
    "context_limit": 2048
  },
  "over_training_context": 37,
  "categories": {
    "AI agents & evals": {
      "n": 154,
      "scored": 154,
      "errors": 0,
      "accuracy": 0.6948051948051948,
      "wilson_95": [
        0.6180692927455592,
        0.7620589729730137
      ],
      "brier": 0.37487566267948963,
      "nll": 0.542818905599451,
      "ece_10bin": 0.049172926839295895,
      "mean_latency_ms": 192.0869365064993,
      "median_latency_ms": 71.77966099970945,
      "mean_input_tokens": 1262.7207792207791,
      "max_input_tokens": 4696
    },
    "Customer support": {
      "n": 67,
      "scored": 67,
      "errors": 0,
      "accuracy": 0.8059701492537313,
      "wilson_95": [
        0.695808877745558,
        0.8829482608358841
      ],
      "brier": 0.25493570008505934,
      "nll": 0.481861330365479,
      "ece_10bin": 0.13217722524457903,
      "mean_latency_ms": 57.37879737313024,
      "median_latency_ms": 57.57680199985771,
      "mean_input_tokens": 535.2835820895523,
      "max_input_tokens": 825
    },
    "Data & analytics": {
      "n": 94,
      "scored": 94,
      "errors": 0,
      "accuracy": 0.7127659574468085,
      "wilson_95": [
        0.6143682600686844,
        0.7944563883178923
      ],
      "brier": 0.41946232755920304,
      "nll": 0.8382055714652954,
      "ece_10bin": 0.11390480272313386,
      "mean_latency_ms": 68.5146179893541,
      "median_latency_ms": 57.99607500011916,
      "mean_input_tokens": 655.6276595744681,
      "max_input_tokens": 2913
    },
    "Documents & meetings": {
      "n": 57,
      "scored": 57,
      "errors": 0,
      "accuracy": 0.38596491228070173,
      "wilson_95": [
        0.2706272388614066,
        0.5157026708419042
      ],
      "brier": 0.6826959456150898,
      "nll": 1.1736475193970026,
      "ece_10bin": 0.10147291265035929,
      "mean_latency_ms": 122.82024449123448,
      "median_latency_ms": 60.9847109999464,
      "mean_input_tokens": 1133.2631578947369,
      "max_input_tokens": 2252
    },
    "Engineering": {
      "n": 150,
      "scored": 150,
      "errors": 0,
      "accuracy": 0.7466666666666667,
      "wilson_95": [
        0.6715059067359275,
        0.8095087727091548
      ],
      "brier": 0.37551092049204415,
      "nll": 0.723327823359612,
      "ece_10bin": 0.09860761245091756,
      "mean_latency_ms": 60.40400776000752,
      "median_latency_ms": 45.56712099974902,
      "mean_input_tokens": 513.5666666666667,
      "max_input_tokens": 1592
    },
    "Finance": {
      "n": 91,
      "scored": 91,
      "errors": 0,
      "accuracy": 0.8351648351648352,
      "wilson_95": [
        0.7456929440655073,
        0.8974856906634852
      ],
      "brier": 0.19699588317777703,
      "nll": 0.33711955341374333,
      "ece_10bin": 0.06922244174139842,
      "mean_latency_ms": 62.35175467030446,
      "median_latency_ms": 56.883475000176986,
      "mean_input_tokens": 578.989010989011,
      "max_input_tokens": 1859
    },
    "Legal": {
      "n": 126,
      "scored": 126,
      "errors": 0,
      "accuracy": 0.753968253968254,
      "wilson_95": [
        0.6719921977689551,
        0.820916620271833
      ],
      "brier": 0.3013085214997986,
      "nll": 0.5249013067995985,
      "ece_10bin": 0.08229180650105548,
      "mean_latency_ms": 93.35907815871212,
      "median_latency_ms": 66.19445100022858,
      "mean_input_tokens": 898.5714285714286,
      "max_input_tokens": 2356
    },
    "Product management": {
      "n": 60,
      "scored": 60,
      "errors": 0,
      "accuracy": 0.5333333333333333,
      "wilson_95": [
        0.4089342702715789,
        0.6537209395856151
      ],
      "brier": 0.5296812775742393,
      "nll": 0.8149562300126921,
      "ece_10bin": 0.13456637958685555,
      "mean_latency_ms": 48.32601779999095,
      "median_latency_ms": 45.57058499995037,
      "mean_input_tokens": 400.8333333333333,
      "max_input_tokens": 1011
    },
    "Sales & commerce": {
      "n": 90,
      "scored": 90,
      "errors": 0,
      "accuracy": 0.7777777777777778,
      "wilson_95": [
        0.6815268960895451,
        0.8512866452636068
      ],
      "brier": 0.2822125936763515,
      "nll": 0.550167074882145,
      "ece_10bin": 0.09292785790231495,
      "mean_latency_ms": 67.23641705555363,
      "median_latency_ms": 60.788372000388335,
      "mean_input_tokens": 642.2222222222222,
      "max_input_tokens": 987
    },
    "Trust & safety": {
      "n": 60,
      "scored": 60,
      "errors": 0,
      "accuracy": 0.8833333333333333,
      "wilson_95": [
        0.7782121197582914,
        0.9423227935994396
      ],
      "brier": 0.2593930852225604,
      "nll": 0.4379730330746877,
      "ece_10bin": 0.1809541270136833,
      "mean_latency_ms": 45.5207765500063,
      "median_latency_ms": 45.40842100004738,
      "mean_input_tokens": 237.16666666666666,
      "max_input_tokens": 538
    }
  },
  "tasks": {
    "AGT-1": {
      "n": 36,
      "scored": 36,
      "errors": 0,
      "accuracy": 0.9722222222222222,
      "wilson_95": [
        0.8583028134672614,
        0.9950795928601897
      ],
      "brier": 0.047137445409462325,
      "nll": 0.13463133286208243,
      "ece_10bin": 0.11122559590472117,
      "mean_latency_ms": 47.875323444448746,
      "median_latency_ms": 45.479337999950076,
      "mean_input_tokens": 398.05555555555554,
      "max_input_tokens": 596
    },
    "AGT-2": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.6333333333333333,
      "wilson_95": [
        0.4551356365479487,
        0.7812607919389929
      ],
      "brier": 0.48240737864434,
      "nll": 0.6769968452017873,
      "ece_10bin": 0.13737173875172934,
      "mean_latency_ms": 137.53539746668744,
      "median_latency_ms": 117.50114100004794,
      "mean_input_tokens": 1333.0,
      "max_input_tokens": 2186
    },
    "AGT-3": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.5,
      "wilson_95": [
        0.33154125640533766,
        0.6684587435946623
      ],
      "brier": 0.5010024332367132,
      "nll": 0.6923456612137429,
      "ece_10bin": 0.16534385681152344,
      "mean_latency_ms": 662.1503154666243,
      "median_latency_ms": 481.16841999990356,
      "mean_input_tokens": 3425.0,
      "max_input_tokens": 4696
    },
    "AGT-4": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.6333333333333333,
      "wilson_95": [
        0.4551356365479487,
        0.7812607919389929
      ],
      "brier": 0.46492787575322597,
      "nll": 0.6569478910517987,
      "ece_10bin": 0.12755708297093712,
      "mean_latency_ms": 62.824255766675684,
      "median_latency_ms": 56.894247000400355,
      "mean_input_tokens": 596.0333333333333,
      "max_input_tokens": 1135
    },
    "AGT-5": {
      "n": 28,
      "scored": 28,
      "errors": 0,
      "accuracy": 0.6785714285714286,
      "wilson_95": [
        0.49338843180857617,
        0.8206675307750504
      ],
      "brier": 0.4494204781740137,
      "nll": 0.6413811269735936,
      "ece_10bin": 0.08698352106979913,
      "mean_latency_ms": 70.80633989289709,
      "median_latency_ms": 71.52832100018713,
      "mean_input_tokens": 696.7142857142857,
      "max_input_tokens": 1018
    },
    "ENG-1": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.9,
      "wilson_95": [
        0.7437891742081593,
        0.9654001112526658
      ],
      "brier": 0.14565384457220348,
      "nll": 0.3026217902749631,
      "ece_10bin": 0.14035690029462178,
      "mean_latency_ms": 103.59838416667723,
      "median_latency_ms": 82.37715299992487,
      "mean_input_tokens": 832.4,
      "max_input_tokens": 1592
    },
    "ENG-2": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.6,
      "wilson_95": [
        0.42320360253322475,
        0.7540937188319815
      ],
      "brier": 0.5305318448663321,
      "nll": 1.0355223976862826,
      "ece_10bin": 0.16309853593508403,
      "mean_latency_ms": 48.742260999976374,
      "median_latency_ms": 45.24634400013383,
      "mean_input_tokens": 444.76666666666665,
      "max_input_tokens": 881
    },
    "ENG-3": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.8,
      "wilson_95": [
        0.6269430358685175,
        0.9049489282271013
      ],
      "brier": 0.3773187637142543,
      "nll": 0.8455486358498006,
      "ece_10bin": 0.13444667557875314,
      "mean_latency_ms": 42.5520824666061,
      "median_latency_ms": 42.502384999806964,
      "mean_input_tokens": 333.4,
      "max_input_tokens": 426
    },
    "ENG-4": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 0.43333333333333335,
      "wilson_95": [
        0.2737748557646652,
        0.6080269299918639
      ],
      "brier": 0.8072208683210879,
      "nll": 1.369341629912595,
      "ece_10bin": 0.34185539782047275,
      "mean_latency_ms": 43.186867766735304,
      "median_latency_ms": 42.640990000109014,
      "mean_input_tokens": 324.2,
      "max_input_tokens": 517
    },
    "ENG-5": {
      "n": 30,
      "scored": 30,
      "errors": 0,
      "accuracy": 1.0,
      "wilson_95": [
        0.8864866068260312,
        0.9999999999999999
      ],
      "brier": 0.016829280986342832,
      "nll": 0.06360466307441823,
      "ece_10bin": 0.05799654920895895,
      "mean_latency_ms": 63.94044340004257,
      "median_latency_ms": 60.36330899996756,
      "mean_input_tokens": 633.0666666666667,
      "max_input_tokens": 985
    }
  },
  "scope": "Saved local text-only evaluation; image-bearing rows excluded. All benchmark metrics refer to selected step 4,153. Extended run raises the runtime context cap to 8,192 tokens without truncating evidence.",
  "metric_note": "Benchmark Brier is reported separately from the mixture half-Brier; benchmark ECE uses 10 bins, mixture ECE uses 15 bins. These are different evaluation populations.",
  "overlap_note": "The pasted prior analysis reports exclusion of 28 literal upstream source matches. That audit was not rerun for this visual report; paraphrase overlap and pretraining exposure remain unassessed here."
}