{ "test_completed": true, "calibration_completed": true, "raw": { "overall": { "count": 3693, "soft_ce": 0.7085598693044783, "kl": 0.3931482496012985, "brier": 0.09769070277607729, "argmax_agreement": 0.7733549959382616, "hard_count": 2134, "hard_ece": 0.042129272019540774, "hard_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 18, "confidence": 0.18404859928386386, "reference_agreement": 0.2222222222222222 }, { "count": 23, "confidence": 0.23661101358223402, "reference_agreement": 0.2608695652173913 }, { "count": 37, "confidence": 0.30428753988952256, "reference_agreement": 0.1891891891891892 }, { "count": 36, "confidence": 0.36932558263805926, "reference_agreement": 0.3888888888888889 }, { "count": 57, "confidence": 0.43265722650360866, "reference_agreement": 0.3333333333333333 }, { "count": 71, "confidence": 0.5035452847115158, "reference_agreement": 0.5070422535211268 }, { "count": 93, "confidence": 0.5702882440958308, "reference_agreement": 0.5053763440860215 }, { "count": 113, "confidence": 0.6289013423061501, "reference_agreement": 0.5398230088495575 }, { "count": 88, "confidence": 0.7035225652548067, "reference_agreement": 0.5909090909090909 }, { "count": 97, "confidence": 0.7663997989026462, "reference_agreement": 0.6804123711340206 }, { "count": 133, "confidence": 0.8331041320580328, "reference_agreement": 0.7218045112781954 }, { "count": 213, "confidence": 0.9010564345576307, "reference_agreement": 0.812206572769953 }, { "count": 1154, "confidence": 0.9892346259941478, "reference_agreement": 0.9809358752166378 } ], "soft_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 22, "confidence": 0.18189946160649148, "reference_agreement": 0.253859649122807 }, { "count": 52, "confidence": 0.24099144225648178, "reference_agreement": 0.2604555167055167 }, { "count": 105, "confidence": 0.30435695062773516, "reference_agreement": 0.2666363084654525 }, { "count": 151, "confidence": 0.36784610221152053, "reference_agreement": 0.38072070275124964 }, { "count": 219, "confidence": 0.4348046806054027, "reference_agreement": 0.4029960976805746 }, { "count": 268, "confidence": 0.5012982178117764, "reference_agreement": 0.4966141654778031 }, { "count": 287, "confidence": 0.5665963085542071, "reference_agreement": 0.5549424985680554 }, { "count": 284, "confidence": 0.6310971171485777, "reference_agreement": 0.600541924578161 }, { "count": 280, "confidence": 0.7030994468429518, "reference_agreement": 0.6497546304223978 }, { "count": 218, "confidence": 0.7682645359325981, "reference_agreement": 0.7226513752826121 }, { "count": 230, "confidence": 0.8322149342184884, "reference_agreement": 0.7612036827968656 }, { "count": 324, "confidence": 0.8996811349428281, "reference_agreement": 0.8247895057612304 }, { "count": 1252, "confidence": 0.9876585496182678, "reference_agreement": 0.9778121922719432 } ], "expected_score_mae": 0.4570759707537796 }, "type": { "choice": { "count": 2526, "soft_ce": 0.6822833143719118, "kl": 0.4189869815862902, "brier": 0.10139614023942027, "argmax_agreement": 0.7862232779097387, "hard_count": 1737, "hard_ece": 0.04277428653676444, "hard_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 5, "confidence": 0.17932821619602474, "reference_agreement": 0.4 }, { "count": 4, "confidence": 0.2380902472918557, "reference_agreement": 0.25 }, { "count": 14, "confidence": 0.3081997411534075, "reference_agreement": 0.2857142857142857 }, { "count": 24, "confidence": 0.3697200232466156, "reference_agreement": 0.4166666666666667 }, { "count": 40, "confidence": 0.43214316524587604, "reference_agreement": 0.3 }, { "count": 58, "confidence": 0.5047682692169494, "reference_agreement": 0.5344827586206896 }, { "count": 76, "confidence": 0.5702475474241634, "reference_agreement": 0.5131578947368421 }, { "count": 85, "confidence": 0.6294861640993207, "reference_agreement": 0.5411764705882353 }, { "count": 61, "confidence": 0.7034878822257229, "reference_agreement": 0.6229508196721312 }, { "count": 73, "confidence": 0.7662886786794736, "reference_agreement": 0.6712328767123288 }, { "count": 109, "confidence": 0.8327800034747108, "reference_agreement": 0.7155963302752294 }, { "count": 163, "confidence": 0.9005930750387896, "reference_agreement": 0.7852760736196319 }, { "count": 1024, "confidence": 0.9905297995078783, "reference_agreement": 0.9814453125 } ], "soft_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 9, "confidence": 0.1761727165792652, "reference_agreement": 0.39832358674463936 }, { "count": 20, "confidence": 0.23727536848341457, "reference_agreement": 0.23968434343434342 }, { "count": 56, "confidence": 0.30649961126616326, "reference_agreement": 0.2928895108131916 }, { "count": 112, "confidence": 0.36821444135421544, "reference_agreement": 0.393456101968026 }, { "count": 148, "confidence": 0.43446487817770657, "reference_agreement": 0.4016674390496405 }, { "count": 173, "confidence": 0.4989247303895505, "reference_agreement": 0.5024153201745094 }, { "count": 161, "confidence": 0.5682084517346793, "reference_agreement": 0.5282341373438668 }, { "count": 170, "confidence": 0.6297812369278754, "reference_agreement": 0.5908322297566935 }, { "count": 148, "confidence": 0.700524650342172, "reference_agreement": 0.6783159155834284 }, { "count": 123, "confidence": 0.7663341138612425, "reference_agreement": 0.708765336806816 }, { "count": 150, "confidence": 0.8331362507431996, "reference_agreement": 0.7423155758796159 }, { "count": 207, "confidence": 0.8989791587743893, "reference_agreement": 0.7958650265103496 }, { "count": 1048, "confidence": 0.9898586616987579, "reference_agreement": 0.9796660513168818 } ] }, "noul": { "count": 582, "soft_ce": 0.485984791408039, "kl": 0.1677006591643438, "brier": 0.056889637384882515, "argmax_agreement": 0.8316151202749141, "hard_count": 154, "hard_ece": 0.05817309278238439, "hard_reliability": [ { "count": 2, "confidence": 0.5156046866868782, "reference_agreement": 0.5 }, { "count": 5, "confidence": 0.5744020664194579, "reference_agreement": 0.8 }, { "count": 15, "confidence": 0.6251505169037502, "reference_agreement": 0.4666666666666667 }, { "count": 15, "confidence": 0.7097643525488307, "reference_agreement": 0.5333333333333333 }, { "count": 11, "confidence": 0.7837242799754484, "reference_agreement": 0.7272727272727273 }, { "count": 10, "confidence": 0.833779143408664, "reference_agreement": 0.9 }, { "count": 22, "confidence": 0.9027589227901665, "reference_agreement": 0.9090909090909091 }, { "count": 74, "confidence": 0.9777466873130052, "reference_agreement": 0.9594594594594594 } ], "soft_reliability": [ { "count": 34, "confidence": 0.5192844401029223, "reference_agreement": 0.502892126954275 }, { "count": 55, "confidence": 0.5665664402627255, "reference_agreement": 0.6365357818181818 }, { "count": 72, "confidence": 0.6353817301417451, "reference_agreement": 0.617865484172874 }, { "count": 84, "confidence": 0.7075900367197564, "reference_agreement": 0.6229389882371539 }, { "count": 58, "confidence": 0.7769928207174944, "reference_agreement": 0.7633095172413793 }, { "count": 55, "confidence": 0.829995783859167, "reference_agreement": 0.8141728722666914 }, { "count": 83, "confidence": 0.9004798674802651, "reference_agreement": 0.8669205460521514 }, { "count": 141, "confidence": 0.975388672803986, "reference_agreement": 0.9554196875487998 } ] }, "score": { "count": 585, "soft_ce": 1.043454353228223, "kl": 0.5058693789008174, "brier": 0.12228264299021577, "argmax_agreement": 0.6598290598290598, "hard_count": 243, "hard_ece": 0.07115621447701632, "hard_reliability": [ { "count": 13, "confidence": 0.18586413124072504, "reference_agreement": 0.15384615384615385 }, { "count": 19, "confidence": 0.2362995959591558, "reference_agreement": 0.2631578947368421 }, { "count": 23, "confidence": 0.3019061999897666, "reference_agreement": 0.13043478260869565 }, { "count": 12, "confidence": 0.3685367014209464, "reference_agreement": 0.3333333333333333 }, { "count": 17, "confidence": 0.4338667824041559, "reference_agreement": 0.4117647058823529 }, { "count": 11, "confidence": 0.49490420241461824, "reference_agreement": 0.36363636363636365 }, { "count": 12, "confidence": 0.5688318970482136, "reference_agreement": 0.3333333333333333 }, { "count": 13, "confidence": 0.6294053829689576, "reference_agreement": 0.6153846153846154 }, { "count": 12, "confidence": 0.6958966365351196, "reference_agreement": 0.5 }, { "count": 13, "confidence": 0.7523646054019367, "reference_agreement": 0.6923076923076923 }, { "count": 14, "confidence": 0.8351455536348746, "reference_agreement": 0.6428571428571429 }, { "count": 28, "confidence": 0.9024161795738924, "reference_agreement": 0.8928571428571429 }, { "count": 56, "confidence": 0.9807319435717284, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 13, "confidence": 0.18586413124072504, "reference_agreement": 0.15384615384615385 }, { "count": 32, "confidence": 0.24331398836464874, "reference_agreement": 0.2734375 }, { "count": 49, "confidence": 0.3019081956123887, "reference_agreement": 0.23663264863946495 }, { "count": 39, "confidence": 0.3667883077504478, "reference_agreement": 0.34414724859025103 }, { "count": 71, "confidence": 0.43551300115891045, "reference_agreement": 0.4057656959535077 }, { "count": 61, "confidence": 0.49800447627318833, "reference_agreement": 0.47666251871173404 }, { "count": 71, "confidence": 0.5629637410045656, "reference_agreement": 0.5523004644601318 }, { "count": 42, "confidence": 0.6290782481488474, "reference_agreement": 0.610145539550307 }, { "count": 48, "confidence": 0.7031798704359478, "reference_agreement": 0.6086180416667305 }, { "count": 37, "confidence": 0.7609997088313216, "reference_agreement": 0.7050786860640831 }, { "count": 25, "confidence": 0.8315691658607272, "reference_agreement": 0.7580001074667474 }, { "count": 34, "confidence": 0.9020050839504612, "reference_agreement": 0.8980392369608144 }, { "count": 63, "confidence": 0.978521012641448, "reference_agreement": 0.9970899523809524 } ], "expected_score_mae": 0.4570759707537796 } }, "category": { "engineering": { "count": 500, "soft_ce": 0.0170732927832887, "kl": 0.0170732927832887, "brier": 0.004504996059548485, "argmax_agreement": 0.996, "hard_count": 500, "hard_ece": 0.005887712376320884, "hard_reliability": [ { "count": 1, "confidence": 0.48051708584209724, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.5901215718541692, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.659498582567833, "reference_agreement": 1.0 }, { "count": 4, "confidence": 0.7592606819949799, "reference_agreement": 0.75 }, { "count": 2, "confidence": 0.809986801374795, "reference_agreement": 1.0 }, { "count": 6, "confidence": 0.9033714425633134, "reference_agreement": 0.8333333333333334 }, { "count": 485, "confidence": 0.9982748543344034, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 1, "confidence": 0.48051708584209724, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.5901215718541692, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.659498582567833, "reference_agreement": 1.0 }, { "count": 4, "confidence": 0.7592606819949799, "reference_agreement": 0.75 }, { "count": 2, "confidence": 0.809986801374795, "reference_agreement": 1.0 }, { "count": 6, "confidence": 0.9033714425633134, "reference_agreement": 0.8333333333333334 }, { "count": 485, "confidence": 0.9982748543344034, "reference_agreement": 1.0 } ], "expected_score_mae": 0.36947776287918127 }, "data_analytics": { "count": 500, "soft_ce": 0.63641646085155, "kl": 0.6331406095589392, "brier": 0.15179326704736298, "argmax_agreement": 0.76, "hard_count": 497, "hard_ece": 0.05635174490474885, "hard_reliability": [ { "count": 8, "confidence": 0.18512452516415256, "reference_agreement": 0.125 }, { "count": 13, "confidence": 0.23685294357226974, "reference_agreement": 0.23076923076923078 }, { "count": 16, "confidence": 0.310619358628449, "reference_agreement": 0.125 }, { "count": 15, "confidence": 0.36840624755576634, "reference_agreement": 0.3333333333333333 }, { "count": 13, "confidence": 0.4338411624493362, "reference_agreement": 0.3076923076923077 }, { "count": 14, "confidence": 0.49562565013910687, "reference_agreement": 0.42857142857142855 }, { "count": 19, "confidence": 0.5716282587437927, "reference_agreement": 0.47368421052631576 }, { "count": 18, "confidence": 0.6307835105473234, "reference_agreement": 0.4444444444444444 }, { "count": 15, "confidence": 0.7034512348597252, "reference_agreement": 0.5333333333333333 }, { "count": 25, "confidence": 0.7610956537573322, "reference_agreement": 0.64 }, { "count": 27, "confidence": 0.8287917395783113, "reference_agreement": 0.7037037037037037 }, { "count": 60, "confidence": 0.9017458338366022, "reference_agreement": 0.8 }, { "count": 254, "confidence": 0.9885657799728477, "reference_agreement": 0.984251968503937 } ], "soft_reliability": [ { "count": 8, "confidence": 0.18512452516415256, "reference_agreement": 0.125 }, { "count": 13, "confidence": 0.23685294357226974, "reference_agreement": 0.23076923076923078 }, { "count": 16, "confidence": 0.310619358628449, "reference_agreement": 0.125 }, { "count": 15, "confidence": 0.36840624755576634, "reference_agreement": 0.3333333333333333 }, { "count": 13, "confidence": 0.4338411624493362, "reference_agreement": 0.3076923076923077 }, { "count": 14, "confidence": 0.49562565013910687, "reference_agreement": 0.42857142857142855 }, { "count": 22, "confidence": 0.5710235531343882, "reference_agreement": 0.4762625909090909 }, { "count": 18, "confidence": 0.6307835105473234, "reference_agreement": 0.4444444444444444 }, { "count": 15, "confidence": 0.7034512348597252, "reference_agreement": 0.5333333333333333 }, { "count": 25, "confidence": 0.7610956537573322, "reference_agreement": 0.64 }, { "count": 27, "confidence": 0.8287917395783113, "reference_agreement": 0.7037037037037037 }, { "count": 60, "confidence": 0.9017458338366022, "reference_agreement": 0.8 }, { "count": 254, "confidence": 0.9885657799728477, "reference_agreement": 0.984251968503937 } ], "expected_score_mae": 0.8379162320369495 }, "finance": { "count": 500, "soft_ce": 0.8325307227564824, "kl": 0.21741036887290283, "brier": 0.05805591006171185, "argmax_agreement": 0.766, "hard_count": 52, "hard_ece": 0.12560551004414863, "hard_reliability": [ { "count": 1, "confidence": 0.42545552219647026, "reference_agreement": 0.0 }, { "count": 3, "confidence": 0.5063873523524167, "reference_agreement": 1.0 }, { "count": 3, "confidence": 0.5575393887409873, "reference_agreement": 0.3333333333333333 }, { "count": 3, "confidence": 0.6185993089535619, "reference_agreement": 0.6666666666666666 }, { "count": 4, "confidence": 0.7072795840269844, "reference_agreement": 0.5 }, { "count": 2, "confidence": 0.7583805784749291, "reference_agreement": 0.0 }, { "count": 5, "confidence": 0.8432613843014515, "reference_agreement": 0.6 }, { "count": 10, "confidence": 0.8980962253086776, "reference_agreement": 0.9 }, { "count": 21, "confidence": 0.9891833973182879, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 11, "confidence": 0.32509882113799443, "reference_agreement": 0.44772725303034394 }, { "count": 31, "confidence": 0.36165303383874403, "reference_agreement": 0.32344085419356344 }, { "count": 56, "confidence": 0.433194585459043, "reference_agreement": 0.39091664392857783 }, { "count": 52, "confidence": 0.49251118240718983, "reference_agreement": 0.5019289031468974 }, { "count": 50, "confidence": 0.5634722409257146, "reference_agreement": 0.6018555810889367 }, { "count": 65, "confidence": 0.6301825914444824, "reference_agreement": 0.6561282074359387 }, { "count": 68, "confidence": 0.7056494781490086, "reference_agreement": 0.627431364951034 }, { "count": 37, "confidence": 0.7683118932263894, "reference_agreement": 0.6799459059550426 }, { "count": 34, "confidence": 0.8290161206378067, "reference_agreement": 0.7909373143137849 }, { "count": 35, "confidence": 0.8969496853528575, "reference_agreement": 0.9005142873333625 }, { "count": 61, "confidence": 0.9760249311137575, "reference_agreement": 0.9830546393442623 } ], "expected_score_mae": 0.28921449751489353 }, "trust_safety": { "count": 500, "soft_ce": 0.6965650575760396, "kl": 0.3262561593110357, "brier": 0.09241847396917179, "argmax_agreement": 0.734, "hard_count": 222, "hard_ece": 0.06416689376484794, "hard_reliability": [ { "count": 2, "confidence": 0.37375993751238135, "reference_agreement": 0.0 }, { "count": 5, "confidence": 0.43472235444626844, "reference_agreement": 0.4 }, { "count": 10, "confidence": 0.502873413716155, "reference_agreement": 0.5 }, { "count": 11, "confidence": 0.5716600619610657, "reference_agreement": 0.8181818181818182 }, { "count": 14, "confidence": 0.6273743231263287, "reference_agreement": 0.5714285714285714 }, { "count": 14, "confidence": 0.7048754950230958, "reference_agreement": 0.6428571428571429 }, { "count": 8, "confidence": 0.772673527518938, "reference_agreement": 0.625 }, { "count": 23, "confidence": 0.8352234500546492, "reference_agreement": 0.6956521739130435 }, { "count": 22, "confidence": 0.9039014670840353, "reference_agreement": 0.8181818181818182 }, { "count": 113, "confidence": 0.9792441416955855, "reference_agreement": 0.9557522123893806 } ], "soft_reliability": [ { "count": 6, "confidence": 0.23458907659811615, "reference_agreement": 0.2075 }, { "count": 8, "confidence": 0.29972115715656333, "reference_agreement": 0.255416625 }, { "count": 17, "confidence": 0.36836999088901834, "reference_agreement": 0.33265160656977494 }, { "count": 24, "confidence": 0.4353651177707092, "reference_agreement": 0.40927912136242695 }, { "count": 42, "confidence": 0.5034541101303811, "reference_agreement": 0.4637698182486976 }, { "count": 44, "confidence": 0.5639370342777291, "reference_agreement": 0.5822456558441558 }, { "count": 43, "confidence": 0.6301712982399027, "reference_agreement": 0.5623823154308876 }, { "count": 45, "confidence": 0.7009662006930144, "reference_agreement": 0.625547008811918 }, { "count": 25, "confidence": 0.7682502104322542, "reference_agreement": 0.6588631578947368 }, { "count": 42, "confidence": 0.8340236494740164, "reference_agreement": 0.7355623967695999 }, { "count": 53, "confidence": 0.9011086418913816, "reference_agreement": 0.8135774190795747 }, { "count": 151, "confidence": 0.9780184864854932, "reference_agreement": 0.9430664742453473 } ], "expected_score_mae": 0.3245526679121535 }, "documents_meetings": { "count": 500, "soft_ce": 0.9479271825455599, "kl": 0.27931837942071347, "brier": 0.06632900701852151, "argmax_agreement": 0.754, "hard_count": 111, "hard_ece": 0.07985648898540725, "hard_reliability": [ { "count": 4, "confidence": 0.5112290740361165, "reference_agreement": 1.0 }, { "count": 8, "confidence": 0.5755079266886977, "reference_agreement": 0.625 }, { "count": 8, "confidence": 0.6215870747073032, "reference_agreement": 0.75 }, { "count": 7, "confidence": 0.7132942799377675, "reference_agreement": 0.5714285714285714 }, { "count": 4, "confidence": 0.7689218976142511, "reference_agreement": 0.5 }, { "count": 13, "confidence": 0.8325792906268336, "reference_agreement": 0.7692307692307693 }, { "count": 13, "confidence": 0.894684750793525, "reference_agreement": 0.8461538461538461 }, { "count": 54, "confidence": 0.980788268448271, "reference_agreement": 0.9444444444444444 } ], "soft_reliability": [ { "count": 16, "confidence": 0.2529896903034424, "reference_agreement": 0.2572032828282828 }, { "count": 26, "confidence": 0.30085803531102356, "reference_agreement": 0.2784615384615385 }, { "count": 37, "confidence": 0.3672361246048308, "reference_agreement": 0.373009828009828 }, { "count": 42, "confidence": 0.43667169969021885, "reference_agreement": 0.49312530062530063 }, { "count": 64, "confidence": 0.5041356739419488, "reference_agreement": 0.576640625 }, { "count": 57, "confidence": 0.5696819547588704, "reference_agreement": 0.6160136452241716 }, { "count": 52, "confidence": 0.6318176118679493, "reference_agreement": 0.6767307692307691 }, { "count": 51, "confidence": 0.7024724227358164, "reference_agreement": 0.7055555555555556 }, { "count": 38, "confidence": 0.7733050801429199, "reference_agreement": 0.7889473684210526 }, { "count": 33, "confidence": 0.8353726880577852, "reference_agreement": 0.8224242424242424 }, { "count": 30, "confidence": 0.8930073021992985, "reference_agreement": 0.8578333333333333 }, { "count": 54, "confidence": 0.980788268448271, "reference_agreement": 0.9444444444444444 } ], "expected_score_mae": 0.3935315866553138 }, "customer_support": { "count": 500, "soft_ce": 1.1372885709258729, "kl": 0.5544979078550764, "brier": 0.11033544189723725, "argmax_agreement": 0.706, "hard_count": 142, "hard_ece": 0.14171122927377042, "hard_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 5, "confidence": 0.17932821619602474, "reference_agreement": 0.4 }, { "count": 1, "confidence": 0.22071972996391087, "reference_agreement": 0.0 }, { "count": 2, "confidence": 0.3112827985653901, "reference_agreement": 0.0 }, { "count": 3, "confidence": 0.37505887333787874, "reference_agreement": 0.3333333333333333 }, { "count": 8, "confidence": 0.4293526333096114, "reference_agreement": 0.25 }, { "count": 2, "confidence": 0.49253722456655336, "reference_agreement": 0.5 }, { "count": 11, "confidence": 0.5647034564825082, "reference_agreement": 0.18181818181818182 }, { "count": 4, "confidence": 0.6399711185141979, "reference_agreement": 0.75 }, { "count": 7, "confidence": 0.6905290585463988, "reference_agreement": 0.5714285714285714 }, { "count": 10, "confidence": 0.7733585610196265, "reference_agreement": 0.6 }, { "count": 17, "confidence": 0.8359032378172329, "reference_agreement": 0.7647058823529411 }, { "count": 20, "confidence": 0.9086037470245399, "reference_agreement": 0.6 }, { "count": 51, "confidence": 0.9778779954883188, "reference_agreement": 0.9411764705882353 } ], "soft_reliability": [ { "count": 1, "confidence": 0.1280767528435369, "reference_agreement": 0.0 }, { "count": 9, "confidence": 0.1761727165792652, "reference_agreement": 0.39832358674463936 }, { "count": 8, "confidence": 0.23185648880822557, "reference_agreement": 0.2729292929292929 }, { "count": 25, "confidence": 0.3010096491709431, "reference_agreement": 0.27153918422154916 }, { "count": 33, "confidence": 0.3716936308219563, "reference_agreement": 0.43249653598146726 }, { "count": 46, "confidence": 0.43566624910724644, "reference_agreement": 0.3784143440302738 }, { "count": 49, "confidence": 0.5004958652365038, "reference_agreement": 0.464408579758434 }, { "count": 64, "confidence": 0.5647801490500414, "reference_agreement": 0.4939270455676706 }, { "count": 35, "confidence": 0.6338884106064454, "reference_agreement": 0.6524762 }, { "count": 49, "confidence": 0.7014723835777643, "reference_agreement": 0.6855101955782364 }, { "count": 34, "confidence": 0.7717612201235851, "reference_agreement": 0.7027941176470588 }, { "count": 37, "confidence": 0.8313773086286068, "reference_agreement": 0.763963945945946 }, { "count": 47, "confidence": 0.9031494358129605, "reference_agreement": 0.7396099361702129 }, { "count": 63, "confidence": 0.9737046850235944, "reference_agreement": 0.9396507939418092 } ], "expected_score_mae": 0.33143847468388904 }, "ai_agents_evals": { "count": 500, "soft_ce": 0.765946732434316, "kl": 0.6764210789434673, "brier": 0.18685146874516365, "argmax_agreement": 0.68, "hard_count": 417, "hard_ece": 0.06936429979471131, "hard_reliability": [ { "count": 5, "confidence": 0.187047500963241, "reference_agreement": 0.2 }, { "count": 8, "confidence": 0.23585635105072375, "reference_agreement": 0.25 }, { "count": 19, "confidence": 0.2982191389855985, "reference_agreement": 0.2631578947368421 }, { "count": 14, "confidence": 0.36932960679831744, "reference_agreement": 0.5714285714285714 }, { "count": 27, "confidence": 0.43134028318033907, "reference_agreement": 0.3333333333333333 }, { "count": 32, "confidence": 0.5068791830358712, "reference_agreement": 0.4375 }, { "count": 33, "confidence": 0.5698085895150488, "reference_agreement": 0.5151515151515151 }, { "count": 53, "confidence": 0.6282809191581149, "reference_agreement": 0.49056603773584906 }, { "count": 34, "confidence": 0.7055160692185535, "reference_agreement": 0.6176470588235294 }, { "count": 33, "confidence": 0.765969600593059, "reference_agreement": 0.8181818181818182 }, { "count": 28, "confidence": 0.8332225392168197, "reference_agreement": 0.7857142857142857 }, { "count": 54, "confidence": 0.8969179427292381, "reference_agreement": 0.8703703703703703 }, { "count": 77, "confidence": 0.9715351814332892, "reference_agreement": 0.922077922077922 } ], "soft_reliability": [ { "count": 5, "confidence": 0.187047500963241, "reference_agreement": 0.2 }, { "count": 8, "confidence": 0.23585635105072375, "reference_agreement": 0.25 }, { "count": 19, "confidence": 0.2982191389855985, "reference_agreement": 0.2631578947368421 }, { "count": 16, "confidence": 0.3728244454729145, "reference_agreement": 0.5458333125 }, { "count": 35, "confidence": 0.43292344830072604, "reference_agreement": 0.35496548479120027 }, { "count": 41, "confidence": 0.5085266662477907, "reference_agreement": 0.45982781975131765 }, { "count": 42, "confidence": 0.5678684243563111, "reference_agreement": 0.5322624638888933 }, { "count": 58, "confidence": 0.6299378167372339, "reference_agreement": 0.5120080436781644 }, { "count": 45, "confidence": 0.7054146769488767, "reference_agreement": 0.6565114530755077 }, { "count": 44, "confidence": 0.7651372249377204, "reference_agreement": 0.8189414169069195 }, { "count": 37, "confidence": 0.833874270098686, "reference_agreement": 0.8076943700618164 }, { "count": 65, "confidence": 0.8966887344775478, "reference_agreement": 0.8630389169038999 }, { "count": 85, "confidence": 0.9707936989016759, "reference_agreement": 0.9253352246481347 } ], "expected_score_mae": 0.8450261299297381 }, "legal": { "count": 122, "soft_ce": 0.5668266819917146, "kl": 0.5668266819917146, "brier": 0.13410281322654918, "argmax_agreement": 0.819672131147541, "hard_count": 122, "hard_ece": 0.07370299961080379, "hard_reliability": [ { "count": 1, "confidence": 0.25539450758217525, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.3631581357093971, "reference_agreement": 0.0 }, { "count": 2, "confidence": 0.4544914341376851, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.503085478379179, "reference_agreement": 0.3333333333333333 }, { "count": 5, "confidence": 0.5760999183430778, "reference_agreement": 0.4 }, { "count": 6, "confidence": 0.6285356936618459, "reference_agreement": 0.6666666666666666 }, { "count": 3, "confidence": 0.6943533513117489, "reference_agreement": 0.6666666666666666 }, { "count": 9, "confidence": 0.7758294858466038, "reference_agreement": 0.6666666666666666 }, { "count": 7, "confidence": 0.8221451030950648, "reference_agreement": 0.42857142857142855 }, { "count": 12, "confidence": 0.9086579271812519, "reference_agreement": 0.75 }, { "count": 72, "confidence": 0.9849048761433361, "reference_agreement": 0.9861111111111112 } ], "soft_reliability": [ { "count": 1, "confidence": 0.25539450758217525, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.3631581357093971, "reference_agreement": 0.0 }, { "count": 2, "confidence": 0.4544914341376851, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.503085478379179, "reference_agreement": 0.3333333333333333 }, { "count": 5, "confidence": 0.5760999183430778, "reference_agreement": 0.4 }, { "count": 6, "confidence": 0.6285356936618459, "reference_agreement": 0.6666666666666666 }, { "count": 3, "confidence": 0.6943533513117489, "reference_agreement": 0.6666666666666666 }, { "count": 9, "confidence": 0.7758294858466038, "reference_agreement": 0.6666666666666666 }, { "count": 7, "confidence": 0.8221451030950648, "reference_agreement": 0.42857142857142855 }, { "count": 12, "confidence": 0.9086579271812519, "reference_agreement": 0.75 }, { "count": 72, "confidence": 0.9849048761433361, "reference_agreement": 0.9861111111111112 } ], "expected_score_mae": 0.05627741100384848 }, "sales_commerce": { "count": 71, "soft_ce": 0.43217932678724524, "kl": 0.43217932678724524, "brier": 0.13052027801486862, "argmax_agreement": 0.8169014084507042, "hard_count": 71, "hard_ece": 0.07210090880407366, "hard_reliability": [ { "count": 1, "confidence": 0.43246792381509247, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.5125808974731219, "reference_agreement": 0.5 }, { "count": 2, "confidence": 0.5624424423010923, "reference_agreement": 0.5 }, { "count": 6, "confidence": 0.6350879180799406, "reference_agreement": 0.5 }, { "count": 4, "confidence": 0.6908680440851511, "reference_agreement": 0.5 }, { "count": 2, "confidence": 0.7547308258492205, "reference_agreement": 0.5 }, { "count": 11, "confidence": 0.8418109162556351, "reference_agreement": 0.7272727272727273 }, { "count": 16, "confidence": 0.8995504132439553, "reference_agreement": 0.875 }, { "count": 27, "confidence": 0.9753556391806691, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 1, "confidence": 0.43246792381509247, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.5125808974731219, "reference_agreement": 0.5 }, { "count": 2, "confidence": 0.5624424423010923, "reference_agreement": 0.5 }, { "count": 6, "confidence": 0.6350879180799406, "reference_agreement": 0.5 }, { "count": 4, "confidence": 0.6908680440851511, "reference_agreement": 0.5 }, { "count": 2, "confidence": 0.7547308258492205, "reference_agreement": 0.5 }, { "count": 11, "confidence": 0.8418109162556351, "reference_agreement": 0.7272727272727273 }, { "count": 16, "confidence": 0.8995504132439553, "reference_agreement": 0.875 }, { "count": 27, "confidence": 0.9753556391806691, "reference_agreement": 1.0 } ], "expected_score_mae": 0.3227189172274408 } } }, "calibrated": { "overall": { "count": 3693, "soft_ce": 0.7041168734088186, "kl": 0.3887052537056387, "brier": 0.09690042536340486, "argmax_agreement": 0.7733549959382616, "hard_count": 2134, "hard_ece": 0.022770969448581948, "hard_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 17, "confidence": 0.17569618187345318, "reference_agreement": 0.17647058823529413 }, { "count": 26, "confidence": 0.23574174336686576, "reference_agreement": 0.23076923076923078 }, { "count": 43, "confidence": 0.30201610783522903, "reference_agreement": 0.23255813953488372 }, { "count": 51, "confidence": 0.3728587574318181, "reference_agreement": 0.35294117647058826 }, { "count": 55, "confidence": 0.43468960041354515, "reference_agreement": 0.36363636363636365 }, { "count": 78, "confidence": 0.5068967125125762, "reference_agreement": 0.47435897435897434 }, { "count": 116, "confidence": 0.5692509240339776, "reference_agreement": 0.5689655172413793 }, { "count": 92, "confidence": 0.6274691893699693, "reference_agreement": 0.5760869565217391 }, { "count": 113, "confidence": 0.6976255782232683, "reference_agreement": 0.584070796460177 }, { "count": 125, "confidence": 0.7655885050647548, "reference_agreement": 0.72 }, { "count": 153, "confidence": 0.8348620973385892, "reference_agreement": 0.7777777777777778 }, { "count": 193, "confidence": 0.9005488041523733, "reference_agreement": 0.8860103626943006 }, { "count": 1069, "confidence": 0.9873898699630395, "reference_agreement": 0.9850327408793265 } ], "soft_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 23, "confidence": 0.1712254393277906, "reference_agreement": 0.22601507061461293 }, { "count": 65, "confidence": 0.241401549839449, "reference_agreement": 0.27101087801087803 }, { "count": 136, "confidence": 0.3047942264115045, "reference_agreement": 0.30327285409683113 }, { "count": 180, "confidence": 0.3704360020830201, "reference_agreement": 0.3727716890584122 }, { "count": 239, "confidence": 0.4341484282824121, "reference_agreement": 0.4200372081322403 }, { "count": 268, "confidence": 0.5028656676116453, "reference_agreement": 0.5155282699000787 }, { "count": 335, "confidence": 0.5660902452493901, "reference_agreement": 0.600639823533436 }, { "count": 269, "confidence": 0.631949793983103, "reference_agreement": 0.6359665011640041 }, { "count": 261, "confidence": 0.6984239785753936, "reference_agreement": 0.6471423835974844 }, { "count": 249, "confidence": 0.7657109724234291, "reference_agreement": 0.7478915785725628 }, { "count": 253, "confidence": 0.8344639775274048, "reference_agreement": 0.8022606606356278 }, { "count": 261, "confidence": 0.8995167793570855, "reference_agreement": 0.8815173885724905 }, { "count": 1151, "confidence": 0.9856462873311037, "reference_agreement": 0.9822842533704442 } ], "expected_score_mae": 0.4584616360574015 }, "type": { "choice": { "count": 2526, "soft_ce": 0.6766963060796009, "kl": 0.4133999732939792, "brier": 0.10042181389472693, "argmax_agreement": 0.7862232779097387, "hard_count": 1737, "hard_ece": 0.024066863233766642, "hard_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 4, "confidence": 0.1466965963178435, "reference_agreement": 0.25 }, { "count": 7, "confidence": 0.23979741242633804, "reference_agreement": 0.14285714285714285 }, { "count": 20, "confidence": 0.30543179732871933, "reference_agreement": 0.35 }, { "count": 39, "confidence": 0.3752333136919975, "reference_agreement": 0.358974358974359 }, { "count": 37, "confidence": 0.43632325318545506, "reference_agreement": 0.35135135135135137 }, { "count": 64, "confidence": 0.5078595631087047, "reference_agreement": 0.484375 }, { "count": 91, "confidence": 0.5663438578130047, "reference_agreement": 0.6043956043956044 }, { "count": 72, "confidence": 0.6259376203313807, "reference_agreement": 0.5555555555555556 }, { "count": 87, "confidence": 0.6985076075501826, "reference_agreement": 0.6091954022988506 }, { "count": 98, "confidence": 0.7672216909611733, "reference_agreement": 0.7142857142857143 }, { "count": 126, "confidence": 0.835107717990711, "reference_agreement": 0.7777777777777778 }, { "count": 140, "confidence": 0.8998423906655577, "reference_agreement": 0.8857142857142857 }, { "count": 949, "confidence": 0.988617468252103, "reference_agreement": 0.9852476290832455 } ], "soft_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 10, "confidence": 0.15381363979618545, "reference_agreement": 0.3198346624136098 }, { "count": 31, "confidence": 0.2398905686918626, "reference_agreement": 0.2732486151840991 }, { "count": 88, "confidence": 0.3066439829194413, "reference_agreement": 0.33710350424812785 }, { "count": 140, "confidence": 0.37159211827431626, "reference_agreement": 0.3838011523963886 }, { "count": 167, "confidence": 0.4341350320260209, "reference_agreement": 0.42457801395752337 }, { "count": 166, "confidence": 0.5007513721327872, "reference_agreement": 0.5253523924620044 }, { "count": 197, "confidence": 0.565477477433576, "reference_agreement": 0.619517122506086 }, { "count": 158, "confidence": 0.6298800441075835, "reference_agreement": 0.6306406317730678 }, { "count": 141, "confidence": 0.6982422404036815, "reference_agreement": 0.6697869686413617 }, { "count": 136, "confidence": 0.7667611071607225, "reference_agreement": 0.7399642556669485 }, { "count": 165, "confidence": 0.8340530433682095, "reference_agreement": 0.7888326080745547 }, { "count": 164, "confidence": 0.899220153771901, "reference_agreement": 0.8851424152139722 }, { "count": 960, "confidence": 0.988268968121057, "reference_agreement": 0.9841545132135424 } ] }, "noul": { "count": 582, "soft_ce": 0.4827658039249084, "kl": 0.16448167168121322, "brier": 0.05629811292264301, "argmax_agreement": 0.8316151202749141, "hard_count": 154, "hard_ece": 0.03838079417551199, "hard_reliability": [ { "count": 2, "confidence": 0.51409214953238, "reference_agreement": 0.5 }, { "count": 12, "confidence": 0.5848970391247416, "reference_agreement": 0.5833333333333334 }, { "count": 10, "confidence": 0.6345947956878246, "reference_agreement": 0.6 }, { "count": 13, "confidence": 0.6959874383006549, "reference_agreement": 0.46153846153846156 }, { "count": 14, "confidence": 0.7675695442215852, "reference_agreement": 0.7857142857142857 }, { "count": 13, "confidence": 0.8339093290833637, "reference_agreement": 0.9230769230769231 }, { "count": 25, "confidence": 0.9051654333633907, "reference_agreement": 0.88 }, { "count": 65, "confidence": 0.9758087375812551, "reference_agreement": 0.9692307692307692 } ], "soft_reliability": [ { "count": 34, "confidence": 0.5174138851791591, "reference_agreement": 0.502892126954275 }, { "count": 71, "confidence": 0.5685861678053349, "reference_agreement": 0.6089361690140845 }, { "count": 72, "confidence": 0.6374148978709203, "reference_agreement": 0.6503378173406389 }, { "count": 72, "confidence": 0.6973135426818078, "reference_agreement": 0.6177620419978036 }, { "count": 76, "confidence": 0.7668883954141004, "reference_agreement": 0.7746528812652387 }, { "count": 63, "confidence": 0.8361374834836659, "reference_agreement": 0.8580618729922205 }, { "count": 65, "confidence": 0.899478685155311, "reference_agreement": 0.8626053588050548 }, { "count": 129, "confidence": 0.9701079851807889, "reference_agreement": 0.9612726817393857 } ] }, "score": { "count": 585, "soft_ce": 1.0427334127476895, "kl": 0.505148438420284, "brier": 0.12208934572306966, "argmax_agreement": 0.6598290598290598, "hard_count": 243, "hard_ece": 0.0736454437457963, "hard_reliability": [ { "count": 13, "confidence": 0.18461913127517923, "reference_agreement": 0.15384615384615385 }, { "count": 19, "confidence": 0.23424754950284965, "reference_agreement": 0.2631578947368421 }, { "count": 23, "confidence": 0.299045943058281, "reference_agreement": 0.13043478260869565 }, { "count": 12, "confidence": 0.365141449586235, "reference_agreement": 0.3333333333333333 }, { "count": 18, "confidence": 0.43133153638239685, "reference_agreement": 0.3888888888888889 }, { "count": 12, "confidence": 0.5005622698299229, "reference_agreement": 0.4166666666666667 }, { "count": 13, "confidence": 0.5751578198046974, "reference_agreement": 0.3076923076923077 }, { "count": 10, "confidence": 0.6313708801299527, "reference_agreement": 0.7 }, { "count": 13, "confidence": 0.6933609064965334, "reference_agreement": 0.5384615384615384 }, { "count": 13, "confidence": 0.7511433692151679, "reference_agreement": 0.6923076923076923 }, { "count": 14, "confidence": 0.8335362248493442, "reference_agreement": 0.6428571428571429 }, { "count": 28, "confidence": 0.8999588812194718, "reference_agreement": 0.8928571428571429 }, { "count": 55, "confidence": 0.9798950122993066, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 13, "confidence": 0.18461913127517923, "reference_agreement": 0.15384615384615385 }, { "count": 34, "confidence": 0.24277920912107187, "reference_agreement": 0.26897058823529413 }, { "count": 48, "confidence": 0.30140300614695376, "reference_agreement": 0.24124999548612047 }, { "count": 40, "confidence": 0.3663895954134836, "reference_agreement": 0.3341685673754948 }, { "count": 72, "confidence": 0.4341795001548752, "reference_agreement": 0.40950506128748676 }, { "count": 68, "confidence": 0.5007529272027481, "reference_agreement": 0.49786392453063255 }, { "count": 67, "confidence": 0.5652470326858571, "reference_agreement": 0.5363432798507779 }, { "count": 39, "confidence": 0.6302455119423146, "reference_agreement": 0.6310114396011891 }, { "count": 48, "confidence": 0.7006234882951765, "reference_agreement": 0.6246944274306163 }, { "count": 37, "confidence": 0.759432527246053, "reference_agreement": 0.7220606842622975 }, { "count": 25, "confidence": 0.8329589079683152, "reference_agreement": 0.7502667524000977 }, { "count": 32, "confidence": 0.9011143643285106, "reference_agreement": 0.9013541875 }, { "count": 62, "confidence": 0.9773667295736088, "reference_agreement": 0.9970430161290323 } ], "expected_score_mae": 0.4584616360574015 } }, "category": { "engineering": { "count": 500, "soft_ce": 0.018681013106648606, "kl": 0.018681013106648606, "brier": 0.004596240452886235, "argmax_agreement": 0.996, "hard_count": 500, "hard_ece": 0.010272557015768891, "hard_reliability": [ { "count": 1, "confidence": 0.4716738957978698, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.5539038973137703, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.655163400891637, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.7052527420032781, "reference_agreement": 0.5 }, { "count": 4, "confidence": 0.7571008671839193, "reference_agreement": 1.0 }, { "count": 3, "confidence": 0.8521442972732137, "reference_agreement": 0.6666666666666666 }, { "count": 9, "confidence": 0.9071717975027496, "reference_agreement": 1.0 }, { "count": 479, "confidence": 0.9978224823124803, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 1, "confidence": 0.4716738957978698, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.5539038973137703, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.655163400891637, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.7052527420032781, "reference_agreement": 0.5 }, { "count": 4, "confidence": 0.7571008671839193, "reference_agreement": 1.0 }, { "count": 3, "confidence": 0.8521442972732137, "reference_agreement": 0.6666666666666666 }, { "count": 9, "confidence": 0.9071717975027496, "reference_agreement": 1.0 }, { "count": 479, "confidence": 0.9978224823124803, "reference_agreement": 1.0 } ], "expected_score_mae": 0.3754904932299786 }, "data_analytics": { "count": 500, "soft_ce": 0.6222487807440032, "kl": 0.6189729294513925, "brier": 0.1492023277961967, "argmax_agreement": 0.76, "hard_count": 497, "hard_ece": 0.04324263356987873, "hard_reliability": [ { "count": 8, "confidence": 0.1836431426040265, "reference_agreement": 0.125 }, { "count": 13, "confidence": 0.2326095606931366, "reference_agreement": 0.23076923076923078 }, { "count": 21, "confidence": 0.3080856226638128, "reference_agreement": 0.19047619047619047 }, { "count": 11, "confidence": 0.37224859620757544, "reference_agreement": 0.36363636363636365 }, { "count": 18, "confidence": 0.43216064330256254, "reference_agreement": 0.2777777777777778 }, { "count": 16, "confidence": 0.49955516847278475, "reference_agreement": 0.375 }, { "count": 20, "confidence": 0.5648218776742052, "reference_agreement": 0.5 }, { "count": 14, "confidence": 0.6284579234052629, "reference_agreement": 0.5 }, { "count": 29, "confidence": 0.7007100933753049, "reference_agreement": 0.5517241379310345 }, { "count": 25, "confidence": 0.7611781134778828, "reference_agreement": 0.68 }, { "count": 42, "confidence": 0.8391754153900324, "reference_agreement": 0.7857142857142857 }, { "count": 45, "confidence": 0.9013655316115832, "reference_agreement": 0.8888888888888888 }, { "count": 235, "confidence": 0.9854872248925048, "reference_agreement": 0.9914893617021276 } ], "soft_reliability": [ { "count": 8, "confidence": 0.1836431426040265, "reference_agreement": 0.125 }, { "count": 13, "confidence": 0.2326095606931366, "reference_agreement": 0.23076923076923078 }, { "count": 21, "confidence": 0.3080856226638128, "reference_agreement": 0.19047619047619047 }, { "count": 11, "confidence": 0.37224859620757544, "reference_agreement": 0.36363636363636365 }, { "count": 18, "confidence": 0.43216064330256254, "reference_agreement": 0.2777777777777778 }, { "count": 16, "confidence": 0.49955516847278475, "reference_agreement": 0.375 }, { "count": 23, "confidence": 0.56429092950471, "reference_agreement": 0.49903378260869563 }, { "count": 14, "confidence": 0.6284579234052629, "reference_agreement": 0.5 }, { "count": 29, "confidence": 0.7007100933753049, "reference_agreement": 0.5517241379310345 }, { "count": 25, "confidence": 0.7611781134778828, "reference_agreement": 0.68 }, { "count": 42, "confidence": 0.8391754153900324, "reference_agreement": 0.7857142857142857 }, { "count": 45, "confidence": 0.9013655316115832, "reference_agreement": 0.8888888888888888 }, { "count": 235, "confidence": 0.9854872248925048, "reference_agreement": 0.9914893617021276 } ], "expected_score_mae": 0.8394948703521851 }, "finance": { "count": 500, "soft_ce": 0.8372534729882183, "kl": 0.22213311910463876, "brier": 0.05824555646969622, "argmax_agreement": 0.766, "hard_count": 52, "hard_ece": 0.11310128402928836, "hard_reliability": [ { "count": 1, "confidence": 0.3723071041294397, "reference_agreement": 0.0 }, { "count": 5, "confidence": 0.50016902311118, "reference_agreement": 0.6 }, { "count": 2, "confidence": 0.5753408310146032, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.6182583123935925, "reference_agreement": 1.0 }, { "count": 3, "confidence": 0.6819092006332838, "reference_agreement": 0.3333333333333333 }, { "count": 5, "confidence": 0.7665259528880326, "reference_agreement": 0.4 }, { "count": 4, "confidence": 0.838982843566272, "reference_agreement": 0.75 }, { "count": 8, "confidence": 0.881833000901765, "reference_agreement": 0.875 }, { "count": 21, "confidence": 0.9797645091294379, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 16, "confidence": 0.3226339567135692, "reference_agreement": 0.3984374629166975 }, { "count": 41, "confidence": 0.36674248561146494, "reference_agreement": 0.33682928918700217 }, { "count": 62, "confidence": 0.4340519346364484, "reference_agreement": 0.4111594596595161 }, { "count": 48, "confidence": 0.49924896849347133, "reference_agreement": 0.5386290065109846 }, { "count": 70, "confidence": 0.5686484601862719, "reference_agreement": 0.655761905714329 }, { "count": 50, "confidence": 0.6308002075172696, "reference_agreement": 0.6414333730000463 }, { "count": 54, "confidence": 0.6999272500653496, "reference_agreement": 0.6493579925926575 }, { "count": 46, "confidence": 0.7648365922391855, "reference_agreement": 0.7116637471812303 }, { "count": 32, "confidence": 0.8315695075811502, "reference_agreement": 0.8268750565625763 }, { "count": 22, "confidence": 0.8916310587032179, "reference_agreement": 0.9127121363636364 }, { "count": 59, "confidence": 0.9675218038967256, "reference_agreement": 0.9836666610169491 } ], "expected_score_mae": 0.29112017853517996 }, "trust_safety": { "count": 500, "soft_ce": 0.6856993182154376, "kl": 0.31539041995043376, "brier": 0.09066598215826954, "argmax_agreement": 0.734, "hard_count": 222, "hard_ece": 0.06329846877237999, "hard_reliability": [ { "count": 2, "confidence": 0.3683034743483865, "reference_agreement": 0.0 }, { "count": 7, "confidence": 0.43137594222756087, "reference_agreement": 0.2857142857142857 }, { "count": 9, "confidence": 0.505779039854673, "reference_agreement": 0.6666666666666666 }, { "count": 16, "confidence": 0.569920504948867, "reference_agreement": 0.75 }, { "count": 14, "confidence": 0.6316407404385947, "reference_agreement": 0.6428571428571429 }, { "count": 8, "confidence": 0.6972827104351492, "reference_agreement": 0.5 }, { "count": 18, "confidence": 0.7672982199470547, "reference_agreement": 0.6666666666666666 }, { "count": 19, "confidence": 0.826724451152989, "reference_agreement": 0.6842105263157895 }, { "count": 25, "confidence": 0.9010283785718083, "reference_agreement": 0.84 }, { "count": 104, "confidence": 0.9729483427736801, "reference_agreement": 0.9711538461538461 } ], "soft_reliability": [ { "count": 9, "confidence": 0.2361890508436565, "reference_agreement": 0.27388888888888885 }, { "count": 7, "confidence": 0.30342299595800304, "reference_agreement": 0.2961904285714286 }, { "count": 18, "confidence": 0.3674761849264621, "reference_agreement": 0.3014857858344078 }, { "count": 33, "confidence": 0.4359383368744807, "reference_agreement": 0.3778090549302529 }, { "count": 35, "confidence": 0.5047182741304667, "reference_agreement": 0.517333303803201 }, { "count": 55, "confidence": 0.5629326069104805, "reference_agreement": 0.5935795651642476 }, { "count": 49, "confidence": 0.6337639434256919, "reference_agreement": 0.6102933127493159 }, { "count": 29, "confidence": 0.693334781004934, "reference_agreement": 0.5768357275211178 }, { "count": 40, "confidence": 0.7684317409650803, "reference_agreement": 0.7117894730881641 }, { "count": 42, "confidence": 0.8344109799487145, "reference_agreement": 0.7692592592237804 }, { "count": 44, "confidence": 0.8999478793240552, "reference_agreement": 0.8112662345983765 }, { "count": 139, "confidence": 0.9715298963628789, "reference_agreement": 0.9553096231010606 } ], "expected_score_mae": 0.3249617918461017 }, "documents_meetings": { "count": 500, "soft_ce": 0.9505308054596932, "kl": 0.2819220023348468, "brier": 0.06778096624550085, "argmax_agreement": 0.754, "hard_count": 111, "hard_ece": 0.0745149993383762, "hard_reliability": [ { "count": 7, "confidence": 0.5130857536757218, "reference_agreement": 0.8571428571428571 }, { "count": 9, "confidence": 0.5759649564141678, "reference_agreement": 0.6666666666666666 }, { "count": 5, "confidence": 0.631365854228681, "reference_agreement": 0.8 }, { "count": 8, "confidence": 0.7057283821613531, "reference_agreement": 0.375 }, { "count": 8, "confidence": 0.7734135275347481, "reference_agreement": 0.75 }, { "count": 14, "confidence": 0.8276341762810223, "reference_agreement": 0.7857142857142857 }, { "count": 11, "confidence": 0.9061937668671157, "reference_agreement": 0.9090909090909091 }, { "count": 49, "confidence": 0.974518301452977, "reference_agreement": 0.9591836734693877 } ], "soft_reliability": [ { "count": 20, "confidence": 0.2507030605824935, "reference_agreement": 0.2815126262626263 }, { "count": 38, "confidence": 0.3027780535641847, "reference_agreement": 0.30187799043062197 }, { "count": 37, "confidence": 0.36910116911685614, "reference_agreement": 0.4304395304395305 }, { "count": 46, "confidence": 0.43197305120456336, "reference_agreement": 0.5305434782608696 }, { "count": 70, "confidence": 0.4998801989425534, "reference_agreement": 0.5903253968253969 }, { "count": 63, "confidence": 0.564395576933704, "reference_agreement": 0.65 }, { "count": 52, "confidence": 0.6360817173907356, "reference_agreement": 0.7249679487179487 }, { "count": 41, "confidence": 0.6999872254915769, "reference_agreement": 0.672560975609756 }, { "count": 34, "confidence": 0.769121243337486, "reference_agreement": 0.8225 }, { "count": 34, "confidence": 0.8294780729236917, "reference_agreement": 0.8451470588235294 }, { "count": 16, "confidence": 0.9032976162782315, "reference_agreement": 0.8896875 }, { "count": 49, "confidence": 0.974518301452977, "reference_agreement": 0.9591836734693877 } ], "expected_score_mae": 0.39489489482681317 }, "customer_support": { "count": 500, "soft_ce": 1.1285135288404535, "kl": 0.545722865769657, "brier": 0.108362788973766, "argmax_agreement": 0.706, "hard_count": 142, "hard_ece": 0.11241881164831904, "hard_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 4, "confidence": 0.1466965963178435, "reference_agreement": 0.25 }, { "count": 2, "confidence": 0.23363389271036505, "reference_agreement": 0.0 }, { "count": 3, "confidence": 0.3018620938804334, "reference_agreement": 0.3333333333333333 }, { "count": 9, "confidence": 0.37797570812443015, "reference_agreement": 0.2222222222222222 }, { "count": 3, "confidence": 0.4272157242978048, "reference_agreement": 0.0 }, { "count": 7, "confidence": 0.5005016740966861, "reference_agreement": 0.14285714285714285 }, { "count": 7, "confidence": 0.5675968327842555, "reference_agreement": 0.8571428571428571 }, { "count": 7, "confidence": 0.6248700358768576, "reference_agreement": 0.5714285714285714 }, { "count": 9, "confidence": 0.7018524259315515, "reference_agreement": 0.5555555555555556 }, { "count": 18, "confidence": 0.763652193538776, "reference_agreement": 0.7222222222222222 }, { "count": 15, "confidence": 0.8335790279408464, "reference_agreement": 0.6666666666666666 }, { "count": 16, "confidence": 0.9023386004425975, "reference_agreement": 0.8125 }, { "count": 39, "confidence": 0.9678176556415846, "reference_agreement": 0.9487179487179487 } ], "soft_reliability": [ { "count": 3, "confidence": 0.11589315469434829, "reference_agreement": 0.3333333333333333 }, { "count": 10, "confidence": 0.15381363979618545, "reference_agreement": 0.3198346624136098 }, { "count": 12, "confidence": 0.24077671068363551, "reference_agreement": 0.29337121212121214 }, { "count": 35, "confidence": 0.3022639463465406, "reference_agreement": 0.3521546318325211 }, { "count": 42, "confidence": 0.37365140372838623, "reference_agreement": 0.35323562863250324 }, { "count": 45, "confidence": 0.4350034398300646, "reference_agreement": 0.36927072142944645 }, { "count": 55, "confidence": 0.5024000185598814, "reference_agreement": 0.4825757636363636 }, { "count": 55, "confidence": 0.5652814585293939, "reference_agreement": 0.631175707569653 }, { "count": 47, "confidence": 0.6319760416224046, "reference_agreement": 0.6904964592198635 }, { "count": 40, "confidence": 0.7024876007617367, "reference_agreement": 0.669458325 }, { "count": 43, "confidence": 0.766737205353825, "reference_agreement": 0.7386433953488373 }, { "count": 33, "confidence": 0.83741488958767, "reference_agreement": 0.7505555454545454 }, { "count": 37, "confidence": 0.8973418194210278, "reference_agreement": 0.8589099459459459 }, { "count": 43, "confidence": 0.965788577174672, "reference_agreement": 0.9501550236821856 } ], "expected_score_mae": 0.33319003384682117 }, "ai_agents_evals": { "count": 500, "soft_ce": 0.7625932921761905, "kl": 0.6730676386853419, "brier": 0.18645192181998924, "argmax_agreement": 0.68, "hard_count": 417, "hard_ece": 0.052463098512104685, "hard_reliability": [ { "count": 5, "confidence": 0.18618071314902357, "reference_agreement": 0.2 }, { "count": 10, "confidence": 0.23838633417547514, "reference_agreement": 0.2 }, { "count": 18, "confidence": 0.295781111577869, "reference_agreement": 0.2777777777777778 }, { "count": 26, "confidence": 0.37139215520169777, "reference_agreement": 0.4230769230769231 }, { "count": 22, "confidence": 0.43633042832674923, "reference_agreement": 0.45454545454545453 }, { "count": 29, "confidence": 0.5119118632329336, "reference_agreement": 0.4482758620689655 }, { "count": 51, "confidence": 0.5699148465657553, "reference_agreement": 0.47058823529411764 }, { "count": 38, "confidence": 0.6258474222785992, "reference_agreement": 0.5263157894736842 }, { "count": 42, "confidence": 0.692690594223072, "reference_agreement": 0.7380952380952381 }, { "count": 35, "confidence": 0.7681140161954323, "reference_agreement": 0.7714285714285715 }, { "count": 40, "confidence": 0.8344200795661016, "reference_agreement": 0.85 }, { "count": 46, "confidence": 0.9006389466100536, "reference_agreement": 0.9347826086956522 }, { "count": 55, "confidence": 0.9717353590779876, "reference_agreement": 0.8909090909090909 } ], "soft_reliability": [ { "count": 5, "confidence": 0.18618071314902357, "reference_agreement": 0.2 }, { "count": 10, "confidence": 0.23838633417547514, "reference_agreement": 0.2 }, { "count": 18, "confidence": 0.295781111577869, "reference_agreement": 0.2777777777777778 }, { "count": 29, "confidence": 0.3733951639465118, "reference_agreement": 0.41724137931034483 }, { "count": 30, "confidence": 0.4336998105448617, "reference_agreement": 0.44690416558973367 }, { "count": 39, "confidence": 0.5120708414698647, "reference_agreement": 0.4701608617898467 }, { "count": 58, "confidence": 0.5690122989604566, "reference_agreement": 0.4917532841954055 }, { "count": 46, "confidence": 0.6283615036372431, "reference_agreement": 0.5640713243854648 }, { "count": 54, "confidence": 0.6932351995199476, "reference_agreement": 0.7547641003856951 }, { "count": 45, "confidence": 0.7641095557632885, "reference_agreement": 0.7842272393489996 }, { "count": 51, "confidence": 0.8318454353186426, "reference_agreement": 0.8372494792826007 }, { "count": 55, "confidence": 0.9002777292404353, "reference_agreement": 0.9324725290016627 }, { "count": 60, "confidence": 0.9709295090707568, "reference_agreement": 0.8988023166666667 } ], "expected_score_mae": 0.8456565717140646 }, "legal": { "count": 122, "soft_ce": 0.5436294554804777, "kl": 0.5436294554804777, "brier": 0.1304449623060641, "argmax_agreement": 0.819672131147541, "hard_count": 122, "hard_ece": 0.08307026110171992, "hard_reliability": [ { "count": 1, "confidence": 0.2542299113522521, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.28724827093183936, "reference_agreement": 0.0 }, { "count": 2, "confidence": 0.377085304774584, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.4441052741464433, "reference_agreement": 0.3333333333333333 }, { "count": 2, "confidence": 0.5174342366971163, "reference_agreement": 0.0 }, { "count": 7, "confidence": 0.5644377124854338, "reference_agreement": 0.5714285714285714 }, { "count": 5, "confidence": 0.6251477859811808, "reference_agreement": 0.8 }, { "count": 8, "confidence": 0.7044969749942575, "reference_agreement": 0.25 }, { "count": 8, "confidence": 0.7587080210898536, "reference_agreement": 0.75 }, { "count": 6, "confidence": 0.8354103639312468, "reference_agreement": 1.0 }, { "count": 13, "confidence": 0.8963652039349991, "reference_agreement": 0.7692307692307693 }, { "count": 66, "confidence": 0.9815239804604637, "reference_agreement": 0.9848484848484849 } ], "soft_reliability": [ { "count": 1, "confidence": 0.2542299113522521, "reference_agreement": 1.0 }, { "count": 1, "confidence": 0.28724827093183936, "reference_agreement": 0.0 }, { "count": 2, "confidence": 0.377085304774584, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.4441052741464433, "reference_agreement": 0.3333333333333333 }, { "count": 2, "confidence": 0.5174342366971163, "reference_agreement": 0.0 }, { "count": 7, "confidence": 0.5644377124854338, "reference_agreement": 0.5714285714285714 }, { "count": 5, "confidence": 0.6251477859811808, "reference_agreement": 0.8 }, { "count": 8, "confidence": 0.7044969749942575, "reference_agreement": 0.25 }, { "count": 8, "confidence": 0.7587080210898536, "reference_agreement": 0.75 }, { "count": 6, "confidence": 0.8354103639312468, "reference_agreement": 1.0 }, { "count": 13, "confidence": 0.8963652039349991, "reference_agreement": 0.7692307692307693 }, { "count": 66, "confidence": 0.9815239804604637, "reference_agreement": 0.9848484848484849 } ], "expected_score_mae": 0.05800823336689319 }, "sales_commerce": { "count": 71, "soft_ce": 0.43972836851867453, "kl": 0.43972836851867453, "brier": 0.13079004940228145, "argmax_agreement": 0.8169014084507042, "hard_count": 71, "hard_ece": 0.08143885493164209, "hard_reliability": [ { "count": 2, "confidence": 0.4480862145923521, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.5225530044071978, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.5799241321545268, "reference_agreement": 0.6666666666666666 }, { "count": 5, "confidence": 0.6273970582753347, "reference_agreement": 0.2 }, { "count": 4, "confidence": 0.6962808034900051, "reference_agreement": 0.75 }, { "count": 4, "confidence": 0.7775016660973879, "reference_agreement": 0.75 }, { "count": 10, "confidence": 0.8388575355152714, "reference_agreement": 0.7 }, { "count": 20, "confidence": 0.9005931196007729, "reference_agreement": 0.9 }, { "count": 21, "confidence": 0.9756813796187571, "reference_agreement": 1.0 } ], "soft_reliability": [ { "count": 2, "confidence": 0.4480862145923521, "reference_agreement": 1.0 }, { "count": 2, "confidence": 0.5225530044071978, "reference_agreement": 0.5 }, { "count": 3, "confidence": 0.5799241321545268, "reference_agreement": 0.6666666666666666 }, { "count": 5, "confidence": 0.6273970582753347, "reference_agreement": 0.2 }, { "count": 4, "confidence": 0.6962808034900051, "reference_agreement": 0.75 }, { "count": 4, "confidence": 0.7775016660973879, "reference_agreement": 0.75 }, { "count": 10, "confidence": 0.8388575355152714, "reference_agreement": 0.7 }, { "count": 20, "confidence": 0.9005931196007729, "reference_agreement": 0.9 }, { "count": 21, "confidence": 0.9756813796187571, "reference_agreement": 1.0 } ], "expected_score_mae": 0.3234410160549905 } } }, "permutation": { "count": 256, "completed": true, "mean_total_variation": 0.061152504018688186, "answer_flip_rate": 0.0625 } }