{
  "paper": "Decision-Aligned Evaluation of Uncertainty Quantification (arXiv:2606.26990v1)",
  "reproduction_scope": "binary decision toy reproduction of metric-vs-utility ranking disagreement",
  "decision_prior": "uniform grid over false-positive cost c in [0.70, 0.85]",
  "seeds": 30,
  "models": {
    "A_calibratedish": {
      "NLL_mean": 0.48110938840686945,
      "NLL_sd": 0.0048582330065389265,
      "Brier_mean": 0.1555247426835071,
      "Brier_sd": 0.001993330588272146,
      "ECE_mean": 0.0956963709866334,
      "ECE_sd": 0.003986004652916124,
      "Accuracy@0.5_mean": 0.7896666666666666,
      "Accuracy@0.5_sd": 0.004463131213608342,
      "PriorWeightedUtility_mean": -0.08564338456284155,
      "PriorWeightedUtility_sd": 0.0016631380167853958,
      "Utility@0.75_mean": -0.0919375,
      "Utility@0.75_sd": 0.002087072793370557,
      "Utility@0.80_mean": -0.08143333333333332,
      "Utility@0.80_sd": 0.0016290114060127205
    },
    "B_decision_oriented": {
      "NLL_mean": 0.32326688342490784,
      "NLL_sd": 0.004818599374556886,
      "Brier_mean": 0.08814292096606918,
      "Brier_sd": 0.001788137857534978,
      "ECE_mean": 0.15075248727404794,
      "ECE_sd": 0.003834963805409071,
      "Accuracy@0.5_mean": 0.9276916666666667,
      "Accuracy@0.5_sd": 0.0035071867922028756,
      "PriorWeightedUtility_mean": -0.06409274214480876,
      "PriorWeightedUtility_sd": 0.0015111598806051595,
      "Utility@0.75_mean": -0.06532916666666666,
      "Utility@0.75_sd": 0.0018386515079851574,
      "Utility@0.80_mean": -0.06596166666666665,
      "Utility@0.80_sd": 0.0015694652797586229
    },
    "C_underconfident": {
      "NLL_mean": 0.4353083416236943,
      "NLL_sd": 0.0013206732017039102,
      "Brier_mean": 0.1262666444036318,
      "Brier_sd": 0.0005936377457488529,
      "ECE_mean": 0.3413898681869887,
      "ECE_sd": 0.001435606147909661,
      "Accuracy@0.5_mean": 1,
      "Accuracy@0.5_sd": 0.0,
      "PriorWeightedUtility_mean": -0.10563196823770495,
      "PriorWeightedUtility_sd": 0.0012836146980261004,
      "Utility@0.75_mean": -0.11866458333333334,
      "Utility@0.75_sd": 0.0015583551346130035,
      "Utility@0.80_mean": -0.10001166666666664,
      "Utility@0.80_sd": 0.0011562188175354125
    }
  },
  "aggregate_rankings": {
    "NLL_lower_better": [
      "B_decision_oriented",
      "C_underconfident",
      "A_calibratedish"
    ],
    "Brier_lower_better": [
      "B_decision_oriented",
      "C_underconfident",
      "A_calibratedish"
    ],
    "ECE_lower_better": [
      "A_calibratedish",
      "B_decision_oriented",
      "C_underconfident"
    ],
    "PriorWeightedUtility_higher_better": [
      "B_decision_oriented",
      "A_calibratedish",
      "C_underconfident"
    ]
  },
  "rank_alignment_spearman_mean": {
    "NLL_mean": 0.5,
    "NLL_sd": 0.0,
    "Brier_mean": 0.5,
    "Brier_sd": 0.0,
    "ECE_mean": 0.5,
    "ECE_sd": 0.0,
    "Accuracy@0.5_mean": -0.5,
    "Accuracy@0.5_sd": 0.0,
    "PriorWeightedUtility_mean": 1.0,
    "PriorWeightedUtility_sd": 0.0
  }
}