{"dataset":{"n_variants":{"train":28608,"val":9072,"test":9680},"n_player_games":{"train":1788,"val":567,"test":605},"n_source_games":{"train":943,"val":297,"test":321},"mean_decisions_per_player_game":21.292905,"bands_test":{"1800-2199":202,"1400-1799":145,"1000-1399":131,"2200-2699":127},"levels":[0.0,0.05,0.1,0.2,0.4,0.7],"strategies":["random","selective","subtle"]},"experiments":{"A0_zt1":{"label":"z(top-1) statistic only","features":["z_t1"],"n_features":1,"classifier":{"roc_auc":0.784662,"pr_auc":0.817773,"tpr_at_fpr":[{"target_fpr":0.01,"threshold":2.461357,"tpr":0.314931,"test_fpr":0.01157},{"target_fpr":0.05,"threshold":1.710974,"tpr":0.439229,"test_fpr":0.061157}],"by_cell":[{"strategy":"random","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.635569,"pr_auc":0.623071,"mean_realized_alpha":0.061363},{"strategy":"random","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.718735,"pr_auc":0.711252,"mean_realized_alpha":0.099557},{"strategy":"random","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.867749,"pr_auc":0.866923,"mean_realized_alpha":0.197908},{"strategy":"random","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.983668,"pr_auc":0.98358,"mean_realized_alpha":0.395342},{"strategy":"random","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999481,"pr_auc":0.999429,"mean_realized_alpha":0.593502},{"strategy":"selective","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.635569,"pr_auc":0.623071,"mean_realized_alpha":0.061363},{"strategy":"selective","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.718735,"pr_auc":0.711252,"mean_realized_alpha":0.099557},{"strategy":"selective","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.867749,"pr_auc":0.866923,"mean_realized_alpha":0.197908},{"strategy":"selective","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.983668,"pr_auc":0.98358,"mean_realized_alpha":0.395342},{"strategy":"selective","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999481,"pr_auc":0.999429,"mean_realized_alpha":0.593502},{"strategy":"subtle","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.541044,"pr_auc":0.525417,"mean_realized_alpha":0.061363},{"strategy":"subtle","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.570466,"pr_auc":0.549837,"mean_realized_alpha":0.099557},{"strategy":"subtle","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.628628,"pr_auc":0.607312,"mean_realized_alpha":0.198161},{"strategy":"subtle","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.749207,"pr_auc":0.733022,"mean_realized_alpha":0.399043},{"strategy":"subtle","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.870181,"pr_auc":0.868801,"mean_realized_alpha":0.679408}]}},"A_stat":{"label":"Statistical baseline","short":"Baseline","features":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard"],"n_features":10,"model":"LogisticRegression / Ridge","classifier":{"roc_auc":0.819662,"roc_auc_ci95":[0.81156,0.829472],"pr_auc":0.845737,"brier":0.171255,"tpr_at_fpr":[{"target_fpr":0.01,"threshold":0.871485,"tpr":0.320661,"test_fpr":0.009917},{"target_fpr":0.05,"threshold":0.708667,"tpr":0.499284,"test_fpr":0.059504}],"calibration":[{"bin_lo":0.0,"bin_hi":0.1,"mean_pred":0.075616,"frac_pos":0.097902,"n":113},{"bin_lo":0.1,"bin_hi":0.2,"mean_pred":0.156942,"frac_pos":0.165733,"n":593},{"bin_lo":0.2,"bin_hi":0.3,"mean_pred":0.248558,"frac_pos":0.273622,"n":818},{"bin_lo":0.3,"bin_hi":0.4,"mean_pred":0.349537,"frac_pos":0.371039,"n":847},{"bin_lo":0.4,"bin_hi":0.5,"mean_pred":0.448848,"frac_pos":0.452491,"n":844},{"bin_lo":0.5,"bin_hi":0.6,"mean_pred":0.550376,"frac_pos":0.485126,"n":908},{"bin_lo":0.6,"bin_hi":0.7,"mean_pred":0.644617,"frac_pos":0.570858,"n":901},{"bin_lo":0.7,"bin_hi":0.8,"mean_pred":0.746693,"frac_pos":0.756003,"n":997},{"bin_lo":0.8,"bin_hi":0.9,"mean_pred":0.850499,"frac_pos":0.822555,"n":1058},{"bin_lo":0.9,"bin_hi":1.0,"mean_pred":0.966718,"frac_pos":0.988589,"n":2601}],"by_cell":[{"strategy":"random","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.648448,"pr_auc":0.629911,"mean_realized_alpha":0.061363},{"strategy":"random","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.72754,"pr_auc":0.711355,"mean_realized_alpha":0.099557},{"strategy":"random","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.875462,"pr_auc":0.869371,"mean_realized_alpha":0.197908},{"strategy":"random","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.981911,"pr_auc":0.982019,"mean_realized_alpha":0.395342},{"strategy":"random","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999123,"pr_auc":0.999105,"mean_realized_alpha":0.593502},{"strategy":"selective","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.72253,"pr_auc":0.696389,"mean_realized_alpha":0.061363},{"strategy":"selective","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.827739,"pr_auc":0.813594,"mean_realized_alpha":0.099557},{"strategy":"selective","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.952615,"pr_auc":0.950047,"mean_realized_alpha":0.197908},{"strategy":"selective","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.995697,"pr_auc":0.995512,"mean_realized_alpha":0.395342},{"strategy":"selective","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999284,"pr_auc":0.999272,"mean_realized_alpha":0.593502},{"strategy":"subtle","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.555668,"pr_auc":0.537837,"mean_realized_alpha":0.061363},{"strategy":"subtle","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.593391,"pr_auc":0.566149,"mean_realized_alpha":0.099557},{"strategy":"subtle","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.666602,"pr_auc":0.630835,"mean_realized_alpha":0.198161},{"strategy":"subtle","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.817624,"pr_auc":0.783786,"mean_realized_alpha":0.399043},{"strategy":"subtle","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.931292,"pr_auc":0.921511,"mean_realized_alpha":0.679408}],"by_band":[{"band":"1000-1399","roc_auc":0.822221},{"band":"1400-1799","roc_auc":0.834042},{"band":"1800-2199","roc_auc":0.822315},{"band":"2200-2699","roc_auc":0.798204}],"best_iteration":null},"regressor":{"mae":0.109332,"rmse":0.140171,"by_level":[{"level":0.0,"mae":0.10893,"mean_estimate":0.10893,"mean_true":0.0},{"level":0.05,"mae":0.100096,"mean_estimate":0.140431,"mean_true":0.061363},{"level":0.1,"mae":0.094857,"mean_estimate":0.16272,"mean_true":0.099557},{"level":0.2,"mae":0.085566,"mean_estimate":0.222035,"mean_true":0.197993},{"level":0.4,"mae":0.096477,"mean_estimate":0.344073,"mean_true":0.396575},{"level":0.7,"mae":0.169796,"mean_estimate":0.478597,"mean_true":0.622137}],"by_strategy":[{"strategy":"none","mae":0.10893},{"strategy":"random","mae":0.098398},{"strategy":"selective","mae":0.098538},{"strategy":"subtle","mae":0.131139}]}},"B_sf":{"label":"Stockfish-XGB","short":"+ nonlinear engine","features":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12"],"n_features":19,"model":"XGBoost","classifier":{"roc_auc":0.826225,"roc_auc_ci95":[0.819321,0.833295],"pr_auc":0.854719,"brier":0.166456,"tpr_at_fpr":[{"target_fpr":0.01,"threshold":0.923963,"tpr":0.370689,"test_fpr":0.016529},{"target_fpr":0.05,"threshold":0.691212,"tpr":0.52,"test_fpr":0.057851}],"calibration":[{"bin_lo":0.0,"bin_hi":0.1,"mean_pred":0.098656,"frac_pos":0.0,"n":1},{"bin_lo":0.1,"bin_hi":0.2,"mean_pred":0.1576,"frac_pos":0.140886,"n":474},{"bin_lo":0.2,"bin_hi":0.3,"mean_pred":0.248691,"frac_pos":0.246044,"n":955},{"bin_lo":0.3,"bin_hi":0.4,"mean_pred":0.347522,"frac_pos":0.364356,"n":1027},{"bin_lo":0.4,"bin_hi":0.5,"mean_pred":0.448386,"frac_pos":0.397208,"n":1034},{"bin_lo":0.5,"bin_hi":0.6,"mean_pred":0.544395,"frac_pos":0.518493,"n":789},{"bin_lo":0.6,"bin_hi":0.7,"mean_pred":0.647163,"frac_pos":0.646226,"n":710},{"bin_lo":0.7,"bin_hi":0.8,"mean_pred":0.748242,"frac_pos":0.717337,"n":586},{"bin_lo":0.8,"bin_hi":0.9,"mean_pred":0.851597,"frac_pos":0.865269,"n":584},{"bin_lo":0.9,"bin_hi":1.0,"mean_pred":0.979116,"frac_pos":0.95509,"n":3520}],"by_cell":[{"strategy":"random","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.644606,"pr_auc":0.631487,"mean_realized_alpha":0.061363},{"strategy":"random","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.723309,"pr_auc":0.714435,"mean_realized_alpha":0.099557},{"strategy":"random","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.876252,"pr_auc":0.872046,"mean_realized_alpha":0.197908},{"strategy":"random","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.982425,"pr_auc":0.982953,"mean_realized_alpha":0.395342},{"strategy":"random","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999773,"pr_auc":0.999772,"mean_realized_alpha":0.593502},{"strategy":"selective","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.720882,"pr_auc":0.705072,"mean_realized_alpha":0.061363},{"strategy":"selective","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.836394,"pr_auc":0.829943,"mean_realized_alpha":0.099557},{"strategy":"selective","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.961314,"pr_auc":0.958626,"mean_realized_alpha":0.197908},{"strategy":"selective","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.99588,"pr_auc":0.995748,"mean_realized_alpha":0.395342},{"strategy":"selective","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999828,"pr_auc":0.999828,"mean_realized_alpha":0.593502},{"strategy":"subtle","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.55913,"pr_auc":0.541569,"mean_realized_alpha":0.061363},{"strategy":"subtle","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.595616,"pr_auc":0.571867,"mean_realized_alpha":0.099557},{"strategy":"subtle","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.677116,"pr_auc":0.642649,"mean_realized_alpha":0.198161},{"strategy":"subtle","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.849374,"pr_auc":0.823571,"mean_realized_alpha":0.399043},{"strategy":"subtle","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.971483,"pr_auc":0.965184,"mean_realized_alpha":0.679408}],"by_band":[{"band":"1000-1399","roc_auc":0.825331},{"band":"1400-1799","roc_auc":0.840439},{"band":"1800-2199","roc_auc":0.827414},{"band":"2200-2699","roc_auc":0.80948}],"best_iteration":168},"regressor":{"mae":0.091148,"rmse":0.123881,"by_level":[{"level":0.0,"mae":0.109929,"mean_estimate":0.109929,"mean_true":0.0},{"level":0.05,"mae":0.06953,"mean_estimate":0.128682,"mean_true":0.061363},{"level":0.1,"mae":0.060409,"mean_estimate":0.145156,"mean_true":0.099557},{"level":0.2,"mae":0.079869,"mean_estimate":0.19763,"mean_true":0.197993},{"level":0.4,"mae":0.122804,"mean_estimate":0.343781,"mean_true":0.396575},{"level":0.7,"mae":0.116869,"mean_estimate":0.549762,"mean_true":0.622137}],"by_strategy":[{"strategy":"none","mae":0.109929},{"strategy":"random","mae":0.085661},{"strategy":"selective","mae":0.079545},{"strategy":"subtle","mae":0.104482}]}},"C_sf_maia":{"label":"Stockfish+Maia-XGB","short":"+ human model","features":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12","mean_surprisal","median_surprisal","mean_maia_p","maia_t1_rate","maia_t3_rate","frac_maia_p_lt05","frac_maia_p_lt02","excess_surprisal","z_maia","nonhuman_engine_rate","mean_log_ratio_maia_top1"],"n_features":30,"model":"XGBoost","classifier":{"roc_auc":0.873027,"roc_auc_ci95":[0.863823,0.881758],"pr_auc":0.891737,"brier":0.142201,"tpr_at_fpr":[{"target_fpr":0.01,"threshold":0.919505,"tpr":0.408595,"test_fpr":0.009917},{"target_fpr":0.05,"threshold":0.762002,"tpr":0.556584,"test_fpr":0.039669}],"calibration":[{"bin_lo":0.0,"bin_hi":0.1,"mean_pred":0.087138,"frac_pos":0.059946,"n":45},{"bin_lo":0.1,"bin_hi":0.2,"mean_pred":0.148335,"frac_pos":0.121739,"n":622},{"bin_lo":0.2,"bin_hi":0.3,"mean_pred":0.248123,"frac_pos":0.229385,"n":858},{"bin_lo":0.3,"bin_hi":0.4,"mean_pred":0.341578,"frac_pos":0.337414,"n":717},{"bin_lo":0.4,"bin_hi":0.5,"mean_pred":0.447222,"frac_pos":0.476226,"n":688},{"bin_lo":0.5,"bin_hi":0.6,"mean_pred":0.545852,"frac_pos":0.525235,"n":651},{"bin_lo":0.6,"bin_hi":0.7,"mean_pred":0.646083,"frac_pos":0.647436,"n":628},{"bin_lo":0.7,"bin_hi":0.8,"mean_pred":0.749565,"frac_pos":0.713805,"n":653},{"bin_lo":0.8,"bin_hi":0.9,"mean_pred":0.852795,"frac_pos":0.850449,"n":863},{"bin_lo":0.9,"bin_hi":1.0,"mean_pred":0.972034,"frac_pos":0.974093,"n":3955}],"by_cell":[{"strategy":"random","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.701867,"pr_auc":0.673852,"mean_realized_alpha":0.061363},{"strategy":"random","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.797142,"pr_auc":0.774011,"mean_realized_alpha":0.099557},{"strategy":"random","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.92683,"pr_auc":0.918893,"mean_realized_alpha":0.197908},{"strategy":"random","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.991364,"pr_auc":0.990761,"mean_realized_alpha":0.395342},{"strategy":"random","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999309,"pr_auc":0.999263,"mean_realized_alpha":0.593502},{"strategy":"selective","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.759757,"pr_auc":0.732779,"mean_realized_alpha":0.061363},{"strategy":"selective","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.874347,"pr_auc":0.861001,"mean_realized_alpha":0.099557},{"strategy":"selective","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.974974,"pr_auc":0.971126,"mean_realized_alpha":0.197908},{"strategy":"selective","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.997271,"pr_auc":0.996964,"mean_realized_alpha":0.395342},{"strategy":"selective","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.99938,"pr_auc":0.999342,"mean_realized_alpha":0.593502},{"strategy":"subtle","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.620128,"pr_auc":0.589264,"mean_realized_alpha":0.061363},{"strategy":"subtle","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.69052,"pr_auc":0.6537,"mean_realized_alpha":0.099557},{"strategy":"subtle","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.819288,"pr_auc":0.784684,"mean_realized_alpha":0.198161},{"strategy":"subtle","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.951998,"pr_auc":0.932349,"mean_realized_alpha":0.399043},{"strategy":"subtle","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.991225,"pr_auc":0.985256,"mean_realized_alpha":0.679408}],"by_band":[{"band":"1000-1399","roc_auc":0.874384},{"band":"1400-1799","roc_auc":0.888802},{"band":"1800-2199","roc_auc":0.879301},{"band":"2200-2699","roc_auc":0.845006}],"best_iteration":140},"regressor":{"mae":0.069297,"rmse":0.09614,"by_level":[{"level":0.0,"mae":0.081875,"mean_estimate":0.081875,"mean_true":0.0},{"level":0.05,"mae":0.048957,"mean_estimate":0.105098,"mean_true":0.061363},{"level":0.1,"mae":0.046618,"mean_estimate":0.124445,"mean_true":0.099557},{"level":0.2,"mae":0.069976,"mean_estimate":0.187801,"mean_true":0.197993},{"level":0.4,"mae":0.096028,"mean_estimate":0.357099,"mean_true":0.396575},{"level":0.7,"mae":0.080712,"mean_estimate":0.581104,"mean_true":0.622137}],"by_strategy":[{"strategy":"none","mae":0.081875},{"strategy":"random","mae":0.066834},{"strategy":"selective","mae":0.06173},{"strategy":"subtle","mae":0.076811}]}},"D_sf_maia_seq":{"label":"Stockfish+Maia+Sequence-XGB","short":"+ sequence/criticality","features":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12","mean_surprisal","median_surprisal","mean_maia_p","maia_t1_rate","maia_t3_rate","frac_maia_p_lt05","frac_maia_p_lt02","excess_surprisal","z_maia","nonhuman_engine_rate","mean_log_ratio_maia_top1","longest_t1_streak","longest_t1_streak_frac","roll_t1_max","roll_t1_std","roll_loss_min","roll_surprisal_max","acc_easy","acc_hard","crit_lift","corr_diff_t1","corr_diff_loss","triple_rate","triple_count","longest_nonhuman_streak"],"n_features":44,"model":"XGBoost","classifier":{"roc_auc":0.879741,"roc_auc_ci95":[0.872697,0.886945],"pr_auc":0.898131,"brier":0.138347,"tpr_at_fpr":[{"target_fpr":0.01,"threshold":0.941473,"tpr":0.403526,"test_fpr":0.006612},{"target_fpr":0.05,"threshold":0.780518,"tpr":0.576088,"test_fpr":0.036364}],"calibration":[{"bin_lo":0.0,"bin_hi":0.1,"mean_pred":0.092088,"frac_pos":0.073529,"n":46},{"bin_lo":0.1,"bin_hi":0.2,"mean_pred":0.145814,"frac_pos":0.126687,"n":740},{"bin_lo":0.2,"bin_hi":0.3,"mean_pred":0.246793,"frac_pos":0.209953,"n":723},{"bin_lo":0.3,"bin_hi":0.4,"mean_pred":0.34563,"frac_pos":0.346734,"n":699},{"bin_lo":0.4,"bin_hi":0.5,"mean_pred":0.449832,"frac_pos":0.503484,"n":616},{"bin_lo":0.5,"bin_hi":0.6,"mean_pred":0.547646,"frac_pos":0.563578,"n":550},{"bin_lo":0.6,"bin_hi":0.7,"mean_pred":0.652266,"frac_pos":0.566667,"n":536},{"bin_lo":0.7,"bin_hi":0.8,"mean_pred":0.750125,"frac_pos":0.710611,"n":681},{"bin_lo":0.8,"bin_hi":0.9,"mean_pred":0.850883,"frac_pos":0.861963,"n":852},{"bin_lo":0.9,"bin_hi":1.0,"mean_pred":0.972153,"frac_pos":0.96573,"n":4237}],"by_cell":[{"strategy":"random","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.716977,"pr_auc":0.6953,"mean_realized_alpha":0.061363},{"strategy":"random","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.809893,"pr_auc":0.794706,"mean_realized_alpha":0.099557},{"strategy":"random","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.932824,"pr_auc":0.927446,"mean_realized_alpha":0.197908},{"strategy":"random","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.990949,"pr_auc":0.990299,"mean_realized_alpha":0.395342},{"strategy":"random","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999205,"pr_auc":0.999103,"mean_realized_alpha":0.593502},{"strategy":"selective","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.810129,"pr_auc":0.790272,"mean_realized_alpha":0.061363},{"strategy":"selective","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.905747,"pr_auc":0.901679,"mean_realized_alpha":0.099557},{"strategy":"selective","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.980766,"pr_auc":0.978511,"mean_realized_alpha":0.197908},{"strategy":"selective","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.997653,"pr_auc":0.997362,"mean_realized_alpha":0.395342},{"strategy":"selective","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.999298,"pr_auc":0.999213,"mean_realized_alpha":0.593502},{"strategy":"subtle","level":0.05,"n_pos":605,"n_neg":605,"roc_auc":0.619179,"pr_auc":0.588683,"mean_realized_alpha":0.061363},{"strategy":"subtle","level":0.1,"n_pos":605,"n_neg":605,"roc_auc":0.684576,"pr_auc":0.652706,"mean_realized_alpha":0.099557},{"strategy":"subtle","level":0.2,"n_pos":605,"n_neg":605,"roc_auc":0.812699,"pr_auc":0.78087,"mean_realized_alpha":0.198161},{"strategy":"subtle","level":0.4,"n_pos":605,"n_neg":605,"roc_auc":0.946206,"pr_auc":0.928364,"mean_realized_alpha":0.399043},{"strategy":"subtle","level":0.7,"n_pos":605,"n_neg":605,"roc_auc":0.990012,"pr_auc":0.98419,"mean_realized_alpha":0.679408}],"by_band":[{"band":"1000-1399","roc_auc":0.886331},{"band":"1400-1799","roc_auc":0.899523},{"band":"1800-2199","roc_auc":0.879583},{"band":"2200-2699","roc_auc":0.851144}],"best_iteration":139},"regressor":{"mae":0.068571,"rmse":0.095321,"by_level":[{"level":0.0,"mae":0.081638,"mean_estimate":0.081638,"mean_true":0.0},{"level":0.05,"mae":0.048147,"mean_estimate":0.104639,"mean_true":0.061363},{"level":0.1,"mae":0.046259,"mean_estimate":0.12429,"mean_true":0.099557},{"level":0.2,"mae":0.069208,"mean_estimate":0.1882,"mean_true":0.197993},{"level":0.4,"mae":0.094871,"mean_estimate":0.358342,"mean_true":0.396575},{"level":0.7,"mae":0.080013,"mean_estimate":0.581123,"mean_true":0.622137}],"by_strategy":[{"strategy":"none","mae":0.081638},{"strategy":"random","mae":0.066746},{"strategy":"selective","mae":0.05984},{"strategy":"subtle","mae":0.076513}]}}},"baseline_calibration":{"z_t1":{"mean":0.020424,"std":1.038163,"frac_gt2":0.033058,"frac_gt3":0.001653,"normal_frac_gt2":0.02275,"normal_frac_gt3":0.00135,"hist_edges":[-5.0,-4.75,-4.5,-4.25,-4.0,-3.75,-3.5,-3.25,-3.0,-2.75,-2.5,-2.25,-2.0,-1.75,-1.5,-1.25,-1.0,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.25,2.5,2.75,3.0,3.25,3.5,3.75,4.0,4.25,4.5,4.75,5.0],"hist":[0,0,0,0,0,0,0,0,1,1,1,7,7,18,38,30,40,56,53,52,63,46,42,43,39,18,15,15,9,6,1,3,0,0,0,0,1,0,0,0],"n":605},"z_t3":{"mean":0.020115,"std":0.965585,"frac_gt2":0.021488,"frac_gt3":0.0,"normal_frac_gt2":0.02275,"normal_frac_gt3":0.00135,"hist_edges":[-5.0,-4.75,-4.5,-4.25,-4.0,-3.75,-3.5,-3.25,-3.0,-2.75,-2.5,-2.25,-2.0,-1.75,-1.5,-1.25,-1.0,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.25,2.5,2.75,3.0,3.25,3.5,3.75,4.0,4.25,4.5,4.75,5.0],"hist":[0,0,0,0,0,0,0,0,1,0,4,5,5,22,16,34,48,39,73,61,57,53,40,43,31,34,19,7,10,2,0,1,0,0,0,0,0,0,0,0],"n":605},"z_loss":{"mean":0.003842,"std":1.074964,"frac_gt2":0.024793,"frac_gt3":0.001653,"normal_frac_gt2":0.02275,"normal_frac_gt3":0.00135,"hist_edges":[-5.0,-4.75,-4.5,-4.25,-4.0,-3.75,-3.5,-3.25,-3.0,-2.75,-2.5,-2.25,-2.0,-1.75,-1.5,-1.25,-1.0,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.25,2.5,2.75,3.0,3.25,3.5,3.75,4.0,4.25,4.5,4.75,5.0],"hist":[0,0,1,0,0,0,1,3,2,3,7,8,14,13,16,34,42,37,46,59,57,63,52,41,43,23,19,6,6,3,5,0,1,0,0,0,0,0,0,0],"n":605},"z_t1_hard":{"mean":0.039655,"std":0.987954,"frac_gt2":0.036364,"frac_gt3":0.009917,"normal_frac_gt2":0.02275,"normal_frac_gt3":0.00135,"hist_edges":[-5.0,-4.75,-4.5,-4.25,-4.0,-3.75,-3.5,-3.25,-3.0,-2.75,-2.5,-2.25,-2.0,-1.75,-1.5,-1.25,-1.0,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.25,2.5,2.75,3.0,3.25,3.5,3.75,4.0,4.25,4.5,4.75,5.0],"hist":[0,0,0,0,0,0,0,0,0,0,0,1,2,9,23,48,65,62,54,41,65,49,42,44,35,20,11,12,7,4,2,3,4,2,0,0,0,0,0,0],"n":605},"z_maia":{"mean":0.049923,"std":0.878246,"frac_gt2":0.02314,"frac_gt3":0.003306,"normal_frac_gt2":0.02275,"normal_frac_gt3":0.00135,"hist_edges":[-5.0,-4.75,-4.5,-4.25,-4.0,-3.75,-3.5,-3.25,-3.0,-2.75,-2.5,-2.25,-2.0,-1.75,-1.5,-1.25,-1.0,-0.75,-0.5,-0.25,0.0,0.25,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.25,2.5,2.75,3.0,3.25,3.5,3.75,4.0,4.25,4.5,4.75,5.0],"hist":[0,0,0,0,0,0,0,0,0,0,0,0,0,9,17,38,49,48,73,80,74,52,48,36,29,14,13,11,3,5,2,2,1,0,0,0,0,0,0,1],"n":605}},"event_power":{"A0_zt1":[{"strategy":"random","level":0.05,"k_games":1,"roc_auc":0.629635},{"strategy":"random","level":0.05,"k_games":3,"roc_auc":0.733647},{"strategy":"random","level":0.05,"k_games":5,"roc_auc":0.787461},{"strategy":"random","level":0.05,"k_games":9,"roc_auc":0.851408},{"strategy":"random","level":0.1,"k_games":1,"roc_auc":0.717852},{"strategy":"random","level":0.1,"k_games":3,"roc_auc":0.841992},{"strategy":"random","level":0.1,"k_games":5,"roc_auc":0.902178},{"strategy":"random","level":0.1,"k_games":9,"roc_auc":0.959182},{"strategy":"random","level":0.2,"k_games":1,"roc_auc":0.864963},{"strategy":"random","level":0.2,"k_games":3,"roc_auc":0.974101},{"strategy":"random","level":0.2,"k_games":5,"roc_auc":0.99391},{"strategy":"random","level":0.2,"k_games":9,"roc_auc":0.999742},{"strategy":"selective","level":0.05,"k_games":1,"roc_auc":0.637907},{"strategy":"selective","level":0.05,"k_games":3,"roc_auc":0.733491},{"strategy":"selective","level":0.05,"k_games":5,"roc_auc":0.780326},{"strategy":"selective","level":0.05,"k_games":9,"roc_auc":0.859252},{"strategy":"selective","level":0.1,"k_games":1,"roc_auc":0.705549},{"strategy":"selective","level":0.1,"k_games":3,"roc_auc":0.837498},{"strategy":"selective","level":0.1,"k_games":5,"roc_auc":0.908485},{"strategy":"selective","level":0.1,"k_games":9,"roc_auc":0.96293},{"strategy":"selective","level":0.2,"k_games":1,"roc_auc":0.867376},{"strategy":"selective","level":0.2,"k_games":3,"roc_auc":0.97565},{"strategy":"selective","level":0.2,"k_games":5,"roc_auc":0.99408},{"strategy":"selective","level":0.2,"k_games":9,"roc_auc":0.999518},{"strategy":"subtle","level":0.05,"k_games":1,"roc_auc":0.552546},{"strategy":"subtle","level":0.05,"k_games":3,"roc_auc":0.574882},{"strategy":"subtle","level":0.05,"k_games":5,"roc_auc":0.610751},{"strategy":"subtle","level":0.05,"k_games":9,"roc_auc":0.619983},{"strategy":"subtle","level":0.1,"k_games":1,"roc_auc":0.573857},{"strategy":"subtle","level":0.1,"k_games":3,"roc_auc":0.621391},{"strategy":"subtle","level":0.1,"k_games":5,"roc_auc":0.649972},{"strategy":"subtle","level":0.1,"k_games":9,"roc_auc":0.696573},{"strategy":"subtle","level":0.2,"k_games":1,"roc_auc":0.638392},{"strategy":"subtle","level":0.2,"k_games":3,"roc_auc":0.709585},{"strategy":"subtle","level":0.2,"k_games":5,"roc_auc":0.763488},{"strategy":"subtle","level":0.2,"k_games":9,"roc_auc":0.823847}],"A_stat":[{"strategy":"random","level":0.05,"k_games":1,"roc_auc":0.647218},{"strategy":"random","level":0.05,"k_games":3,"roc_auc":0.736555},{"strategy":"random","level":0.05,"k_games":5,"roc_auc":0.778719},{"strategy":"random","level":0.05,"k_games":9,"roc_auc":0.854836},{"strategy":"random","level":0.1,"k_games":1,"roc_auc":0.744289},{"strategy":"random","level":0.1,"k_games":3,"roc_auc":0.847772},{"strategy":"random","level":0.1,"k_games":5,"roc_auc":0.908567},{"strategy":"random","level":0.1,"k_games":9,"roc_auc":0.958506},{"strategy":"random","level":0.2,"k_games":1,"roc_auc":0.887915},{"strategy":"random","level":0.2,"k_games":3,"roc_auc":0.976295},{"strategy":"random","level":0.2,"k_games":5,"roc_auc":0.994273},{"strategy":"random","level":0.2,"k_games":9,"roc_auc":0.999633},{"strategy":"selective","level":0.05,"k_games":1,"roc_auc":0.720688},{"strategy":"selective","level":0.05,"k_games":3,"roc_auc":0.832936},{"strategy":"selective","level":0.05,"k_games":5,"roc_auc":0.895207},{"strategy":"selective","level":0.05,"k_games":9,"roc_auc":0.959449},{"strategy":"selective","level":0.1,"k_games":1,"roc_auc":0.819656},{"strategy":"selective","level":0.1,"k_games":3,"roc_auc":0.949178},{"strategy":"selective","level":0.1,"k_games":5,"roc_auc":0.980626},{"strategy":"selective","level":0.1,"k_games":9,"roc_auc":0.997664},{"strategy":"selective","level":0.2,"k_games":1,"roc_auc":0.949978},{"strategy":"selective","level":0.2,"k_games":3,"roc_auc":0.997799},{"strategy":"selective","level":0.2,"k_games":5,"roc_auc":0.999952},{"strategy":"selective","level":0.2,"k_games":9,"roc_auc":1.0},{"strategy":"subtle","level":0.05,"k_games":1,"roc_auc":0.562391},{"strategy":"subtle","level":0.05,"k_games":3,"roc_auc":0.587959},{"strategy":"subtle","level":0.05,"k_games":5,"roc_auc":0.609923},{"strategy":"subtle","level":0.05,"k_games":9,"roc_auc":0.634369},{"strategy":"subtle","level":0.1,"k_games":1,"roc_auc":0.605845},{"strategy":"subtle","level":0.1,"k_games":3,"roc_auc":0.648448},{"strategy":"subtle","level":0.1,"k_games":5,"roc_auc":0.698883},{"strategy":"subtle","level":0.1,"k_games":9,"roc_auc":0.746441},{"strategy":"subtle","level":0.2,"k_games":1,"roc_auc":0.670191},{"strategy":"subtle","level":0.2,"k_games":3,"roc_auc":0.760176},{"strategy":"subtle","level":0.2,"k_games":5,"roc_auc":0.815904},{"strategy":"subtle","level":0.2,"k_games":9,"roc_auc":0.897979}],"D_sf_maia_seq":[{"strategy":"random","level":0.05,"k_games":1,"roc_auc":0.730211},{"strategy":"random","level":0.05,"k_games":3,"roc_auc":0.840195},{"strategy":"random","level":0.05,"k_games":5,"roc_auc":0.883718},{"strategy":"random","level":0.05,"k_games":9,"roc_auc":0.942611},{"strategy":"random","level":0.1,"k_games":1,"roc_auc":0.811176},{"strategy":"random","level":0.1,"k_games":3,"roc_auc":0.918929},{"strategy":"random","level":0.1,"k_games":5,"roc_auc":0.966833},{"strategy":"random","level":0.1,"k_games":9,"roc_auc":0.992551},{"strategy":"random","level":0.2,"k_games":1,"roc_auc":0.933017},{"strategy":"random","level":0.2,"k_games":3,"roc_auc":0.99291},{"strategy":"random","level":0.2,"k_games":5,"roc_auc":0.999198},{"strategy":"random","level":0.2,"k_games":9,"roc_auc":0.999996},{"strategy":"selective","level":0.05,"k_games":1,"roc_auc":0.81335},{"strategy":"selective","level":0.05,"k_games":3,"roc_auc":0.928457},{"strategy":"selective","level":0.05,"k_games":5,"roc_auc":0.967375},{"strategy":"selective","level":0.05,"k_games":9,"roc_auc":0.994246},{"strategy":"selective","level":0.1,"k_games":1,"roc_auc":0.899581},{"strategy":"selective","level":0.1,"k_games":3,"roc_auc":0.987068},{"strategy":"selective","level":0.1,"k_games":5,"roc_auc":0.996957},{"strategy":"selective","level":0.1,"k_games":9,"roc_auc":0.99994},{"strategy":"selective","level":0.2,"k_games":1,"roc_auc":0.980427},{"strategy":"selective","level":0.2,"k_games":3,"roc_auc":0.999679},{"strategy":"selective","level":0.2,"k_games":5,"roc_auc":0.999993},{"strategy":"selective","level":0.2,"k_games":9,"roc_auc":1.0},{"strategy":"subtle","level":0.05,"k_games":1,"roc_auc":0.628875},{"strategy":"subtle","level":0.05,"k_games":3,"roc_auc":0.6693},{"strategy":"subtle","level":0.05,"k_games":5,"roc_auc":0.715137},{"strategy":"subtle","level":0.05,"k_games":9,"roc_auc":0.780548},{"strategy":"subtle","level":0.1,"k_games":1,"roc_auc":0.696539},{"strategy":"subtle","level":0.1,"k_games":3,"roc_auc":0.774418},{"strategy":"subtle","level":0.1,"k_games":5,"roc_auc":0.832132},{"strategy":"subtle","level":0.1,"k_games":9,"roc_auc":0.903034},{"strategy":"subtle","level":0.2,"k_games":1,"roc_auc":0.800713},{"strategy":"subtle","level":0.2,"k_games":3,"roc_auc":0.920423},{"strategy":"subtle","level":0.2,"k_games":5,"roc_auc":0.964573},{"strategy":"subtle","level":0.2,"k_games":9,"roc_auc":0.991141}]},"shap_global":{"classifier":[{"feature":"nonhuman_engine_rate","mean_abs":0.674001},{"feature":"triple_rate","mean_abs":0.619968},{"feature":"z_t1_hard","mean_abs":0.298513},{"feature":"z_t3","mean_abs":0.220655},{"feature":"z_t1","mean_abs":0.212664},{"feature":"triple_count","mean_abs":0.172428},{"feature":"mean_surprisal","mean_abs":0.087482},{"feature":"t1_rate","mean_abs":0.081404},{"feature":"z_maia","mean_abs":0.081239},{"feature":"frac_maia_p_lt05","mean_abs":0.078503},{"feature":"frac_near_opt","mean_abs":0.066505},{"feature":"mean_log_ratio_maia_top1","mean_abs":0.063012},{"feature":"t3_rate","mean_abs":0.060059},{"feature":"corr_diff_t1","mean_abs":0.059989},{"feature":"acc_hard","mean_abs":0.051225},{"feature":"roll_surprisal_max","mean_abs":0.047601},{"feature":"elo","mean_abs":0.039042},{"feature":"excess_surprisal","mean_abs":0.034367},{"feature":"frac_maia_p_lt02","mean_abs":0.030718},{"feature":"opp_elo","mean_abs":0.029291},{"feature":"mean_rank","mean_abs":0.026719},{"feature":"median_surprisal","mean_abs":0.022735},{"feature":"mean_maia_p","mean_abs":0.016368},{"feature":"crit_lift","mean_abs":0.015825},{"feature":"mean_gap12","mean_abs":0.015736},{"feature":"maia_t3_rate","mean_abs":0.015383},{"feature":"longest_t1_streak_frac","mean_abs":0.015238},{"feature":"median_loss","mean_abs":0.014158},{"feature":"maia_t1_rate","mean_abs":0.013946},{"feature":"roll_t1_std","mean_abs":0.012429},{"feature":"corr_diff_loss","mean_abs":0.009735},{"feature":"n_dec","mean_abs":0.009041},{"feature":"t5_rate","mean_abs":0.007629},{"feature":"longest_nonhuman_streak","mean_abs":0.007585},{"feature":"mean_loss","mean_abs":0.006981},{"feature":"z_loss","mean_abs":0.005719},{"feature":"mean_abs_eval","mean_abs":0.005093},{"feature":"roll_loss_min","mean_abs":0.005005},{"feature":"std_loss","mean_abs":0.004019},{"feature":"frac_loss_ge100","mean_abs":0.003576},{"feature":"acc_easy","mean_abs":0.002686},{"feature":"longest_t1_streak","mean_abs":0.001188},{"feature":"roll_t1_max","mean_abs":0.000859},{"feature":"frac_loss_ge300","mean_abs":0.000638}],"regressor":[{"feature":"z_t3","mean_abs":0.057825},{"feature":"t3_rate","mean_abs":0.034886},{"feature":"nonhuman_engine_rate","mean_abs":0.031876},{"feature":"mean_surprisal","mean_abs":0.025045},{"feature":"frac_near_opt","mean_abs":0.024863},{"feature":"mean_maia_p","mean_abs":0.020523},{"feature":"elo","mean_abs":0.008178},{"feature":"t1_rate","mean_abs":0.007804},{"feature":"mean_log_ratio_maia_top1","mean_abs":0.007294},{"feature":"std_loss","mean_abs":0.007261},{"feature":"opp_elo","mean_abs":0.005285},{"feature":"corr_diff_t1","mean_abs":0.004683},{"feature":"n_dec","mean_abs":0.004491},{"feature":"mean_gap12","mean_abs":0.00448},{"feature":"mean_rank","mean_abs":0.004015},{"feature":"median_surprisal","mean_abs":0.003693},{"feature":"frac_maia_p_lt05","mean_abs":0.003353},{"feature":"triple_rate","mean_abs":0.003261},{"feature":"mean_abs_eval","mean_abs":0.003206},{"feature":"z_t1","mean_abs":0.003015},{"feature":"acc_hard","mean_abs":0.002904},{"feature":"z_t1_hard","mean_abs":0.001998},{"feature":"excess_surprisal","mean_abs":0.001625},{"feature":"maia_t1_rate","mean_abs":0.001559},{"feature":"crit_lift","mean_abs":0.001547},{"feature":"frac_loss_ge100","mean_abs":0.00114},{"feature":"roll_loss_min","mean_abs":0.001103},{"feature":"longest_t1_streak_frac","mean_abs":0.00109},{"feature":"longest_nonhuman_streak","mean_abs":0.00101},{"feature":"median_loss","mean_abs":0.001008},{"feature":"t5_rate","mean_abs":0.000931},{"feature":"maia_t3_rate","mean_abs":0.000903},{"feature":"mean_loss","mean_abs":0.000844},{"feature":"z_loss","mean_abs":0.000811},{"feature":"frac_maia_p_lt02","mean_abs":0.000747},{"feature":"z_maia","mean_abs":0.000684},{"feature":"roll_t1_std","mean_abs":0.000442},{"feature":"corr_diff_loss","mean_abs":0.00038},{"feature":"roll_surprisal_max","mean_abs":0.000318},{"feature":"longest_t1_streak","mean_abs":0.000191},{"feature":"acc_easy","mean_abs":0.000171},{"feature":"roll_t1_max","mean_abs":9.7e-05},{"feature":"frac_loss_ge300","mean_abs":6.6e-05},{"feature":"triple_count","mean_abs":6e-05}],"classifier_C":[{"feature":"nonhuman_engine_rate","mean_abs":0.776061},{"feature":"z_t1_hard","mean_abs":0.600149},{"feature":"z_t1","mean_abs":0.314012},{"feature":"z_t3","mean_abs":0.262913},{"feature":"mean_surprisal","mean_abs":0.120196},{"feature":"frac_maia_p_lt05","mean_abs":0.104045},{"feature":"mean_log_ratio_maia_top1","mean_abs":0.098566},{"feature":"z_maia","mean_abs":0.073581},{"feature":"excess_surprisal","mean_abs":0.067837},{"feature":"frac_near_opt","mean_abs":0.059658},{"feature":"elo","mean_abs":0.051914},{"feature":"t1_rate","mean_abs":0.050076},{"feature":"t3_rate","mean_abs":0.04931},{"feature":"maia_t3_rate","mean_abs":0.046529},{"feature":"opp_elo","mean_abs":0.04379},{"feature":"frac_maia_p_lt02","mean_abs":0.038052},{"feature":"median_surprisal","mean_abs":0.032666},{"feature":"mean_rank","mean_abs":0.027889},{"feature":"std_loss","mean_abs":0.024774},{"feature":"mean_maia_p","mean_abs":0.021482},{"feature":"median_loss","mean_abs":0.016044},{"feature":"n_dec","mean_abs":0.015563},{"feature":"maia_t1_rate","mean_abs":0.014991},{"feature":"t5_rate","mean_abs":0.012764},{"feature":"mean_abs_eval","mean_abs":0.012483},{"feature":"mean_gap12","mean_abs":0.011058},{"feature":"mean_loss","mean_abs":0.009783},{"feature":"z_loss","mean_abs":0.006245},{"feature":"frac_loss_ge100","mean_abs":0.004141},{"feature":"frac_loss_ge300","mean_abs":0.00343}]},"elapsed_s":15.324081,"decision_model":{"features":["sf_rank","cpl","gap12","near_opt","difficulty","n_legal","abs_eval","maia_p","maia_rank","surprisal","maia_entropy","maia_p_best","maia_sf_agree","elo"],"levels_used":[0.0,0.2,0.4],"test_roc_auc":0.926145,"test_pr_auc":0.808278,"test_pos_rate":0.255772,"by_strategy":{"random":0.920567,"selective":0.949713,"subtle":0.915224}},"projection":{"kappa":{"t1":1.288778,"t3":1.32437,"loss":1.864335,"t1_hard":1.081946,"maia":1.503292,"ev":1.463061},"hard_threshold":0.731664,"easy_threshold":0.576598,"n_cells":40},"analysis":{"engine":"Stockfish 19","depth":12,"multipv":5,"maia_model":"Maia3 5M"},"config":{"opening_skip_plies":16,"decided_eval_cp":300,"near_optimal_cp":25,"subtle_max_loss_cp":30,"prefilter_depth":8,"prefilter_decided_cp":500,"split_fractions":{"train":0.6,"val":0.2,"test":0.2},"seed":20261003,"levels":[0.0,0.05,0.1,0.2,0.4,0.7],"eval_cap_cp":1000,"max_analyzed_ply":160},"hyperparameters":{"xgb_classifier":{"n_estimators":1500,"learning_rate":0.03,"max_depth":4,"subsample":0.8,"colsample_bytree":0.8,"min_child_weight":5,"reg_lambda":1.0,"eval_metric":"logloss","early_stopping_rounds":100,"tree_method":"hist","random_state":20261003},"xgb_regressor":{"n_estimators":1500,"learning_rate":0.03,"max_depth":4,"subsample":0.8,"colsample_bytree":0.8,"min_child_weight":5,"reg_lambda":1.0,"eval_metric":"mae","early_stopping_rounds":100,"tree_method":"hist","random_state":20261003,"objective":"reg:squarederror"},"baseline_classifier":"StandardScaler + LogisticRegression(C=1.0, max_iter=5000), balanced sample weights","baseline_regressor":"StandardScaler + Ridge(alpha=1.0)","decision_model":{"n_estimators":800,"learning_rate":0.05,"max_depth":5,"subsample":0.8,"colsample_bytree":0.8,"min_child_weight":10,"eval_metric":"logloss","early_stopping_rounds":50,"tree_method":"hist","random_state":20261003},"decision_model_levels":[0.0,0.2,0.4],"class_weighting":"balanced: each class carries half of the total sample weight","early_stopping":"validation split only","bootstrap":"200 resamples of test source games (cluster bootstrap)","rolling_window":8,"low_human_likelihood":0.1,"min_decisions_per_player_game":10,"projection_min_cell":30,"loss_cap_cp":300},"feature_groups":{"A_stat":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard"],"B_sf":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12"],"C_sf_maia":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12","mean_surprisal","median_surprisal","mean_maia_p","maia_t1_rate","maia_t3_rate","frac_maia_p_lt05","frac_maia_p_lt02","excess_surprisal","z_maia","nonhuman_engine_rate","mean_log_ratio_maia_top1"],"D_sf_maia_seq":["n_dec","elo","opp_elo","t1_rate","t3_rate","mean_loss","z_t1","z_t3","z_loss","z_t1_hard","t5_rate","median_loss","std_loss","frac_loss_ge100","frac_loss_ge300","mean_rank","frac_near_opt","mean_abs_eval","mean_gap12","mean_surprisal","median_surprisal","mean_maia_p","maia_t1_rate","maia_t3_rate","frac_maia_p_lt05","frac_maia_p_lt02","excess_surprisal","z_maia","nonhuman_engine_rate","mean_log_ratio_maia_top1","longest_t1_streak","longest_t1_streak_frac","roll_t1_max","roll_t1_std","roll_loss_min","roll_surprisal_max","acc_easy","acc_hard","crit_lift","corr_diff_t1","corr_diff_loss","triple_rate","triple_count","longest_nonhuman_streak"]},"versions":{"python":"3.13.7","xgboost":"3.4.1","scikit_learn":"1.9.1","shap":"0.52.0","torch":"2.14.1","python_chess":"1.11.2","maia3_commit":"1e13597c42d4858b7cfd7cfdae01e297263364b2","maia3_checkpoint":"UofTCSSLab/Maia3-5M","data":"Lichess open database, standard rated, August 2026 (first ~420 MB of the dump)"},"score_histograms":{"A_stat":{"edges":[0.0,0.04,0.08,0.12,0.16,0.2,0.24,0.28,0.32,0.36,0.4,0.44,0.48,0.52,0.56,0.6,0.64,0.68,0.72,0.76,0.8,0.84,0.88,0.92,0.96,1.0],"clean":[3,19,36,65,69,50,52,39,36,32,27,23,25,24,24,27,13,8,11,5,4,7,5,1,0],"by_level":{"0.05":[2,15,46,82,116,107,139,134,119,100,109,91,96,87,96,72,85,68,75,58,47,31,23,15,2],"0.1":[0,9,27,51,82,86,88,104,96,106,94,93,112,102,90,78,93,87,92,88,72,63,57,39,6],"0.2":[0,1,16,24,33,41,54,47,60,55,69,68,71,73,87,88,100,116,102,99,111,121,128,149,102],"0.4":[0,0,0,1,8,9,13,29,29,24,30,43,37,42,50,52,64,58,62,82,100,97,171,268,546],"0.7":[0,0,0,0,1,0,4,2,2,8,12,16,16,13,32,29,28,36,63,59,84,91,82,192,1045]}},"B_sf":{"edges":[0.0,0.04,0.08,0.12,0.16,0.2,0.24,0.28,0.32,0.36,0.4,0.44,0.48,0.52,0.56,0.6,0.64,0.68,0.72,0.76,0.8,0.84,0.88,0.92,0.96,1.0],"clean":[0,0,21,49,68,74,61,53,46,35,38,41,29,20,13,13,9,6,6,6,1,5,1,6,4],"by_level":{"0.05":[0,0,22,46,113,130,150,166,157,124,121,109,111,71,85,86,54,50,28,28,36,35,23,30,40],"0.1":[0,0,7,31,73,95,105,121,131,117,127,130,105,72,85,70,76,64,68,44,49,43,39,57,106],"0.2":[0,0,1,16,27,34,59,68,69,79,95,74,98,78,78,66,80,74,81,59,54,63,59,118,385],"0.4":[0,0,0,0,1,7,10,22,29,34,36,45,55,35,44,40,48,39,44,54,63,52,58,118,981],"0.7":[0,0,0,0,0,0,1,2,2,1,2,5,7,5,9,13,21,15,22,32,25,27,68,161,1397]}},"C_sf_maia":{"edges":[0.0,0.04,0.08,0.12,0.16,0.2,0.24,0.28,0.32,0.36,0.4,0.44,0.48,0.52,0.56,0.6,0.64,0.68,0.72,0.76,0.8,0.84,0.88,0.92,0.96,1.0],"clean":[0,5,63,86,71,72,60,56,34,18,25,14,18,17,10,11,11,3,7,7,3,5,3,1,5],"by_level":{"0.05":[0,0,47,87,121,141,171,135,109,113,98,90,92,80,80,64,67,54,55,46,37,38,39,27,24],"0.1":[0,0,22,49,74,84,106,84,94,90,92,97,92,89,93,83,86,78,71,76,60,69,74,74,78],"0.2":[0,0,3,10,26,31,27,40,45,41,47,64,49,66,60,81,66,75,77,72,97,129,164,176,369],"0.4":[0,0,0,0,3,1,4,8,2,9,10,7,11,17,16,14,25,31,53,54,91,89,131,216,1023],"0.7":[0,0,0,0,0,0,0,0,0,0,0,2,1,1,1,1,3,0,2,8,20,18,52,200,1506]}},"D_sf_maia_seq":{"edges":[0.0,0.04,0.08,0.12,0.16,0.2,0.24,0.28,0.32,0.36,0.4,0.44,0.48,0.52,0.56,0.6,0.64,0.68,0.72,0.76,0.8,0.84,0.88,0.92,0.96,1.0],"clean":[0,2,77,109,66,66,57,40,31,29,14,17,11,15,8,8,13,10,6,7,6,3,5,3,2],"by_level":{"0.05":[0,0,60,119,121,110,123,129,112,102,88,99,76,70,60,40,78,54,63,63,46,67,35,60,40],"0.1":[0,0,32,79,68,78,81,85,83,72,72,89,84,80,56,52,73,63,83,73,74,73,98,135,132],"0.2":[0,0,4,16,31,29,26,50,36,52,48,40,61,43,59,65,48,61,76,56,94,88,120,195,517],"0.4":[0,0,0,0,2,1,4,9,8,9,11,7,15,21,17,22,22,35,50,67,75,81,101,180,1078],"0.7":[0,0,0,0,0,0,0,0,0,0,2,1,0,2,0,3,3,3,2,18,16,32,62,213,1458]}}},"paired_auc_bootstrap":{"method":"Paired cluster bootstrap over test source games (resampled with replacement; the same resample for every model). Balanced-weight ROC-AUC as in the main evaluation. Percentile 95% interval of the paired difference. Post hoc analysis of saved held-out predictions; no refitting or tuning.","n_boot":2000,"seed":20261012,"cluster":"game_id (source game)","n_clusters":321,"n_rows":9680,"n_player_games":605,"n_positive_rows":9075,"source":"data/artifacts/test_predictions.parquet","reproduced_auc":{"A0_zt1":0.784662,"A_stat":0.819662,"B_sf":0.826225,"C_sf_maia":0.873027,"D_sf_maia_seq":0.879741},"comparisons":[{"from":"A0_zt1","to":"A_stat","delta":0.035,"ci95":[0.02912,0.040549],"share_le_0":0.0},{"from":"A_stat","to":"B_sf","delta":0.006564,"ci95":[0.002563,0.010464],"share_le_0":0.001},{"from":"B_sf","to":"C_sf_maia","delta":0.046801,"ci95":[0.039536,0.053612],"share_le_0":0.0},{"from":"C_sf_maia","to":"D_sf_maia_seq","delta":0.006714,"ci95":[0.002457,0.011041],"share_le_0":0.001},{"from":"A_stat","to":"C_sf_maia","delta":0.053365,"ci95":[0.045799,0.060716],"share_le_0":0.0},{"from":"A_stat","to":"D_sf_maia_seq","delta":0.060079,"ci95":[0.051575,0.068144],"share_le_0":0.0},{"from":"B_sf","to":"D_sf_maia_seq","delta":0.053515,"ci95":[0.04509,0.061201],"share_le_0":0.0}],"generated_utc":"2026-10-04T21:05:31Z"}}