{"schema":"movelens.regan_checks.v1","generator":"experiments/24_export_regan_checks.py","generated_utc":"2026-10-08T02:56:55Z","game":{"game_id":"li_aVMHaGzT","color":"black","elo":1956,"n_decisions":36,"kappa_t1":1.288778,"formula":"z = (actual - projected) / sqrt(kappa * sum p(1-p)); kappa estimated on validation original games","decisions":[{"ply":17,"move_no":9,"san":"Bxc4","p_t1":0.473365,"engine_match":false,"engine_first":"Nc6","maia_p":0.257114},{"ply":19,"move_no":10,"san":"Be7","p_t1":0.264975,"engine_match":true,"engine_first":"Be7","maia_p":0.628587},{"ply":21,"move_no":11,"san":"O-O","p_t1":0.279176,"engine_match":false,"engine_first":"Qd7","maia_p":0.91613},{"ply":23,"move_no":12,"san":"c5","p_t1":0.264975,"engine_match":false,"engine_first":"Nc6","maia_p":0.446093},{"ply":25,"move_no":13,"san":"Bd6","p_t1":0.216621,"engine_match":true,"engine_first":"Bd6","maia_p":0.060012},{"ply":27,"move_no":14,"san":"Qxd6","p_t1":0.853127,"engine_match":true,"engine_first":"Qxd6","maia_p":0.999649},{"ply":29,"move_no":15,"san":"Nc6","p_t1":0.279176,"engine_match":true,"engine_first":"Nc6","maia_p":0.820207},{"ply":31,"move_no":16,"san":"Rad8","p_t1":0.216621,"engine_match":false,"engine_first":"b6","maia_p":0.557},{"ply":33,"move_no":17,"san":"Nxd4","p_t1":0.325908,"engine_match":true,"engine_first":"Nxd4","maia_p":0.664242},{"ply":35,"move_no":18,"san":"Qxd4","p_t1":0.39801,"engine_match":false,"engine_first":"cxd4","maia_p":0.750748},{"ply":37,"move_no":19,"san":"b6","p_t1":0.17023,"engine_match":true,"engine_first":"b6","maia_p":0.919176},{"ply":39,"move_no":20,"san":"Qf6","p_t1":0.264975,"engine_match":false,"engine_first":"Rfe8","maia_p":0.678074},{"ply":41,"move_no":21,"san":"Rfe8","p_t1":0.17023,"engine_match":false,"engine_first":"h6","maia_p":0.064195},{"ply":45,"move_no":23,"san":"Qe7","p_t1":0.17023,"engine_match":false,"engine_first":"h6","maia_p":0.019166},{"ply":47,"move_no":24,"san":"h6","p_t1":0.17023,"engine_match":false,"engine_first":"Qe6","maia_p":0.336765},{"ply":49,"move_no":25,"san":"Rd8","p_t1":0.216621,"engine_match":false,"engine_first":"a5","maia_p":0.15975},{"ply":51,"move_no":26,"san":"Qxd8","p_t1":0.853127,"engine_match":true,"engine_first":"Qxd8","maia_p":0.999471},{"ply":53,"move_no":27,"san":"Qc7+","p_t1":0.264975,"engine_match":false,"engine_first":"Qd6+","maia_p":0.123919},{"ply":55,"move_no":28,"san":"Kf8","p_t1":0.264975,"engine_match":false,"engine_first":"a5","maia_p":0.029615},{"ply":57,"move_no":29,"san":"Qe5","p_t1":0.39801,"engine_match":false,"engine_first":"f6","maia_p":0.582865},{"ply":61,"move_no":31,"san":"Qc7","p_t1":0.853127,"engine_match":true,"engine_first":"Qc7","maia_p":0.699637},{"ply":63,"move_no":32,"san":"Kf8","p_t1":0.346074,"engine_match":false,"engine_first":"Kf6","maia_p":0.703035},{"ply":65,"move_no":33,"san":"Ke7","p_t1":0.853127,"engine_match":true,"engine_first":"Ke7","maia_p":0.999584},{"ply":67,"move_no":34,"san":"Qd6","p_t1":0.216621,"engine_match":false,"engine_first":"g6","maia_p":0.137136},{"ply":69,"move_no":35,"san":"Qd7","p_t1":0.853127,"engine_match":true,"engine_first":"Qd7","maia_p":0.958148},{"ply":71,"move_no":36,"san":"Kf6","p_t1":0.325908,"engine_match":true,"engine_first":"Kf6","maia_p":0.133029},{"ply":73,"move_no":37,"san":"Qd2+","p_t1":0.264975,"engine_match":true,"engine_first":"Qd2+","maia_p":0.628065},{"ply":75,"move_no":38,"san":"Qxg2+","p_t1":0.325908,"engine_match":false,"engine_first":"Qd3","maia_p":0.670266},{"ply":77,"move_no":39,"san":"Ke6","p_t1":0.279176,"engine_match":false,"engine_first":"Kf5","maia_p":0.04757},{"ply":79,"move_no":40,"san":"f5","p_t1":0.279176,"engine_match":true,"engine_first":"f5","maia_p":0.685368},{"ply":81,"move_no":41,"san":"fxg4+","p_t1":0.279176,"engine_match":false,"engine_first":"g5","maia_p":0.055464},{"ply":83,"move_no":42,"san":"g6","p_t1":0.279176,"engine_match":false,"engine_first":"Kf6","maia_p":0.59408},{"ply":85,"move_no":43,"san":"h5","p_t1":0.325908,"engine_match":false,"engine_first":"a6","maia_p":0.644664},{"ply":87,"move_no":44,"san":"gxf5+","p_t1":0.853127,"engine_match":true,"engine_first":"gxf5+","maia_p":0.867116},{"ply":115,"move_no":58,"san":"Kb3","p_t1":0.279176,"engine_match":false,"engine_first":"a6","maia_p":0.572941},{"ply":117,"move_no":59,"san":"a5","p_t1":0.279176,"engine_match":false,"engine_first":"Ka3","maia_p":0.306824}],"variants":[{"strategy":"none","level":0.0,"realized_alpha":0.0,"n_altered":0,"projected_t1":13.40852,"actual_t1":14,"var_t1":6.607637,"z_t1":0.202688,"mean_maia_surprisal":1.056439,"altered_plies":[],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,1,1,1,0,0,1,0,0,0,1,0,0]},{"strategy":"random","level":0.05,"realized_alpha":0.055556,"n_altered":2,"projected_t1":13.40852,"actual_t1":16,"var_t1":6.607637,"z_t1":0.888047,"mean_maia_surprisal":1.142377,"altered_plies":[23,75],"t1":[0,1,0,1,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,1,1,1,1,0,1,0,0,0,1,0,0]},{"strategy":"random","level":0.1,"realized_alpha":0.111111,"n_altered":4,"projected_t1":13.40852,"actual_t1":18,"var_t1":6.607637,"z_t1":1.573405,"mean_maia_surprisal":1.307742,"altered_plies":[23,47,75,83],"t1":[0,1,0,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,0,1,0,1,0,1,1,1,1,0,1,0,1,0,1,0,0]},{"strategy":"random","level":0.2,"realized_alpha":0.194444,"n_altered":7,"projected_t1":13.40852,"actual_t1":21,"var_t1":6.607637,"z_t1":2.601443,"mean_maia_surprisal":1.258697,"altered_plies":[23,47,75,77,83,85,117],"t1":[0,1,0,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,0,1,0,1,0,1,1,1,1,1,1,0,1,1,1,0,1]},{"strategy":"random","level":0.4,"realized_alpha":0.388889,"n_altered":14,"projected_t1":13.40852,"actual_t1":28,"var_t1":6.607637,"z_t1":5.000198,"mean_maia_surprisal":1.606131,"altered_plies":[17,21,23,35,39,41,47,55,63,75,77,83,85,117],"t1":[1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,0,1,0,1,1,1,0,1,1,1,1,1,1,0,1,1,1,0,1]},{"strategy":"random","level":0.7,"realized_alpha":0.611111,"n_altered":22,"projected_t1":13.40852,"actual_t1":36,"var_t1":6.607637,"z_t1":7.741633,"mean_maia_surprisal":1.754133,"altered_plies":[17,21,23,31,35,39,41,45,47,49,53,55,57,63,67,75,77,81,83,85,115,117],"t1":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"strategy":"selective","level":0.05,"realized_alpha":0.055556,"n_altered":2,"projected_t1":13.40852,"actual_t1":16,"var_t1":6.607637,"z_t1":0.888047,"mean_maia_surprisal":0.929485,"altered_plies":[41,45],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,1,1,0,0,1,0,0,0,1,0,1,0,1,1,1,0,0,1,0,0,0,1,0,0]},{"strategy":"selective","level":0.1,"realized_alpha":0.111111,"n_altered":4,"projected_t1":13.40852,"actual_t1":18,"var_t1":6.607637,"z_t1":1.573405,"mean_maia_surprisal":1.117854,"altered_plies":[41,45,47,49],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,1,1,1,1,1,0,0,0,1,0,1,0,1,1,1,0,0,1,0,0,0,1,0,0]},{"strategy":"selective","level":0.2,"realized_alpha":0.194444,"n_altered":7,"projected_t1":13.40852,"actual_t1":21,"var_t1":6.607637,"z_t1":2.601443,"mean_maia_surprisal":1.374143,"altered_plies":[31,39,41,45,47,49,67],"t1":[0,1,0,0,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,0,1,0,1,1,1,1,1,0,0,1,0,0,0,1,0,0]},{"strategy":"selective","level":0.4,"realized_alpha":0.388889,"n_altered":14,"projected_t1":13.40852,"actual_t1":28,"var_t1":6.607637,"z_t1":5.000198,"mean_maia_surprisal":1.514116,"altered_plies":[21,23,31,39,41,45,47,49,53,55,67,77,81,115],"t1":[0,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1,1,1,0,1,1,1,0,0,1,1,0]},{"strategy":"selective","level":0.7,"realized_alpha":0.611111,"n_altered":22,"projected_t1":13.40852,"actual_t1":36,"var_t1":6.607637,"z_t1":7.741633,"mean_maia_surprisal":1.754133,"altered_plies":[17,21,23,31,35,39,41,45,47,49,53,55,57,63,67,75,77,81,83,85,115,117],"t1":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"strategy":"subtle","level":0.05,"realized_alpha":0.055556,"n_altered":2,"projected_t1":13.40852,"actual_t1":15,"var_t1":6.607637,"z_t1":0.545367,"mean_maia_surprisal":1.209647,"altered_plies":[23,81],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,1,1,1,0,0,1,1,0,0,1,0,0]},{"strategy":"subtle","level":0.1,"realized_alpha":0.111111,"n_altered":4,"projected_t1":13.40852,"actual_t1":15,"var_t1":6.607637,"z_t1":0.545367,"mean_maia_surprisal":1.386687,"altered_plies":[23,55,81,83],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,1,1,1,0,0,1,1,0,0,1,0,0]},{"strategy":"subtle","level":0.2,"realized_alpha":0.194444,"n_altered":7,"projected_t1":13.40852,"actual_t1":15,"var_t1":6.607637,"z_t1":0.545367,"mean_maia_surprisal":1.454178,"altered_plies":[23,49,55,79,81,83,117],"t1":[0,1,0,0,1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,1,1,1,0,0,0,1,0,0,1,0,1]},{"strategy":"subtle","level":0.4,"realized_alpha":0.388889,"n_altered":14,"projected_t1":13.40852,"actual_t1":19,"var_t1":6.607637,"z_t1":1.916085,"mean_maia_surprisal":1.874941,"altered_plies":[21,23,31,35,45,49,55,63,77,79,81,83,85,117],"t1":[0,1,0,0,1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,1,1,1,0,1,1,1,0,1,0,1,0,1,1,0,1]},{"strategy":"subtle","level":0.7,"realized_alpha":0.694444,"n_altered":25,"projected_t1":13.40852,"actual_t1":18,"var_t1":6.607637,"z_t1":1.573405,"mean_maia_surprisal":2.717358,"altered_plies":[17,19,21,23,25,29,31,35,39,41,45,49,55,63,67,71,73,75,77,79,81,83,85,115,117],"t1":[1,0,0,0,0,1,0,1,1,0,1,1,0,0,0,0,1,0,0,0,1,1,1,0,1,0,0,1,1,0,1,0,1,1,1,1]}],"check":"z(top-1) recomputed from the per-decision projections equals the stored value for every variant (|diff| < 1e-6)"},"calibration":{"n":12863,"projected_mean":0.388372,"actual_mean":0.393299,"by_bin":[{"bin":"0.1-0.2","n":1215,"projected":0.170832,"actual":0.155556},{"bin":"0.2-0.3","n":3955,"projected":0.245922,"actual":0.258407},{"bin":"0.3-0.4","n":3386,"projected":0.348553,"actual":0.342882},{"bin":"0.4-0.5","n":1789,"projected":0.454696,"actual":0.471213},{"bin":"0.5-0.6","n":448,"projected":0.586801,"actual":0.591518},{"bin":"0.6-0.7","n":792,"projected":0.612984,"actual":0.618687},{"bin":"0.8-0.9","n":1278,"projected":0.839931,"actual":0.852113}],"by_band":[{"band":"1000-1399","n":2397,"projected":0.333232,"actual":0.356279},{"band":"1400-1799","n":2942,"projected":0.36963,"actual":0.370496},{"band":"1800-2199","n":4435,"projected":0.397451,"actual":0.394589},{"band":"2200-2699","n":3089,"projected":0.435976,"actual":0.441891}],"brier":0.200705,"note":"Held-out test split, original moves only; the projection table was fitted on training decisions."},"clean_z":{"n":605,"mean":0.020424,"sd":1.038163,"hist_edges":[-4.0,-3.5,-3.0,-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0],"hist":[0,0,2,8,25,68,96,105,109,85,57,30,15,4,0,1],"tails":[{"threshold":2.0,"observed":0.033058,"count":20,"normal":0.02275},{"threshold":2.5,"observed":0.008264,"count":5,"normal":0.00621},{"threshold":3.0,"observed":0.001653,"count":1,"normal":0.00135}],"note":"Source games are presumed human; kappa was calibrated on validation games so that clean z has about unit variance."},"screening":{"rows":[{"strategy":"none","level":0.0,"n":605,"mean_realized_alpha":0.0,"share_z_gt2":0.033058,"share_z_gt3":0.001653,"median_z":-0.01294},{"strategy":"random","level":0.05,"n":605,"mean_realized_alpha":0.061363,"share_z_gt2":0.089256,"share_z_gt3":0.009917,"median_z":0.502705},{"strategy":"random","level":0.1,"n":605,"mean_realized_alpha":0.099557,"share_z_gt2":0.157025,"share_z_gt3":0.034711,"median_z":0.829099},{"strategy":"random","level":0.2,"n":605,"mean_realized_alpha":0.197908,"share_z_gt2":0.396694,"share_z_gt3":0.14876,"median_z":1.703561},{"strategy":"random","level":0.4,"n":605,"mean_realized_alpha":0.395342,"share_z_gt2":0.87438,"share_z_gt3":0.624793,"median_z":3.38772},{"strategy":"random","level":0.7,"n":605,"mean_realized_alpha":0.593502,"share_z_gt2":1.0,"share_z_gt3":0.975207,"median_z":5.090785},{"strategy":"selective","level":0.05,"n":605,"mean_realized_alpha":0.061363,"share_z_gt2":0.089256,"share_z_gt3":0.009917,"median_z":0.502705},{"strategy":"selective","level":0.1,"n":605,"mean_realized_alpha":0.099557,"share_z_gt2":0.157025,"share_z_gt3":0.034711,"median_z":0.829099},{"strategy":"selective","level":0.2,"n":605,"mean_realized_alpha":0.197908,"share_z_gt2":0.396694,"share_z_gt3":0.14876,"median_z":1.703561},{"strategy":"selective","level":0.4,"n":605,"mean_realized_alpha":0.395342,"share_z_gt2":0.87438,"share_z_gt3":0.624793,"median_z":3.38772},{"strategy":"selective","level":0.7,"n":605,"mean_realized_alpha":0.593502,"share_z_gt2":1.0,"share_z_gt3":0.975207,"median_z":5.090785},{"strategy":"subtle","level":0.05,"n":605,"mean_realized_alpha":0.061363,"share_z_gt2":0.033058,"share_z_gt3":0.003306,"median_z":0.110596},{"strategy":"subtle","level":0.1,"n":605,"mean_realized_alpha":0.099557,"share_z_gt2":0.046281,"share_z_gt3":0.003306,"median_z":0.219576},{"strategy":"subtle","level":0.2,"n":605,"mean_realized_alpha":0.198161,"share_z_gt2":0.066116,"share_z_gt3":0.009917,"median_z":0.470089},{"strategy":"subtle","level":0.4,"n":605,"mean_realized_alpha":0.399043,"share_z_gt2":0.175207,"share_z_gt3":0.02314,"median_z":0.962732},{"strategy":"subtle","level":0.7,"n":605,"mean_realized_alpha":0.679408,"share_z_gt2":0.424793,"share_z_gt3":0.109091,"median_z":1.731666}],"note":"Each row: held-out player-games (605 per cell). Substitutions are evaluated in the original positions."},"maia_by_band":[{"band":"1000-1399","n":2397,"top1_accuracy":0.5335836462244472,"mean_nll":1.3942462828775513},{"band":"1400-1799","n":2942,"top1_accuracy":0.5625424881033311,"mean_nll":1.2977616865593287},{"band":"1800-2199","n":4435,"top1_accuracy":0.580608793686584,"mean_nll":1.2308066548916359},{"band":"2200-2699","n":3089,"top1_accuracy":0.5875687924894788,"mean_nll":1.2444286973788836}],"qwen_ablation":{"status":"precomputed-local","n_positions":7,"rows":[{"position_id":"li_05BKPNaA:24","played_san":"Bf3","engine_first_san":"O-O","maia_top_san":"O-O","regan_aligned":{"top_san":"O-O","top_p":0.199372,"top_is_engine_first":true,"p_engine_first":0.199372,"rank_engine_first":1,"played_rank":3,"played_p":0.107288,"maia_rank_of_top":1},"position_only":{"top_san":"Bc3","top_p":0.098407,"top_is_engine_first":false,"p_engine_first":0.038208,"rank_engine_first":12,"played_rank":5,"played_p":0.063554,"maia_rank_of_top":26}},{"position_id":"li_ItVPTIJx:20","played_san":"hxg3","engine_first_san":"hxg3","maia_top_san":"hxg3","regan_aligned":{"top_san":"d5","top_p":0.204878,"top_is_engine_first":false,"p_engine_first":0.080548,"rank_engine_first":4,"played_rank":4,"played_p":0.080548,"maia_rank_of_top":30},"position_only":{"top_san":"Ng5","top_p":0.101424,"top_is_engine_first":false,"p_engine_first":0.025269,"rank_engine_first":16,"played_rank":16,"played_p":0.025269,"maia_rank_of_top":29}},{"position_id":"li_MY3MVlQx:40","played_san":"Rc7","engine_first_san":"Kf2","maia_top_san":"Rc7","regan_aligned":{"top_san":"g3","top_p":0.22122,"top_is_engine_first":false,"p_engine_first":0.149518,"rank_engine_first":2,"played_rank":33,"played_p":0.000426,"maia_rank_of_top":11},"position_only":{"top_san":"e5","top_p":0.208702,"top_is_engine_first":false,"p_engine_first":0.012896,"rank_engine_first":19,"played_rank":24,"played_p":0.004292,"maia_rank_of_top":5}},{"position_id":"li_Xoxn2IZI:58","played_san":"Rxg5","engine_first_san":"Rxg5","maia_top_san":"Rxg5","regan_aligned":{"top_san":"f3","top_p":0.236057,"top_is_engine_first":false,"p_engine_first":0.07454,"rank_engine_first":5,"played_rank":5,"played_p":0.07454,"maia_rank_of_top":11},"position_only":{"top_san":"a4","top_p":0.153737,"top_is_engine_first":false,"p_engine_first":0.016617,"rank_engine_first":19,"played_rank":19,"played_p":0.016617,"maia_rank_of_top":6}},{"position_id":"li_aVMHaGzT:47","played_san":"h6","engine_first_san":"Qe6","maia_top_san":"g6","regan_aligned":{"top_san":"Qe6","top_p":0.272324,"top_is_engine_first":true,"p_engine_first":0.272324,"rank_engine_first":1,"played_rank":3,"played_p":0.147027,"maia_rank_of_top":13},"position_only":{"top_san":"Qd6","top_p":0.128608,"top_is_engine_first":false,"p_engine_first":0.033766,"rank_engine_first":10,"played_rank":11,"played_p":0.033246,"maia_rank_of_top":27}},{"position_id":"li_gJTPcrY0:40","played_san":"fxe4","engine_first_san":"Rce1","maia_top_san":"fxe4","regan_aligned":{"top_san":"Rc5","top_p":0.161425,"top_is_engine_first":false,"p_engine_first":0.00215,"rank_engine_first":40,"played_rank":16,"played_p":0.019361,"maia_rank_of_top":26},"position_only":{"top_san":"fxe4","top_p":0.083631,"top_is_engine_first":false,"p_engine_first":0.002319,"rank_engine_first":43,"played_rank":1,"played_p":0.083631,"maia_rank_of_top":1}},{"position_id":"li_nhSbU99G:29","played_san":"Nf6","engine_first_san":"Nf6","maia_top_san":"Qd7","regan_aligned":{"top_san":"Nf6","top_p":0.388302,"top_is_engine_first":true,"p_engine_first":0.388302,"rank_engine_first":1,"played_rank":1,"played_p":0.388302,"maia_rank_of_top":4},"position_only":{"top_san":"Nc3","top_p":0.093912,"top_is_engine_first":false,"p_engine_first":0.032346,"rank_engine_first":13,"played_rank":13,"played_p":0.032346,"maia_rank_of_top":6}}],"summary":{"regan_aligned":{"top_is_engine_first":3,"median_rank_engine_first":2.0,"median_played_rank":4.0},"position_only":{"top_is_engine_first":0,"median_rank_engine_first":16.0,"median_played_rank":13.0}},"runtime_position_only":{"model_id":"Qwen/Qwen3-8B-Base","revision":"49e3418fbbbca6ecbdf9608b4d22e5a407081db4","dtype":"bfloat16","quantization":null,"hardware":"accelerate device_map='auto' on NVIDIA GeForce RTX 3050 Ti Laptop GPU machine; see modules_per_device for where layers ran","modules_per_device":{"cpu":16,"disk":24},"candidate_batch":4,"torch":"2.14.1+cu126","transformers":"5.18.0","condition":"position_only","prompt_hash":"ba79104a5919a07344052908c02a551346fa41841887c5504e957f23b2debcc1","scoring":"sum of log-probabilities of every token of ' <uci>\\n'; softmax over legal moves; nothing generated","scope":"7 MoveLens demo positions; not the held-out evaluation","consistency_check":{"position_id":"li_ItVPTIJx:20","moves":["h2g3","a1b1"],"max_abs_diff_nats":0.11157413525506854},"generated_utc":"2026-10-07T18:31:56Z"},"note":"7 demo positions only, scored locally; a qualitative check, not a test."}}