{"generator":{"script":"experiments/14_three_model_eval.py","plan_version":"three-model-plan.v1","plan_sha256":"58637622b3a881238d122acd0b73718f992123c038cb5b925baac23c1f9d9d36","generated_utc":"2026-10-05T06:28:41Z"},"task":"Task 1 \u2014 prediction of original (presumed-human) moves, held-out test split","data":{"positions":12863,"source_games":321,"player_games":605},"plan":{"data":"eligible decisions of ORIGINAL (presumed-human) test player-games only; no synthetic variants","metrics_distribution_models":["top1_accuracy","top3_accuracy","mean_nll","brier_multiclass","mean_rank","median_rank"],"metrics_all_models":["t1_event_logloss","t1_event_brier","t3_event_logloss","t3_event_brier"],"t1_event":"played move == Stockfish MultiPV line 1; model prediction = P_model(that move) (R: p_t1)","t3_event":"played move among Stockfish MultiPV lines 1-3; prediction = sum of P_model over those moves (R: p_t3)","rank_rule":"1 + number of legal moves with strictly higher probability","probability_floor":1e-06,"brier_definition":"sum over legal moves of (p_i - y_i)^2 per position, then mean (valid for varying legal sets)"},"compatibility":[{"metric":"Probability for every legal move","R":false,"Q":true,"M":true},{"metric":"Top-1 / top-3 move prediction","R":false,"Q":true,"M":true},{"metric":"Negative log-likelihood of the played move","R":false,"Q":true,"M":true},{"metric":"Multi-class Brier score","R":false,"Q":true,"M":true},{"metric":"Rank of the played move","R":false,"Q":true,"M":true},{"metric":"P(played move = engine's first line) \u2014 T1 event","R":true,"Q":true,"M":true},{"metric":"P(played move in engine's top 3) \u2014 T3 event","R":true,"Q":true,"M":true},{"metric":"Game-level assistance-sensitive score","R":true,"Q":true,"M":true}],"models":{"R":{"label":"Regan-inspired reconstruction","status":"ok","metrics":{"top1_accuracy":null,"top3_accuracy":null,"mean_nll":null,"brier_multiclass":null,"mean_rank":null,"t1_event_logloss":0.5878987621067446,"t1_event_brier":0.2007048919366821,"t3_event_logloss":0.5927675538019839,"t3_event_brier":0.2049751889894624},"ci95":{"t1_event_logloss":[0.5799187715186733,0.5965839434959006],"t1_event_brier":[0.1971634807820497,0.20459664425810303],"t3_event_logloss":[0.5842417013824363,0.6010334974925536],"t3_event_brier":[0.20135909347759223,0.20841674586236836]},"observed_t1_rate":0.39329860841172354,"mean_pred_t1":0.38837242335490924},"Q":{"label":"Qwen3-8B-Base","status":"pending","metrics":{"top1_accuracy":null,"top3_accuracy":null,"mean_nll":null,"brier_multiclass":null,"mean_rank":null,"t1_event_logloss":null,"t1_event_brier":null,"t3_event_logloss":null,"t3_event_brier":null,"median_rank":null},"ci95":{}},"Q0":{"label":"Qwen3-8B-Base (position-only ablation)","status":"pending","metrics":{"top1_accuracy":null,"top3_accuracy":null,"mean_nll":null,"brier_multiclass":null,"mean_rank":null,"t1_event_logloss":null,"t1_event_brier":null,"t3_event_logloss":null,"t3_event_brier":null,"median_rank":null},"ci95":{}},"M":{"label":"Maia-3 79M","status":"ok","metrics":{"top1_accuracy":0.5693850579180596,"top3_accuracy":0.8438933374795926,"mean_nll":1.2798484787812183,"brier_multiclass":0.5586745961246925,"mean_rank":2.234859675036928,"t1_event_logloss":0.4224938075569124,"t1_event_brier":0.13517787661135416,"t3_event_logloss":0.4226857887229042,"t3_event_brier":0.13987266077897975,"median_rank":1.0,"mean_entropy":1.2660095520917982},"ci95":{"top1_accuracy":[0.5596850848692518,0.5792920191530658],"top3_accuracy":[0.8364744983294716,0.8513879826706915],"mean_nll":[1.250935736376125,1.3087480742035156],"brier_multiclass":[0.5488384885088472,0.5686572866791366],"mean_rank":[2.182411520022295,2.2910110348416155],"t1_event_logloss":[0.40924638310357186,0.43676923876518164],"t1_event_brier":[0.13085535053241706,0.13983081506445466],"t3_event_logloss":[0.4127660431302018,0.43229663039861677],"t3_event_brier":[0.13621867208674976,0.14348226992677557]},"by_band":[{"band":"1000-1399","n":2397,"top1_accuracy":0.5335836462244472,"mean_nll":1.3942462828775513},{"band":"1400-1799","n":2942,"top1_accuracy":0.5625424881033311,"mean_nll":1.2977616865593287},{"band":"1800-2199","n":4435,"top1_accuracy":0.580608793686584,"mean_nll":1.2308066548916359},{"band":"2200-2699","n":3089,"top1_accuracy":0.5875687924894788,"mean_nll":1.2444286973788836}],"observed_t1_rate":0.39329860841172354,"mean_pred_t1":0.3682693037656633},"M5":{"label":"Maia-3 5M","status":"ok","metrics":{"top1_accuracy":0.5554691751535412,"top3_accuracy":0.8248464588354194,"mean_nll":1.3501596768028683,"brier_multiclass":0.578607917937292,"mean_rank":2.368265567907953,"t1_event_logloss":0.4414609244227981,"t1_event_brier":0.14114516568630273,"t3_event_logloss":0.4407341000205279,"t3_event_brier":0.14609762034313645,"median_rank":1.0,"mean_entropy":1.3394349940055066},"ci95":{"top1_accuracy":[0.5451524254474203,0.5651894876383764],"top3_accuracy":[0.8173982484690333,0.8324571216339112],"mean_nll":[1.3202778153412702,1.3790900929024164],"brier_multiclass":[0.5687203302626375,0.588701601578859],"mean_rank":[2.307284894705213,2.4275690358022515],"t1_event_logloss":[0.4271768717507915,0.45706906158431],"t1_event_brier":[0.13639794552042292,0.1463079941402938],"t3_event_logloss":[0.43133230054646987,0.44991868039294475],"t3_event_brier":[0.14254079904306963,0.149619388295698]},"by_band":[{"band":"1000-1399","n":2397,"top1_accuracy":0.5235711305798916,"mean_nll":1.4359612936947812},{"band":"1400-1799","n":2942,"top1_accuracy":0.5516655336505778,"mean_nll":1.3588618188378667},{"band":"1800-2199","n":4435,"top1_accuracy":0.5697857948139797,"mean_nll":1.3069666236556825},{"band":"2200-2699","n":3089,"top1_accuracy":0.563289090320492,"mean_nll":1.3373053592732111}],"observed_t1_rate":0.39329860841172354,"mean_pred_t1":0.3543437653735521}},"bootstrap":{"n_boot":2000,"seed":20261017,"cluster":"source game"},"caveats":["Source games are presumed human; their absence of real assistance is not established.","Substitutions are controlled decision-level substitutions evaluated in the original positions; games are not replayed.","Bootstrap resamples source games with the fitted/frozen models held fixed; refit variability is not captured.","Several comparisons are reported; no multiplicity correction is applied.","The Regan-inspired reconstruction is not Professor Regan's analyzer and outputs no per-move distribution."]}