{
 "generated_at": "2026-11-04T09:18:52",
 "game": "lotus_game",
 "question": {
  "type": "choice",
  "instructions": "This is a screenshot of a racing game. You drive the red sports car at the bottom centre of the screen. Stay on the grey road, do not hit the other cars or the trees and signs beside the road, and drive as fast as you can. What should the driver do now?",
  "criteria": {
   "left": "Steer left: the road bends to the left, the red car is too far right, or a slower car ahead should be passed on its left",
   "straight": "Keep straight at full throttle: the road ahead is straight and the red car is well placed on it",
   "right": "Steer right: the road bends to the right, the red car is too far left, or a slower car ahead should be passed on its right",
   "brake": "Brake: a sharp bend or a slower car right in front is too close to take at this speed"
  }
 },
 "models": [
  {
   "model": "oracle",
   "api": "baseline",
   "run_id": "20261103-160733-4160",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": null,
   "finished": 3,
   "races": 3,
   "score": 128.77222222222295,
   "progress": 1,
   "finish_time": 61.68333333333111,
   "position": 1,
   "avg_speed_kmh": 210.49838492251192,
   "collisions": 0.3333333333333333,
   "crashes": 0,
   "offroad_ratio": 0.005392933235485155,
   "oracle_agreement": 1,
   "latency_ms_median": null,
   "latency_ms_p95": null,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 58.53333333333129,
     "race_time": 58.53333333333129,
     "position": 1,
     "avg_speed_kmh": 217.45733788395904,
     "collisions": 0,
     "crashes": 0,
     "offroad_ratio": 0.0,
     "decisions": 586,
     "errors": 0,
     "oracle_agreement": 1,
     "actions": {
      "left": 83,
      "straight": 380,
      "right": 82,
      "brake": 41
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 130.48888888888956
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 60.96666666666449,
     "race_time": 60.96666666666449,
     "position": 1,
     "avg_speed_kmh": 208.74726775956285,
     "collisions": 0,
     "crashes": 0,
     "offroad_ratio": 0.009568069983597594,
     "decisions": 610,
     "errors": 0,
     "oracle_agreement": 1,
     "actions": {
      "left": 66,
      "straight": 378,
      "right": 121,
      "brake": 45
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 129.6777777777785
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": true,
     "progress": 1,
     "finish_time": 65.54999999999757,
     "race_time": 65.54999999999757,
     "position": 1,
     "avg_speed_kmh": 205.29054912401384,
     "collisions": 1,
     "crashes": 0,
     "offroad_ratio": 0.006610729722857869,
     "decisions": 656,
     "errors": 0,
     "oracle_agreement": 1,
     "actions": {
      "left": 12,
      "straight": 379,
      "right": 220,
      "brake": 45
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 126.1500000000008
    }
   ],
   "score_vs_straight": 101.97403199930639
  },
  {
   "model": "basal-omni-1.0-max-FP8",
   "api": "basal",
   "run_id": "20261104-090350-b62a",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": 48.286799999914365,
   "finished": 3,
   "races": 3,
   "score": 60.83703703703873,
   "progress": 1,
   "finish_time": 119.48888888888378,
   "position": 12,
   "avg_speed_kmh": 142.09247177542426,
   "collisions": 1.3333333333333333,
   "crashes": 9.333333333333334,
   "offroad_ratio": 0.20034096866917742,
   "oracle_agreement": 0.20180270069844927,
   "latency_ms_median": 253.3562666662874,
   "latency_ms_p95": 278.0852666671005,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 108.13333333332848,
     "race_time": 108.13333333332848,
     "position": 12,
     "avg_speed_kmh": 146.16309287773004,
     "collisions": 3,
     "crashes": 7,
     "offroad_ratio": 0.218711467324291,
     "decisions": 873,
     "errors": 0,
     "oracle_agreement": 0.13172966781214204,
     "actions": {
      "left": 96,
      "straight": 689,
      "right": 88,
      "brake": 0
     },
     "latency_ms_median": 256.21119999959774,
     "latency_ms_p95": 278.2193000002735,
     "score": 72.95555555555717
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 129.38333333332844,
     "race_time": 129.38333333332844,
     "position": 12,
     "avg_speed_kmh": 132.08979662185453,
     "collisions": 0,
     "crashes": 11,
     "offroad_ratio": 0.2283910859203916,
     "decisions": 967,
     "errors": 0,
     "oracle_agreement": 0.22750775594622544,
     "actions": {
      "left": 129,
      "straight": 716,
      "right": 122,
      "brake": 0
     },
     "latency_ms_median": 256.4122999992833,
     "latency_ms_p95": 288.98580000077345,
     "score": 51.872222222223854
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": true,
     "progress": 1,
     "finish_time": 120.94999999999442,
     "race_time": 120.94999999999442,
     "position": 12,
     "avg_speed_kmh": 148.02452582668818,
     "collisions": 1,
     "crashes": 10,
     "offroad_ratio": 0.15392035276284965,
     "decisions": 914,
     "errors": 0,
     "oracle_agreement": 0.2461706783369803,
     "actions": {
      "left": 25,
      "straight": 703,
      "right": 186,
      "brake": 0
     },
     "latency_ms_median": 247.44529999998122,
     "latency_ms_p95": 267.05070000025444,
     "score": 57.683333333335185
    }
   ],
   "score_vs_straight": 34.03884681412217
  },
  {
   "model": "d3:4b",
   "api": "ollama",
   "run_id": "20261103-164355-0af9",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": 11149.982300004922,
   "finished": 2,
   "races": 3,
   "score": 57.5165944342416,
   "progress": 0.9776289073053677,
   "finish_time": 112.10833333332826,
   "position": 11.333333333333334,
   "avg_speed_kmh": 132.8986932477042,
   "collisions": 1,
   "crashes": 9.333333333333334,
   "offroad_ratio": 0.24264257021294894,
   "oracle_agreement": 0.25290149226550357,
   "latency_ms_median": 142.2249333348494,
   "latency_ms_p95": 161.6565666651392,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.9328867219161029,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 109.92197109471094,
     "collisions": 0,
     "crashes": 14,
     "offroad_ratio": 0.3248888888888889,
     "decisions": 1084,
     "errors": 0,
     "oracle_agreement": 0.11900369003690037,
     "actions": {
      "left": 338,
      "straight": 590,
      "right": 151,
      "brake": 5
     },
     "latency_ms_median": 143.11480000105803,
     "latency_ms_p95": 163.6075999995228,
     "score": 23.28867219161029
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 116.91666666666131,
     "race_time": 116.91666666666131,
     "position": 12,
     "avg_speed_kmh": 132.6902528169799,
     "collisions": 1,
     "crashes": 7,
     "offroad_ratio": 0.26557377049180325,
     "decisions": 962,
     "errors": 0,
     "oracle_agreement": 0.26715176715176714,
     "actions": {
      "left": 206,
      "straight": 574,
      "right": 166,
      "brake": 16
     },
     "latency_ms_median": 141.48710000154097,
     "latency_ms_p95": 160.85439999733353,
     "score": 74.02777777777956
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": true,
     "progress": 1,
     "finish_time": 107.2999999999952,
     "race_time": 107.2999999999952,
     "position": 10,
     "avg_speed_kmh": 156.08385583142172,
     "collisions": 2,
     "crashes": 7,
     "offroad_ratio": 0.1374650512581547,
     "decisions": 867,
     "errors": 0,
     "oracle_agreement": 0.37254901960784315,
     "actions": {
      "left": 46,
      "straight": 528,
      "right": 283,
      "brake": 10
     },
     "latency_ms_median": 142.07290000194917,
     "latency_ms_p95": 160.5076999985613,
     "score": 75.23333333333494
    }
   ],
   "score_vs_straight": 30.718404211325034
  },
  {
   "model": "d3:9b",
   "api": "ollama",
   "run_id": "20261103-164355-0af9",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": 11208.79650000279,
   "finished": 1,
   "races": 3,
   "score": 44.27452463047283,
   "progress": 0.9049674685269465,
   "finish_time": 130.9999999999964,
   "position": 12,
   "avg_speed_kmh": 102.65073668384801,
   "collisions": 0,
   "crashes": 9.666666666666666,
   "offroad_ratio": 0.41042239185750634,
   "oracle_agreement": 0.2645690088194347,
   "latency_ms_median": 200.07191666809376,
   "latency_ms_p95": 216.1038666672539,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.7731632246483433,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 86.19063228438229,
     "collisions": 0,
     "crashes": 12,
     "offroad_ratio": 0.4398888888888889,
     "decisions": 1144,
     "errors": 0,
     "oracle_agreement": 0.17045454545454544,
     "actions": {
      "left": 336,
      "straight": 533,
      "right": 189,
      "brake": 86
     },
     "latency_ms_median": 199.70165000268025,
     "latency_ms_p95": 217.21410000463948,
     "score": 17.31632246483433
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.9417391809324963,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 102.55927030096537,
     "collisions": 0,
     "crashes": 11,
     "offroad_ratio": 0.3927777777777778,
     "decisions": 1174,
     "errors": 0,
     "oracle_agreement": 0.30579216354344124,
     "actions": {
      "left": 304,
      "straight": 612,
      "right": 247,
      "brake": 11
     },
     "latency_ms_median": 201.39275000110501,
     "latency_ms_p95": 215.23329999763519,
     "score": 39.173918093249625
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": true,
     "progress": 1,
     "finish_time": 130.9999999999964,
     "race_time": 130.9999999999964,
     "position": 12,
     "avg_speed_kmh": 119.20230746619636,
     "collisions": 0,
     "crashes": 6,
     "offroad_ratio": 0.3986005089058524,
     "decisions": 1134,
     "errors": 0,
     "oracle_agreement": 0.31746031746031744,
     "actions": {
      "left": 117,
      "straight": 710,
      "right": 306,
      "brake": 1
     },
     "latency_ms_median": 199.12135000049602,
     "latency_ms_p95": 215.86419999948703,
     "score": 76.33333333333454
    }
   ],
   "score_vs_straight": 17.47633440755627
  },
  {
   "model": "straight",
   "api": "baseline",
   "run_id": "20261103-160733-4160",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": null,
   "finished": 1,
   "races": 3,
   "score": 26.798190222916563,
   "progress": 0.8962967170439893,
   "finish_time": 145.48333333334125,
   "position": 12,
   "avg_speed_kmh": 104.54333989283981,
   "collisions": 0,
   "crashes": 12.666666666666666,
   "offroad_ratio": 0.36043447766703585,
   "oracle_agreement": 0.10582725354193456,
   "latency_ms_median": null,
   "latency_ms_p95": null,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 145.48333333334125,
     "race_time": 145.48333333334125,
     "position": 12,
     "avg_speed_kmh": 116.30654219793564,
     "collisions": 0,
     "crashes": 12,
     "offroad_ratio": 0.3089700996677741,
     "decisions": 1098,
     "errors": 0,
     "oracle_agreement": 0.08014571948998178,
     "actions": {
      "left": 0,
      "straight": 1098,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 41.50555555555292
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.8340670497849558,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 90.63903097470872,
     "collisions": 0,
     "crashes": 11,
     "offroad_ratio": 0.45322222222222225,
     "decisions": 1173,
     "errors": 0,
     "oracle_agreement": 0.09633418584825235,
     "actions": {
      "left": 0,
      "straight": 1173,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 28.406704978495583
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": false,
     "progress": 0.854823101347012,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 106.68444650587507,
     "collisions": 0,
     "crashes": 15,
     "offroad_ratio": 0.3191111111111111,
     "decisions": 1078,
     "errors": 0,
     "oracle_agreement": 0.14100185528756956,
     "actions": {
      "left": 0,
      "straight": 1078,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 10.482310134701194
    }
   ],
   "score_vs_straight": 0.0
  },
  {
   "model": "random",
   "api": "baseline",
   "run_id": "20261103-160733-4160",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": null,
   "finished": 0,
   "races": 3,
   "score": 20.64250884419433,
   "progress": 0.2564250884419433,
   "finish_time": null,
   "position": 12,
   "avg_speed_kmh": 22.68872291938271,
   "collisions": 0,
   "crashes": 1,
   "offroad_ratio": 0.2762962962962963,
   "oracle_agreement": 0.23754875901314204,
   "latency_ms_median": null,
   "latency_ms_p95": null,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.27977360716200383,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 24.48157596371882,
     "collisions": 0,
     "crashes": 1,
     "offroad_ratio": 0.17577777777777778,
     "decisions": 1470,
     "errors": 0,
     "oracle_agreement": 0.24829931972789115,
     "actions": {
      "left": 368,
      "straight": 357,
      "right": 383,
      "brake": 362
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 22.977360716200383
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.2766572137193815,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 24.225703905540417,
     "collisions": 0,
     "crashes": 2,
     "offroad_ratio": 0.5757777777777778,
     "decisions": 1468,
     "errors": 0,
     "oracle_agreement": 0.23501362397820164,
     "actions": {
      "left": 367,
      "straight": 394,
      "right": 342,
      "brake": 365
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 17.66572137193815
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": false,
     "progress": 0.21284444444444456,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 19.358888888888888,
     "collisions": 0,
     "crashes": 0,
     "offroad_ratio": 0.07733333333333334,
     "decisions": 1500,
     "errors": 0,
     "oracle_agreement": 0.22933333333333333,
     "actions": {
      "left": 375,
      "straight": 358,
      "right": 381,
      "brake": 386
     },
     "latency_ms_median": null,
     "latency_ms_p95": null,
     "score": 21.284444444444457
    }
   ],
   "score_vs_straight": -6.155681378722232
  },
  {
   "model": "basal-omni-1.0-mini",
   "api": "basal",
   "run_id": "20261103-170950-3f9f",
   "game_version": 3,
   "hold_frames": 6,
   "realtime": false,
   "telemetry": false,
   "laps": 1,
   "seeds": [
    1,
    2,
    3
   ],
   "load_ms": 989.2646000007517,
   "finished": 1,
   "races": 3,
   "score": 17.239445641506805,
   "progress": 0.8543574193780427,
   "finish_time": 148.7666666666772,
   "position": 12,
   "avg_speed_kmh": 101.28673037388505,
   "collisions": 0,
   "crashes": 13.666666666666666,
   "offroad_ratio": 0.3321941145716633,
   "oracle_agreement": 0.10346837846274566,
   "latency_ms_median": 283.7553833342099,
   "latency_ms_p95": 356.6630666682613,
   "errors": 0,
   "per_seed": [
    {
     "seed": 1,
     "game_version": 3,
     "segments": 3050,
     "finished": true,
     "progress": 1,
     "finish_time": 148.7666666666772,
     "race_time": 148.7666666666772,
     "position": 12,
     "avg_speed_kmh": 112.88501177856301,
     "collisions": 0,
     "crashes": 12,
     "offroad_ratio": 0.3275823437149899,
     "decisions": 1132,
     "errors": 0,
     "oracle_agreement": 0.09098939929328621,
     "actions": {
      "left": 56,
      "straight": 1076,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": 294.70579999906477,
     "latency_ms_p95": 448.024000004807,
     "score": 40.4111111111076
    },
    {
     "seed": 2,
     "game_version": 3,
     "segments": 3050,
     "finished": false,
     "progress": 0.8706801645390537,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 99.75613405146619,
     "collisions": 0,
     "crashes": 13,
     "offroad_ratio": 0.3497777777777778,
     "decisions": 1114,
     "errors": 0,
     "oracle_agreement": 0.11579892280071813,
     "actions": {
      "left": 60,
      "straight": 1054,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": 278.9839500001108,
     "latency_ms_p95": 311.33499999850756,
     "score": 22.068016453905372
    },
    {
     "seed": 3,
     "game_version": 3,
     "segments": 3225,
     "finished": false,
     "progress": 0.6923920935950744,
     "finish_time": null,
     "race_time": 150.0000000000115,
     "position": 12,
     "avg_speed_kmh": 91.21904529162593,
     "collisions": 0,
     "crashes": 16,
     "offroad_ratio": 0.31922222222222224,
     "decisions": 1023,
     "errors": 0,
     "oracle_agreement": 0.10361681329423265,
     "actions": {
      "left": 98,
      "straight": 925,
      "right": 0,
      "brake": 0
     },
     "latency_ms_median": 277.5764000034542,
     "latency_ms_p95": 310.63020000146935,
     "score": -10.760790640492559
    }
   ],
   "score_vs_straight": -9.558744581409758
  }
 ]
}