{
  "kind": "historical-example",
  "date": "2026-09-21",
  "protocol": "historical-local-playwright",
  "notice": "Real historical measurements, not browser-choice-v1 or current live results. Aggregates use all 1200 attempts; only 20 selected attempt records are published here.",
  "selection": "For each experiment: all four APIs on the first scheduled question version; the valid attempt nearest each API median latency (ties by sequence); plus every failed attempt. Deduplicated by sequence.",
  "experiments": [
    {
      "id": "calculus",
      "title": "Calculus — five sets",
      "runId": "calculus-expanded-2026-09-21T09-28-48-078Z",
      "sourceTrialsSha256": "1378143334cc7fa44e1d6ea1ed904310d01984714f7798520a6248bbb57bad03",
      "baseQuestions": 50,
      "rounds": 3,
      "totalAttempts": 600,
      "results": [
        {
          "arm": "jev_text",
          "label": "Jev text",
          "n": 150,
          "correct": 127,
          "accuracy": 0.8466666666666667,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.8466666666666667,
          "http": {
            "meanMs": 863.0106921933299,
            "p50Ms": 668.1504790000035,
            "p95Ms": 1393.2947876499943
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 44
            },
            {
              "round": 2,
              "n": 50,
              "correct": 40
            },
            {
              "round": 3,
              "n": 50,
              "correct": 43
            }
          ]
        },
        {
          "arm": "openjev_text",
          "label": "OpenJev text",
          "n": 150,
          "correct": 79,
          "accuracy": 0.5266666666666666,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.5266666666666666,
          "http": {
            "meanMs": 1597.1571914133349,
            "p50Ms": 1601.3201875000377,
            "p95Ms": 1713.148231249966
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 27
            },
            {
              "round": 2,
              "n": 50,
              "correct": 24
            },
            {
              "round": 3,
              "n": 50,
              "correct": 28
            }
          ]
        },
        {
          "arm": "openjev_image",
          "label": "OpenJev image",
          "n": 150,
          "correct": 72,
          "accuracy": 0.48,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.48,
          "http": {
            "meanMs": 4163.076032993328,
            "p50Ms": 4228.583000000042,
            "p95Ms": 4362.398054199986
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 24
            },
            {
              "round": 2,
              "n": 50,
              "correct": 24
            },
            {
              "round": 3,
              "n": 50,
              "correct": 24
            }
          ]
        },
        {
          "arm": "openjev_text_image",
          "label": "OpenJev text + image",
          "n": 150,
          "correct": 83,
          "accuracy": 0.5533333333333333,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.5533333333333333,
          "http": {
            "meanMs": 4411.684193666674,
            "p50Ms": 4482.086333500018,
            "p95Ms": 4615.527283149948
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 28
            },
            {
              "round": 2,
              "n": 50,
              "correct": 26
            },
            {
              "round": 3,
              "n": 50,
              "correct": 29
            }
          ]
        }
      ],
      "exampleQuestion": {
        "id": "S2-Q01-R1",
        "baseId": "S2-Q01",
        "set": 2,
        "round": 1,
        "topic": "Polynomial derivative",
        "prompt": "For f(x) = 3x³ + 4x² − 5x, find f′(1).",
        "options": {
          "A": "8",
          "B": "2",
          "C": "12",
          "D": "17"
        }
      },
      "exampleGold": "C",
      "sample": [
        {
          "sequence": 1,
          "arm": "jev_text",
          "questionId": "S2-Q01-R1",
          "baseId": "S2-Q01",
          "set": 2,
          "round": 1,
          "choice": "D",
          "gold": "C",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:28:55.283Z",
          "receivedAt": "2026-09-21T09:28:58.361Z",
          "latencyMs": 3077.7552909999995,
          "actionMs": 51.772375000000466,
          "inputCaptureMs": 73.63224999999966,
          "screenshotMs": 47.85466599999927,
          "trialTotalMs": 3258.0996670000013,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 2,
          "arm": "openjev_text",
          "questionId": "S2-Q01-R1",
          "baseId": "S2-Q01",
          "set": 2,
          "round": 1,
          "choice": "A",
          "gold": "C",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:28:58.531Z",
          "receivedAt": "2026-09-21T09:29:00.347Z",
          "latencyMs": 1816.3088749999988,
          "actionMs": 15.434291999999914,
          "inputCaptureMs": 63.35416599999917,
          "screenshotMs": 24.191791999999623,
          "trialTotalMs": 1922.6075,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 3,
          "arm": "openjev_image",
          "questionId": "S2-Q01-R1",
          "baseId": "S2-Q01",
          "set": 2,
          "round": 1,
          "choice": "A",
          "gold": "C",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:29:00.460Z",
          "receivedAt": "2026-09-21T09:29:04.713Z",
          "latencyMs": 4252.544790999998,
          "actionMs": 16.05412499999875,
          "inputCaptureMs": 69.60449999999946,
          "screenshotMs": 25.43304200000057,
          "trialTotalMs": 4366.730416999999,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 4,
          "arm": "openjev_text_image",
          "questionId": "S2-Q01-R1",
          "baseId": "S2-Q01",
          "set": 2,
          "round": 1,
          "choice": "A",
          "gold": "C",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:29:04.820Z",
          "receivedAt": "2026-09-21T09:29:09.433Z",
          "latencyMs": 4613.546000000002,
          "actionMs": 12.141707999999198,
          "inputCaptureMs": 61.488875000002736,
          "screenshotMs": 27.639917000000423,
          "trialTotalMs": 4717.791999999998,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 138,
          "arm": "openjev_text_image",
          "questionId": "S1-Q04-R1",
          "baseId": "S1-Q04",
          "set": 1,
          "round": 1,
          "choice": "D",
          "gold": "D",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:35:26.146Z",
          "receivedAt": "2026-09-21T09:35:30.630Z",
          "latencyMs": 4484.470792000007,
          "actionMs": 12.722750000015367,
          "inputCaptureMs": 61.63758399995277,
          "screenshotMs": 24.552832999965176,
          "trialTotalMs": 4586.28662499995,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 158,
          "arm": "jev_text",
          "questionId": "S4-Q04-R1",
          "baseId": "S4-Q04",
          "set": 4,
          "round": 1,
          "choice": "B",
          "gold": "B",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:36:26.462Z",
          "receivedAt": "2026-09-21T09:36:27.131Z",
          "latencyMs": 668.4007909999928,
          "actionMs": 18.159832999983337,
          "inputCaptureMs": 61.38620800001081,
          "screenshotMs": 32.18183299998054,
          "trialTotalMs": 785.217249999987,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 172,
          "arm": "openjev_text",
          "questionId": "S5-Q05-R1",
          "baseId": "S5-Q05",
          "set": 5,
          "round": 1,
          "choice": "B",
          "gold": "A",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:37:08.667Z",
          "receivedAt": "2026-09-21T09:37:10.270Z",
          "latencyMs": 1602.8634170000441,
          "actionMs": 9.525250000006054,
          "inputCaptureMs": 61.7987500000163,
          "screenshotMs": 25.13949999999022,
          "trialTotalMs": 1702.2735000000102,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 198,
          "arm": "openjev_image",
          "questionId": "S5-Q07-R1",
          "baseId": "S5-Q07",
          "set": 5,
          "round": 1,
          "choice": "C",
          "gold": "C",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T09:38:20.102Z",
          "receivedAt": "2026-09-21T09:38:24.331Z",
          "latencyMs": 4228.927833000082,
          "actionMs": 16.559499999973923,
          "inputCaptureMs": 63.93083299999125,
          "screenshotMs": 25.485708999913186,
          "trialTotalMs": 4337.716707999934,
          "selectionReason": "nearest valid latency median"
        }
      ]
    },
    {
      "id": "scenarios",
      "title": "Decision scenarios — five sets",
      "runId": "scenarios-2026-09-21T10-10-33-638Z",
      "sourceTrialsSha256": "93ac2cf38d2f53dfef6a996f373ebd72d2a7e854749a415f23a613942a35f2f5",
      "baseQuestions": 50,
      "rounds": 3,
      "totalAttempts": 600,
      "results": [
        {
          "arm": "jev_text",
          "label": "Jev text",
          "n": 150,
          "correct": 122,
          "accuracy": 0.8133333333333334,
          "valid": 146,
          "errors": 4,
          "validAccuracy": 0.8356164383561644,
          "http": {
            "meanMs": 1147.888283306666,
            "p50Ms": 724.5596039999509,
            "p95Ms": 2194.3414586999947
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 39
            },
            {
              "round": 2,
              "n": 50,
              "correct": 43
            },
            {
              "round": 3,
              "n": 50,
              "correct": 40
            }
          ]
        },
        {
          "arm": "openjev_text",
          "label": "OpenJev text",
          "n": 150,
          "correct": 93,
          "accuracy": 0.62,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.62,
          "http": {
            "meanMs": 2331.741099986677,
            "p50Ms": 2204.164208000002,
            "p95Ms": 2887.7617125000106
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 32
            },
            {
              "round": 2,
              "n": 50,
              "correct": 30
            },
            {
              "round": 3,
              "n": 50,
              "correct": 31
            }
          ]
        },
        {
          "arm": "openjev_image",
          "label": "OpenJev image",
          "n": 150,
          "correct": 90,
          "accuracy": 0.6,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.6,
          "http": {
            "meanMs": 4815.153669953336,
            "p50Ms": 4779.887042000017,
            "p95Ms": 4962.790609750001
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 30
            },
            {
              "round": 2,
              "n": 50,
              "correct": 32
            },
            {
              "round": 3,
              "n": 50,
              "correct": 28
            }
          ]
        },
        {
          "arm": "openjev_text_image",
          "label": "OpenJev text + image",
          "n": 150,
          "correct": 105,
          "accuracy": 0.7,
          "valid": 150,
          "errors": 0,
          "validAccuracy": 0.7,
          "http": {
            "meanMs": 5874.591054393328,
            "p50Ms": 5734.7184789999155,
            "p95Ms": 6421.586345949986
          },
          "rounds": [
            {
              "round": 1,
              "n": 50,
              "correct": 37
            },
            {
              "round": 2,
              "n": 50,
              "correct": 35
            },
            {
              "round": 3,
              "n": 50,
              "correct": 33
            }
          ]
        }
      ],
      "exampleQuestion": {
        "id": "D1-Q08-R1",
        "baseId": "D1-Q08",
        "set": 1,
        "round": 1,
        "topic": "Parallel planning",
        "prompt": "School poster. One person; hands-on steps cannot overlap or pause. Alone phases can overlap anything.\nDesign: 2m hands.\nPrint: 1m hands, then 5m alone; after Design fully finish.\nFrame: 4m hands.\nMount: 2m hands; after Print + Frame fully finish.\nTidy: 2m hands.\nFollow each listed order, waiting if needed. Which finishes all work earliest?",
        "options": {
          "A": "Design → Tidy → Print → Frame → Mount",
          "B": "Design → Print → Frame → Mount → Tidy",
          "C": "Tidy → Design → Print → Frame → Mount",
          "D": "Design → Print → Tidy → Frame → Mount"
        }
      },
      "exampleGold": "D",
      "sample": [
        {
          "sequence": 1,
          "arm": "jev_text",
          "questionId": "D1-Q08-R1",
          "baseId": "D1-Q08",
          "set": 1,
          "round": 1,
          "choice": "B",
          "gold": "D",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:10:43.082Z",
          "receivedAt": "2026-09-21T10:10:44.019Z",
          "latencyMs": 937.4352500000005,
          "actionMs": 19.502291999999215,
          "inputCaptureMs": 84.66012500000033,
          "screenshotMs": 80.1568329999991,
          "trialTotalMs": 1127.225375,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 2,
          "arm": "openjev_text",
          "questionId": "D1-Q08-R1",
          "baseId": "D1-Q08",
          "set": 1,
          "round": 1,
          "choice": "B",
          "gold": "D",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:10:44.205Z",
          "receivedAt": "2026-09-21T10:10:48.147Z",
          "latencyMs": 3942.377875,
          "actionMs": 14.41229100000055,
          "inputCaptureMs": 80.11954200000037,
          "screenshotMs": 32.583457999999155,
          "trialTotalMs": 4073.0130420000005,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 3,
          "arm": "openjev_image",
          "questionId": "D1-Q08-R1",
          "baseId": "D1-Q08",
          "set": 1,
          "round": 1,
          "choice": "B",
          "gold": "D",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:10:48.279Z",
          "receivedAt": "2026-09-21T10:10:53.657Z",
          "latencyMs": 5378.661791,
          "actionMs": 12.844333000000915,
          "inputCaptureMs": 80.0222079999985,
          "screenshotMs": 32.376791999999114,
          "trialTotalMs": 5507.171082999999,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 4,
          "arm": "openjev_text_image",
          "questionId": "D1-Q08-R1",
          "baseId": "D1-Q08",
          "set": 1,
          "round": 1,
          "choice": "B",
          "gold": "D",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:10:53.787Z",
          "receivedAt": "2026-09-21T10:11:00.900Z",
          "latencyMs": 7111.950165999999,
          "actionMs": 16.0256249999984,
          "inputCaptureMs": 80.7297499999986,
          "screenshotMs": 34.974333000001934,
          "trialTotalMs": 7246.594459,
          "selectionReason": "first shared question"
        },
        {
          "sequence": 78,
          "arm": "jev_text",
          "questionId": "D2-Q03-R1",
          "baseId": "D2-Q03",
          "set": 2,
          "round": 1,
          "choice": null,
          "gold": "A",
          "correct": false,
          "status": null,
          "error": "fetch failed [UND_ERR_CONNECT_TIMEOUT]",
          "requestedAt": "2026-09-21T10:15:27.380Z",
          "receivedAt": "2026-09-21T10:15:37.760Z",
          "latencyMs": 10380.249083000002,
          "actionMs": null,
          "inputCaptureMs": 71.40712500002701,
          "screenshotMs": 44.01399999996647,
          "trialTotalMs": 10502.658791999973,
          "selectionReason": "all failed attempts"
        },
        {
          "sequence": 108,
          "arm": "openjev_text",
          "questionId": "D2-Q02-R1",
          "baseId": "D2-Q02",
          "set": 2,
          "round": 1,
          "choice": "D",
          "gold": "D",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:17:32.678Z",
          "receivedAt": "2026-09-21T10:17:34.882Z",
          "latencyMs": 2203.4226659999695,
          "actionMs": 20.45566699997289,
          "inputCaptureMs": 83.42037499998696,
          "screenshotMs": 43.75345899997046,
          "trialTotalMs": 2356.4290000000037,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 150,
          "arm": "openjev_image",
          "questionId": "D4-Q10-R1",
          "baseId": "D4-Q10",
          "set": 4,
          "round": 1,
          "choice": "C",
          "gold": "D",
          "correct": false,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:20:02.389Z",
          "receivedAt": "2026-09-21T10:20:07.170Z",
          "latencyMs": 4780.264250000007,
          "actionMs": 19.363417000044137,
          "inputCaptureMs": 75.26829200005159,
          "screenshotMs": 42.09025000000838,
          "trialTotalMs": 4922.246041999897,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 152,
          "arm": "jev_text",
          "questionId": "D4-Q10-R1",
          "baseId": "D4-Q10",
          "set": 4,
          "round": 1,
          "choice": "D",
          "gold": "D",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:20:13.132Z",
          "receivedAt": "2026-09-21T10:20:13.849Z",
          "latencyMs": 716.7815410000039,
          "actionMs": 15.824208999983966,
          "inputCaptureMs": 75.98391700000502,
          "screenshotMs": 45.99866700009443,
          "trialTotalMs": 859.3987079999642,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 263,
          "arm": "openjev_text_image",
          "questionId": "D5-Q09-R2",
          "baseId": "D5-Q09",
          "set": 5,
          "round": 2,
          "choice": "B",
          "gold": "B",
          "correct": true,
          "status": 200,
          "error": null,
          "requestedAt": "2026-09-21T10:26:47.452Z",
          "receivedAt": "2026-09-21T10:26:53.189Z",
          "latencyMs": 5736.381749999942,
          "actionMs": 17.813709000009112,
          "inputCaptureMs": 76.81449999997858,
          "screenshotMs": 45.55845799995586,
          "trialTotalMs": 5881.531709000003,
          "selectionReason": "nearest valid latency median"
        },
        {
          "sequence": 568,
          "arm": "jev_text",
          "questionId": "D2-Q06-R3",
          "baseId": "D2-Q06",
          "set": 2,
          "round": 3,
          "choice": null,
          "gold": "B",
          "correct": false,
          "status": null,
          "error": "fetch failed [UND_ERR_CONNECT_TIMEOUT]",
          "requestedAt": "2026-09-21T10:45:09.126Z",
          "receivedAt": "2026-09-21T10:45:19.488Z",
          "latencyMs": 10361.398040999891,
          "actionMs": null,
          "inputCaptureMs": 76.80475000012666,
          "screenshotMs": 37.26558400015347,
          "trialTotalMs": 10479.638375000097,
          "selectionReason": "all failed attempts"
        },
        {
          "sequence": 587,
          "arm": "jev_text",
          "questionId": "D1-Q08-R3",
          "baseId": "D1-Q08",
          "set": 1,
          "round": 3,
          "choice": null,
          "gold": "B",
          "correct": false,
          "status": null,
          "error": "fetch failed [UND_ERR_CONNECT_TIMEOUT]",
          "requestedAt": "2026-09-21T10:46:28.076Z",
          "receivedAt": "2026-09-21T10:46:38.445Z",
          "latencyMs": 10368.177916999906,
          "actionMs": null,
          "inputCaptureMs": 71.9618330001831,
          "screenshotMs": 43.79054199997336,
          "trialTotalMs": 10491.288457999937,
          "selectionReason": "all failed attempts"
        },
        {
          "sequence": 600,
          "arm": "jev_text",
          "questionId": "D3-Q02-R3",
          "baseId": "D3-Q02",
          "set": 3,
          "round": 3,
          "choice": null,
          "gold": "D",
          "correct": false,
          "status": null,
          "error": "fetch failed [UND_ERR_CONNECT_TIMEOUT]",
          "requestedAt": "2026-09-21T10:47:23.642Z",
          "receivedAt": "2026-09-21T10:47:34.028Z",
          "latencyMs": 10385.34895899985,
          "actionMs": null,
          "inputCaptureMs": 64.27691700030118,
          "screenshotMs": 37.150375000201166,
          "trialTotalMs": 10490.461165999994,
          "selectionReason": "all failed attempts"
        }
      ]
    }
  ]
}