{
  "version": 1,
  "evaluatedAt": "2026-09-21",
  "model": "jev-lite",
  "benchmark": "JevBench public subset",
  "scope": "231 public tasks only; no official full-suite composite or rank",
  "benchmarkRepository": "https://github.com/fstandhartinger/jevbench",
  "benchmarkCommit": "a1799db9673bf59010bc698753e729b04132431f",
  "sourceAggregateSha256": "a74f1ccfa7676d8e743cdebff83b5cc4925ed38342ac8ddff036d8e4ec076083",
  "historicalEndpoint": "https://api.autograd.run/v1/chat/completions",
  "reproductionEndpoint": "https://instinct.zoowork.ai/v1/chat/completions",
  "hardware": "1 × NVIDIA H200",
  "precision": "BF16",
  "inputLimitTokens": 8192,
  "quality": {
    "correct": 182,
    "cases": 231,
    "accuracy": 0.7878787878787878,
    "brier": 0.2971855507015192,
    "ece": 0.11840704646482116,
    "tiers": [
      {
        "name": "easy",
        "correct": 48,
        "cases": 48,
        "accuracy": 1
      },
      {
        "name": "original",
        "correct": 66,
        "cases": 72,
        "accuracy": 0.9166666666666666
      },
      {
        "name": "hard",
        "correct": 68,
        "cases": 111,
        "accuracy": 0.6126126126126126
      }
    ]
  },
  "sweep": [
    {
      "concurrency": 1,
      "requests": 231,
      "correct": 182,
      "accuracy": 0.7878787878787878,
      "successfulRps": 10.573799432369992,
      "p50Ms": 77.31503201648593,
      "p95Ms": 149.58279253914952,
      "elapsedSeconds": 21.846451833844185,
      "httpFailures": 0
    },
    {
      "concurrency": 8,
      "requests": 231,
      "correct": 183,
      "accuracy": 0.7922077922077922,
      "successfulRps": 25.80553017321013,
      "p50Ms": 228.85851189494133,
      "p95Ms": 588.3519493509084,
      "elapsedSeconds": 8.951569622848183,
      "httpFailures": 0
    },
    {
      "concurrency": 32,
      "requests": 231,
      "correct": 184,
      "accuracy": 0.7965367965367965,
      "successfulRps": 36.18672789032203,
      "p50Ms": 831.7904490977526,
      "p95Ms": 1350.5295254290104,
      "elapsedSeconds": 6.383555890992284,
      "httpFailures": 0
    },
    {
      "concurrency": 64,
      "requests": 231,
      "correct": 184,
      "accuracy": 0.7965367965367965,
      "successfulRps": 37.98025117005709,
      "p50Ms": 1362.9488600417972,
      "p95Ms": 2481.3536070287228,
      "elapsedSeconds": 6.082108276896179,
      "httpFailures": 0
    }
  ],
  "extendedLoad": {
    "concurrency": 64,
    "requests": 2310,
    "correct": 1840,
    "accuracy": 0.7965367965367965,
    "successfulRps": 40.82350372969255,
    "p50Ms": 1613.1590795703232,
    "p95Ms": 2209.5728620653963,
    "elapsedSeconds": 56.58505000686273,
    "httpFailures": 0
  },
  "caveats": [
    "Only the 231 public cases, not the full 534-case leaderboard. No official rank, composite score, or cost score.",
    "Quality uses the fixed serial baseline; concurrent accuracy is not substituted for it.",
    "Historical measurements used the previous public hostname. The Instinct endpoint uses the same serving system; these are not latency measurements of the new hostname.",
    "Public HTTPS round trip from one client host. Pooled connections, no retries, no artificial latency adjustment.",
    "Repeated inputs may benefit from prefix caching. Load results do not establish a serving capacity ceiling.",
    "Batching changed some close or tied decisions. Probabilities are not calibrated confidence guarantees."
  ]
}