{
  "title": "MATO Jev routing study",
  "reviewedAt": "2026-09-29",
  "model": "jev-1.13.0",
  "status": "isolated-experiment",
  "methodology": {
    "language": "Korean",
    "inputs": "synthetic; no customer conversations",
    "repeatsPerCase": 2,
    "warmupsExcluded": true,
    "time": "client measured request to response",
    "cost": "public Standard API price conversion, not subscription billing or customer credits",
    "scope": "classification and fixed-policy selection, not execution success"
  },
  "commonClassification": {
    "n": 72,
    "completed": 72,
    "classification_correct": 72,
    "p50_s": 0.233,
    "p95_s": 0.327,
    "reference_usd": 0.02036286
  },
  "fixedPolicy": {
    "attempts": 84,
    "completed": 84,
    "rawCorrect": 84,
    "direct": 39,
    "directCorrect": 39,
    "escalated": 45,
    "expectedReview": 24,
    "extraEscalations": 21,
    "fallbackCompleted": 45,
    "fallbackCorrect": 45,
    "hybridCorrect": 84,
    "p50Ms": 211,
    "p95Ms": 266,
    "hybridP50Ms": 1838,
    "hybridP95Ms": 4540,
    "jevReferenceUsd": 0.0025630079999999994,
    "fallbackReferenceUsd": 0.2006868
  },
  "thresholds": {
    "confidence": 0.9,
    "probabilityMargin": 0.2,
    "timeoutMs": 2000
  },
  "limitations": [
    "Repeated synthetic cases, not a representative traffic sample",
    "Fixed-policy agreement does not prove optimal model assignment",
    "Worker tasks were not executed in this selection experiment",
    "No production deployment",
    "No same-input full Sonnet-only control for the 84 selection calls"
  ],
  "sources": [
    "https://docs.typesafe.ai/models",
    "https://docs.typesafe.ai/confidence"
  ],
  "followup": {
    "jev-detailed": {
      "attempts": 98,
      "rawCorrect": 84,
      "direct": 36,
      "directCorrect": 36,
      "escalated": 62,
      "fallbackCorrect": 62,
      "finalCorrect": 98,
      "p50Ms": 215,
      "p95Ms": 324,
      "jevReferenceUsd": 0.005427071999999998,
      "fallbackReferenceUsd": 0.4180500000000002,
      "workSamples": {
        "attempted": 20,
        "passed": 20,
        "skipped": 0,
        "costUsd": 0.08890580000000001
      },
      "cohorts": {
        "selection": {
          "attempts": 78,
          "rawCorrect": 64,
          "direct": 20,
          "finalCorrect": 78
        },
        "work": {
          "attempts": 20,
          "rawCorrect": 20,
          "direct": 16,
          "finalCorrect": 20
        }
      }
    },
    "jev-detailed-holdout": {
      "attempts": 56,
      "rawCorrect": 54,
      "direct": 28,
      "directCorrect": 28,
      "escalated": 28,
      "fallbackCorrect": 28,
      "finalCorrect": 56,
      "p50Ms": 204,
      "p95Ms": 259,
      "jevReferenceUsd": 0.004055352,
      "fallbackReferenceUsd": 0.22394000000000003,
      "workSamples": {
        "attempted": 0,
        "passed": 0,
        "skipped": 0,
        "costUsd": 0
      },
      "cohorts": {
        "selection": {
          "attempts": 56,
          "rawCorrect": 54,
          "direct": 28,
          "finalCorrect": 56
        },
        "work": {
          "attempts": 0,
          "rawCorrect": 0,
          "direct": 0,
          "finalCorrect": 0
        }
      }
    },
    "methodology": {
      "questions": [
        "task kind (13 options)",
        "environment",
        "difficulty",
        "handling"
      ],
      "minMarginalAssignmentProbability": 0.85,
      "minProbabilityMargin": 0.2,
      "calibration": "39 new selection cases and 10 small work cases, twice each",
      "holdout": "28 separate cases twice after prompt/policy clarification; no repeated worker execution here",
      "workSamples": "Data/text answers and pure generated JS functions, not customer PC tools or app building",
      "productionRoutingChanged": false
    }
  }
}