{
  "schema": "causal-showcase-manifest/1",
  "slug": "agentic-ai-oversight",
  "version": "1.0",
  "title": "Agentic AI Workflow Reliability and Human Oversight",
  "question": "Which routing, tool, memory and human-review policy maximizes task success while controlling hallucination, latency, cost and operational risk?",
  "lastValidated": "2026-11-18",
  "evidence": "Reviewed",
  "scenario": {
    "params": [
      {
        "key": "route",
        "label": "Tasks routed to the stronger model",
        "min": 0,
        "max": 100,
        "step": 10,
        "default": 0,
        "unit": "%"
      },
      {
        "key": "verifier",
        "label": "Verifier rejection threshold",
        "min": 0,
        "max": 0.9,
        "step": 0.1,
        "default": 0,
        "unit": "score"
      },
      {
        "key": "review",
        "label": "Tasks escalated to human review",
        "min": 0,
        "max": 30,
        "step": 5,
        "default": 0,
        "unit": "%"
      }
    ],
    "alternatives": [
      {
        "name": "Single model",
        "description": "One model, no verifier, no review.",
        "params": {
          "route": 0,
          "verifier": 0,
          "review": 0
        }
      },
      {
        "name": "Routed models",
        "description": "Route 60% of tasks to the stronger model.",
        "params": {
          "route": 60,
          "verifier": 0,
          "review": 0
        }
      },
      {
        "name": "Verifier",
        "description": "Strict verifier, no routing or review.",
        "params": {
          "route": 0,
          "verifier": 0.8,
          "review": 0
        }
      },
      {
        "name": "Heavy human review",
        "description": "Escalate 30% of tasks to reviewers.",
        "params": {
          "route": 0,
          "verifier": 0,
          "review": 30
        }
      },
      {
        "name": "Verifier and review",
        "description": "Strict verifier plus 10% review, no routing.",
        "params": {
          "route": 0,
          "verifier": 0.8,
          "review": 10
        }
      },
      {
        "name": "Optimized policy",
        "description": "Recommended: partial routing, moderate verifier, 10% review.",
        "params": {
          "route": 40,
          "verifier": 0.6,
          "review": 10
        }
      }
    ],
    "metrics": [
      {
        "key": "success",
        "label": "Task success",
        "unit": "%",
        "base": 78,
        "sd": 1.5,
        "higherIsBetter": true,
        "sensitivity": {
          "route": 0.08,
          "verifier": 6,
          "review": 0.25
        },
        "curvature": {
          "verifier": -4
        }
      },
      {
        "key": "severe",
        "label": "Severe-error rate",
        "unit": "per 1k",
        "base": 9,
        "sd": 1,
        "higherIsBetter": false,
        "sensitivity": {
          "route": -0.03,
          "verifier": -8,
          "review": -0.18
        },
        "constraint": {
          "max": 2,
          "label": "Limit 2 per 1k"
        }
      },
      {
        "key": "latency",
        "label": "p95 latency",
        "unit": "s",
        "base": 6,
        "sd": 0.4,
        "higherIsBetter": false,
        "sensitivity": {
          "route": 0.03,
          "verifier": 2.5,
          "review": 0.9
        },
        "constraint": {
          "max": 20,
          "label": "Limit 20 s"
        }
      },
      {
        "key": "cost",
        "label": "Cost per 1k tasks",
        "unit": "$",
        "base": 40,
        "sd": 3,
        "higherIsBetter": false,
        "sensitivity": {
          "route": 1.1,
          "verifier": 18,
          "review": 9
        }
      }
    ],
    "objective": "Maximize task success while keeping severe errors at or below 2 per 1,000 tasks and p95 latency at or below 20 seconds.",
    "runs": 2000
  },
  "technical": {
    "dag": [
      [
        "Task difficulty",
        "Routing"
      ],
      [
        "Task difficulty",
        "Error"
      ],
      [
        "Routing",
        "Output quality"
      ],
      [
        "Retrieval",
        "Output quality"
      ],
      [
        "Output quality",
        "Verifier score"
      ],
      [
        "Verifier score",
        "Rejection"
      ],
      [
        "Rejection",
        "Retry"
      ],
      [
        "Retry",
        "Latency"
      ],
      [
        "Review",
        "Error"
      ],
      [
        "Output quality",
        "Error"
      ]
    ],
    "estimand": "ATE of enabling the verifier at threshold 0.6 on severe errors per 1,000 tasks, from a randomized A/B test",
    "assumptions": [
      "Synthetic or sanitized traces only",
      "Success rubric defined before comparison",
      "Evaluator error modeled explicitly",
      "Provider behavior valid only for tested model and prompt versions"
    ],
    "equations": [
      "Severe rate = p_err · (1 − p_catch_verifier) · (1 − p_catch_review)",
      "Effective load = λ · (1 + retry rate)",
      "Review wait ≈ Kingman approximation for the reviewer queue"
    ],
    "engines": {
      "Causal": "causal-core 3.4.1",
      "Digital twins": "twin-runtime 2.8.0",
      "DES": "des-kernel 5.1.2",
      "Monte Carlo": "mc-sampler 4.0.0",
      "Optimization": "policy-opt 2.2.5"
    },
    "seed": 20261120,
    "validation": [
      {
        "name": "Trace replay agreement",
        "value": "97%"
      },
      {
        "name": "p95 latency error",
        "value": "7.6%"
      },
      {
        "name": "Evaluator agreement (κ)",
        "value": "0.86"
      }
    ]
  },
  "provenance": {
    "authors": [
      "Platform Education Team"
    ],
    "reviewers": [
      "AI Platform Review",
      "Red Team and Compliance"
    ],
    "rights": "CC BY 4.0. Synthetic data generated by the platform.",
    "sourceData": "Synthetic tasks, prompts, model responses, tool calls, latency, tokens, costs, verifier scores, human decisions, failures and outcomes (seed 20261120). No secrets or customer data.",
    "changelog": [
      {
        "version": "1.0",
        "date": "2026-11-18",
        "note": "Initial publication from the extended catalog specification (packages 8 to 20)."
      }
    ]
  },
  "exportedAt": "2026-11-18T23:20:19.095Z"
}