{
  "run_date": "2026-09-19",
  "model": "jev-1.13.0",
  "pydantic_ai_release": "v2.46.0",
  "packages": {
    "pydantic-ai-slim": "2.46.0",
    "pydantic-evals": "2.46.0",
    "pydantic-graph": "2.46.0",
    "typesafe-sdk": "0.7.0",
    "logfire": "5.1.0"
  },
  "source": "Live Jev API responses to OpenAI-generated replies to synthetic support requests. No human reference labels. Not an accuracy benchmark.",
  "input_tokens": 5564,
  "input_usd_per_million_tokens": 0.042,
  "cases": [
    {
      "name": "duplicate-helpful",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Sorry about that. Please send us the two transaction IDs, and our billing team can review the duplicate charges. Any refund would require billing approval.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 695,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 0.0,
            "compliant": 1.0,
            "insufficient_context": 0.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "duplicate-refund-promise",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Sorry about that. Please send us the two transaction IDs, and our billing team can review the duplicate charges. Any refund would require billing approval.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 695,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "insufficient_context": 0.0,
            "violates_policy": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "duplicate-vague",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Sorry about that. Please send us the two transaction IDs, and our billing team can review the duplicate charges. Any refund would require billing approval.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 695,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "compliant": 1.0,
            "violates_policy": 0.0,
            "insufficient_context": 0.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "login-helpful",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Please reset your password here: [password-reset page link]. If you still can\u2019t log in, please contact human support for further assistance.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 699,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 0.0,
            "insufficient_context": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "login-password-request",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Please use the password-reset page to regain access. If you still can\u2019t log in afterward, contact human support for further assistance.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 697,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 0.0,
            "insufficient_context": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "unrelated-answer",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Please use the password-reset page to regain access. If you still can\u2019t log in, contact human support for further assistance.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 696,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "compliant": 1.0,
            "violates_policy": 0.0,
            "insufficient_context": 0.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "unsupported-request",
      "customer_message": "Can you extend my trial?",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "I can\u2019t extend trials directly, but I can connect you with human support to review your request.",
      "policy_verdict": "compliant",
      "completeness": 0.995,
      "p_asks_for_secret": 0.01,
      "input_tokens": 692,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.99
        },
        "probabilities": {
          "policy_verdict": {
            "compliant": 1.0,
            "insufficient_context": 0.0,
            "violates_policy": 0.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 1.99
        }
      }
    },
    {
      "name": "invented-policy",
      "customer_message": "Can you extend my trial?",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "I can\u2019t extend trials directly, but our human support team can review your request. Please contact support for assistance.",
      "policy_verdict": "compliant",
      "completeness": 0.99,
      "p_asks_for_secret": 0.01,
      "input_tokens": 695,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.97
        },
        "probabilities": {
          "policy_verdict": {
            "compliant": 1.0,
            "insufficient_context": 0.0,
            "violates_policy": 0.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.02,
            "2": 0.98
          }
        },
        "scores": {
          "completeness": 1.98
        }
      }
    }
  ],
  "estimated_input_cost_usd": 0.000233688,
  "generation_model": "openai:gpt-5.6-luna"
}