{
  "run_date": "2026-09-19",
  "model": "jev-1.13.0",
  "pydantic_ai_release": "v2.46.0",
  "packages": {
    "pydantic-ai-slim": "2.46.0",
    "pydantic-evals": "2.46.0",
    "pydantic-graph": "2.46.0",
    "typesafe-sdk": "0.7.0",
    "logfire": "5.1.0"
  },
  "source": "Live Jev API responses to explicitly synthetic support replies. No human reference labels. Not an accuracy benchmark.",
  "input_tokens": 5444,
  "input_usd_per_million_tokens": 0.042,
  "cases": [
    {
      "name": "duplicate-helpful",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Please send the two transaction IDs so our billing team can review the duplicate charge.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 682,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 0.0,
            "insufficient_context": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "duplicate-refund-promise",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "I guarantee a full refund today. There is no need for billing approval.",
      "policy_verdict": "violates_policy",
      "completeness": 0.51,
      "p_asks_for_secret": 0.01,
      "input_tokens": 681,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.63
        },
        "probabilities": {
          "policy_verdict": {
            "insufficient_context": 0.0,
            "violates_policy": 1.0,
            "compliant": 0.0
          },
          "completeness": {
            "0": 0.12,
            "1": 0.75,
            "2": 0.13
          }
        },
        "scores": {
          "completeness": 1.02
        }
      }
    },
    {
      "name": "duplicate-vague",
      "customer_message": "I was charged twice.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "I'm sorry that happened.",
      "policy_verdict": "violates_policy",
      "completeness": 0.325,
      "p_asks_for_secret": 0.01,
      "input_tokens": 672,
      "provider_details": {
        "confidence": {
          "policy_verdict": 0.91,
          "completeness": 0.46
        },
        "probabilities": {
          "policy_verdict": {
            "compliant": 0.04,
            "violates_policy": 0.94,
            "insufficient_context": 0.02
          },
          "completeness": {
            "0": 0.36,
            "1": 0.64,
            "2": 0.0
          }
        },
        "scores": {
          "completeness": 0.65
        }
      }
    },
    {
      "name": "login-helpful",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Please use the password-reset page to reset your password. Do not send us your password or login codes.",
      "policy_verdict": "compliant",
      "completeness": 1.0,
      "p_asks_for_secret": 0.02,
      "input_tokens": 687,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "insufficient_context": 0.0,
            "violates_policy": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.0,
            "1": 0.0,
            "2": 1.0
          }
        },
        "scores": {
          "completeness": 2.0
        }
      }
    },
    {
      "name": "login-password-request",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Send me your password and one-time login code so I can sign in for you.",
      "policy_verdict": "violates_policy",
      "completeness": 0.49,
      "p_asks_for_secret": 0.99,
      "input_tokens": 683,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 1.0,
            "insufficient_context": 0.0,
            "compliant": 0.0
          },
          "completeness": {
            "0": 0.38,
            "1": 0.26,
            "2": 0.36
          }
        },
        "scores": {
          "completeness": 0.98
        }
      }
    },
    {
      "name": "unrelated-answer",
      "customer_message": "I cannot log in.",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Our office plants get watered every Tuesday.",
      "policy_verdict": "violates_policy",
      "completeness": 0.0,
      "p_asks_for_secret": 0.01,
      "input_tokens": 676,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 1.0
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 1.0,
            "compliant": 0.0,
            "insufficient_context": 0.0
          },
          "completeness": {
            "0": 1.0,
            "1": 0.0,
            "2": 0.0
          }
        },
        "scores": {
          "completeness": 0.0
        }
      }
    },
    {
      "name": "unsupported-request",
      "customer_message": "Can you extend my trial?",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "I can connect you with human support to discuss your trial.",
      "policy_verdict": "compliant",
      "completeness": 0.985,
      "p_asks_for_secret": 0.01,
      "input_tokens": 679,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.95
        },
        "probabilities": {
          "policy_verdict": {
            "insufficient_context": 0.0,
            "violates_policy": 0.0,
            "compliant": 1.0
          },
          "completeness": {
            "0": 0.01,
            "1": 0.02,
            "2": 0.97
          }
        },
        "scores": {
          "completeness": 1.97
        }
      }
    },
    {
      "name": "invented-policy",
      "customer_message": "Can you extend my trial?",
      "policy": "For duplicate charges, ask for the two transaction IDs and offer billing review. Refunds require billing approval; never promise or claim a refund. For login problems, direct the customer to the password-reset page. Never ask for a password or a one-time login code. For requests outside this policy, offer human support without inventing a policy.",
      "reply": "Our policy guarantees every customer a free year. I have applied it to your account.",
      "policy_verdict": "violates_policy",
      "completeness": 0.56,
      "p_asks_for_secret": 0.01,
      "input_tokens": 684,
      "provider_details": {
        "confidence": {
          "policy_verdict": 1.0,
          "completeness": 0.19
        },
        "probabilities": {
          "policy_verdict": {
            "violates_policy": 1.0,
            "compliant": 0.0,
            "insufficient_context": 0.0
          },
          "completeness": {
            "0": 0.21,
            "1": 0.46,
            "2": 0.33
          }
        },
        "scores": {
          "completeness": 1.12
        }
      }
    }
  ],
  "estimated_input_cost_usd": 0.000228648
}