Python

Add Bench using https://docs.usebench.ai/sdk/python and the setup key from my Bench SDK page. Keep the key server-side. Label the
environment and start with metadata-only capture. Wrap one real request and
its tools, including async context propagation. Flush a synthetic interaction
and confirm it arrived in Bench. Show changed files and remaining setup steps.

python -m pip install trybench-sdk==0.2.3

import os
from bench_sdk import Bench

bench = Bench(
    api_key=os.environ["BENCH_API_KEY"],
    repository=os.environ["BENCH_REPOSITORY"],
    branch=os.environ["BENCH_BRANCH"],
    endpoint=os.environ["BENCH_API_BASE_URL"],
    system_name="Support agent",
    environment="staging",
)

with bench.trace("connection-check", kind="AGENT") as span:
    span.set_output({"ok": True})

bench.shutdown()

from bench_sdk import Bench
from bench_sdk.otel import attach

bench = Bench(api_key=..., repository="owner/repo", branch="main", system_name="Bank support")
attach(bench)                 # global tracer provider; pass the framework's provider if it owns one
# Pydantic AI additionally: Agent.instrument_all()
# Strands: attach(bench, StrandsTelemetry().tracer_provider)

report = await bench.evaluate_system(
    source_revision=os.environ["GIT_COMMIT_SHA"],  # Full 40-character commit SHA
    context_revision="refund-policy-v1",
    cases=[{
        "id": "outside-refund-policy", "split": "regression",
        "input": {"days": 45}, "expected_output": {"refunded": False},
        "forbidden_tools": ["issue-refund"],
    }],
    run=lambda request, context: handle_request(request),
)
assert report["summary"]["status"] == "completed"
assert all(case["status"] == "passed" for case in report["cases"])
# Explicit upload, only when you want this report saved in Bench:
await bench.publish_system_evaluation(int(os.environ["BENCH_SYSTEM_ID"]), report)