Run tests in CI

Set up Bench in CI using https://docs.usebench.ai/sdk/ci.
Use my existing SDK installation and repository-scoped key. Keep keys in CI secrets.
For TypeScript or JavaScript, run evaluateSystem or simulateSystem against the real
application with isolated test dependencies. Include capability, regression and
incident cases. Assert actual tool effects, not just the final answer.
Fail the job for a failed, errored, unscored, missing or incomplete case. Save the
redacted report even when checks fail and flush telemetry before exit.
For Python, Go or Rust, instrument the existing test suite and keep its assertions
and exit status. Native Bench application-test helpers are coming soon.
Use synthetic data and show a passing run, a deliberately failing run, and the
setup still required in my CI provider. Do not expose secrets to forked PR code.

import { mkdir, writeFile } from 'node:fs/promises'
import { execFileSync } from 'node:child_process'
import { Bench } from '@trybench/sdk'
import { createBenchSuite } from '../tests/bench-suite.mjs'

for (const name of ['BENCH_API_KEY', 'BENCH_REPOSITORY', 'BENCH_BRANCH']) {
  if (!process.env[name]) throw new Error(`Set ${name} in CI`)
}

const bench = new Bench({
  apiKey: process.env.BENCH_API_KEY,
  repository: process.env.BENCH_REPOSITORY,
  branch: process.env.BENCH_BRANCH,
  endpoint: process.env.BENCH_API_BASE_URL,
  environment: 'ci',
  captureContent: false,
})

try {
  const suite = await createBenchSuite(bench)
  const report = await bench.evaluateSystem({
    ...suite,
    sourceRevision: execFileSync('git', ['rev-parse', 'HEAD'], {
      encoding: 'utf8',
    }).trim(),
  })

  await mkdir('bench-results', { recursive: true })
  await writeFile('bench-results/report.json', JSON.stringify(report, null, 2))

  const passed = report.summary.status === 'completed'
    && report.planned_case_count > 0
    && report.cases.length === report.planned_case_count
    && report.cases.every((item) => item.status === 'passed')

  console.log(JSON.stringify(report.summary))
  if (!passed) process.exitCode = 1
} finally {
  await bench.shutdown()
}

name: Bench application tests
on:
  pull_request:
  workflow_dispatch:

permissions:
  contents: read

jobs:
  application-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
      - run: npm ci
      # Add your app's build and test-dependency setup here if needed.
      - name: Check application outcomes
        run: node scripts/bench-ci.mjs
        env:
          BENCH_API_KEY: ${{ secrets.BENCH_API_KEY }}
          BENCH_API_BASE_URL: ${{ vars.BENCH_API_BASE_URL }}
          BENCH_REPOSITORY: ${{ github.repository }}
          BENCH_BRANCH: ${{ github.head_ref || github.ref_name }}
      - name: Save the report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: bench-results
          path: bench-results/report.json
          if-no-files-found: warn
          retention-days: 7