Regression eval suites in CI

import Anthropic from '@anthropic-ai/sdk';
import { readFile, writeFile } from 'node:fs/promises';

const client = new Anthropic();

const SMOKE = process.argv.includes('--smoke');
const UPDATE_BASELINE = process.argv.includes('--update-baseline');
const CONCURRENCY = 4; // Stay well inside your rate limits.
const TOLERANCE = 0.03; // Allow three points of run-to-run noise before calling it a regression.

interface Case {
  id: string;
  category: string;
  input: string;
  mustInclude: string[];
  smoke?: boolean;
}
type Scores = Record<string, number>;

/** The feature under test. Returns null if the response didn't finish normally, which fails the case. */
async function feature(input: string): Promise<string | null> {
  const response = await client.messages.create({
    model: 'claude-opus-5-5',
    max_tokens: 4096,
    output_config: { effort: 'low' },
    system: 'Summarize the email thread in at most five short bullet points. Name the decision that is needed, if any.',
    messages: [{ role: 'user', content: input }],
  });
  if (response.stop_reason !== 'end_turn') return null;
  return response.content.flatMap((b) => (b.type === 'text' ? [b.text] : [])).join('');
}

const passes = (output: string | null, c: Case) =>
  output !== null && c.mustInclude.every((fact) => output.toLowerCase().includes(fact.toLowerCase()));

/** Runs tasks with at most `limit` in flight. */
async function mapLimited<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]> {
  const results: R[] = [];
  let next = 0;
  const worker = async () => {
    while (next < items.length) {
      const i = next++;
      results[i] = await fn(items[i] as T);
    }
  };
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
  return results;
}

const all: Case[] = JSON.parse(await readFile('evals/cases.json', 'utf8'));
const cases = SMOKE ? all.filter((c) => c.smoke) : all;

const results = await mapLimited(cases, CONCURRENCY, async (c) => {
  try {
    return { c, passed: passes(await feature(c.input), c) };
  } catch {
    // An API error fails the case rather than crashing the whole run.
    return { c, passed: false };
  }
});

const scores: Scores = {};
for (const category of new Set(cases.map((c) => c.category))) {
  const inCategory = results.filter((r) => r.c.category === category);
  scores[category] = inCategory.filter((r) => r.passed).length / inCategory.length;
}

const baseline: Scores = JSON.parse(await readFile('evals/baseline.json', 'utf8').catch(() => '{}'));
const problems: string[] = [];

for (const [category, score] of Object.entries(scores)) {
  const before = baseline[category];
  const line = `${category.padEnd(20)} ${(score * 100).toFixed(0)}%${before === undefined ? '' : ` (baseline ${(before * 100).toFixed(0)}%)`}`;
  console.log(line);
  if (category === 'must-never-fail' && score < 1) problems.push(`must-never-fail: ${(score * 100).toFixed(0)}%, expected 100%`);
  else if (before !== undefined && score < before - TOLERANCE) problems.push(`${category} dropped from ${(before * 100).toFixed(0)}% to ${(score * 100).toFixed(0)}%`);
}

await writeFile('evals/latest.json', JSON.stringify(scores, null, 2));
if (UPDATE_BASELINE) await writeFile('evals/baseline.json', JSON.stringify(scores, null, 2));

if (problems.length > 0) {
  console.error(`\nRegressions:\n- ${problems.join('\n- ')}`);
  process.exitCode = 1;
}

name: Evals

on:
  pull_request:
    paths: ['prompts/**', 'lib/ai/**', 'evals/**']
  schedule:
    - cron: '0 3 * * *' # Nightly full suite at 03:00 UTC

jobs:
  evals:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
      - run: npm ci
      - name: Smoke evals
        if: github.event_name == 'pull_request'
        run: npx tsx evals/run-suite.ts --smoke
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      - name: Full eval suite
        if: github.event_name == 'schedule'
        run: npx tsx evals/run-suite.ts
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      - name: Keep results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: evals/latest.json