Agent Benchmarks

Standardized benchmarks for evaluating agent capabilities across tasks and domains.

Benchmark Taxonomy

Key Benchmarks Detail

1. SWE-bench (Software Engineering)

# Real GitHub issues from 12 popular Python repos
# Task: Generate patch that fixes issue
# Metrics: Resolved rate, patch quality

class SWEBenchEvaluator:
    def __init__(self, dataset: str = "princeton-nlp/SWE-bench"):
        self.dataset = load_dataset(dataset)

    async def evaluate(self, agent: CodingAgent) -> SWEBenchResult:
        results = []
        for instance in self.dataset["test"]:
            # Setup repo at base commit
            repo = await self.setup_repo(instance)

            # Agent generates patch
            patch = await agent.generate_fix(
                repo=repo,
                issue_text=instance["problem_statement"],
                hints=instance.get("hints")
            )

            # Apply & test
            result = await self.apply_and_test(repo, patch, instance["FAIL_TO_PASS"], instance["PASS_TO_PASS"])
            results.append(result)

        return SWEBenchResult(
            resolved_rate=sum(r.resolved for r in results) / len(results),
            avg_patch_lines=statistics.mean(len(r.patch.splitlines()) for r in results),
            by_repo=self.group_by_repo(results)
        )

2. AgentBench

# 8 environments: OS, DB, KG, WebShop, DigitalCardGame, Household, Web, API
# Metrics: Success rate, steps, efficiency

class AgentBenchEvaluator:
    def __init__(self):
        self.environments = {
            "os": OSEnvironment(),
            "db": DBEnvironment(),
            "kg": KGEnvironment(),
            "webshop": WebShopEnv(),
            "card_game": CardGameEnv(),
            "household": HouseholdEnv(),
            "web": WebEnv(),
            "api": APIEnv()
        }

    async def evaluate(self, agent: Agent) -> AgentBenchResult:
        results = {}
        for name, env in self.environments.items():
            result = await self.run_env(agent, env)
            results[name] = result

        return AgentBenchResult(
            overall_success=statistics.mean(r.success_rate for r in results.values()),
            by_env=results,
            total_steps=sum(r.avg_steps for r in results.values())
        )

3. GAIA (General AI Assistants)

# 466 questions across 3 levels
# Level 1: Simple (5-10 min human)
# Level 2: Moderate (10-30 min)
# Level 3: Hard (30+ min)
# Requires: Web search, tools, reasoning

class GAIAEvaluator:
    def __init__(self):
        self.dataset = load_dataset("gaia-benchmark/GAIA", "2023_all")

    async def evaluate(self, agent: Agent) -> GAIAResult:
        results = {"level1": [], "level2": [], "level3": []}

        for instance in self.dataset:
            level = instance["Level"]
            answer = await agent.run(instance["Question"])

            correct = self.check_answer(answer, instance["Final answer"])
            results[f"level{level}"].append(correct)

        return GAIAResult(
            level1_acc=sum(results["level1"]) / len(results["level1"]),
            level2_acc=sum(results["level2"]) / len(results["level2"]),
            level3_acc=sum(results["level3"]) / len(results["level3"]),
            overall_acc=sum(sum(v) for v in results.values()) / sum(len(v) for v in results.values())
        )

4. τ-bench (Tool Use)

# Realistic tool-use scenarios
# Airline booking, retail, etc.
# Metrics: Task completion, API correctness

class TauBenchEvaluator:
    def __init__(self):
        self.environments = {
            "airline": AirlineEnv(),
            "retail": RetailEnv()
        }

    async def evaluate(self, agent: Agent) -> TauBenchResult:
        results = {}
        for name, env in self.environments.items():
            result = await self.run_episodes(agent, env, num_episodes=100)
            results[name] = result

        return TauBenchResult(
            airline_success=results["airline"].success_rate,
            retail_success=results["retail"].success_rate,
            api_correctness=self.compute_api_correctness(results)
        )

5. Coding Benchmarks

HumanEval / MBPP

class CodeGenerationEvaluator:
    def __init__(self):
        self.humaneval = load_dataset("openai/humaneval")
        self.mbpp = load_dataset("mbpp")

    async def evaluate(self, agent: CodingAgent) -> CodeGenResult:
        results = {"humaneval": [], "mbpp": []}

        for dataset_name, dataset in [("humaneval", self.humaneval), ("mbpp", self.mbpp)]:
            for problem in dataset["test"]:
                solution = await agent.generate_code(
                    prompt=problem["prompt"],
                    signature=problem.get("signature")
                )

                # Execute tests
                passed = await self.run_tests(solution, problem["test"])
                results[dataset_name].append(passed)

        return CodeGenResult(
            humaneval_pass=sum(results["humaneval"]) / len(results["humaneval"]),
            mbpp_pass=sum(results["mbpp"]) / len(results["mbpp"]),
            pass_at_k=self.compute_pass_at_k(results)
        )

LiveCodeBench

class LiveCodeBenchEvaluator:
    def __init__(self):
        self.dataset = load_dataset("livecodebench/livecodebench")

    async def evaluate(self, agent: CodingAgent) -> LiveCodeBenchResult:
        # Problems from recent contests (no training contamination)
        # Categories: Code Generation, Self-Repair, Code Execution, Test Output Prediction
        ...

6. Web Automation Benchmarks

WebShop

class WebShopEvaluator:
    def __init__(self):
        self.env = WebShopEnvironment()

    async def evaluate(self, agent: Agent) -> WebShopResult:
        # 12,000+ products, 1.18M instructions
        # Metrics: SR (Success Rate), Reward, Subtask completion
        ...

Mind2Web

class Mind2WebEvaluator:
    def __init__(self):
        self.dataset = load_dataset("mind2web")

    async def evaluate(self, agent: Agent) -> Mind2WebResult:
        # 2000+ tasks across 137 websites
        # Metrics: Step SR, Task SR, Efficiency
        ...

7. Safety Benchmarks

HarmBench

class HarmBenchEvaluator:
    def __init__(self):
        self.categories = [
            "copyright", "harassment", "hate_speech", "violence",
            "illegal_activities", "pII", "sexual_content", "self_harm"
        ]

    async def evaluate(self, agent: Agent) -> HarmBenchResult:
        results = {}
        for cat in self.categories:
            prompts = self.load_prompts(cat)
            results[cat] = await self.test_category(agent, prompts)

        return HarmBenchResult(
            overall_refusal_rate=statistics.mean(r.refusal_rate for r in results.values()),
            by_category=results
        )

SafeBench

class SafeBenchEvaluator:
    def __init__(self):
        self.tasks = {
            "adversarial": AdversarialTasks(),
            "jailbreak": JailbreakTasks(),
            "alignment": AlignmentTasks(),
            "robustness": RobustnessTasks()
        }

    async def evaluate(self, agent: Agent) -> SafeBenchResult:
        ...

7. Multi-Agent Benchmarks

MAgent / MARS

class MultiAgentBenchmark:
    def __init__(self):
        self.scenarios = {
            "cooperative": CooperativeScenarios(),
            "competitive": CompetitiveScenarios(),
            "mixed": MixedScenarios()
        }

    async def evaluate(self, agents: list[Agent]) -> MultiAgentResult:
        ...

Evaluation Framework

Unified Evaluation Harness

class AgentEvaluator:
    def __init__(self):
        self.benchmarks = {
            "swe_bench": SWEBenchEvaluator(),
            "agent_bench": AgentBenchEvaluator(),
            "gaia": GAIAEvaluator(),
            "tau_bench": TauBenchEvaluator(),
            "humaneval": CodeGenerationEvaluator(),
            "webshop": WebShopEvaluator(),
            "harmbench": HarmBenchEvaluator(),
        }

    async def evaluate(self, agent: Agent, benchmarks: list[str] = None) -> EvaluationReport:
        benchmarks = benchmarks or list(self.benchmarks.keys())
        results = {}

        for name in benchmarks:
            evaluator = self.benchmarks[name]
            print(f"Running {name}...")
            try:
                results[name] = await evaluator.evaluate(agent)
            except Exception as e:
                results[name] = EvaluationError(str(e))

        return EvaluationReport(
            agent=agent.name,
            timestamp=datetime.utcnow(),
            results=results,
            summary=self.summarize(results)
        )

Evaluation Config

@dataclass
class EvaluationConfig:
    benchmarks: list[str] = field(default_factory=lambda: ["swe_bench", "agent_bench", "gaia"])
    num_samples: int | None = None  # None = full dataset
    timeout_per_task: int = 300
    max_concurrent: int = 5
    retry_failed: bool = True
    save_trajectories: bool = True
    output_dir: str = "./eval_results"

Metrics & Reporting

Standard Metrics

Reporting

class EvaluationReporter:
    def __init__(self, output_dir: str):
        self.output_dir = Path(output_dir)

    def generate_report(self, report: EvaluationReport) -> str:
        # HTML report with charts
        html = self.generate_html(report)
        (self.output_dir / "report.html").write_text(html)

        # JSON for programmatic access
        (self.output_dir / "results.json").write_text(json.dumps(asdict(report), indent=2))

        # Markdown summary
        md = self.generate_markdown(report)
        (self.output_dir / "summary.md").write_text(md)

        return str(self.output_dir / "report.html")

    def generate_markdown(self, report: EvaluationReport) -> str:
        md = f"# Evaluation Report: {report.agent}\n\n"
        md += f"**Date:** {report.timestamp}\n\n"

        for name, result in report.results.items():
            md += f"## {name}\n"
            if isinstance(result, EvaluationError):
                md += f"❌ Error: {result.error}\n\n"
            else:
                md += self.format_result(name, result) + "\n\n"

        return md

CI/CD Integration

# .github/workflows/agent-eval.yml
name: Agent Evaluation
on: [push, pull_request]
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v5
        with: {python-version: "3.11"}
      - name: Install deps
        run: pip install -r requirements-eval.txt
      - name: Run evaluation
        run: python -m agent_eval --agent my_agent --benchmarks swe_bench,gaia,agent_bench
      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: ./eval_results/
      - name: Comment PR
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const summary = fs.readFileSync('./eval_results/summary.md', 'utf8');
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: summary
            })

Related Notes

• Evaluation Frameworks — Evaluation frameworks
• Metrics — Detailed metrics
• Offline Eval — Offline evaluation
• Online Eval — Online evaluation
• Red Teaming — Red teaming
• Coding Agents — Coding benchmarks
• Research Agents — Research benchmarks
• Creative Agents — Creative benchmarks
• LangGraph — LangGraph evaluation
• AutoGen — AutoGen evaluation
• CrewAI — CrewAI evaluation
• Observability — Production monitoring
• Reliability — Reliability testing
• Secure Coding — Security benchmarks
• Research Directions — Benchmark research

Sources: SWE-bench, AgentBench, GAIA, τ-bench, HumanEval, MBPP, LiveCodeBench, WebShop, Mind2Web, ALFWorld, MMLU, GPQA, MMLU-Pro, BBH, ARC, HELM, LMSYS Chatbot Arena, LMSYS MT-Bench, AlpacaEval, WildBench, Safety Benchmarks (HarmBench, SafeBench, RedBench), Multi-Agent Benchmarks (MAgent, MARS), LLM Leaderboard (Open LLM Leaderboard, LMSYS), Evaluation Frameworks (LangSmith, LangFuse, Arize, Phoenix, Weights & Biases, MLflow, ClearML, Neptune, Comet).