Evaluation Metrics

Standardized metrics for evaluating agent performance across dimensions.

Metric Taxonomy

Evaluation Metrics
├── Task Completion
│   ├── Success Rate
│   ├── Partial Credit
│   ├── Goal Achievement
│   └── Constraint Satisfaction
├── Quality
│   ├── Output Quality (LLM Judge)
│   ├── Human Preference
│   ├── Coherence & Fluency
│   ├── Factual Accuracy
│   └── Instruction Following
├── Efficiency
│   ├── Steps to Completion
│   ├── Token Usage
│   ├── Latency (TTFT, TPOT)
│   ├── API Calls
│   └── Cost (USD)
├── Safety
│   ├── Refusal Rate (Benign)
│   ├── Attack Success Rate (Adversarial)
│   ├── Hallucination Rate
│   ├── PII Leakage
│   └── Toxicity
├── Robustness
│   ├── Adversarial Accuracy
│   ├── OOD Generalization
│   ├── Consistency
│   └── Calibration (ECE)
├── User Experience
│   ├── CSAT / NPS
│   ├── Task Completion Rate
│   ├── Escalation Rate
│   └── Time to Resolution
└── Multi-Agent
    ├── Coordination Efficiency
    ├── Communication Overhead
    ├── Load Balancing
    └── Conflict Rate

Task Completion Metrics

Success Rate

def success_rate(results: list[TaskResult]) -> float:
    """Binary success: task fully completed per spec"""
    return sum(1 for r in results if r.success) / len(results)

def partial_credit(results: list[TaskResult], rubric: Rubric) -> float:
    """Weighted score based on rubric"""
    total = 0
    for r in results:
        score = 0
        for criterion, weight in rubric.items():
            if r.meets_criterion(criterion):
                score += weight
        total += score
    return total / len(results)

Goal Achievement

def goal_achievement_rate(results: list[TaskResult]) -> float:
    """Did the agent achieve the stated goal?"""
    return sum(1 for r in results if r.goal_achieved) / len(results)

def constraint_satisfaction(results: list[TaskResult]) -> float:
    """Percentage of constraints satisfied"""
    satisfied = sum(1 for r in results if r.constraints_satisfied)
    total_constraints = sum(len(r.constraints) for r in results)
    satisfied_constraints = sum(r.satisfied_constraints for r in results)
    return satisfied_constraints / total_constraints if total_constraints > 0 else 1.0

Quality Metrics

LLM-as-Judge

class LLMJudge:
    def __init__(self, model: str = "gpt-4o"):
        self.model = model

    async def evaluate(self, criteria: list[str], input: str, output: str, 
                       reference: str = None, context: str = None) -> dict[str, float]:
        prompt = self.build_prompt(criteria, input, output, reference, context)
        response = await self.llm.generate(prompt, response_format="json")
        return self.parse_scores(response)

    def build_prompt(self, criteria, input, output, reference, context):
        return f"""
        Evaluate the following output on a scale of 1-10 for each criterion:

        Input: {input}
        Output: {output}
        {f'Reference: {reference}' if reference else ''}
        {f'Context: {context}' if context else ''}

        Criteria:
        {chr(10).join(f'- {c}' for c in criteria)}

        Return JSON: {{"criterion": score, ...}}
        """

Standard Quality Dimensions

QUALITY_CRITERIA = {
    "accuracy": "Factual correctness of information",
    "completeness": "Addresses all parts of the request",
    "relevance": "Stays on topic, addresses user need",
    "coherence": "Logical flow, consistent reasoning",
    "clarity": "Clear, understandable, well-structured",
    "conciseness": "Concise without losing important detail",
    "tone": "Appropriate tone for context/audience",
    "creativity": "Novelty, originality where appropriate",
    "instruction_following": "Follows all explicit instructions",
    "format_compliance": "Matches requested format exactly"
}

Human Preference (Pairwise)

async def pairwise_preference(output_a: str, output_b: str, 
                              prompt: str, judge: LLMJudge) -> PreferenceResult:
    """A/B preference test"""
    prompt = f"""
    Compare two outputs for the same task.

    Task: {prompt}

    Output A: {output_a}
    Output B: {output_b}

    Which is better? Consider: accuracy, helpfulness, clarity, tone.
    Return: "A", "B", or "Tie" with reasoning.
    """
    result = await llm.generate(prompt, response_format="json")
    return PreferenceResult(preference=result["preference"], reasoning=result["reasoning"])

Factual Accuracy (RAG)

def factual_accuracy(answer: str, contexts: list[str]) -> float:
    """Faithfulness: answer supported by contexts"""
    claims = extract_claims(answer)
    supported = 0
    for claim in claims:
        if any(claim_supported(claim, ctx) for ctx in contexts):
            supported += 1
    return supported / len(claims) if claims else 1.0

def answer_relevancy(answer: str, question: str) -> float:
    """Does answer address the question?"""
    # Use LLM judge or embedding similarity
    return embedding_similarity(embed(answer), embed(question))

Efficiency Metrics

Token Metrics

def token_efficiency(result: TaskResult) -> TokenMetrics:
    return TokenMetrics(
        input_tokens=result.input_tokens,
        output_tokens=result.output_tokens,
        total_tokens=result.input_tokens + result.output_tokens,
        tokens_per_step=result.total_tokens / max(1, result.steps),
        cost_usd=calculate_cost(result.input_tokens, result.output_tokens, model)
    )

def token_efficiency_ratio(result: TaskResult) -> float:
    """Output tokens per input token (higher = more efficient)"""
    return result.output_tokens / max(1, result.input_tokens)

Latency Metrics

def latency_metrics(result: TaskResult) -> LatencyMetrics:
    return LatencyMetrics(
        ttft_ms=result.ttft_ms,           # Time to First Token
        tpot_ms=result.tpot_ms,           # Time Per Output Token
        total_latency_ms=result.latency_ms,
        e2e_latency_ms=result.e2e_latency_ms
    )

def throughput(tokens_per_second: float) -> float:
    return tokens_per_second

Cost Metrics

def cost_metrics(result: TaskResult, pricing: ModelPricing) -> CostMetrics:
    input_cost = result.input_tokens * pricing.input_per_1k / 1000
    output_cost = result.output_tokens * pricing.output_per_1k / 1000
    total_cost = input_cost + output_cost

    return CostMetrics(
        input_cost_usd=input_cost,
        output_cost_usd=output_cost,
        total_cost_usd=total_cost,
        cost_per_step=total_cost / max(1, result.steps),
        cost_per_success=total_cost if result.success else float('inf')
    )

Safety Metrics

Refusal & Attack Metrics

def refusal_rate(benign_prompts: list[str], responses: list[str]) -> float:
    """Percentage of benign requests incorrectly refused"""
    refusals = sum(1 for r in responses if is_refusal(r))
    return refusals / len(responses)

def attack_success_rate(adversarial_prompts: list[str], responses: list[str]) -> float:
    """Percentage of attacks that succeeded"""
    successes = sum(1 for r in responses if attack_succeeded(r))
    return successes / len(responses)

def false_positive_rate(benign_prompts: list[str], responses: list[str]) -> float:
    """Benign requests flagged as malicious"""
    false_positives = sum(1 for r in responses if is_false_positive(r))
    return false_positives / len(responses)

Hallucination & Faithfulness

def hallucination_rate(answers: list[str], contexts: list[list[str]]) -> float:
    """Percentage of unsupported claims"""
    total_claims = 0
    unsupported = 0
    for answer, ctx in zip(answers, contexts):
        claims = extract_claims(answers)
        for claim in claims:
            total_claims += 1
            if not any(claim_supported(claim, ctx) for ctx in contexts):
                unsupported += 1
    return unsupported / total_claims if total_claims > 0 else 0.0

def faithfulness_score(answer: str, context: str) -> float:
    """How well answer is grounded in context"""
    claims = extract_claims(answer)
    supported = sum(1 for c in claims if claim_supported(c, context))
    return supported / len(claims) if claims else 1.0

PII & Toxicity

def pii_leakage_rate(outputs: list[str]) -> float:
    """Percentage of outputs containing PII"""
    leaks = sum(1 for o in outputs if contains_pii(o))
    return leaks / len(outputs)

def toxicity_score(outputs: list[str]) -> float:
    """Average toxicity score (0-1)"""
    scores = [toxicity_classifier.score(o) for o in outputs]
    return statistics.mean(scores)

Robustness Metrics

Adversarial Accuracy

def adversarial_accuracy(clean_acc: float, adv_acc: float) -> RobustnessMetrics:
    return RobustMetrics(
        clean_accuracy=clean_acc,
        adversarial_accuracy=adv_acc,
        robustness_gap=clean_acc - adv_acc,
        relative_robustness=adv_acc / clean_acc if clean_acc > 0 else 0
    )

Consistency

def consistency_score(outputs: list[str]) -> float:
    """Self-consistency across multiple generations"""
    if len(outputs) < 2:
        return 1.0

    # Pairwise similarity
    similarities = []
    for i in range(len(outputs)):
        for j in range(i+1, len(outputs)):
            sim = semantic_similarity(outputs[i], outputs[j])
            similarities.append(sim)

    return statistics.mean(similarities) if similarities else 1.0

Calibration (ECE)

def expected_calibration_error(predictions: list[float], outcomes: list[int], n_bins: int = 10) -> float:
    """Expected Calibration Error"""
    bin_boundaries = np.linspace(0, 1, n_bins + 1)
    ece = 0.0

    for i in range(n_bins):
        bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i+1]
        in_bin = [(p, o) for p, o in zip(predictions, outcomes) if bin_lower <= p < bin_upper]

        if in_bin:
            bin_confidence = np.mean([p for p, _ in in_bin])
            bin_accuracy = np.mean([o for _, o in in_bin])
            bin_weight = len(in_bin) / len(predictions)
            ece += bin_weight * abs(bin_confidence - bin_accuracy)

    return ece

Multi-Agent Metrics

Coordination Efficiency

def coordination_efficiency(trajectory: MultiAgentTrajectory) -> CoordinationMetrics:
    return CoordinationMetrics(
        total_messages=len(trajectory.messages),
        unique_senders=len(set(m.sender for m in trajectory.messages)),
        avg_message_length=statistics.mean(len(m.content) for m in trajectory.messages),
        coordination_overhead=trajectory.coordination_time / trajectory.total_time,
        redundant_messages=count_redundant(trajectory.messages),
        conflicts=count_conflicts(trajectory.messages)
    )

Load Balancing

def load_balance(agent_loads: dict[str, int]) -> LoadBalanceMetrics:
    loads = list(agent_loads.values())
    return LoadBalanceMetrics(
        mean_load=statistics.mean(loads),
        std_load=statistics.stdev(loads) if len(loads) > 1 else 0,
        max_load=max(loads),
        min_load=min(loads),
        gini_coefficient=gini_coefficient(loads)
    )

Calibration & Confidence

Expected Calibration Error (ECE)

def expected_calibration_error(predictions: list[float], outcomes: list[int], n_bins: int = 10) -> float:
    bin_boundaries = np.linspace(0, 1, n_bins + 1)
    ece = 0.0

    for i in range(n_bins):
        bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i+1]
        in_bin = [(p, o) for p, o in zip(predictions, outcomes) if bin_lower <= p < bin_upper]

        if in_bin:
            bin_confidence = np.mean([p for p, _ in in_bin])
            bin_accuracy = np.mean([o for _, o in in_bin])
            bin_weight = len(in_bin) / len(predictions)
            ece += bin_weight * abs(bin_confidence - bin_accuracy)

    return ece

Reliability Diagram

def reliability_diagram(predictions: list[float], outcomes: list[int], n_bins: int = 10) -> ReliabilityData:
    bin_boundaries = np.linspace(0, 1, n_bins + 1)
    data = []

    for i in range(n_bins):
        bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i+1]
        in_bin = [(p, o) for p, o in zip(predictions, outcomes) if bin_lower <= p < bin_upper]

        if in_bin:
            data.append({
                "bin_center": (bin_lower + bin_upper) / 2,
                "confidence": np.mean([p for p, _ in in_bin]),
                "accuracy": np.mean([o for _, o in in_bin]),
                "count": len(in_bin)
            })

    return ReliabilityData(bins=data)

Statistical Significance

Bootstrap Confidence Intervals

def bootstrap_ci(metric_fn, data: list, n_bootstrap: int = 1000, confidence: float = 0.95) -> tuple[float, float]:
    """Bootstrap confidence interval for any metric"""
    bootstrapped = []
    for _ in range(n_bootstrap):
        sample = np.random.choice(data, size=len(data), replace=True)
        bootstrapped.append(metric_fn(sample))

    alpha = (1 - confidence) / 2
    lower = np.percentile(bootstrapped, alpha * 100)
    upper = np.percentile(bootstrapped, (1 - alpha) * 100)

    return (lower, upper)

Statistical Significance Testing

def paired_ttest(metric_a: list[float], metric_b: list[float]) -> TTestResult:
    """Paired t-test for comparing two systems on same tasks"""
    from scipy import stats
    t_stat, p_value = stats.ttest_rel(metric_a, metric_b)
    return TTestResult(t_stat=t_stat, p_value=p_value, significant=p_value < 0.05)

def mcnemar_test(outcomes_a: list[bool], outcomes_b: list[bool]) -> McNemarResult:
    """McNemar's test for paired binary outcomes"""
    from statsmodels.stats.contingency_tables import mcnemar

    # Contingency table: [[both_correct, a_only], [b_only, both_wrong]]
    both_correct = sum(a and b for a, b in zip(outcomes_a, outcomes_b))
    a_only = sum(a and not b for a, b in zip(outcomes_a, outcomes_b))
    b_only = sum(not a and b for a, b in zip(outcomes_a, outcomes_b))
    both_wrong = sum(not a and not b for a, b in zip(outcomes_a, outcomes_b))

    table = [[both_correct, a_only], [b_only, both_wrong]]
    result = mcnemar(table, exact=False)

    return McNemarResult(statistic=result.statistic, p_value=result.pvalue)

Reporting & Visualization

Metric Dashboard

class MetricDashboard:
    def __init__(self, output_dir: str):
        self.output_dir = Path(output_dir)

    def generate_report(self, results: dict[str, EvaluationResult]) -> str:
        html = self.generate_html_report(results)
        (self.output_dir / "dashboard.html").write_text(html)
        return str(self.output_dir / "dashboard.html")

    def generate_html(self, results: dict) -> str:
        # Generate HTML with Chart.js charts
        # Include: summary table, bar charts, radar charts, trend lines
        pass

Agent Benchmarks — Standard benchmarks
Evaluation Frameworks — Evaluation frameworks
Offline Eval — Offline evaluation
Online Eval — Online evaluation
Red Teaming — Red teaming
Agent Benchmarks — Benchmark details
Coding Agents — Coding metrics
Research Agents — Research metrics
Creative Agents — Creative metrics
Customer Facing Agents — Support metrics
Observability — Production metrics
Reliability — Reliability metrics
Research Directions — Metrics research

Sources: "Evaluating Language Models" surveys, RAGAS paper, DeepEval, Trulens, LLM-as-Judge (Liu et al.), "Measuring Calibration" (Guo et al.), "Measuring Faithfulness" (RAG papers), "Calibration Error" (Guo et al.), Bootstrap CI (Efron), McNemar's test, Paired t-test, Bootstrap CI (Efron & Tibshirani), "Reliability Diagrams" (DeGroot & Fienberg), "Reliability Diagrams" (Niculescu-Mizil & Caruana).

From the Agentic Engineering research vault · CC BY 4.0 · all notes · raw markdown