evaluator.py

import asyncio

from metrics.answer_correctness import evaluate as eval_answer_correctness
from metrics.hallucination import evaluate as eval_hallucination
from metrics.entity_accuracy import evaluate as eval_entity_accuracy
from metrics.date_accuracy import evaluate as eval_date_accuracy
from metrics.refusal_correctness import evaluate as eval_refusal_correctness


async def _evaluate_async(document_id, document_text, samples, run_agent):
    """Evaluate each sample using separate LLM metric evaluators."""

    results = []

    print("\nRunning Evaluation\n")

    for sample in samples:
        question = sample.get("question", "")
        expected = sample.get("expected_answer", "")

        if document_id:
            result = run_agent(document_id, question)
        else:
            result = run_agent(document_text, question)

        if isinstance(result, dict):
            predicted = result.get("answer", "")
        else:
            predicted = result

        (
            answer_result,
            hallucination_result,
            entity_result,
            date_result,
            refusal_result,
        ) = await asyncio.gather(
            eval_answer_correctness(predicted, expected, document_text),
            eval_hallucination(predicted, document_text),
            eval_entity_accuracy(predicted, expected, document_text),
            eval_date_accuracy(predicted, expected, document_text),
            eval_refusal_correctness(predicted, expected, document_text),
        )

        metrics = {
            "answer_correctness": answer_result,
            "hallucination": hallucination_result,
            "entity_accuracy": entity_result,
            "date_accuracy": date_result,
            "refusal_correctness": refusal_result,
        }

        metric_scores = [m.get("score", 0.0) for m in metrics.values() if isinstance(m, dict)]
        sample_score = sum(metric_scores) / len(metric_scores) if metric_scores else 0.0

        print(f"Q: {question}")
        print(f"Expected: {expected}")
        print(f"Predicted: {predicted}")
        print(f"Sample score: {sample_score}\n")

        results.append({
            "question": question,
            "expected_answer": expected,
            "predicted_answer": predicted,
            "sample_score": sample_score,
            "metrics": metrics,
        })

    overall_score = 0.0
    if results:
        overall_score = sum(r.get("sample_score", 0.0) for r in results) / len(results)

    return {"overall_score": overall_score, "results": results}


def evaluate(documentId, document_text, samples, run_agent):
    """Synchronous entrypoint for running evaluation from CLI/API."""
    return asyncio.run(_evaluate_async(documentId, document_text, samples, run_agent))