Use run-level evaluators

import asyncio
import os

from mistralai.evaluations import (
    Evaluation, Evaluator, Mistral, RunEvaluator, RunEvaluatorContext,
    ScorerContext, System, TaskContext,
)

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

dataset = [
    {"prompt": "What is the capital of France?", "expected": "Paris"},
    {"prompt": "Which planet is known as the Red Planet?", "expected": "Mars"},
]

def accuracy_scorer(ctx: ScorerContext) -> int:
    return 1 if ctx.input_record["expected"].lower() in str(ctx.output).lower() else 0

def accuracy_gate(ctx: RunEvaluatorContext):
    return ctx.statistics["accuracy"].avg > 0.8

async def task(ctx: TaskContext) -> str:
    r = await client.chat.complete_async(
        model="mistral-small-latest",
        messages=[{"role": "user", "content": ctx.input_record["prompt"]}],
    )
    return str(r.choices[0].message.content)

async def main():
    run = await client.evaluation.run(
        evaluation=Evaluation(name="Gated Eval"),
        dataset=dataset,
        task=task,
        evaluators=[Evaluator(name="accuracy", description="1 if the expected answer appears in the output.", scorer=accuracy_scorer)],
        run_evaluators=[
            RunEvaluator(
                name="accuracy_above_80pct",
                description="True if average accuracy exceeds 80%.",
                scorer=accuracy_gate,
            ),
        ],
    )
    print(run.run_scores["accuracy_above_80pct"].value)  # True

asyncio.run(main())

from mistralai.evaluations import RunEvaluator, RunEvaluatorContext, Score, get_score

def f1_scorer(ctx: RunEvaluatorContext) -> Score:
    tp = fp = fn = 0
    for record in ctx.records:
        # Failed records are passed too: skip any record whose generation or scoring failed.
        if any(
            g.status != "success" or any(s.status != "success" for scores in g.scores.values() for s in scores)
            for g in record.output.generations
        ):
            continue
        predicted = get_score(record, "predicted_label").value
        expected = get_score(record, "expected_label").value
        if predicted == 1 and expected == 1:
            tp += 1
        elif predicted == 1 and expected == 0:
            fp += 1
        elif predicted == 0 and expected == 1:
            fn += 1

    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

    return Score(
        value=f1,
        rationale=f"precision={precision:.2f}, recall={recall:.2f}",
    )

run = await client.evaluation.run(
    # ...
    run_evaluators=[RunEvaluator(name="f1", description="Harmonic mean of precision and recall across all records.", scorer=f1_scorer)],
)

from mistralai.evaluations import Goal

RunEvaluator(
    name="f1",
    description="Harmonic mean of precision and recall.",
    scorer=f1_scorer,
    goal=Goal.gte(0.85),
)