uv init first-evaluation && cd first-evaluation
uv add mistralai-evaluations
export MISTRAL_API_KEY="your-api-key"

import asyncio
import os

from mistralai.evaluations import (
    Evaluation, Evaluator, Goal, Mistral, Project, ScorerContext, System, TaskContext,
)

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

# 1. The dataset: each record is a test case with its expected answer.
dataset = [
    {"sentence": "Hello, how are you?", "groundtruth": "English"},
    {"sentence": "Bonjour, comment ça va?", "groundtruth": "French"},
    {"sentence": "Hola, ¿cómo estás?", "groundtruth": "Spanish"},
]

# 2. The task: the code you evaluate. It reads the model and the prompt from the system params.
async def task(ctx: TaskContext) -> str:
    response = await client.chat.complete_async(
        model=str(ctx.system.params["model"]),
        messages=[
            {"role": "system", "content": str(ctx.system.params["system_prompt"])},
            {"role": "user", "content": ctx.input_record["sentence"]},
        ],
    )
    return str(response.choices[0].message.content)

# 3. The scorer: 1 if the answer matches the expected language, 0 otherwise.
def scorer(ctx: ScorerContext) -> int:
    return 1 if ctx.input_record["groundtruth"].lower() == str(ctx.output).lower() else 0

# 4. The run: ties the dataset, task, and evaluator together and uploads the results to Studio.
async def main():
    run = await client.evaluation.run(
        project=Project(name="Language Detection"),
        evaluation=Evaluation(name="Accuracy Eval"),
        system=System(name="mistral-small", params={
            "model": "mistral-small-latest",
            "system_prompt": "What language is this sentence in? Reply with ONLY the language name.",
        }),
        dataset=dataset,
        task=task,
        evaluators=[
            Evaluator(
                name="accuracy",
                description="1 if the detected language matches the groundtruth, 0 otherwise.",
                scorer=scorer,
                goal=Goal.gte(0.8),
            ),
        ],
    )
    run.show(level="records")

asyncio.run(main())

uv run eval.py