Skip to main content

Outcome verification

metis.library.graders.OutcomeVerificationGrader
Measure
outcome_verification
Scopes
session
Grader type
llm
Reference type
rubric_based
Judgment form
binary
Aggregation
none_single_score

Assess whether the agent checks relevant results before claiming success.

class TaskArgs(BaseModel):
task_description: NonEmptyText | None = None
"""What the agent was asked to do. When unset, the judge infers the goal from the user's
messages. A criterion runs in every task that selects its behaviour, so this text applies
to the scenarios of all those tasks."""


class OutcomeVerificationArgs(JudgeArgs, TaskArgs):
verification_requirements: tuple[str, ...] = Field(
default=(),
description=(
"Specific results or states the agent should check before claiming "
"success. When empty, the judge assesses general verification behaviour."
),
)


class OutcomeVerificationGrader(Grader[OutcomeVerificationArgs]):
def criteria(self) -> str:
return (
"You are assessing whether the agent checks relevant results before "
"claiming success or taking a dependent action."
+ self.args.task_block()
+ section("verification_requirements", self.args.verification_requirements)
+ "\n\n<assessment_boundaries>\n"
"Verification is the checking action; error detection is recognition "
"of an error. A successful claim is not evidence that verification "
"occurred."
"\n</assessment_boundaries>"
)