Task success
metis.library.graders.TaskSuccessGraderAssess whether the requested end goal was achieved under declared success conditions.
Adapted from DeepEval Task Completion (verdict).
class TaskArgs(BaseModel):
task_description: NonEmptyText | None = None
"""What the agent was asked to do. When unset, the judge infers the goal from the user's
messages. A criterion runs in every task that selects its behaviour, so this text applies
to the scenarios of all those tasks."""
class ArtifactArgs(BaseModel):
artifact: NonEmptyText | None = None
"""The name of the file to judge. When the target revised the file, the judge reads the
latest version."""
class TaskSuccessArgs(JudgeArgs, ArtifactArgs, TaskArgs):
success_conditions: tuple[str, ...] = Field(
default=(),
description=(
"Observable conditions that define success. When empty, "
"the judge infers conditions from the task description."
),
)
class TaskSuccessGrader(Grader[TaskSuccessArgs]):
def criteria(self) -> str:
return (
"You are assessing whether the requested end goal was achieved."
+ self._task_block()
+ section("success_conditions", self.args.success_conditions)
+ self._steps()
+ "\n\n<assessment_boundaries>\n"
"Assess whether the declared end goals and mandatory conditions were achieved. "
+ self._evidence_rule()
+ "\n</assessment_boundaries>"
)
def _task_block(self) -> str:
if self.scope == Scope.ARTIFACT and self.args.task_description is None:
return (
"\n\nNo task description is given. Judge the file against the "
"success conditions only."
)
return self.args.task_block()
def _steps(self) -> str:
if self.scope == Scope.ARTIFACT:
task = "1. Identify the task: the goal and success conditions given above."
outcome = (
"2. Describe the outcome: a strictly factual description of what the "
"file contains, based only on the file."
)
enumerate_ = "Enumerate each relevant part of the file, in plain language."
else:
task = (
"1. Identify the task: the task or objective expressed by the user, "
"together with any task description and success conditions given above."
)
outcome = (
"2. Describe the outcome: a strictly factual description of what the "
"system did, based only on the evidence."
)
enumerate_ = (
"Enumerate each relevant action or output the evidence shows, in plain "
"language."
)
return (
"\n\n<steps>\n"
"Follow these steps before you score.\n"
f"{task}\n"
f"{outcome} The outcome should be solely factual, derived strictly from the "
'evidence. Do not include subjective language such as "successfully", '
f'"efficiently", or "well". {enumerate_}\n'
"3. Compare how well the actual outcome aligns with the desired task. "
"Score 1 only when the actual outcome achieves the task and every success "
"condition. Otherwise, score 0."
"\n</steps>"
)
def _evidence_rule(self) -> str:
if self.scope == Scope.ARTIFACT:
return (
"Judge only the attached file. It is the only evidence, so decide "
"whether the file itself achieves the goal."
)
return (
"For action tasks, require recorded outcome evidence such as tool results that "
"show state changes or confirmations from external systems. An agent's claim of "
'completion alone (e.g. "I have completed the task") is insufficient without '
"supporting evidence in tool results or system responses. "
"For answer-only tasks, a correct and complete answer may establish success."
)