Can Jev replace your LLM judge? Evaluating quality, cost, and latency

uv add mlflow typesafe-sdk
uv run mlflow server --host 127.0.0.1 --port 5000

export MLFLOW_TRACKING_URI="http://127.0.0.1:5000"
export TYPESAFE_API_KEY="your-typesafe-api-key"

from
time
import
perf_counter
import
mlflow
from
mlflow
.
entities
import
Feedback
from
mlflow
.
genai
.
scorers
import
scorer
from
typesafe_sdk
import
Noul
,
RetryPolicy
,
TypeSafeClient
mlflow
.
set_experiment
(
"jev-judge-comparison"
)
client
=
TypeSafeClient
(
model
=
"jev-1.13.0"
,
retry
=
RetryPolicy
(
max_retries
=
0
)
,
timeout
=
30
)
THRESHOLD
=
0.5
INPUT_USD_PER_MILLION
=
0.042
CRITERION
=
(
"Is the candidate answer factually and technically correct for the question, "
"using the supplied MLflow documentation and respecting any version specified "
"in the question? Answer yes if there is no material factual or API error. "
"Answer no if at least one claim or instruction is materially wrong or misleading. "
"Do not penalize style or harmless omissions. An omission is material when it "
"makes the requested technical answer misleading."
)
def
judge_answer
(
question
:
str
,
context
:
str
,
answer
:
str
)
-
>
dict
:
started
=
perf_counter
(
)
response
=
client
.
system_one
(
state
=
{
"question"
:
question
,
"context"
:
context
,
"answer"
:
answer
}
,
questions
=
{
"correct"
:
Noul
(
instructions
=
CRITERION
)
}
,
)
latency_ms
=
(
perf_counter
(
)
-
started
)
*
1000
probability
=
response
.
nouls
[
"correct"
]
.
noul
tokens
=
response
.
usage
.
input_tokens
cost
=
None
if
tokens
is
None
else
tokens
*
INPUT_USD_PER_MILLION
/
1_000_000
return
{
"prediction"
:
"correct"
if
probability
>=
THRESHOLD
else
"incorrect"
,
"p_correct"
:
probability
,
"model"
:
response
.
model
,
"input_tokens"
:
tokens
,
"latency_ms"
:
latency_ms
,
"estimated_cost_usd"
:
cost
,
}
@scorer
def
jev_correctness
(
inputs
:
dict
)
-
>
Feedback
:
result
=
judge_answer
(
**
inputs
)
return
Feedback
(
value
=
result
[
"prediction"
]
==
"correct"
,
metadata
=
{
**
result
,
"threshold"
:
THRESHOLD
}
,
)

{
"inputs"
:
{
"question"
:
(
'My `@scorer` function is called `check_answer`, but it returns '
'`Feedback(name="answer_quality", value=True)`. '
'Which name appears as the metric?'
)
,
"context"
:
(
"1. If the scorer returns one or more `Feedback` objects, then "
"`Feedback.name` fields take precedence, if specified.\n"
"2. For primitive return values or unnamed `Feedback`s, the "
"function name (for the `@scorer` decorator) or the `Scorer.name` "
"field (for the `Scorer` class) are used."
)
,
}
,
"outputs"
:
(
"The metric is named `check_answer`, because decorated scorers "
"always use the Python function name. To show `answer_quality` "
"in the results, rename the function to that name."
)
,
# This field is not passed to the Jev judge.
"expectations"
:
{
"human_correctness"
:
"incorrect"
}
,
}

mlflow
.
genai
.
evaluate
(
data
=
data
,
scorers
=
[
judge_answer
]
)

查看 MLflow 评估快速入门和自定义评分指南以适配示例。与 MLflow 社区分享你的学习成果。