Part 2: Evals as a deployment gate — and how to know when they drift

{
  "id": "case-0412",
  "slice": "tier1/en",
  "input": { "request": "...", "context": { } },
  "expect": { "decision": "approve", "min_confidence": 0.80 },
  "must_not": { "decision": "auto_resolve" },
  "tags": ["edge-case", "previously-broke"]
}

from dataclasses import dataclass

@dataclass
class CaseResult:
    id: str; slice: str; passed: bool; checks: list

def score_case(case, output) -> CaseResult:
    exp = case.get("expect", {})
    checks = []
    if "decision" in exp:                                                       # symmetric .get on both sides
        checks.append(("decision", output.get("decision") == exp["decision"]))
    if "min_confidence" in exp:
        checks.append(("min_conf", output.get("confidence", 0) >= exp["min_confidence"]))
    for k, v in case.get("must_not", {}).items():
        checks.append((f"not_{k}", output.get(k) != v))
    assert checks, "empty expect + no must_not would vacuously pass"            # now this CAN actually fire
    return CaseResult(case["id"], case["slice"], all(ok for _, ok in checks), checks)

def test_scorer_catches_regression():                  # sensitivity: it MUST fail a wrong decision
    case = {"id": "c1", "slice": "tier1/en", "expect": {"decision": "approve", "min_confidence": 0.8}}
    assert     score_case(case, {"decision": "approve", "confidence": 0.9}).passed
    assert not score_case(case, {"decision": "reject",  "confidence": 0.9}).passed

# .ci/eval-gate.yml  (示例)
eval-gate:
  steps:
    - run: eval run --suite golden --gate --min-pass 0.98 --baseline artifacts/baseline.json
  # 退出码不等于0会失败流水线 → 部署被阻断

$ eval run --suite golden --gate --min-pass 0.98
  cases: 240   passed: 236   failed: 4   pass-rate: 98.3%
  gate(min-pass>=0.98): PASS
  failures: case-0412 (decision), case-0511 (min_conf), ...

// baseline.json — 在确认某个版本为已知良好版本时，需要主动捕获该文件
{
  "release": "2026-01-09",
  "metrics": {
    "exact_match":        { "mean": 0.942, "n": 2400, "std": 0.012 },
    "mean_confidence":    { "mean": 0.871, "n": 2400, "std": 0.030 },
    "human_override_rate":{ "mean": 0.060, "n": 2400 }
  }
}

import math
# 每项指标的最小有意义变化——0.06的指标变化中，0.02的变动具有意义，而对置信度指标则微不足道
MIN_EFFECT = {"exact_match": 0.01, "human_override_rate": 0.02, "mean_confidence": 0.03}
HIGHER_IS_WORSE = {"human_override_rate", "judge_disagreement_ratio", "abstention_rate"}  # 对这些指标而言，数值上升表示恶化

def compare_to_baseline(metric, cur, base, is_proportion=True):
    # cur/base 是 {"mean":.., "n":.., "std":..}。使用两个运行的 n — 不仅限于基线
    delta = cur["mean"] - base["mean"]
    if is_proportion:                                   # 率指标 → 两比例 z 检验
        p  = (base["mean"]*base["n"] + cur["mean"]*cur["n"]) / (base["n"] + cur["n"])   # 池化
        se = math.sqrt(p*(1-p) * (1/base["n"] + 1/cur["n"]))
    else:                                               # 连续值 → 均值差的标准误
        se = math.sqrt(base["std"]**2/base["n"] + cur["std"]**2/cur["n"])
    z = delta/se if se else 0.0
    worse = delta > 0 if metric in HIGHER_IS_WORSE else delta < 0   # 方向按指标定义，不一定是“更低”
    drifted = worse and abs(delta) >= MIN_EFFECT.get(metric, 0.02) and abs(z) > 2   # 有意义且约2σ朝坏方向
    return {"metric": metric, "delta": round(delta, 4), "z": round(z, 1), "drift": drifted}

# 常见错误：用基线标准差除以√n 会忽略当前运行自身的规模和方差 — 用50个实时决策对比2400行基线时，会误将纯噪声标记为异常
# 差值的标准误使用两者数据；比例使用池化两比例形式，而非存储的std

# 改进可能高于阈值但仍显著低于基线 — 这就是门控机制会漏掉的信号：

SHADOW_SAMPLE_RATE = 0.05    # 评估5%的实时决策（配置参数，非硬编码）

def maybe_shadow(decision, sample_rate=SHADOW_SAMPLE_RATE):
    # deterministic_hash = 你的稳定ID哈希；score_against = 你的规则/评委评分入口
    # 使用% 10000（非% 100）使低于1%的采样率不会静默四舍五入为零
    if deterministic_hash(decision["decision_id"]) % 10000 < sample_rate * 10000:
        verdict = score_against(rules_or_judge, decision)   # 异步执行，非关键路径
        emit("shadow_eval_pass_ratio", 1.0 if verdict.ok else 0.0, slice=decision["slice"])
        if not verdict.ok:
            add_to_review_queue(decision)                   # 新黄金案例候选

# 失败案例会成为新的黄金案例 — 生产环境会在现实最严峻的地方强化你的测试套件。使用ID的确定性哈希而非随机数，确保采样可复现且不会导致测试不稳定

### 早期预警信号（关注趋势而非单点）

### 触发时 — 调查，不要自动回滚

- 确认是真实问题（有意义且约2σ，而非噪声）