test(qa): craft 成稿质量前后对照评估回路(灵感 F2)
固定 3 题材金标准输入(仙侠/都市/悬疑,各含最小 spec+world+outline 及开篇章), 经真实 assemble()+build_write_request() 组装 craft-on / craft-off 两版写章请求: - dry 模式(默认,零成本、免真实 LLM,可进 CI):断言 craft 教条块、genre 片段、 开篇黄金三章 marker 确实按 fixture 注入正确内容,且 craft-off 基线不含这三者; 并把两版将发送给模型的 prompt 并排 dump 供人眼看。这些断言即 tests/test_craft_eval.py, 是 craft/审查注入路径的回归门禁。 - --live 模式(真实付费 LLM,绝不进 CI):经 gateway 各生成一章 + analyst 五维裁判, 凭据读 DB、路由走 tier_routing,与生产写章同一网关路径。 入口 `uv run python -m tests.craft_eval [--live]`;用法/成本/门禁说明见 README.md。
This commit is contained in:
104
tests/craft_eval/__main__.py
Normal file
104
tests/craft_eval/__main__.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""craft A/B 评估脚本入口(灵感 F2)。
|
||||
|
||||
# dry(默认,零成本、免真实 LLM)——组装+校验+并排 dump prompt
|
||||
uv run python -m tests.craft_eval
|
||||
|
||||
# live(真实付费生成 + LLM 五维裁判)
|
||||
uv run python -m tests.craft_eval --live
|
||||
|
||||
选项:`--genre 仙侠` 只跑某题材;`--out PATH` 指定报告路径;`--quiet` 少打印。
|
||||
退出码:dry 任一断言失败 → 非零(可当回归门禁)。详见 README.md。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from .fixtures import FIXTURES, FixtureCase
|
||||
from .harness import (
|
||||
CaseReport,
|
||||
all_passed,
|
||||
build_prompt_pair,
|
||||
render_report,
|
||||
verify_case,
|
||||
write_report,
|
||||
)
|
||||
|
||||
|
||||
def _select(genre: str | None) -> list[FixtureCase]:
|
||||
if genre is None:
|
||||
return list(FIXTURES)
|
||||
picked = [c for c in FIXTURES if c.genre == genre]
|
||||
if not picked:
|
||||
raise SystemExit(f"没有题材为 {genre!r} 的 fixture;可选:{[c.genre for c in FIXTURES]}")
|
||||
return picked
|
||||
|
||||
|
||||
def _default_out(live: bool) -> Path:
|
||||
name = "craft_ab_live.md" if live else "craft_ab_dry.md"
|
||||
return Path(tempfile.gettempdir()) / name
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> int:
|
||||
cases = _select(args.genre)
|
||||
|
||||
reports: list[CaseReport] = []
|
||||
for case in cases:
|
||||
pair = await build_prompt_pair(case)
|
||||
reports.append(CaseReport(case=case, pair=pair, checks=verify_case(case, pair)))
|
||||
|
||||
failed = [(r.case.slug, c) for r in reports for c in r.checks if not c.passed]
|
||||
if not args.quiet:
|
||||
for r in reports:
|
||||
status = "OK" if all_passed(r.checks) else "FAIL"
|
||||
print(f"[dry] {r.case.slug:10s} {r.case.genre} 第{r.pair.chapter_no}章 {status}")
|
||||
for slug, chk in failed:
|
||||
print(f"[dry][FAIL] {slug} · {chk.name}: {chk.detail}", file=sys.stderr)
|
||||
|
||||
if args.live:
|
||||
if failed:
|
||||
print("[live] dry 校验未全绿,拒绝进入付费生成——先修正注入。", file=sys.stderr)
|
||||
return 1
|
||||
from .live import run_live, summarize
|
||||
|
||||
print("[live] 开始真实生成(付费)……", flush=True)
|
||||
reports = await run_live(reports)
|
||||
print("\n[live] 摘要:\n" + summarize(reports))
|
||||
|
||||
out = Path(args.out) if args.out else _default_out(args.live)
|
||||
write_report(render_report(reports, live=args.live), out)
|
||||
print(f"\n[out] 报告写至 {out}")
|
||||
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m tests.craft_eval",
|
||||
description="craft A/B 成稿质量前后对照评估(灵感 F2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--live",
|
||||
action="store_true",
|
||||
help="真实调用付费 LLM 生成两版正文 + 五维裁判(默认 dry,免真实 LLM)",
|
||||
)
|
||||
parser.add_argument("--genre", default=None, help="只跑指定题材(如 仙侠/都市/悬疑)")
|
||||
parser.add_argument("--out", default=None, help="报告输出路径(默认写系统临时目录)")
|
||||
parser.add_argument("--quiet", action="store_true", help="减少 stdout 打印")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.live:
|
||||
# get_settings() 按 cwd 读 .env;确保 cwd = 仓库根(本文件在 tests/craft_eval/ 下三层)。
|
||||
import os
|
||||
|
||||
os.chdir(Path(__file__).resolve().parents[2])
|
||||
|
||||
raise SystemExit(asyncio.run(_run(args)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user