51bfb6e320
固定 3 题材金标准输入（仙侠/都市/悬疑，各含最小 spec+world+outline 及开篇章）， 经真实 assemble()+build_write_request() 组装 craft-on / craft-off 两版写章请求： - dry 模式（默认，零成本、免真实 LLM，可进 CI）：断言 craft 教条块、genre 片段、 开篇黄金三章 marker 确实按 fixture 注入正确内容，且 craft-off 基线不含这三者； 并把两版将发送给模型的 prompt 并排 dump 供人眼看。这些断言即 tests/test_craft_eval.py， 是 craft/审查注入路径的回归门禁。 - --live 模式（真实付费 LLM，绝不进 CI）：经 gateway 各生成一章 + analyst 五维裁判， 凭据读 DB、路由走 tier_routing，与生产写章同一网关路径。 入口 `uv run python -m tests.craft_eval [--live]`；用法/成本/门禁说明见 README.md。
122 lines
5.0 KiB
Python
122 lines
5.0 KiB
Python
"""craft A/B 评估回路的 dry 模式确定性断言（灵感 F2）。
|
||
|
||
这些断言 = 评估脚本 dry 模式的回归门禁：验证 craft 教条块 / genre 片段 / 开篇 marker
|
||
**确实按 fixture 注入了正确内容**，且 craft-off 基线**不含**这三者。dry 路径不碰 LLM，
|
||
故可进常规 pytest / CI（`--live` 真实生成路径不在此、不进 CI，见 tests/craft_eval/README.md）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
from ww_agents import load_craft_doctrine
|
||
from ww_config import genre_craft_fragment
|
||
|
||
from tests.craft_eval import (
|
||
FIXTURES,
|
||
FixtureCase,
|
||
all_passed,
|
||
build_prompt_pair,
|
||
input_text,
|
||
render_report,
|
||
system_text,
|
||
verify_case,
|
||
write_report,
|
||
)
|
||
from tests.craft_eval.harness import OPENING_CHAPTER_LIMIT, CaseReport
|
||
|
||
_OPENING_HEADER = "开篇黄金三章"
|
||
|
||
|
||
def test_fixtures_cover_three_distinct_genres() -> None:
|
||
genres = {c.genre for c in FIXTURES}
|
||
assert {"仙侠", "都市", "悬疑"} <= genres
|
||
assert len(FIXTURES) == len(genres), "每个 fixture 应是不同题材"
|
||
for genre in genres:
|
||
assert genre_craft_fragment(genre), f"题材 {genre!r} 缺 GENRE_CRAFT 片段"
|
||
|
||
|
||
@pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug)
|
||
async def test_dry_verification_all_pass(case: FixtureCase) -> None:
|
||
"""每个金标准 fixture 的 dry 校验全绿——这是 craft 改动的回归门禁本体。"""
|
||
pair = await build_prompt_pair(case)
|
||
checks = verify_case(case, pair)
|
||
failed = [c for c in checks if not c.passed]
|
||
assert not failed, "dry 校验失败：\n" + "\n".join(f" {c.name}: {c.detail}" for c in failed)
|
||
assert all_passed(checks)
|
||
|
||
|
||
@pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug)
|
||
async def test_craft_on_injects_doctrine_genre_and_marker(case: FixtureCase) -> None:
|
||
pair = await build_prompt_pair(case)
|
||
on = pair.craft_on
|
||
|
||
# 教条块：第一个 system 块、字节等于 load_craft_doctrine()、cache=True（不变量 #9）。
|
||
assert on.system[0].text == load_craft_doctrine()
|
||
assert on.system[0].cache is True
|
||
assert len(on.system) >= 2, "craft-on 应有 [教条, stable_core] 两块"
|
||
|
||
# genre 片段：进 stable_core 缓存块（book-stable → 缓存前缀），不进易变 input（#9）。
|
||
fragment = genre_craft_fragment(case.genre)
|
||
assert fragment is not None
|
||
assert fragment in on.system[1].text
|
||
assert fragment not in on.system[0].text, "genre 片段不应污染教条块"
|
||
assert fragment not in input_text(on), "genre 片段是 book-stable，不应进 volatile input"
|
||
|
||
# 开篇 marker：开篇章进 volatile input、不进缓存 system（含章号→易变，#9）。
|
||
if case.chapter_no <= OPENING_CHAPTER_LIMIT:
|
||
assert _OPENING_HEADER in input_text(on)
|
||
assert _OPENING_HEADER not in system_text(on)
|
||
|
||
# 不变量 #2：只声明 tier，不写 model。
|
||
assert on.tier == "writer"
|
||
|
||
|
||
@pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug)
|
||
async def test_craft_off_baseline_excludes_craft(case: FixtureCase) -> None:
|
||
pair = await build_prompt_pair(case)
|
||
off = pair.craft_off
|
||
|
||
assert load_craft_doctrine() not in system_text(off)
|
||
fragment = genre_craft_fragment(case.genre)
|
||
assert fragment is not None
|
||
assert fragment not in system_text(off)
|
||
assert fragment not in input_text(off)
|
||
assert _OPENING_HEADER not in input_text(off)
|
||
# 基线仍是可用写章请求：保留书名与写作指令（stripping 未误伤本体）。
|
||
assert case.spec.title in system_text(off)
|
||
assert "请创作第" in input_text(off)
|
||
|
||
|
||
async def test_opening_marker_absent_beyond_opening_limit() -> None:
|
||
"""章号 > 开篇上限时 craft-on 不注入开篇 marker（黄金三章边界）。"""
|
||
case = FIXTURES[0]
|
||
pair = await build_prompt_pair(case, chapter_no=OPENING_CHAPTER_LIMIT + 1)
|
||
assert _OPENING_HEADER not in input_text(pair.craft_on)
|
||
assert _OPENING_HEADER not in system_text(pair.craft_on)
|
||
|
||
|
||
async def test_render_and_write_report(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||
reports: list[CaseReport] = []
|
||
for case in FIXTURES:
|
||
pair = await build_prompt_pair(case)
|
||
reports.append(CaseReport(case=case, pair=pair, checks=verify_case(case, pair)))
|
||
text = render_report(reports, live=False)
|
||
assert "craft-on" in text and "craft-off" in text
|
||
for case in FIXTURES:
|
||
assert case.slug in text
|
||
out = write_report(text, tmp_path / "report.md")
|
||
assert out.exists()
|
||
assert out.read_text("utf-8") == text
|
||
|
||
|
||
def test_build_prompt_pair_is_deterministic_across_runs() -> None:
|
||
"""同一 fixture 两次组装字节一致（缓存前缀稳定 / 无时间戳-UUID，#9）。"""
|
||
import asyncio
|
||
|
||
case = FIXTURES[0]
|
||
a = asyncio.run(build_prompt_pair(case))
|
||
b = asyncio.run(build_prompt_pair(case))
|
||
assert system_text(a.craft_on) == system_text(b.craft_on)
|
||
assert input_text(a.craft_on) == input_text(b.craft_on)
|
||
assert system_text(a.craft_off) == system_text(b.craft_off)
|