"""craft A/B 评估回路的 dry 模式确定性断言(灵感 F2)。 这些断言 = 评估脚本 dry 模式的回归门禁:验证 craft 教条块 / genre 片段 / 开篇 marker **确实按 fixture 注入了正确内容**,且 craft-off 基线**不含**这三者。dry 路径不碰 LLM, 故可进常规 pytest / CI(`--live` 真实生成路径不在此、不进 CI,见 tests/craft_eval/README.md)。 """ from __future__ import annotations import pytest from ww_agents import load_craft_doctrine from ww_config import genre_craft_fragment from tests.craft_eval import ( FIXTURES, FixtureCase, all_passed, build_prompt_pair, input_text, render_report, system_text, verify_case, write_report, ) from tests.craft_eval.harness import OPENING_CHAPTER_LIMIT, CaseReport _OPENING_HEADER = "开篇黄金三章" def test_fixtures_cover_three_distinct_genres() -> None: genres = {c.genre for c in FIXTURES} assert {"仙侠", "都市", "悬疑"} <= genres assert len(FIXTURES) == len(genres), "每个 fixture 应是不同题材" for genre in genres: assert genre_craft_fragment(genre), f"题材 {genre!r} 缺 GENRE_CRAFT 片段" @pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug) async def test_dry_verification_all_pass(case: FixtureCase) -> None: """每个金标准 fixture 的 dry 校验全绿——这是 craft 改动的回归门禁本体。""" pair = await build_prompt_pair(case) checks = verify_case(case, pair) failed = [c for c in checks if not c.passed] assert not failed, "dry 校验失败:\n" + "\n".join(f" {c.name}: {c.detail}" for c in failed) assert all_passed(checks) @pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug) async def test_craft_on_injects_doctrine_genre_and_marker(case: FixtureCase) -> None: pair = await build_prompt_pair(case) on = pair.craft_on # 教条块:第一个 system 块、字节等于 load_craft_doctrine()、cache=True(不变量 #9)。 assert on.system[0].text == load_craft_doctrine() assert on.system[0].cache is True assert len(on.system) >= 2, "craft-on 应有 [教条, stable_core] 两块" # genre 片段:进 stable_core 缓存块(book-stable → 缓存前缀),不进易变 input(#9)。 fragment = genre_craft_fragment(case.genre) assert fragment is not None assert fragment in on.system[1].text assert fragment not in on.system[0].text, "genre 片段不应污染教条块" assert fragment not in input_text(on), "genre 片段是 book-stable,不应进 volatile input" # 开篇 marker:开篇章进 volatile input、不进缓存 system(含章号→易变,#9)。 if case.chapter_no <= OPENING_CHAPTER_LIMIT: assert _OPENING_HEADER in input_text(on) assert _OPENING_HEADER not in system_text(on) # 不变量 #2:只声明 tier,不写 model。 assert on.tier == "writer" @pytest.mark.parametrize("case", FIXTURES, ids=lambda c: c.slug) async def test_craft_off_baseline_excludes_craft(case: FixtureCase) -> None: pair = await build_prompt_pair(case) off = pair.craft_off assert load_craft_doctrine() not in system_text(off) fragment = genre_craft_fragment(case.genre) assert fragment is not None assert fragment not in system_text(off) assert fragment not in input_text(off) assert _OPENING_HEADER not in input_text(off) # 基线仍是可用写章请求:保留书名与写作指令(stripping 未误伤本体)。 assert case.spec.title in system_text(off) assert "请创作第" in input_text(off) async def test_opening_marker_absent_beyond_opening_limit() -> None: """章号 > 开篇上限时 craft-on 不注入开篇 marker(黄金三章边界)。""" case = FIXTURES[0] pair = await build_prompt_pair(case, chapter_no=OPENING_CHAPTER_LIMIT + 1) assert _OPENING_HEADER not in input_text(pair.craft_on) assert _OPENING_HEADER not in system_text(pair.craft_on) async def test_render_and_write_report(tmp_path) -> None: # type: ignore[no-untyped-def] reports: list[CaseReport] = [] for case in FIXTURES: pair = await build_prompt_pair(case) reports.append(CaseReport(case=case, pair=pair, checks=verify_case(case, pair))) text = render_report(reports, live=False) assert "craft-on" in text and "craft-off" in text for case in FIXTURES: assert case.slug in text out = write_report(text, tmp_path / "report.md") assert out.exists() assert out.read_text("utf-8") == text def test_build_prompt_pair_is_deterministic_across_runs() -> None: """同一 fixture 两次组装字节一致(缓存前缀稳定 / 无时间戳-UUID,#9)。""" import asyncio case = FIXTURES[0] a = asyncio.run(build_prompt_pair(case)) b = asyncio.run(build_prompt_pair(case)) assert system_text(a.craft_on) == system_text(b.craft_on) assert input_text(a.craft_on) == input_text(b.craft_on) assert system_text(a.craft_off) == system_text(b.craft_off)