perf(backend): 角色/世界观入库改批量——create_many 一次 flush 落全部行
角色入库端点与 toolbox world_entities 入库端点原逐卡/逐条 create(每行一次 SELECT/flush/refresh,N+1 round-trip)。新增 create_many:世界观用 add_all, 角色用一次 IN 查已有名 + 一次 flush(保持 (project_id,name) upsert 幂等与批内 重名去重、保序返回)。单条 create 薄封装 create_many,行为不变(既有调用方/E2E 复用)。复活并复用 CharacterWriteFields、新增 WorldEntityFields 作批量入参。 补两端点「多行一次落库」测试;既有幂等/冲突 gate 测试与 m5 真 pg E2E 全绿。
This commit is contained in:
@@ -28,7 +28,7 @@ from ww_agents import (
|
||||
WorldEntityCard,
|
||||
WorldGenResult,
|
||||
)
|
||||
from ww_core.domain.character_repo import CharacterWriteView
|
||||
from ww_core.domain.character_repo import CharacterWriteFields, CharacterWriteView
|
||||
from ww_core.domain.project_repo import ProjectCreate
|
||||
from ww_core.domain.rule_repo import RuleListItemView
|
||||
from ww_llm_gateway.types import LlmRequest, LlmResponse, ServedBy, Usage
|
||||
@@ -120,6 +120,27 @@ class _FakeCharacterWriteRepo:
|
||||
)
|
||||
return CharacterWriteView(id=row_id, name=name, role=role)
|
||||
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, cards: list[CharacterWriteFields]
|
||||
) -> list[CharacterWriteView]:
|
||||
# 镜像 Sql:逐卡 upsert(含批内重名去重),保序返回逐卡 View。
|
||||
return [
|
||||
await self.create(
|
||||
project_id,
|
||||
name=c.name,
|
||||
role=c.role,
|
||||
traits=list(c.traits),
|
||||
appearance=c.appearance,
|
||||
motive=c.motive,
|
||||
backstory=c.backstory,
|
||||
arc=c.arc,
|
||||
speech_tics=list(c.speech_tics),
|
||||
tags=list(c.tags),
|
||||
relations=[dict(r) for r in c.relations],
|
||||
)
|
||||
for c in cards
|
||||
]
|
||||
|
||||
|
||||
class _FakeRulesReadRepo:
|
||||
"""实现规则 repo 的读侧 `list_for_project`(带 id,供 list_rules 端点)。"""
|
||||
@@ -352,6 +373,39 @@ async def test_ingest_characters_no_conflict_writes_and_201() -> None:
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ingest_multiple_characters_all_land_in_one_batch() -> None:
|
||||
# 批量入库:多张卡一次 create_many 全部落库,created 保序,单次 commit。
|
||||
repo = FakeProjectRepo()
|
||||
pid = await _seed_project(repo)
|
||||
gateway = _SchemaRoutingGateway({ContinuityReview: _no_conflicts()})
|
||||
char_repo = _FakeCharacterWriteRepo()
|
||||
app, session = _make_app(project_repo=repo, gateway=gateway, char_repo=char_repo)
|
||||
|
||||
def _card(name: str) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"role": "配角",
|
||||
"traits": [],
|
||||
"backstory": "x",
|
||||
"arc": "y",
|
||||
"speech_tics": [],
|
||||
"tags": [],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
async with _client(app) as client:
|
||||
resp = await client.post(
|
||||
f"/projects/{pid}/characters",
|
||||
json={"cards": [_card("甲"), _card("乙"), _card("丙")]},
|
||||
)
|
||||
|
||||
assert resp.status_code == 201
|
||||
assert resp.json()["created"] == ["甲", "乙", "丙"]
|
||||
assert [r["name"] for r in char_repo.rows] == ["甲", "乙", "丙"]
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ingest_characters_with_conflict_blocks_409() -> None:
|
||||
repo = FakeProjectRepo()
|
||||
|
||||
@@ -32,7 +32,7 @@ from ww_core.domain.outline_write_repo import OutlineWriteView
|
||||
from ww_core.domain.project_repo import ProjectCreate
|
||||
from ww_core.domain.repositories import OutlineView
|
||||
from ww_core.domain.rule_repo import RuleWriteView
|
||||
from ww_core.domain.world_entity_repo import WorldEntityWriteView
|
||||
from ww_core.domain.world_entity_repo import WorldEntityFields, WorldEntityWriteView
|
||||
from ww_llm_gateway.types import LlmRequest, LlmResponse, ServedBy, Usage
|
||||
from ww_shared import AppError, ErrorCode
|
||||
|
||||
@@ -75,6 +75,15 @@ class _FakeWorldWriteRepo:
|
||||
self.rows.append({"type": type, "name": name, "rules": list(rules)})
|
||||
return WorldEntityWriteView(id=uuid.uuid4(), type=type, name=name)
|
||||
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, entities: list[WorldEntityFields]
|
||||
) -> list[WorldEntityWriteView]:
|
||||
# 镜像 Sql:批量插入,保序返回逐条 View。
|
||||
return [
|
||||
await self.create(project_id, type=e.type, name=e.name, rules=list(e.rules))
|
||||
for e in entities
|
||||
]
|
||||
|
||||
|
||||
class _FakeOutlineWriteRepo:
|
||||
def __init__(self) -> None:
|
||||
@@ -569,6 +578,30 @@ async def test_ingest_world_entities_no_conflict_writes_201() -> None:
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ingest_multiple_world_entities_all_land_in_one_batch() -> None:
|
||||
# 批量入库:多条实体一次 create_many 全部落库,created 保序,单次 commit。
|
||||
repo = FakeProjectRepo()
|
||||
pid = await _seed_project(repo)
|
||||
gateway = _SchemaRoutingGateway({ContinuityReview: _no_conflicts()})
|
||||
app, session, world_repo, _ = _make_app(project_repo=repo, gateway=gateway)
|
||||
payload = {
|
||||
"world_entities": [
|
||||
{"type": "力量体系", "name": "吞噬系统", "rules": ["每日上限"]},
|
||||
{"type": "势力", "name": "天机阁", "rules": ["中立"]},
|
||||
{"type": "地点", "name": "无尽海", "rules": []},
|
||||
]
|
||||
}
|
||||
|
||||
async with _client(app) as client:
|
||||
resp = await client.post(f"/projects/{pid}/skills/golden-finger/ingest", json=payload)
|
||||
|
||||
assert resp.status_code == 201
|
||||
assert resp.json()["created"] == ["吞噬系统", "天机阁", "无尽海"]
|
||||
assert [r["name"] for r in world_repo.rows] == ["吞噬系统", "天机阁", "无尽海"]
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ingest_world_entities_conflict_blocks_409() -> None:
|
||||
repo = FakeProjectRepo()
|
||||
|
||||
@@ -33,6 +33,7 @@ from ww_agents import (
|
||||
worldbuilder_spec,
|
||||
)
|
||||
from ww_core.domain import (
|
||||
CharacterWriteFields,
|
||||
CharacterWriteRepo,
|
||||
ProjectRepo,
|
||||
RuleWriteRepo,
|
||||
@@ -372,10 +373,8 @@ async def ingest_characters(
|
||||
rejected_tables=rejected,
|
||||
)
|
||||
|
||||
created: list[str] = []
|
||||
for card in allowed.get("characters", []):
|
||||
view = await char_repo.create(
|
||||
project_id,
|
||||
fields = [
|
||||
CharacterWriteFields(
|
||||
name=card.name,
|
||||
role=card.role,
|
||||
traits=list(card.traits),
|
||||
@@ -387,7 +386,10 @@ async def ingest_characters(
|
||||
tags=list(card.tags),
|
||||
relations=[r.model_dump() for r in card.relations],
|
||||
)
|
||||
created.append(view.name)
|
||||
for card in allowed.get("characters", [])
|
||||
]
|
||||
# 批量 upsert(一次 flush 落全部),保序返回逐卡 View。
|
||||
created = [view.name for view in await char_repo.create_many(project_id, fields)]
|
||||
|
||||
# 提交边界:网关 ledger(precheck)+ 角色写侧均只 flush → 端点末尾一次 commit。
|
||||
await session.commit()
|
||||
|
||||
@@ -25,7 +25,7 @@ from ww_core.domain.outline_write_repo import OutlineWriteRepo
|
||||
from ww_core.domain.project_repo import ProjectRepo
|
||||
from ww_core.domain.repositories import MemoryRepos, OutlineRepo
|
||||
from ww_core.domain.rule_repo import RuleWriteRepo
|
||||
from ww_core.domain.world_entity_repo import WorldEntityWriteRepo
|
||||
from ww_core.domain.world_entity_repo import WorldEntityFields, WorldEntityWriteRepo
|
||||
from ww_core.orchestrator import run_generator
|
||||
from ww_db import get_session
|
||||
from ww_llm_gateway import Gateway
|
||||
@@ -427,12 +427,12 @@ async def _ingest_world_entities(
|
||||
rejected_tables=rejected,
|
||||
)
|
||||
|
||||
created: list[str] = []
|
||||
for entity in allowed.get("world_entities", []):
|
||||
view = await world_write_repo.create(
|
||||
project_id, type=entity.type, name=entity.name, rules=list(entity.rules)
|
||||
)
|
||||
created.append(view.name)
|
||||
fields = [
|
||||
WorldEntityFields(type=entity.type, name=entity.name, rules=list(entity.rules))
|
||||
for entity in allowed.get("world_entities", [])
|
||||
]
|
||||
# 批量插入(一次 flush 落全部),保序返回逐条 View。
|
||||
created = [view.name for view in await world_write_repo.create_many(project_id, fields)]
|
||||
|
||||
await session.commit()
|
||||
log.info(
|
||||
|
||||
@@ -9,6 +9,7 @@ from ww_core.domain.chapter_repo import (
|
||||
SqlChapterRepo,
|
||||
)
|
||||
from ww_core.domain.character_repo import (
|
||||
CharacterWriteFields,
|
||||
CharacterWriteRepo,
|
||||
CharacterWriteView,
|
||||
SqlCharacterWriteRepo,
|
||||
@@ -73,6 +74,7 @@ from ww_core.domain.template_repo import (
|
||||
)
|
||||
from ww_core.domain.world_entity_repo import (
|
||||
SqlWorldEntityWriteRepo,
|
||||
WorldEntityFields,
|
||||
WorldEntityWriteRepo,
|
||||
WorldEntityWriteView,
|
||||
)
|
||||
@@ -82,9 +84,11 @@ __all__ = [
|
||||
"ChapterRepo",
|
||||
"ChapterView",
|
||||
"SqlChapterRepo",
|
||||
"CharacterWriteFields",
|
||||
"CharacterWriteRepo",
|
||||
"CharacterWriteView",
|
||||
"SqlCharacterWriteRepo",
|
||||
"WorldEntityFields",
|
||||
"WorldEntityWriteRepo",
|
||||
"WorldEntityWriteView",
|
||||
"SqlWorldEntityWriteRepo",
|
||||
|
||||
@@ -75,12 +75,35 @@ class CharacterWriteRepo(Protocol):
|
||||
relations: list[dict[str, Any]],
|
||||
) -> CharacterWriteView: ...
|
||||
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, cards: list[CharacterWriteFields]
|
||||
) -> list[CharacterWriteView]:
|
||||
"""批量 upsert 角色卡(一次 flush 落全部),逐卡返回 View(保序、含重名重复)。"""
|
||||
...
|
||||
|
||||
|
||||
class CharacterWriteFields(BaseModel):
|
||||
"""从 `CharacterCard` 拆出的入库字段(端点把 schema 卡转成此形喂 create_many)。"""
|
||||
|
||||
name: str
|
||||
role: str
|
||||
traits: list[str] = Field(default_factory=list)
|
||||
appearance: str = ""
|
||||
motive: str = ""
|
||||
backstory: str
|
||||
arc: str
|
||||
speech_tics: list[str] = Field(default_factory=list)
|
||||
tags: list[Any] = Field(default_factory=list)
|
||||
relations: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SqlCharacterWriteRepo:
|
||||
"""SQLAlchemy 实现:upsert 一行 `characters`(schema→DB 形变;只 flush 不 commit)。
|
||||
"""SQLAlchemy 实现:upsert `characters`(schema→DB 形变;只 flush 不 commit)。
|
||||
|
||||
幂等:先按 (project_id, name) 查既有行——命中则原地更新该行字段,否则插新行。
|
||||
幂等:按 (project_id, name) 查既有行——命中则原地更新该行字段,否则插新行。
|
||||
避免同名角色重复入库产生重复卡(QA #8;不加 DB 约束,纯 app 层)。
|
||||
批量入库走 `create_many`:一次 `IN` 查已有名 + 一次 `flush` 落全部新行(少 N 次
|
||||
round-trip),语义与逐卡 `create` 完全一致(含批内重名去重与保序返回)。
|
||||
"""
|
||||
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
@@ -101,56 +124,61 @@ class SqlCharacterWriteRepo:
|
||||
tags: list[Any],
|
||||
relations: list[dict[str, Any]],
|
||||
) -> CharacterWriteView:
|
||||
existing = (
|
||||
await self._s.execute(
|
||||
select(Character).where(Character.project_id == project_id, Character.name == name)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
existing.role = role
|
||||
existing.traits = _traits_to_jsonb(traits)
|
||||
existing.appearance = appearance
|
||||
existing.motive = motive
|
||||
existing.backstory = backstory
|
||||
existing.arc = _arc_to_jsonb(arc)
|
||||
existing.speech_tics = _traits_to_jsonb(speech_tics)
|
||||
existing.tags = list(tags)
|
||||
existing.relations = [dict(r) for r in relations]
|
||||
await self._s.flush()
|
||||
await self._s.refresh(existing)
|
||||
return CharacterWriteView(id=existing.id, name=existing.name, role=existing.role)
|
||||
row = Character(
|
||||
project_id=project_id,
|
||||
"""单卡 upsert——薄封装 `create_many`,行为不变(供既有调用方/E2E 复用)。"""
|
||||
fields = CharacterWriteFields(
|
||||
name=name,
|
||||
role=role,
|
||||
traits=_traits_to_jsonb(traits),
|
||||
traits=list(traits),
|
||||
appearance=appearance,
|
||||
motive=motive,
|
||||
backstory=backstory,
|
||||
arc=_arc_to_jsonb(arc),
|
||||
speech_tics=_traits_to_jsonb(speech_tics),
|
||||
arc=arc,
|
||||
speech_tics=list(speech_tics),
|
||||
tags=list(tags),
|
||||
relations=[dict(r) for r in relations],
|
||||
relations=list(relations),
|
||||
)
|
||||
self._s.add(row)
|
||||
await self._s.flush()
|
||||
await self._s.refresh(row)
|
||||
return CharacterWriteView(id=row.id, name=row.name, role=row.role)
|
||||
return (await self.create_many(project_id, [fields]))[0]
|
||||
|
||||
def _apply_fields(self, row: Character, card: CharacterWriteFields) -> None:
|
||||
"""把入库字段写到 ORM 行(schema→DB JSONB 形变;新旧行共用)。"""
|
||||
row.role = card.role
|
||||
row.traits = _traits_to_jsonb(card.traits)
|
||||
row.appearance = card.appearance
|
||||
row.motive = card.motive
|
||||
row.backstory = card.backstory
|
||||
row.arc = _arc_to_jsonb(card.arc)
|
||||
row.speech_tics = _traits_to_jsonb(card.speech_tics)
|
||||
row.tags = list(card.tags)
|
||||
row.relations = [dict(r) for r in card.relations]
|
||||
|
||||
# ---- request-shaping helper(schema 字段名导出供测试/端点共用)----
|
||||
|
||||
|
||||
class CharacterWriteFields(BaseModel):
|
||||
"""从 `CharacterCard` 拆出的入库字段(端点把 schema 卡转成此 kwargs 形)。"""
|
||||
|
||||
name: str
|
||||
role: str
|
||||
traits: list[str] = Field(default_factory=list)
|
||||
appearance: str = ""
|
||||
motive: str = ""
|
||||
backstory: str
|
||||
arc: str
|
||||
speech_tics: list[str] = Field(default_factory=list)
|
||||
tags: list[Any] = Field(default_factory=list)
|
||||
relations: list[dict[str, Any]] = Field(default_factory=list)
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, cards: list[CharacterWriteFields]
|
||||
) -> list[CharacterWriteView]:
|
||||
if not cards:
|
||||
return []
|
||||
names = [c.name for c in cards]
|
||||
existing = (
|
||||
(
|
||||
await self._s.execute(
|
||||
select(Character).where(
|
||||
Character.project_id == project_id, Character.name.in_(names)
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
by_name: dict[str, Character] = {r.name: r for r in existing}
|
||||
rows: list[Character] = []
|
||||
for card in cards:
|
||||
row = by_name.get(card.name)
|
||||
if row is None:
|
||||
row = Character(project_id=project_id, name=card.name)
|
||||
self._apply_fields(row, card)
|
||||
self._s.add(row)
|
||||
by_name[card.name] = row
|
||||
else:
|
||||
self._apply_fields(row, card)
|
||||
rows.append(row) # 逐卡收集(同名多卡指向同一行 → 保序返回含重复)。
|
||||
await self._s.flush() # 一次 flush 落全部新行;server_default id 经 RETURNING 回填。
|
||||
return [CharacterWriteView(id=r.id, name=r.name, role=r.role) for r in rows]
|
||||
|
||||
@@ -36,6 +36,14 @@ def _rules_to_jsonb(rules: list[str]) -> dict[str, Any]:
|
||||
return {"rules": list(rules)}
|
||||
|
||||
|
||||
class WorldEntityFields(BaseModel):
|
||||
"""单条世界观实体入库字段(端点把 schema 卡转成此形喂 create_many)。"""
|
||||
|
||||
type: str
|
||||
name: str
|
||||
rules: list[str]
|
||||
|
||||
|
||||
class WorldEntityWriteRepo(Protocol):
|
||||
"""世界观实体写侧接口(按 project_id 隔离;只 flush 不 commit)。"""
|
||||
|
||||
@@ -48,9 +56,19 @@ class WorldEntityWriteRepo(Protocol):
|
||||
rules: list[str],
|
||||
) -> WorldEntityWriteView: ...
|
||||
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, entities: list[WorldEntityFields]
|
||||
) -> list[WorldEntityWriteView]:
|
||||
"""批量插入世界观实体(一次 flush 落全部),逐条保序返回 View。"""
|
||||
...
|
||||
|
||||
|
||||
class SqlWorldEntityWriteRepo:
|
||||
"""SQLAlchemy 实现:插一行 `world_entities`(rules list→dict;只 flush 不 commit)。"""
|
||||
"""SQLAlchemy 实现:插 `world_entities`(rules list→dict;只 flush 不 commit)。
|
||||
|
||||
批量入库走 `create_many`:`add_all` + 一次 `flush`(少 N 次 round-trip),
|
||||
语义与逐条 `create` 一致;server_default id 经 RETURNING 回填。
|
||||
"""
|
||||
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self._s = session
|
||||
@@ -63,13 +81,24 @@ class SqlWorldEntityWriteRepo:
|
||||
name: str,
|
||||
rules: list[str],
|
||||
) -> WorldEntityWriteView:
|
||||
row = WorldEntity(
|
||||
project_id=project_id,
|
||||
type=type,
|
||||
name=name,
|
||||
rules=_rules_to_jsonb(rules),
|
||||
)
|
||||
self._s.add(row)
|
||||
await self._s.flush()
|
||||
await self._s.refresh(row)
|
||||
return WorldEntityWriteView(id=row.id, type=row.type, name=row.name)
|
||||
"""单条插入——薄封装 `create_many`,行为不变(供既有调用方复用)。"""
|
||||
entity = WorldEntityFields(type=type, name=name, rules=list(rules))
|
||||
return (await self.create_many(project_id, [entity]))[0]
|
||||
|
||||
async def create_many(
|
||||
self, project_id: uuid.UUID, entities: list[WorldEntityFields]
|
||||
) -> list[WorldEntityWriteView]:
|
||||
if not entities:
|
||||
return []
|
||||
rows = [
|
||||
WorldEntity(
|
||||
project_id=project_id,
|
||||
type=e.type,
|
||||
name=e.name,
|
||||
rules=_rules_to_jsonb(e.rules),
|
||||
)
|
||||
for e in entities
|
||||
]
|
||||
self._s.add_all(rows)
|
||||
await self._s.flush() # 一次 flush 落全部;server_default id 经 RETURNING 回填。
|
||||
return [WorldEntityWriteView(id=r.id, type=r.type, name=r.name) for r in rows]
|
||||
|
||||
Reference in New Issue
Block a user