feat: M1 — 立项→写章草稿(SSE)→自动保存;连一家 provider
- 薄自建 LLM 网关:OpenAI 兼容适配器(DeepSeek) + instructor 结构化输出 + usage_ledger 记账 + 档位路由 - 记忆服务 assemble:确定性选择(显式+主角+近况) + 渲染卡 + 缓存断点(中性文本) - LangGraph 写章节点 + Postgres checkpointer + SSE 归一(token/done/error) - API:立项 + 写章 draft(SSE) + PUT 自动保存 + 提供商凭据(Fernet 加密/测试连接) - 前端:AppShell + 作品库 + 5 步立项向导 + 写作工作台(流式打字机+自动保存) + 设置页 - M1 E2E:真实 DB + mock 网关零 token 走通闭环
This commit is contained in:
2
packages/llm_gateway/README.md
Normal file
2
packages/llm_gateway/README.md
Normal file
@@ -0,0 +1,2 @@
|
||||
# core (@llm/@backend) — 占位骨架
|
||||
领域核心(domain/状态机)、编排器(orchestrator/LangGraph 图)、记忆服务(memory)。Phase 1+ 由对应 owner 填充。
|
||||
24
packages/llm_gateway/pyproject.toml
Normal file
24
packages/llm_gateway/pyproject.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
[project]
|
||||
name = "ww-llm-gateway"
|
||||
version = "0.0.0"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"openai>=1.40",
|
||||
"instructor>=1.5",
|
||||
"pydantic>=2.7",
|
||||
"ww-shared",
|
||||
"ww-config",
|
||||
"ww-db",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["ww_llm_gateway"]
|
||||
|
||||
[tool.uv.sources]
|
||||
ww-shared = { workspace = true }
|
||||
ww-config = { workspace = true }
|
||||
ww-db = { workspace = true }
|
||||
18
packages/llm_gateway/tests/conftest.py
Normal file
18
packages/llm_gateway/tests/conftest.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""网关单测公用 fixtures。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from ww_llm_gateway.types import LlmRequest, Scope
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scope() -> Scope:
|
||||
return Scope(user_id=uuid.UUID(int=1), project_id=uuid.UUID(int=2))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def req(scope: Scope) -> LlmRequest:
|
||||
return LlmRequest(tier="writer", input="写第 1 章", scope=scope)
|
||||
56
packages/llm_gateway/tests/fakes.py
Normal file
56
packages/llm_gateway/tests/fakes.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""网关单测替身(mock provider + 内存 ledger)——不碰真实 API / DB。
|
||||
|
||||
放在独立模块(非 conftest)以便测试用绝对导入 `from fakes import ...`;
|
||||
本目录无 __init__.py(避免与顶层 tests 包同名冲突),故走 pytest 路径注入。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from ww_llm_gateway.adapters.base import (
|
||||
Capabilities,
|
||||
ProviderResult,
|
||||
ProviderUsage,
|
||||
StreamChunk,
|
||||
)
|
||||
from ww_llm_gateway.routing import Route
|
||||
from ww_llm_gateway.types import LlmRequest, Scope, Tier, Usage
|
||||
|
||||
|
||||
class FakeAdapter:
|
||||
provider = "deepseek"
|
||||
|
||||
def __init__(self, text: str = "hello world", deltas: list[str] | None = None) -> None:
|
||||
self.text = text
|
||||
self.deltas = deltas if deltas is not None else ["hel", "lo ", "world"]
|
||||
self.complete_calls = 0
|
||||
self.stream_calls = 0
|
||||
|
||||
def capabilities(self) -> Capabilities:
|
||||
return Capabilities(structured_output=True, prefix_cache=True)
|
||||
|
||||
async def complete(self, req: LlmRequest, model: str) -> ProviderResult:
|
||||
self.complete_calls += 1
|
||||
return ProviderResult(
|
||||
text=self.text,
|
||||
usage=ProviderUsage(input_tokens=100, output_tokens=50),
|
||||
)
|
||||
|
||||
async def stream(self, req: LlmRequest, model: str) -> AsyncIterator[StreamChunk]:
|
||||
self.stream_calls += 1
|
||||
for d in self.deltas:
|
||||
yield StreamChunk(text=d)
|
||||
yield StreamChunk(usage=ProviderUsage(input_tokens=100, output_tokens=50))
|
||||
|
||||
|
||||
class FakeLedger:
|
||||
def __init__(self) -> None:
|
||||
self.records: list[Usage] = []
|
||||
|
||||
async def record(self, scope: Scope, usage: Usage) -> None:
|
||||
self.records.append(usage)
|
||||
|
||||
|
||||
def fake_route(tier: Tier) -> Route:
|
||||
return Route(provider="deepseek", model="deepseek-chat")
|
||||
82
packages/llm_gateway/tests/test_gateway.py
Normal file
82
packages/llm_gateway/tests/test_gateway.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""T1.1 网关核心单测:run / stream / 记账 / 路由 / 错误。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from fakes import FakeAdapter, FakeLedger, fake_route
|
||||
from ww_llm_gateway.gateway import Gateway
|
||||
from ww_llm_gateway.routing import resolve_route
|
||||
from ww_llm_gateway.types import LlmRequest, Scope
|
||||
from ww_shared import AppError, ErrorCode
|
||||
|
||||
|
||||
async def test_run_returns_text_usage_and_served_by(req: LlmRequest) -> None:
|
||||
adapter = FakeAdapter(text="第一章正文")
|
||||
ledger = FakeLedger()
|
||||
gw = Gateway({"deepseek": adapter}, ledger, resolver=fake_route)
|
||||
|
||||
resp = await gw.run(req)
|
||||
|
||||
assert resp.text == "第一章正文"
|
||||
assert resp.served_by.provider == "deepseek"
|
||||
assert resp.served_by.fell_back is False
|
||||
assert resp.usage.input_tokens == 100
|
||||
assert resp.usage.output_tokens == 50
|
||||
assert adapter.complete_calls == 1
|
||||
|
||||
|
||||
async def test_run_writes_exactly_one_ledger_record(req: LlmRequest) -> None:
|
||||
ledger = FakeLedger()
|
||||
gw = Gateway({"deepseek": FakeAdapter()}, ledger, resolver=fake_route)
|
||||
|
||||
await gw.run(req)
|
||||
|
||||
assert len(ledger.records) == 1
|
||||
assert ledger.records[0].provider == "deepseek"
|
||||
assert ledger.records[0].model == "deepseek-chat"
|
||||
|
||||
|
||||
async def test_run_computes_cost_from_pricing(req: LlmRequest) -> None:
|
||||
ledger = FakeLedger()
|
||||
gw = Gateway({"deepseek": FakeAdapter()}, ledger, resolver=fake_route)
|
||||
|
||||
resp = await gw.run(req)
|
||||
|
||||
# ceil(100/1e6*27 + 50/1e6*110) == 1 cent
|
||||
assert resp.usage.cost_minor == 1
|
||||
assert resp.usage.currency == "USD"
|
||||
|
||||
|
||||
async def test_stream_yields_deltas_and_records_once(req: LlmRequest) -> None:
|
||||
ledger = FakeLedger()
|
||||
gw = Gateway({"deepseek": FakeAdapter(deltas=["a", "b", "c"])}, ledger, resolver=fake_route)
|
||||
|
||||
collected = [d.text async for d in gw.stream(req)]
|
||||
|
||||
assert "".join(collected) == "abc"
|
||||
assert len(ledger.records) == 1
|
||||
assert ledger.records[0].output_tokens == 50
|
||||
|
||||
|
||||
async def test_unknown_provider_raises_llm_unavailable(req: LlmRequest) -> None:
|
||||
gw = Gateway({}, FakeLedger(), resolver=fake_route)
|
||||
|
||||
with pytest.raises(AppError) as exc:
|
||||
await gw.run(req)
|
||||
|
||||
assert exc.value.code == ErrorCode.LLM_UNAVAILABLE
|
||||
|
||||
|
||||
def test_resolve_route_parses_tier_defaults() -> None:
|
||||
route = resolve_route("writer")
|
||||
assert route.provider == "deepseek"
|
||||
assert route.model == "deepseek-chat"
|
||||
|
||||
|
||||
def test_agent_passes_only_tier_never_model() -> None:
|
||||
# 不变量 ②:LlmRequest 无 model 字段,只有 tier
|
||||
req = LlmRequest(tier="analyst", input="x", scope=Scope(user_id=uuid.UUID(int=9)))
|
||||
assert "model" not in LlmRequest.model_fields
|
||||
assert req.tier == "analyst"
|
||||
102
packages/llm_gateway/tests/test_openai_adapter.py
Normal file
102
packages/llm_gateway/tests/test_openai_adapter.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""OpenAI 兼容适配器单测:用替身 client 验证消息翻译 + usage 映射(不联网)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from ww_llm_gateway.adapters.openai_compat import OpenAICompatAdapter, _messages
|
||||
from ww_llm_gateway.types import Block, LlmRequest, Scope
|
||||
|
||||
|
||||
def _req(**kw: Any) -> LlmRequest:
|
||||
kw.setdefault("tier", "writer")
|
||||
return LlmRequest(scope=Scope(user_id=uuid.UUID(int=1)), **kw)
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, chunks: list[Any]) -> None:
|
||||
self._chunks = chunks
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[Any]:
|
||||
for c in self._chunks:
|
||||
yield c
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, response: Any, stream_chunks: list[Any]) -> None:
|
||||
self._response = response
|
||||
self._stream_chunks = stream_chunks
|
||||
self.last_messages: Any = None
|
||||
|
||||
async def create(self, **kw: Any) -> Any:
|
||||
self.last_messages = kw.get("messages")
|
||||
if kw.get("stream"):
|
||||
return _FakeStream(self._stream_chunks)
|
||||
return self._response
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, completions: _FakeCompletions) -> None:
|
||||
self.chat = SimpleNamespace(completions=completions)
|
||||
|
||||
|
||||
def test_messages_put_system_first_then_user() -> None:
|
||||
req = _req(input="正文", system=[Block(text="世界观硬规则", cache=True)])
|
||||
msgs = _messages(req)
|
||||
assert msgs[0]["role"] == "system"
|
||||
assert msgs[0]["content"] == "世界观硬规则"
|
||||
assert msgs[1]["role"] == "user"
|
||||
assert msgs[1]["content"] == "正文"
|
||||
|
||||
|
||||
async def test_complete_maps_text_and_usage() -> None:
|
||||
response = SimpleNamespace(
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content="草稿"))],
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=120,
|
||||
completion_tokens=80,
|
||||
prompt_tokens_details=SimpleNamespace(cached_tokens=30),
|
||||
),
|
||||
)
|
||||
completions = _FakeCompletions(response, [])
|
||||
adapter = OpenAICompatAdapter("deepseek", cast(AsyncOpenAI, _FakeClient(completions)))
|
||||
|
||||
result = await adapter.complete(_req(input="x"), "deepseek-chat")
|
||||
|
||||
assert result.text == "草稿"
|
||||
assert result.usage.input_tokens == 120
|
||||
assert result.usage.output_tokens == 80
|
||||
assert result.usage.cache_read_tokens == 30
|
||||
|
||||
|
||||
async def test_stream_yields_text_then_final_usage() -> None:
|
||||
chunks = [
|
||||
SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="他"))], usage=None),
|
||||
SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="说"))], usage=None),
|
||||
SimpleNamespace(
|
||||
choices=[],
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=10,
|
||||
completion_tokens=2,
|
||||
prompt_tokens_details=None,
|
||||
),
|
||||
),
|
||||
]
|
||||
completions = _FakeCompletions(None, chunks)
|
||||
adapter = OpenAICompatAdapter("deepseek", cast(AsyncOpenAI, _FakeClient(completions)))
|
||||
|
||||
texts: list[str] = []
|
||||
usage_seen = None
|
||||
async for ch in adapter.stream(_req(input="x"), "deepseek-chat"):
|
||||
if ch.text:
|
||||
texts.append(ch.text)
|
||||
if ch.usage is not None:
|
||||
usage_seen = ch.usage
|
||||
|
||||
assert "".join(texts) == "他说"
|
||||
assert usage_seen is not None
|
||||
assert usage_seen.output_tokens == 2
|
||||
123
packages/llm_gateway/tests/test_structured_output.py
Normal file
123
packages/llm_gateway/tests/test_structured_output.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""T2.1 结构化输出接线单测:output_schema → instructor → LlmResponse.parsed。
|
||||
|
||||
注入 fake instructor client(返回 (parsed, raw_completion)),不联网。
|
||||
覆盖:① 带 schema → parsed 是实例且字段正确 + 仍记 1 条 ledger;
|
||||
② 无 schema → parsed is None 且纯文本路径不变。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import BaseModel
|
||||
from ww_llm_gateway.adapters.openai_compat import OpenAICompatAdapter
|
||||
from ww_llm_gateway.gateway import Gateway
|
||||
from ww_llm_gateway.routing import Route
|
||||
from ww_llm_gateway.types import LlmRequest, Scope, Tier
|
||||
|
||||
|
||||
class _Out(BaseModel):
|
||||
label: str
|
||||
score: int
|
||||
|
||||
|
||||
def _req(**kw: Any) -> LlmRequest:
|
||||
kw.setdefault("tier", "analyst")
|
||||
return LlmRequest(scope=Scope(user_id=uuid.UUID(int=1)), **kw)
|
||||
|
||||
|
||||
def _route(_tier: Tier) -> Route:
|
||||
return Route(provider="deepseek", model="deepseek-chat")
|
||||
|
||||
|
||||
class _FakeStructured:
|
||||
"""模拟 instructor.AsyncInstructor:create_with_completion 返回 (parsed, raw)。"""
|
||||
|
||||
def __init__(self, parsed: BaseModel, raw: Any) -> None:
|
||||
self._parsed = parsed
|
||||
self._raw = raw
|
||||
self.calls = 0
|
||||
self.last_response_model: Any = None
|
||||
|
||||
async def create_with_completion(
|
||||
self, *, messages: Any, response_model: Any, **kw: Any
|
||||
) -> tuple[BaseModel, Any]:
|
||||
self.calls += 1
|
||||
self.last_response_model = response_model
|
||||
return self._parsed, self._raw
|
||||
|
||||
|
||||
def _raw_with_usage() -> Any:
|
||||
return SimpleNamespace(
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=42,
|
||||
completion_tokens=7,
|
||||
prompt_tokens_details=SimpleNamespace(cached_tokens=5),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _client() -> AsyncOpenAI:
|
||||
return cast(AsyncOpenAI, SimpleNamespace(chat=SimpleNamespace(completions=None)))
|
||||
|
||||
|
||||
async def test_complete_with_schema_returns_parsed_instance() -> None:
|
||||
structured = _FakeStructured(_Out(label="ok", score=9), _raw_with_usage())
|
||||
adapter = OpenAICompatAdapter("deepseek", _client(), structured_client=structured)
|
||||
|
||||
result = await adapter.complete(_req(input="x", output_schema=_Out), "deepseek-chat")
|
||||
|
||||
assert isinstance(result.parsed, _Out)
|
||||
assert result.parsed.label == "ok"
|
||||
assert result.parsed.score == 9
|
||||
assert structured.last_response_model is _Out
|
||||
# usage 从 raw completion 提取
|
||||
assert result.usage.input_tokens == 42
|
||||
assert result.usage.output_tokens == 7
|
||||
assert result.usage.cache_read_tokens == 5
|
||||
|
||||
|
||||
async def test_complete_without_schema_keeps_parsed_none() -> None:
|
||||
response = SimpleNamespace(
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content="纯文本"))],
|
||||
usage=SimpleNamespace(prompt_tokens=3, completion_tokens=2, prompt_tokens_details=None),
|
||||
)
|
||||
|
||||
class _Completions:
|
||||
async def create(self, **kw: Any) -> Any:
|
||||
return response
|
||||
|
||||
client = cast(
|
||||
AsyncOpenAI,
|
||||
SimpleNamespace(chat=SimpleNamespace(completions=_Completions())),
|
||||
)
|
||||
adapter = OpenAICompatAdapter("deepseek", client)
|
||||
|
||||
result = await adapter.complete(_req(input="x"), "deepseek-chat")
|
||||
|
||||
assert result.parsed is None
|
||||
assert result.text == "纯文本"
|
||||
|
||||
|
||||
class _FakeLedger:
|
||||
def __init__(self) -> None:
|
||||
self.records: list[Any] = []
|
||||
|
||||
async def record(self, scope: Any, usage: Any) -> None:
|
||||
self.records.append(usage)
|
||||
|
||||
|
||||
async def test_gateway_run_passes_parsed_through_and_records_once() -> None:
|
||||
structured = _FakeStructured(_Out(label="hit", score=1), _raw_with_usage())
|
||||
adapter = OpenAICompatAdapter("deepseek", _client(), structured_client=structured)
|
||||
ledger = _FakeLedger()
|
||||
gw = Gateway({"deepseek": adapter}, ledger, resolver=_route)
|
||||
|
||||
resp = await gw.run(_req(input="x", output_schema=_Out))
|
||||
|
||||
assert isinstance(resp.parsed, _Out)
|
||||
assert resp.parsed.label == "hit"
|
||||
assert len(ledger.records) == 1
|
||||
47
packages/llm_gateway/ww_llm_gateway/__init__.py
Normal file
47
packages/llm_gateway/ww_llm_gateway/__init__.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""LLM 网关(C1 / ARCH §4):薄自建,tier→provider+model,屏蔽厂商差异。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .adapters.base import (
|
||||
Capabilities,
|
||||
ProviderAdapter,
|
||||
ProviderResult,
|
||||
ProviderUsage,
|
||||
StreamChunk,
|
||||
)
|
||||
from .adapters.openai_compat import OpenAICompatAdapter
|
||||
from .gateway import Gateway
|
||||
from .ledger import LedgerSink, SqlAlchemyLedgerSink
|
||||
from .routing import Route, resolve_route
|
||||
from .types import (
|
||||
Block,
|
||||
Delta,
|
||||
LlmRequest,
|
||||
LlmResponse,
|
||||
Scope,
|
||||
ServedBy,
|
||||
Tier,
|
||||
Usage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Block",
|
||||
"Capabilities",
|
||||
"Delta",
|
||||
"Gateway",
|
||||
"LedgerSink",
|
||||
"LlmRequest",
|
||||
"LlmResponse",
|
||||
"OpenAICompatAdapter",
|
||||
"ProviderAdapter",
|
||||
"ProviderResult",
|
||||
"ProviderUsage",
|
||||
"Route",
|
||||
"Scope",
|
||||
"ServedBy",
|
||||
"SqlAlchemyLedgerSink",
|
||||
"StreamChunk",
|
||||
"Tier",
|
||||
"Usage",
|
||||
"resolve_route",
|
||||
]
|
||||
51
packages/llm_gateway/ww_llm_gateway/adapters/base.py
Normal file
51
packages/llm_gateway/ww_llm_gateway/adapters/base.py
Normal file
@@ -0,0 +1,51 @@
|
||||
"""适配器接口与中间数据形(ARCH §4.2/§4.4)。
|
||||
|
||||
适配器把 `LlmRequest` 翻译成目标厂商请求,并把响应/流/usage 翻译回统一中间形。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from ..types import LlmRequest
|
||||
|
||||
|
||||
class Capabilities(BaseModel):
|
||||
structured_output: bool = False
|
||||
prefix_cache: bool = False
|
||||
thinking: bool = False
|
||||
|
||||
|
||||
class ProviderUsage(BaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_read_tokens: int = 0
|
||||
|
||||
|
||||
class ProviderResult(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
text: str
|
||||
usage: ProviderUsage
|
||||
parsed: BaseModel | None = None # output_schema 命中时的结构化结果(§4.4)
|
||||
|
||||
|
||||
class StreamChunk(BaseModel):
|
||||
"""流式块:文本增量(usage=None),或末尾用量块(text="")。"""
|
||||
|
||||
text: str = ""
|
||||
usage: ProviderUsage | None = None
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ProviderAdapter(Protocol):
|
||||
provider: str
|
||||
|
||||
def capabilities(self) -> Capabilities: ...
|
||||
|
||||
async def complete(self, req: LlmRequest, model: str) -> ProviderResult: ...
|
||||
|
||||
def stream(self, req: LlmRequest, model: str) -> AsyncIterator[StreamChunk]: ...
|
||||
129
packages/llm_gateway/ww_llm_gateway/adapters/openai_compat.py
Normal file
129
packages/llm_gateway/ww_llm_gateway/adapters/openai_compat.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""OpenAI 兼容适配器:一套覆盖 DeepSeek/Kimi/Qwen/GLM/OpenAI(ARCH §4.2)。
|
||||
|
||||
仅 base_url + model + key 不同。注入 `AsyncOpenAI` 客户端以便测试用替身。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Protocol
|
||||
|
||||
import instructor
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.chat import ChatCompletionMessageParam
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..types import LlmRequest
|
||||
from .base import Capabilities, ProviderResult, ProviderUsage, StreamChunk
|
||||
|
||||
|
||||
class StructuredClient(Protocol):
|
||||
"""instructor 风格的结构化客户端缝(`AsyncInstructor` 即满足此协议)。
|
||||
|
||||
抽成 Protocol 以便测试注入 fake,绝不联网(不变量:测试零真实 LLM)。
|
||||
"""
|
||||
|
||||
async def create_with_completion(
|
||||
self, *, messages: Any, response_model: type[BaseModel], **kwargs: Any
|
||||
) -> tuple[BaseModel, Any]: ...
|
||||
|
||||
|
||||
def _system_text(req: LlmRequest) -> str:
|
||||
return "\n\n".join(b.text for b in req.system)
|
||||
|
||||
|
||||
def _input_text(req: LlmRequest) -> str:
|
||||
if isinstance(req.input, str):
|
||||
return req.input
|
||||
return "\n\n".join(b.text for b in req.input)
|
||||
|
||||
|
||||
def _messages(req: LlmRequest) -> list[ChatCompletionMessageParam]:
|
||||
msgs: list[ChatCompletionMessageParam] = []
|
||||
system = _system_text(req)
|
||||
if system:
|
||||
msgs.append({"role": "system", "content": system})
|
||||
msgs.append({"role": "user", "content": _input_text(req)})
|
||||
return msgs
|
||||
|
||||
|
||||
def _cache_read(usage: Any) -> int:
|
||||
details = getattr(usage, "prompt_tokens_details", None)
|
||||
if details is None:
|
||||
return 0
|
||||
return int(getattr(details, "cached_tokens", 0) or 0)
|
||||
|
||||
|
||||
def _usage_from(usage: Any) -> ProviderUsage:
|
||||
if usage is None:
|
||||
return ProviderUsage(input_tokens=0, output_tokens=0)
|
||||
return ProviderUsage(
|
||||
input_tokens=getattr(usage, "prompt_tokens", 0) or 0,
|
||||
output_tokens=getattr(usage, "completion_tokens", 0) or 0,
|
||||
cache_read_tokens=_cache_read(usage),
|
||||
)
|
||||
|
||||
|
||||
class OpenAICompatAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
provider: str,
|
||||
client: AsyncOpenAI,
|
||||
*,
|
||||
structured_client: StructuredClient | None = None,
|
||||
) -> None:
|
||||
self.provider = provider
|
||||
self._client = client
|
||||
# 结构化输出走 instructor(Pydantic 校验 + 重试,锁定栈);可注入便于测试。
|
||||
self._structured_client = structured_client
|
||||
|
||||
def capabilities(self) -> Capabilities:
|
||||
return Capabilities(structured_output=True, prefix_cache=True, thinking=False)
|
||||
|
||||
def _structured(self) -> StructuredClient:
|
||||
if self._structured_client is None:
|
||||
# 懒构建:从同一 AsyncOpenAI client patch 出 instructor 客户端。
|
||||
self._structured_client = instructor.from_openai(self._client)
|
||||
return self._structured_client
|
||||
|
||||
async def complete(self, req: LlmRequest, model: str) -> ProviderResult:
|
||||
if req.output_schema is not None:
|
||||
return await self._complete_structured(req, model)
|
||||
return await self._complete_text(req, model)
|
||||
|
||||
async def _complete_text(self, req: LlmRequest, model: str) -> ProviderResult:
|
||||
resp = await self._client.chat.completions.create(
|
||||
model=model,
|
||||
messages=_messages(req),
|
||||
max_tokens=req.max_tokens,
|
||||
)
|
||||
text = resp.choices[0].message.content or ""
|
||||
return ProviderResult(text=text, usage=_usage_from(resp.usage))
|
||||
|
||||
async def _complete_structured(self, req: LlmRequest, model: str) -> ProviderResult:
|
||||
assert req.output_schema is not None
|
||||
parsed, raw = await self._structured().create_with_completion(
|
||||
messages=_messages(req),
|
||||
response_model=req.output_schema,
|
||||
model=model,
|
||||
max_tokens=req.max_tokens,
|
||||
)
|
||||
usage = _usage_from(getattr(raw, "usage", None))
|
||||
# 文本载体保留校验后的 JSON(便于日志/留痕);程序消费走 parsed。
|
||||
return ProviderResult(text=parsed.model_dump_json(), usage=usage, parsed=parsed)
|
||||
|
||||
async def stream(self, req: LlmRequest, model: str) -> AsyncIterator[StreamChunk]:
|
||||
stream = await self._client.chat.completions.create(
|
||||
model=model,
|
||||
messages=_messages(req),
|
||||
max_tokens=req.max_tokens,
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
async for chunk in stream:
|
||||
if chunk.choices:
|
||||
delta = chunk.choices[0].delta
|
||||
if delta and delta.content:
|
||||
yield StreamChunk(text=delta.content)
|
||||
if getattr(chunk, "usage", None):
|
||||
yield StreamChunk(usage=_usage_from(chunk.usage))
|
||||
99
packages/llm_gateway/ww_llm_gateway/gateway.py
Normal file
99
packages/llm_gateway/ww_llm_gateway/gateway.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""网关核心:路由 → 调用适配器 → 记账 → 返回(ARCH §4.1–4.3/§4.8)。
|
||||
|
||||
M1 单 provider,无回退/熔断(那是 M5/T5.4)。流式经 `stream()` 归一为 `Delta`。
|
||||
日志脱敏:只记长度,绝不记原文/api key(不变量、§9.3)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
|
||||
import structlog
|
||||
from ww_shared import AppError, ErrorCode
|
||||
|
||||
from .adapters.base import ProviderAdapter, ProviderUsage
|
||||
from .ledger import LedgerSink
|
||||
from .pricing import cost_minor
|
||||
from .routing import Route, resolve_route
|
||||
from .types import Delta, LlmRequest, LlmResponse, ServedBy, Tier, Usage
|
||||
|
||||
log = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
def _input_len(req: LlmRequest) -> int:
|
||||
if isinstance(req.input, str):
|
||||
return len(req.input)
|
||||
return sum(len(b.text) for b in req.input)
|
||||
|
||||
|
||||
class Gateway:
|
||||
def __init__(
|
||||
self,
|
||||
adapters: dict[str, ProviderAdapter],
|
||||
ledger: LedgerSink,
|
||||
resolver: Callable[[Tier], Route] = resolve_route,
|
||||
) -> None:
|
||||
self._adapters = adapters
|
||||
self._ledger = ledger
|
||||
self._resolve = resolver
|
||||
|
||||
def _adapter_for(self, provider: str) -> ProviderAdapter:
|
||||
adapter = self._adapters.get(provider)
|
||||
if adapter is None:
|
||||
raise AppError(ErrorCode.LLM_UNAVAILABLE, f"no adapter for provider {provider!r}")
|
||||
return adapter
|
||||
|
||||
def _usage(self, route: Route, pu: ProviderUsage) -> Usage:
|
||||
cost, currency = cost_minor(route.provider, route.model, pu.input_tokens, pu.output_tokens)
|
||||
return Usage(
|
||||
provider=route.provider,
|
||||
model=route.model,
|
||||
input_tokens=pu.input_tokens,
|
||||
output_tokens=pu.output_tokens,
|
||||
cache_read_tokens=pu.cache_read_tokens,
|
||||
cost_minor=cost,
|
||||
currency=currency,
|
||||
)
|
||||
|
||||
def _log_call(self, req: LlmRequest, usage: Usage, *, stream: bool) -> None:
|
||||
log.info(
|
||||
"llm_call",
|
||||
provider=usage.provider,
|
||||
model=usage.model,
|
||||
tier=req.tier,
|
||||
input_tokens=usage.input_tokens,
|
||||
output_tokens=usage.output_tokens,
|
||||
cache_read_tokens=usage.cache_read_tokens,
|
||||
cost_minor=usage.cost_minor,
|
||||
currency=usage.currency,
|
||||
stream=stream,
|
||||
input_chars=_input_len(req),
|
||||
project_id=str(req.scope.project_id) if req.scope.project_id else None,
|
||||
)
|
||||
|
||||
async def run(self, req: LlmRequest) -> LlmResponse:
|
||||
route = self._resolve(req.tier)
|
||||
adapter = self._adapter_for(route.provider)
|
||||
result = await adapter.complete(req, route.model)
|
||||
usage = self._usage(route, result.usage)
|
||||
await self._ledger.record(req.scope, usage)
|
||||
self._log_call(req, usage, stream=False)
|
||||
return LlmResponse(
|
||||
text=result.text,
|
||||
parsed=result.parsed,
|
||||
usage=usage,
|
||||
served_by=ServedBy(provider=route.provider, model=route.model),
|
||||
)
|
||||
|
||||
async def stream(self, req: LlmRequest) -> AsyncIterator[Delta]:
|
||||
route = self._resolve(req.tier)
|
||||
adapter = self._adapter_for(route.provider)
|
||||
final = ProviderUsage(input_tokens=0, output_tokens=0)
|
||||
async for chunk in adapter.stream(req, route.model):
|
||||
if chunk.text:
|
||||
yield Delta(text=chunk.text)
|
||||
if chunk.usage is not None:
|
||||
final = chunk.usage
|
||||
usage = self._usage(route, final)
|
||||
await self._ledger.record(req.scope, usage)
|
||||
self._log_call(req, usage, stream=True)
|
||||
39
packages/llm_gateway/ww_llm_gateway/ledger.py
Normal file
39
packages/llm_gateway/ww_llm_gateway/ledger.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""用量记账落库(ARCH §4.8)。
|
||||
|
||||
`LedgerSink` 为接口,便于测试注入内存替身;生产用 SQLAlchemy 实现写 usage_ledger。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from ww_db.models import UsageLedger
|
||||
|
||||
from .types import Scope, Usage
|
||||
|
||||
|
||||
class LedgerSink(Protocol):
|
||||
async def record(self, scope: Scope, usage: Usage) -> None: ...
|
||||
|
||||
|
||||
class SqlAlchemyLedgerSink:
|
||||
"""把每次调用写入 usage_ledger(owner_id 取 scope.user_id,单用户 stub)。"""
|
||||
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
self._session = session
|
||||
|
||||
async def record(self, scope: Scope, usage: Usage) -> None:
|
||||
row = UsageLedger(
|
||||
owner_id=scope.user_id,
|
||||
project_id=scope.project_id,
|
||||
provider=usage.provider,
|
||||
model=usage.model,
|
||||
input_tokens=usage.input_tokens,
|
||||
output_tokens=usage.output_tokens,
|
||||
cache_read=usage.cache_read_tokens,
|
||||
cost_minor=usage.cost_minor,
|
||||
currency=usage.currency,
|
||||
)
|
||||
self._session.add(row)
|
||||
await self._session.flush()
|
||||
34
packages/llm_gateway/ww_llm_gateway/pricing.py
Normal file
34
packages/llm_gateway/ww_llm_gateway/pricing.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""提供商价格表与成本换算(ARCH §4.8)。
|
||||
|
||||
价格以「每百万 token 的最小货币单位(如分/cent)」表示,随 provider 配置维护;
|
||||
未知 (provider, model) 则成本计 0(仍记账,便于观测)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Price:
|
||||
in_per_mtok: int
|
||||
out_per_mtok: int
|
||||
currency: str
|
||||
|
||||
|
||||
# 近似价(可后续移入 config / provider 配置维护)
|
||||
_PRICING: dict[tuple[str, str], Price] = {
|
||||
("deepseek", "deepseek-chat"): Price(in_per_mtok=27, out_per_mtok=110, currency="USD"),
|
||||
}
|
||||
|
||||
|
||||
def cost_minor(provider: str, model: str, input_tokens: int, output_tokens: int) -> tuple[int, str]:
|
||||
price = _PRICING.get((provider, model))
|
||||
if price is None:
|
||||
return 0, "USD"
|
||||
cost = math.ceil(
|
||||
input_tokens / 1_000_000 * price.in_per_mtok
|
||||
+ output_tokens / 1_000_000 * price.out_per_mtok
|
||||
)
|
||||
return cost, price.currency
|
||||
29
packages/llm_gateway/ww_llm_gateway/routing.py
Normal file
29
packages/llm_gateway/ww_llm_gateway/routing.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""档位路由:tier -> (provider, model)(ARCH §4.3)。
|
||||
|
||||
M1 只读全局默认(config.tier_defaults,形如 "deepseek:deepseek-chat");
|
||||
作品级 / Skill 级覆盖留待后续(§4.3 三级解析)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ww_config import get_settings
|
||||
|
||||
from .types import Tier
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Route:
|
||||
provider: str
|
||||
model: str
|
||||
|
||||
|
||||
def resolve_route(tier: Tier) -> Route:
|
||||
spec = get_settings().tier_defaults.get(tier)
|
||||
if not spec:
|
||||
raise ValueError(f"no tier_defaults entry for tier={tier!r}")
|
||||
provider, sep, model = spec.partition(":")
|
||||
if not sep or not provider or not model:
|
||||
raise ValueError(f"invalid tier route {spec!r}; expected 'provider:model'")
|
||||
return Route(provider=provider, model=model)
|
||||
78
packages/llm_gateway/ww_llm_gateway/types.py
Normal file
78
packages/llm_gateway/ww_llm_gateway/types.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""LLM 网关统一接口契约(C1 / ARCH §4.1)——snake_case,Pydantic v2。
|
||||
|
||||
上层(编排器/Agent)只碰这些类型,永不接触具体厂商字段。Agent 只声明 `tier`,
|
||||
不传具体 model(不变量 ②)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
Tier = Literal["writer", "analyst", "light"]
|
||||
|
||||
|
||||
class Block(BaseModel):
|
||||
"""一个 prompt 文本块;`cache=True` 标记缓存断点前的稳定块(ARCH §4.6)。"""
|
||||
|
||||
text: str
|
||||
cache: bool = False
|
||||
|
||||
|
||||
class Scope(BaseModel):
|
||||
"""调用作用域。原型单用户:`user_id` 可固定 stub,`project_id` 可空。"""
|
||||
|
||||
user_id: uuid.UUID
|
||||
project_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
class LlmRequest(BaseModel):
|
||||
"""统一请求。`system` 稳定块在前(断点前),`input` 易变内容在后。"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
tier: Tier
|
||||
input: str | list[Block]
|
||||
system: list[Block] = Field(default_factory=list)
|
||||
stream: bool = False
|
||||
output_schema: type[BaseModel] | None = None
|
||||
thinking: bool = False
|
||||
max_tokens: int | None = None
|
||||
scope: Scope
|
||||
|
||||
|
||||
class Usage(BaseModel):
|
||||
"""一次调用的用量与成本(落 usage_ledger,ARCH §4.8)。"""
|
||||
|
||||
provider: str
|
||||
model: str
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_read_tokens: int = 0
|
||||
cost_minor: int # 最小货币单位(如分/cent)
|
||||
currency: str
|
||||
|
||||
|
||||
class ServedBy(BaseModel):
|
||||
"""实际服务方;`fell_back` 标记是否走了回退链(M5 才有回退,M1 恒 False)。"""
|
||||
|
||||
provider: str
|
||||
model: str
|
||||
fell_back: bool = False
|
||||
|
||||
|
||||
class LlmResponse(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
text: str
|
||||
parsed: BaseModel | None = None
|
||||
usage: Usage
|
||||
served_by: ServedBy
|
||||
|
||||
|
||||
class Delta(BaseModel):
|
||||
"""流式增量:归一各家 SSE 的统一 token 块。"""
|
||||
|
||||
text: str
|
||||
Reference in New Issue
Block a user