fix(extract): 实体提取改走流式通道 + 锁定豆包2.0Pro,根治"未返回有效JSON"
问题:进资产阶段点"提取人物/场景"间歇报「提取结果解析失败(模型未返回有效 JSON)」。根因——提取走非流式只读 content,而默认文本模型是会思考的推理模型 (豆包2.0Pro think:True),思考期常把内容留在 reasoning_content、content 返空 → 正则抽不到 JSON。脚本生成不犯此病是因其走流式、早已分离思考/正文。 改动: - 锁模型:提取固定用 doubao-seed-2-0-pro-260215(_resolve_extract_model_config), 取不到再回落默认,不再受各环境 DB 创建序漂移影响路由。 - 根治:提取复用与脚本生成同一条已验证稳定的流式通道(_collect_extract_text)—— 丢弃 reasoning 事件、只收正文 delta;正文为空时兜底回退 reasoning 里的 JSON。 response_payload 改存精简流式存档(streamed/model/content/had_reasoning)。 - 测试:提取步原零测试,新增 2 个——流式正文JSON正确落库+计费一次、content空→ 回退reasoning。其余链路(脚本生成/单镜重跑)未动。 验收:本机 Python3.9 跑不了 Django5 全套,核心逻辑(事件过滤/回退/JSON抽取)已 独立验证通过;py_compile 绿;完整套件由 CI 跑。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
612ab6864d
commit
71af0ae166
@@ -295,6 +295,55 @@ def _normalize_extracted_segment_refs(items: object, valid_ids: set[str]) -> lis
|
||||
return out
|
||||
|
||||
|
||||
# 提取步固定锁定豆包 2.0 Pro(与脚本生成同款),不靠 get_default_model 的「最早创建」排序——
|
||||
# 避免不同环境 DB 创建序漂移把提取路由到别的(中转站)推理模型。取不到再回落默认文本模型。
|
||||
EXTRACT_TEXT_MODEL_NAME = "doubao-seed-2-0-pro-260215"
|
||||
|
||||
|
||||
def _resolve_extract_model_config():
|
||||
"""提取实体用的文本模型:优先豆包 2.0 Pro(active 且 provider active),否则回落默认文本模型。"""
|
||||
pinned = (
|
||||
ModelConfig.objects.select_related("provider")
|
||||
.filter(
|
||||
name=EXTRACT_TEXT_MODEL_NAME,
|
||||
capability=ModelConfig.Capability.TEXT,
|
||||
status=ModelConfig.Status.ACTIVE,
|
||||
provider__status="active",
|
||||
)
|
||||
.order_by("created_at")
|
||||
.first()
|
||||
)
|
||||
return pinned or get_default_model(ModelConfig.Capability.TEXT)
|
||||
|
||||
|
||||
def _collect_extract_text(provider, model_config, messages) -> tuple[str, dict]:
|
||||
"""走与「脚本生成」同一条已在生产验证稳定的流式通道把模型输出收全。
|
||||
|
||||
豆包 seed-pro / GPT / Gemini 等思考模型,思考期只发 reasoning_content、正文期才发 content,
|
||||
流式分支(chat_completion_stream)已分别转发为 `reasoning` / `delta` 事件。这里**只收正文 delta**;
|
||||
仅当正文为空(整轮被思考占满的极端兜底)才回退已收到的 reasoning。如此从根上避免非流式只读
|
||||
content 拿到空串 → 解析不到 JSON 的老问题。返回 (text, 存档用精简 payload)。
|
||||
"""
|
||||
content_parts: list[str] = []
|
||||
reasoning_parts: list[str] = []
|
||||
for ev in provider.chat_completion_stream(
|
||||
model=model_config.name,
|
||||
endpoint=model_config.endpoint,
|
||||
messages=messages,
|
||||
temperature=0.3, # 结构化抽取要稳:低温降低 JSON 漂移
|
||||
):
|
||||
etype = ev.get("type")
|
||||
if etype == "delta":
|
||||
content_parts.append(ev.get("text") or "")
|
||||
elif etype == "reasoning":
|
||||
reasoning_parts.append(ev.get("text") or "")
|
||||
content = "".join(content_parts).strip()
|
||||
reasoning = "".join(reasoning_parts).strip()
|
||||
text = content or reasoning
|
||||
payload = {"streamed": True, "model": model_config.name, "content": content, "had_reasoning": bool(reasoning)}
|
||||
return text, payload
|
||||
|
||||
|
||||
def extract_entities_for_project(*, project, user) -> dict:
|
||||
"""独立实体提取步(剧本定稿后、进资产阶段时跑)。读已定稿(或最新)脚本 → 调 ecommerce-entity-extract
|
||||
skill(一个文本模型)→ 产出角色 / 场景 entities + 每镜 entity_refs。**不提商品**(商品由参考图层用真实
|
||||
@@ -315,7 +364,7 @@ def extract_entities_for_project(*, project, user) -> dict:
|
||||
segments = list(script.segments.order_by("sort_order"))
|
||||
if not segments:
|
||||
raise ValueError("脚本没有分镜,无法提取")
|
||||
model_config = get_default_model(ModelConfig.Capability.TEXT)
|
||||
model_config = _resolve_extract_model_config()
|
||||
if model_config is None:
|
||||
raise ValueError("没有可用的文本模型")
|
||||
|
||||
@@ -352,8 +401,7 @@ def extract_entities_for_project(*, project, user) -> dict:
|
||||
task.submitted_at = timezone.now()
|
||||
task.save(update_fields=["status", "submitted_at", "updated_at"])
|
||||
provider = build_provider(model_config)
|
||||
response = provider.chat_completion(model=model_config.name, endpoint=model_config.endpoint, messages=messages)
|
||||
text = provider.extract_text(response)
|
||||
text, response = _collect_extract_text(provider, model_config, messages)
|
||||
with transaction.atomic():
|
||||
task.status = AITask.Status.SUCCEEDED
|
||||
task.response_payload = response
|
||||
|
||||
Reference in New Issue
Block a user