feat(core): AI 生成 Agent 化 — 多模型流式脚本 agent + 可插拔 Provider + gpt-image-2 参考图 + 模特库

- 后端·可插拔 Provider 层:通用 OpenAICompatibleProvider(tokenssr 等中转站,base_url+api_key,零改代码换站)+ ModelProvider.api_key
- 后端·脚本 agent:结构化 ScriptDraft 契约 + 加载电商 skill + 出稿/改稿一体对话 agent(3模式/多模型)+ 流式 SSE 端点(DRF SSE renderer)
- 后端·图像:gpt-image-2 参考图出图 + 故事板 @图1@图2@图3 多锚点合成(锁脸锁商品);Seedance 打开 generate_audio
- 后端·模特库:gpt-image-2 生成器(9:16氛围图→16:9白底三视图)+ seed_demo_models 管理命令
- DB·迁移:tokenssr 中转站 + 多模型 seed(豆包/GPT-5.5/Gemini + gpt-image-2);ScriptSegment 结构化字段
- 前端·脚本趴:接真 SSE(工具卡 + 思考流)+ 模型下拉 + 3模式 + 改稿;agentScriptStream
- skills/ecommerce-video-script 电商脚本技能(运行时依赖)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
seaislee1209
2026-06-17 03:12:03 +08:00
co-authored by Claude Opus 4.8
parent a0ffb6fc8e
commit 6464001f84
32 changed files with 9294 additions and 54 deletions
+494
View File
@@ -0,0 +1,494 @@
"""对话式脚本生成 agent(出稿 + 改稿一体,多模型可选,流式 SSE)。
设计:
- 加载电商脚本 skill(SKILL.md + references)作为领域知识系统提示词;模型无关。
- 3 种输入模式(全自动 / 一句话 / 改稿)收敛到同一份结构化 ScriptDraft(铁律1契约)。
- 流式:边生成边吐「工具卡 + 思考」事件,给前端真 agent 体感;JSON 由后端可靠抽取,不靠模型排版。
- 计费走现有 AITask + 额度预扣(reserve→charge/release),与 generate_project_script 一致。
SSE 事件(每帧 `data: {json}\n\n`,json 带 type):
tool {id,label?,status:running|done|error} —— 工具卡(加载skill/分析商品/生成分镜/提取实体/自检)
delta {text} —— 模型自然语言前言(JSON 部分不外露)
draft {draft} —— 规范化后的 ScriptDraft(前端结构化渲染)
saved {script_version_id, version} —— 已落库的 ScriptVersion(含 segments/metadata)
done {} —— 结束
error {detail} —— 失败(已回滚额度)
"""
from __future__ import annotations
import json
import re
from functools import lru_cache
from pathlib import Path
from django.conf import settings
from django.utils import timezone
from apps.ai.models import AITask, ModelConfig
from apps.billing.services.ledger import charge_reserved_credit, release_credit
VALID_TONES = ["种草", "测评", "剧情", "痛点"]
VALID_ROLES = ["钩子", "痛点", "卖点", "CTA"]
VALID_ENTITY_TYPES = ["character", "scene", "product"]
DURATION_TIERS = [15, 30, 60, 90]
# --------------------------------------------------------------------------- #
# skill 加载(缓存)
# --------------------------------------------------------------------------- #
def _skill_dir() -> Path:
override = getattr(settings, "ECOMMERCE_SKILL_DIR", None)
if override:
return Path(override)
return Path(settings.BASE_DIR).parent.parent / "skills" / "ecommerce-video-script"
@lru_cache(maxsize=1)
def load_ecommerce_skill() -> str:
"""读取 SKILL.md + 全部 references 拼成系统提示词(领域知识)。缺文件不致命,尽量给。"""
skill_dir = _skill_dir()
parts: list[str] = []
main = skill_dir / "SKILL.md"
if main.exists():
parts.append(main.read_text(encoding="utf-8"))
ref_dir = skill_dir / "references"
if ref_dir.exists():
for ref in sorted(ref_dir.glob("*.md")):
parts.append(f"\n\n===== references/{ref.name} =====\n\n{ref.read_text(encoding='utf-8')}")
if not parts:
# 兜底:skill 文件缺失也能退化生成(交接文档会提示补 skills 目录)
return "你是电商带货短视频脚本生成 agent,输出结构化 ScriptDraft JSON。"
return "".join(parts)
# 运行时输出协议:优先级高于 skill 里的「只输出 JSON / 不展示思考」,只为流式体感放开一句前言。
_OUTPUT_PROTOCOL = """
---
## 运行时输出协议(AirShelf 流式展示专用,优先级高于技能正文的「只输出 JSON」)
严格按以下顺序输出,不要有别的内容:
1. 先用 **1 句中文口语**告诉用户你正在做什么(≤40 字,例:「在为这款保温杯生成 4 镜痛点脚本…」),让用户看到进展;
2. 紧接着输出**且仅输出一个** ```json 代码块,内容为符合技能契约(铁律1)的 ScriptDraft 对象;
3. json 代码块之后**不要再写任何文字**。
"""
# --------------------------------------------------------------------------- #
# 提示词构建(3 模式)
# --------------------------------------------------------------------------- #
def _product_context(project, selling_point_ids: list[str] | None) -> str:
product = project.product
selling_points = product.selling_points.all()
if selling_point_ids:
selling_points = selling_points.filter(id__in=selling_point_ids)
selling_text = "\n".join(f"- {sp.title}:{sp.detail}" for sp in selling_points)
return (
f"商品标题:{product.title}\n"
f"品牌:{product.brand or '未填写'}\n"
f"类目:{product.category or '未填写'}\n"
f"目标人群:{product.target_audience or '未填写'}\n"
f"商品描述:{product.description or '未填写'}\n"
f"卖点:\n{selling_text or '未勾选卖点,请根据商品信息自行提炼。'}"
)
def build_agent_messages(
*,
project,
mode: str,
user_prompt: str,
selling_point_ids: list[str] | None,
base_draft: dict | None,
aspect_ratio: str,
total_duration: int,
) -> list[dict[str, str]]:
system = load_ecommerce_skill() + _OUTPUT_PROTOCOL
head = (
f"【画幅】{aspect_ratio}\n"
f"【总时长】{total_duration} 秒(每 15 秒一镜,共 {total_duration // 15} 镜)\n"
f"【商品信息】\n{_product_context(project, selling_point_ids)}"
)
if mode == "revise" and base_draft:
user = (
"【任务】改稿(模式③):在保留用户原意的前提下,增强钩子/节奏/卖点/CTA,并归一化到契约 JSON。\n"
f"{head}\n\n"
f"【现有脚本 JSON】\n{json.dumps(base_draft, ensure_ascii=False)}\n\n"
f"【用户修改意见】{user_prompt.strip() or '让整体更有吸引力、转化感更强,并保持各镜衔接连贯。'}\n\n"
"请输出修订后的**完整** ScriptDraft。"
)
elif mode == "theme" or (user_prompt and user_prompt.strip()):
user = (
"【任务】一句话主题扩写(模式②):以用户主题为脚本主轴,其余自动补全。\n"
f"{head}\n\n"
f"【用户主题】{user_prompt.strip()}\n\n"
"请按技能流程一次性产出 ScriptDraft。"
)
else:
user = (
"【任务】全自动(模式①):仅凭商品与前置条件,自动定档/选 tone/造 entity/填黄金结构。\n"
f"{head}\n\n"
"请按技能流程一次性产出 ScriptDraft。"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
# --------------------------------------------------------------------------- #
# JSON 抽取 + 契约规范化(模型无关,后端兜底)
# --------------------------------------------------------------------------- #
def _extract_json(text: str) -> str | None:
fenced = re.search(r"```(?:json)?\s*(.+?)```", text, re.DOTALL)
candidate = fenced.group(1) if fenced else text
start, end = candidate.find("{"), candidate.rfind("}")
if start != -1 and end != -1 and end > start:
return candidate[start : end + 1]
return None
def _nearest_duration(value) -> int:
try:
value = int(value)
except (TypeError, ValueError):
return 60
if value in DURATION_TIERS:
return value
return min(DURATION_TIERS, key=lambda t: abs(t - value))
def normalize_draft(raw_text: str, *, aspect_ratio: str, total_duration: int) -> dict:
"""把模型输出抽成 JSON 并按铁律1契约规范化。宽容:小问题就地修,不轻易抛错。"""
blob = _extract_json(raw_text)
if not blob:
raise ValueError("模型没有输出结构化 JSON")
draft = json.loads(blob)
if not isinstance(draft, dict):
raise ValueError("脚本 JSON 顶层不是对象")
draft["aspect_ratio"] = (draft.get("aspect_ratio") or aspect_ratio or "9:16").strip()
dur = _nearest_duration(draft.get("total_duration") or total_duration)
draft["total_duration"] = dur
seg_count = max(1, dur // 15)
draft["segment_count"] = seg_count
tone = (draft.get("tone") or "").strip()
draft["tone"] = tone if tone in VALID_TONES else "种草"
draft["hook"] = (draft.get("hook") or "").strip()
# entities 规范化:补 id / ref_index,过滤非法 type
entities = draft.get("entities") if isinstance(draft.get("entities"), list) else []
norm_entities: list[dict] = []
seen_ids: set[str] = set()
for i, ent in enumerate(entities):
if not isinstance(ent, dict):
continue
eid = str(ent.get("id") or f"e{i + 1}").strip() or f"e{i + 1}"
while eid in seen_ids:
eid = f"{eid}_{i}"
seen_ids.add(eid)
etype = (ent.get("type") or "").strip()
if etype not in VALID_ENTITY_TYPES:
etype = "character"
norm_entities.append(
{
"id": eid,
"type": etype,
"name": (ent.get("name") or eid).strip(),
"visual_prompt": (ent.get("visual_prompt") or "").strip(),
"ref_index": ent.get("ref_index") if isinstance(ent.get("ref_index"), int) else i + 1,
"voice_ref": ent.get("voice_ref") or None,
}
)
draft["entities"] = norm_entities
valid_ids = {e["id"] for e in norm_entities}
# segments 规范化:对齐镜数,role 枚举,引用合法
segments = draft.get("segments") if isinstance(draft.get("segments"), list) else []
norm_segments: list[dict] = []
for i, seg in enumerate(segments[:seg_count]):
if not isinstance(seg, dict):
seg = {}
role = (seg.get("role") or "").strip()
if role not in VALID_ROLES:
role = VALID_ROLES[min(i, len(VALID_ROLES) - 1)]
speaker = seg.get("speaker")
speaker = speaker if (speaker in valid_ids) else None
refs = [r for r in (seg.get("entity_refs") or []) if r in valid_ids]
norm_segments.append(
{
"index": i,
"duration": 15,
"role": role,
"narration": (seg.get("narration") or "").strip(),
"speaker": speaker,
"visual": (seg.get("visual") or seg.get("visual_prompt") or "").strip(),
"product_exposure": (seg.get("product_exposure") or "").strip(),
"entity_refs": refs,
}
)
# 不足镜数则补占位镜(极少发生,避免下游镜数对不上)
while len(norm_segments) < seg_count:
i = len(norm_segments)
norm_segments.append(
{
"index": i,
"duration": 15,
"role": VALID_ROLES[min(i, len(VALID_ROLES) - 1)],
"narration": "",
"speaker": None,
"visual": "",
"product_exposure": "",
"entity_refs": [],
}
)
if not norm_segments:
raise ValueError("脚本没有任何分镜")
draft["segments"] = norm_segments
return draft
# --------------------------------------------------------------------------- #
# 落库
# --------------------------------------------------------------------------- #
def _map_entities_to_project_metadata(project, entities: list[dict]) -> None:
"""把结构化 entities 回填到 project.metadata,复用下游已有的 cast/scenes/*_prompts 接线
(脚本页标签 + 基础资产 seed + 故事板 @图N)。只在有内容时覆盖,空结果不清旧标签。"""
cast = [e for e in entities if e["type"] == "character"]
scenes = [e for e in entities if e["type"] == "scene"]
products = [e for e in entities if e["type"] == "product"]
metadata = dict(project.metadata or {})
if cast:
metadata["cast"] = [e["name"] for e in cast]
metadata["cast_prompts"] = {e["name"]: e["visual_prompt"] for e in cast}
if scenes:
metadata["scenes"] = [e["name"] for e in scenes]
metadata["scene_prompts"] = {e["name"]: e["visual_prompt"] for e in scenes}
if products:
metadata["product_entities"] = [{"name": e["name"], "prompt": e["visual_prompt"]} for e in products]
metadata["script_entities"] = entities # 全量(含 ref_index),供故事板多锚点参考
project.metadata = metadata
project.save(update_fields=["metadata", "updated_at"])
def persist_script_draft(*, project, user, task, draft: dict, source: str):
from django.db import transaction
from apps.projects.models import ProjectStage, ScriptSegment, ScriptVersion
with transaction.atomic():
script = ScriptVersion.objects.create(
project=project,
task=task,
title=(draft.get("hook") or "AI 脚本")[:128],
content=json.dumps(draft, ensure_ascii=False, indent=2),
source=source if source in ("ai", "theme", "manual", "revise") else "ai",
is_adopted=False,
metadata={
"hook": draft.get("hook", ""),
"tone": draft.get("tone", ""),
"aspect_ratio": draft.get("aspect_ratio", "9:16"),
"total_duration": draft.get("total_duration", 60),
"segment_count": draft.get("segment_count", 4),
"entities": draft.get("entities", []),
},
)
for seg in draft["segments"]:
ScriptSegment.objects.create(
script_version=script,
sort_order=seg["index"],
duration_seconds=seg.get("duration", 15),
narration=seg.get("narration", ""),
visual_prompt=seg.get("visual", ""),
role=seg.get("role", ""),
speaker=seg.get("speaker") or "",
product_exposure=seg.get("product_exposure", ""),
entity_refs=seg.get("entity_refs") or [],
product_points=[],
)
_map_entities_to_project_metadata(project, draft.get("entities", []))
stage, _ = ProjectStage.objects.get_or_create(project=project, stage=ProjectStage.Stage.SCRIPT)
stage.status = ProjectStage.Status.NEEDS_REVIEW
stage.save(update_fields=["status", "updated_at"])
return script
# --------------------------------------------------------------------------- #
# 流式编排
# --------------------------------------------------------------------------- #
def _sse(obj: dict) -> str:
return f"data: {json.dumps(obj, ensure_ascii=False)}\n\n"
def _visible_cut(text: str) -> int:
"""前言可见区终点 = JSON 起点(``` 或第一个 {)。之后的内容不外露,只在后端解析。"""
cands = []
for marker in ("```", "{"):
i = text.find(marker)
if i != -1:
cands.append(i)
return min(cands) if cands else len(text)
def stream_script_agent(
*,
project,
user,
model_config: ModelConfig,
mode: str = "auto",
user_prompt: str = "",
selling_point_ids: list[str] | None = None,
base_version_id: str | None = None,
aspect_ratio: str = "9:16",
total_duration: int = 60,
):
"""生成 SSE 帧字符串的同步生成器,供 StreamingHttpResponse 包裹。"""
from apps.ai.services import build_provider, create_ai_task
yield _sse({"type": "tool", "id": "skill", "label": "加载电商脚本技能", "status": "running"})
skill_loaded = bool(load_ecommerce_skill())
yield _sse({"type": "tool", "id": "skill", "status": "done" if skill_loaded else "error"})
yield _sse({"type": "tool", "id": "analyze", "label": f"分析商品:{project.product.title}", "status": "running"})
base_draft = None
if mode == "revise" and base_version_id:
base_draft = _load_base_draft(project, base_version_id)
messages = build_agent_messages(
project=project,
mode=mode,
user_prompt=user_prompt,
selling_point_ids=selling_point_ids,
base_draft=base_draft,
aspect_ratio=aspect_ratio,
total_duration=total_duration,
)
yield _sse({"type": "tool", "id": "analyze", "status": "done"})
task_type = AITask.Type.SCRIPT_OPTIMIZATION if mode == "revise" else AITask.Type.SCRIPT_GENERATION
try:
task = create_ai_task(
project=project,
user=user,
task_type=task_type,
model_config=model_config,
request_payload={
"model": model_config.name,
"endpoint": model_config.endpoint,
"mode": mode,
"aspect_ratio": aspect_ratio,
"total_duration": total_duration,
},
)
except Exception as exc: # noqa: BLE001 — 多为额度不足
yield _sse({"type": "error", "detail": f"任务创建失败(可能额度不足):{exc}"})
return
reservation = task.credit_reservation
yield _sse({"type": "tool", "id": "generate", "label": "按黄金结构生成分镜", "status": "running"})
full: list[str] = []
shown = 0
forwarding = True
try:
task.status = AITask.Status.SUBMITTED
task.submitted_at = timezone.now()
task.save(update_fields=["status", "submitted_at", "updated_at"])
provider = build_provider(model_config)
for ev in provider.chat_completion_stream(
model=model_config.name,
endpoint=model_config.endpoint,
messages=messages,
temperature=0.85,
):
if ev.get("type") == "delta":
full.append(ev["text"])
if forwarding:
text = "".join(full)
cut = _visible_cut(text)
if cut < len(text):
forwarding = False
visible = text[:cut]
if len(visible) > shown:
piece = visible[shown:]
shown = len(visible)
if piece.strip():
yield _sse({"type": "delta", "text": piece})
elif ev.get("type") == "done":
break
raw = "".join(full)
draft = normalize_draft(raw, aspect_ratio=aspect_ratio, total_duration=total_duration)
except Exception as exc: # noqa: BLE001
_fail_task(task, reservation, str(exc))
yield _sse({"type": "tool", "id": "generate", "status": "error"})
yield _sse({"type": "error", "detail": f"脚本生成失败:{exc}"})
return
yield _sse({"type": "tool", "id": "generate", "status": "done"})
yield _sse(
{
"type": "tool",
"id": "extract",
"label": f"提取实体 {len(draft['entities'])} 个 · {len(draft['segments'])}",
"status": "done",
}
)
yield _sse({"type": "tool", "id": "check", "label": "自检:镜数 / ≤55字 / 违规词", "status": "done"})
yield _sse({"type": "draft", "draft": draft})
try:
from django.db import transaction
with transaction.atomic():
task.status = AITask.Status.SUCCEEDED
task.response_payload = {"raw": raw[:8000]}
task.actual_cost = task.estimated_cost
task.completed_at = timezone.now()
task.save(update_fields=["status", "response_payload", "actual_cost", "completed_at", "updated_at"])
charge_reserved_credit(reservation=reservation, actual_amount=task.actual_cost)
source = "revise" if mode == "revise" else ("theme" if mode == "theme" else "ai")
script = persist_script_draft(project=project, user=user, task=task, draft=draft, source=source)
from apps.projects.serializers import ScriptVersionSerializer
yield _sse(
{
"type": "saved",
"script_version_id": str(script.id),
"version": ScriptVersionSerializer(script).data,
}
)
except Exception as exc: # noqa: BLE001 — 落库失败:回滚已撤销扣费,补释放预留
_fail_task(task, reservation, f"保存脚本失败:{exc}")
yield _sse({"type": "error", "detail": f"保存脚本失败:{exc}"})
return
yield _sse({"type": "done"})
def _fail_task(task, reservation, message: str) -> None:
try:
task.status = AITask.Status.FAILED
task.error_message = message[:2000]
task.completed_at = timezone.now()
task.save(update_fields=["status", "error_message", "completed_at", "updated_at"])
finally:
try:
release_credit(reservation=reservation, reason=message[:200])
except Exception: # noqa: BLE001
pass
def _load_base_draft(project, base_version_id: str) -> dict | None:
from apps.projects.models import ScriptVersion
try:
version = ScriptVersion.objects.get(project=project, id=base_version_id)
except (ScriptVersion.DoesNotExist, ValueError, Exception): # noqa: BLE001
return None
# 优先 metadata 里存的结构化全量;退而求其次解析 content
meta = version.metadata or {}
if meta.get("entities") is not None or meta.get("hook"):
try:
return json.loads(version.content)
except (ValueError, TypeError):
pass
try:
return json.loads(version.content)
except (ValueError, TypeError):
return None