feat(core): 角色对白(剧情向按需)+ 聊天精准改一镜 + 模型下拉移位 + 全流程e2e + 交叉验证修复
新能力(均已端到端验证):
- 角色对白 dialogue:[{speaker,line}](speaker=null 即旁白)。默认口播,用户在聊天提要求才出多角色对白(剧情向);narration 保留扁平拼接兼容下游字幕/配音。skill 同步:默认不强制对白。
- 聊天精准改一镜:聊天说「第N镜改XX」→ agent 读全脚本上下文、只重写那一镜(target_index + 后端 _merge_single_segment 强制保其余镜原样),保衔接。
- 前端:分镜卡渲染对白/镜型/露出;模型下拉从顶部移到输入框下方小按钮(对齐 ChatGPT/Lovart)。
全流程 e2e 真跑通:商品→脚本→基础资产(gpt-image-2)→故事板(@图N 多图合成锁脸锁商品)→视频(Seedance 6.4分钟出片,带音效+人声)。
对抗式交叉验证(3 审查员)修复:
- 改稿用基准稿时长 effective_duration(prompt head + normalize + merge),避免请求默认 60 把 90s/6镜稿尾镜截掉/单镜改空转。
- target_index 越界服务端早返回报错、不计费;模型未产出目标镜时抛错释放额度(不静默空转)。
- 前端「第N镜」正则收窄为 镜|场(去掉会误判「第3个卖点」的 个|段)。
- 回归:测试 setUp 停用 seed 中转站文本模型,保证命中可 mock 的 provider(18/18 过)。
CLAUDE.md 加「AI 生成 Agent 化架构」一节供后续维护。
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
163a6627ba
commit
3d38c173e6
@@ -103,6 +103,7 @@ def build_agent_messages(
|
||||
base_draft: dict | None,
|
||||
aspect_ratio: str,
|
||||
total_duration: int,
|
||||
target_index: int | None = None,
|
||||
) -> list[dict[str, str]]:
|
||||
system = load_ecommerce_skill() + _OUTPUT_PROTOCOL
|
||||
head = (
|
||||
@@ -110,7 +111,17 @@ def build_agent_messages(
|
||||
f"【总时长】{total_duration} 秒(每 15 秒一镜,共 {total_duration // 15} 镜)\n"
|
||||
f"【商品信息】\n{_product_context(project, selling_point_ids)}"
|
||||
)
|
||||
if mode == "revise" and base_draft:
|
||||
if mode == "revise" and base_draft and target_index is not None:
|
||||
# 精准改一镜:读全脚本上下文,只重写第 N 镜,强制与前后镜衔接;其余镜后端会强制保持原样。
|
||||
user = (
|
||||
f"【任务】只重写第 {target_index + 1} 镜(共 {len(base_draft.get('segments', []))} 镜),其余镜保持不变。\n"
|
||||
f"{head}\n\n"
|
||||
f"【现有完整脚本 JSON(读它保证与前后镜衔接)】\n{json.dumps(base_draft, ensure_ascii=False)}\n\n"
|
||||
f"【对第 {target_index + 1} 镜的修改意见】{user_prompt.strip() or '让这一镜更有吸引力、表达更清晰。'}\n\n"
|
||||
"仍输出**完整** ScriptDraft(我只会采用第 "
|
||||
f"{target_index + 1} 镜的改动);若意见涉及角色对白,就给这一镜填 dialogue。"
|
||||
)
|
||||
elif mode == "revise" and base_draft:
|
||||
user = (
|
||||
"【任务】改稿(模式③):在保留用户原意的前提下,增强钩子/节奏/卖点/CTA,并归一化到契约 JSON。\n"
|
||||
f"{head}\n\n"
|
||||
@@ -257,16 +268,30 @@ def normalize_draft(raw_text: str, *, aspect_ratio: str, total_duration: int) ->
|
||||
speaker = seg.get("speaker")
|
||||
speaker = speaker if (speaker in valid_ids) else None
|
||||
refs = [r for r in (seg.get("entity_refs") or []) if r in valid_ids]
|
||||
# 对白(剧情向):[{speaker(合法 entity id 或 null=旁白), line}];默认空 = 纯口播
|
||||
dialogue = []
|
||||
for d in (seg.get("dialogue") or []):
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
line = (d.get("line") or d.get("text") or "").strip()
|
||||
if not line:
|
||||
continue
|
||||
sp = d.get("speaker")
|
||||
dialogue.append({"speaker": sp if sp in valid_ids else None, "line": line})
|
||||
narration = (seg.get("narration") or "").strip()
|
||||
if not narration and dialogue:
|
||||
narration = " ".join(d["line"] for d in dialogue) # 扁平拼接,兼容下游字幕/配音
|
||||
norm_segments.append(
|
||||
{
|
||||
"index": i,
|
||||
"duration": 15,
|
||||
"role": role,
|
||||
"narration": (seg.get("narration") or "").strip(),
|
||||
"narration": narration,
|
||||
"speaker": speaker,
|
||||
"visual": (seg.get("visual") or seg.get("visual_prompt") or "").strip(),
|
||||
"product_exposure": (seg.get("product_exposure") or "").strip(),
|
||||
"entity_refs": refs,
|
||||
"dialogue": dialogue,
|
||||
}
|
||||
)
|
||||
# 不足镜数则补占位镜(极少发生,避免下游镜数对不上)
|
||||
@@ -282,6 +307,7 @@ def normalize_draft(raw_text: str, *, aspect_ratio: str, total_duration: int) ->
|
||||
"visual": "",
|
||||
"product_exposure": "",
|
||||
"entity_refs": [],
|
||||
"dialogue": [],
|
||||
}
|
||||
)
|
||||
if not norm_segments:
|
||||
@@ -290,6 +316,30 @@ def normalize_draft(raw_text: str, *, aspect_ratio: str, total_duration: int) ->
|
||||
return draft
|
||||
|
||||
|
||||
def _merge_single_segment(base: dict, new: dict, idx: int, aspect_ratio: str, total_duration: int) -> dict:
|
||||
"""精准改一镜:以基准稿为底,只用新稿的第 idx 镜替换,其余镜逐字保持;合并新稿引入的新 entity(对白可能加角色)。再整体规范化。"""
|
||||
merged = json.loads(json.dumps(base)) # 深拷贝
|
||||
base_ids = {e.get("id") for e in merged.get("entities", []) if isinstance(e, dict)}
|
||||
for e in new.get("entities", []):
|
||||
if isinstance(e, dict) and e.get("id") and e["id"] not in base_ids:
|
||||
merged.setdefault("entities", []).append(e)
|
||||
base_ids.add(e["id"])
|
||||
new_segs = new.get("segments", [])
|
||||
target = next((s for s in new_segs if isinstance(s, dict) and s.get("index") == idx), None)
|
||||
if target is None and 0 <= idx < len(new_segs):
|
||||
target = new_segs[idx]
|
||||
if not isinstance(target, dict):
|
||||
# 模型没产出目标镜(没按 N 镜输出)→ 抛错让上层释放额度+报错,而不是静默返回 base 空转计费
|
||||
raise ValueError(f"模型未产出第 {idx + 1} 镜的改动,请重试")
|
||||
segs = merged.get("segments", [])
|
||||
if isinstance(target, dict) and 0 <= idx < len(segs):
|
||||
target = dict(target)
|
||||
target["index"] = idx
|
||||
segs[idx] = target
|
||||
merged["segments"] = segs
|
||||
return normalize_draft(json.dumps(merged, ensure_ascii=False), aspect_ratio=aspect_ratio, total_duration=total_duration)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 落库
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -346,6 +396,7 @@ def persist_script_draft(*, project, user, task, draft: dict, source: str):
|
||||
speaker=seg.get("speaker") or "",
|
||||
product_exposure=seg.get("product_exposure", ""),
|
||||
entity_refs=seg.get("entity_refs") or [],
|
||||
dialogue=seg.get("dialogue") or [],
|
||||
product_points=[],
|
||||
)
|
||||
_map_entities_to_project_metadata(project, draft.get("entities", []))
|
||||
@@ -383,8 +434,10 @@ def stream_script_agent(
|
||||
base_version_id: str | None = None,
|
||||
aspect_ratio: str = "9:16",
|
||||
total_duration: int = 60,
|
||||
target_index: int | None = None,
|
||||
):
|
||||
"""生成 SSE 帧字符串的同步生成器,供 StreamingHttpResponse 包裹。"""
|
||||
"""生成 SSE 帧字符串的同步生成器,供 StreamingHttpResponse 包裹。
|
||||
target_index 非空 = 精准只改第 N 镜(读全脚本上下文,后端强制保留其余镜原样)。"""
|
||||
from apps.ai.services import build_provider, create_ai_task
|
||||
|
||||
yield _sse({"type": "tool", "id": "skill", "label": "加载电商脚本技能", "status": "running"})
|
||||
@@ -395,6 +448,16 @@ def stream_script_agent(
|
||||
base_draft = None
|
||||
if mode == "revise" and base_version_id:
|
||||
base_draft = _load_base_draft(project, base_version_id)
|
||||
if base_draft is None:
|
||||
target_index = None # 没有基准稿就退回整版生成,单镜改无从谈起
|
||||
# 改稿以基准稿的时长/镜数为准,避免请求侧默认值(前端可能硬编码 60)把 90s/6镜稿的尾镜挤掉
|
||||
effective_duration = (base_draft.get("total_duration") if base_draft else None) or total_duration
|
||||
# 精准改一镜:镜号越界直接报错返回,绝不建任务/扣费(避免计费空转的静默 no-op)
|
||||
if target_index is not None and base_draft is not None:
|
||||
seg_n = len(base_draft.get("segments", []))
|
||||
if not (0 <= target_index < seg_n):
|
||||
yield _sse({"type": "error", "detail": f"镜号越界:第 {target_index + 1} 镜(共 {seg_n} 镜)"})
|
||||
return
|
||||
messages = build_agent_messages(
|
||||
project=project,
|
||||
mode=mode,
|
||||
@@ -402,7 +465,8 @@ def stream_script_agent(
|
||||
selling_point_ids=selling_point_ids,
|
||||
base_draft=base_draft,
|
||||
aspect_ratio=aspect_ratio,
|
||||
total_duration=total_duration,
|
||||
total_duration=effective_duration, # 改稿用基准稿时长,prompt head 才不会误导模型镜数(否则模型按60s只出4镜)
|
||||
target_index=target_index,
|
||||
)
|
||||
yield _sse({"type": "tool", "id": "analyze", "status": "done"})
|
||||
|
||||
@@ -461,7 +525,10 @@ def stream_script_agent(
|
||||
elif ev.get("type") == "done":
|
||||
break
|
||||
raw = "".join(full)
|
||||
draft = normalize_draft(raw, aspect_ratio=aspect_ratio, total_duration=total_duration)
|
||||
draft = normalize_draft(raw, aspect_ratio=aspect_ratio, total_duration=effective_duration)
|
||||
if target_index is not None and base_draft:
|
||||
# 精准改一镜:只采用新稿的第 target_index 镜,其余镜强制保持基准稿原样
|
||||
draft = _merge_single_segment(base_draft, draft, target_index, aspect_ratio, effective_duration)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_fail_task(task, reservation, str(exc))
|
||||
settled = True
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.1.15 on 2026-06-16 20:25
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('projects', '0002_scriptsegment_structured_fields'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='scriptsegment',
|
||||
name='dialogue',
|
||||
field=models.JSONField(blank=True, default=list),
|
||||
),
|
||||
]
|
||||
@@ -85,6 +85,9 @@ class ScriptSegment(TimeStampedModel):
|
||||
speaker = models.CharField(max_length=32, blank=True) # 指向某 entity id;画外旁白为空
|
||||
product_exposure = models.CharField(max_length=64, blank=True) # 手持/特写/使用中…
|
||||
entity_refs = models.JSONField(default=list, blank=True) # 本镜引用的 entity id 列表(→ 故事板 @图N)
|
||||
# 角色对白(剧情向):[{speaker, line}],speaker=entity id 即角色台词、null 即旁白。
|
||||
# 默认空 = 纯口播/旁白(用 narration);用户在聊天里明确要对白时 agent 才填。narration 仍存扁平拼接兼容下游。
|
||||
dialogue = models.JSONField(default=list, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["sort_order", "created_at"]
|
||||
|
||||
@@ -230,7 +230,7 @@ class ScriptSegmentSerializer(serializers.ModelSerializer):
|
||||
model = ScriptSegment
|
||||
fields = [
|
||||
"id", "sort_order", "duration_seconds", "narration", "visual_prompt", "product_points",
|
||||
"role", "speaker", "product_exposure", "entity_refs",
|
||||
"role", "speaker", "product_exposure", "entity_refs", "dialogue",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
|
||||
@@ -41,6 +41,10 @@ class ProjectApiTests(TestCase):
|
||||
endpoint="chat/completions",
|
||||
unit_price="2.0000",
|
||||
)
|
||||
# 数据迁移(0006)会 seed 中转站文本模型,created_at 更早 → get_default_model 会选到它,
|
||||
# 从而路由到未被 patch 的 OpenAICompatibleProvider 打真网络。这里只保留自建 volcengine 模型为
|
||||
# active,确保命中可 mock 的 VolcanoArkProvider(生产有 bootstrap 豆包在前,默认仍是豆包,不受影响)。
|
||||
ModelConfig.objects.filter(capability=ModelConfig.Capability.TEXT).exclude(id=self.model.id).update(status="disabled")
|
||||
self.client = APIClient()
|
||||
self.client.force_authenticate(self.user)
|
||||
|
||||
|
||||
@@ -166,6 +166,11 @@ class ProjectViewSet(TeamScopedViewSetMixin, ModelViewSet):
|
||||
total_duration = int(request.data.get("total_duration") or 60)
|
||||
except (TypeError, ValueError):
|
||||
total_duration = 60
|
||||
target_index = request.data.get("target_index")
|
||||
try:
|
||||
target_index = int(target_index) if target_index is not None else None
|
||||
except (TypeError, ValueError):
|
||||
target_index = None
|
||||
|
||||
model_config = None
|
||||
requested = request.data.get("model_config_id")
|
||||
@@ -191,6 +196,7 @@ class ProjectViewSet(TeamScopedViewSetMixin, ModelViewSet):
|
||||
base_version_id=base_version_id,
|
||||
aspect_ratio=aspect_ratio,
|
||||
total_duration=total_duration,
|
||||
target_index=target_index,
|
||||
)
|
||||
response = StreamingHttpResponse(stream, content_type="text/event-stream")
|
||||
response["Cache-Control"] = "no-cache"
|
||||
|
||||
@@ -235,6 +235,7 @@ export const api = {
|
||||
base_version_id?: string;
|
||||
aspect_ratio?: string;
|
||||
total_duration?: number;
|
||||
target_index?: number;
|
||||
},
|
||||
onEvent: (evt: { type: string; [k: string]: unknown }) => void
|
||||
): Promise<void> {
|
||||
|
||||
@@ -421,6 +421,9 @@ export function PipelinePage(props: {
|
||||
null;
|
||||
const scriptAdopted = Boolean(currentScript?.is_adopted);
|
||||
const shots = [...(currentScript?.segments ?? [])].sort((a, b) => a.sort_order - b.sort_order);
|
||||
// 对白 speaker(entity id)→ 角色名;null = 旁白。entities 在 ScriptVersion.metadata
|
||||
const scriptEntities = ((currentScript?.metadata as { entities?: Array<{ id: string; name: string }> } | undefined)?.entities) ?? [];
|
||||
const entityName = (id: string | null | undefined) => (id ? (scriptEntities.find((e) => e.id === id)?.name || id) : "旁白");
|
||||
// 行34 · 脚本里的「人物 / 场景」标签:持久化进 project.metadata.cast / .scenes。
|
||||
// 初值从 metadata 读,刷新/重进后仍在;增删后 onSaveProjectMeta 合并落库。
|
||||
const [castTags, setCastTags] = useState<string[]>(() => project.metadata?.cast ?? []);
|
||||
@@ -649,11 +652,12 @@ export function PipelinePage(props: {
|
||||
if (src === "revise") return "revise";
|
||||
return "auto"; // ai / manual / 默认
|
||||
}
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string, mode?: "auto" | "theme" | "revise") {
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string, mode?: "auto" | "theme" | "revise", targetIndex?: number) {
|
||||
pushMsg("user", userLabel || prompt);
|
||||
const progressId = nextMsgId();
|
||||
setChatMsgs((list) => [...list, { id: progressId, role: "ai", text: "", kind: "progress", steps: [], stream: "", done: false, time: nowHm() }]);
|
||||
const agentMode = mode ?? mapSourceToMode(source ?? chatMode);
|
||||
// 指定镜号 = 精准改一镜,强制走 revise(后端读全脚本上下文、只动那一镜)
|
||||
const agentMode = targetIndex != null ? "revise" : (mode ?? mapSourceToMode(source ?? chatMode));
|
||||
const baseVersionId = agentMode === "revise" ? currentScript?.id : undefined;
|
||||
let ok = false;
|
||||
let receivedEvent = false;
|
||||
@@ -666,7 +670,8 @@ export function PipelinePage(props: {
|
||||
model_config_id: activeScriptModelId || undefined,
|
||||
base_version_id: baseVersionId,
|
||||
aspect_ratio: "9:16",
|
||||
total_duration: 60
|
||||
total_duration: 60,
|
||||
target_index: targetIndex
|
||||
},
|
||||
(evt) => {
|
||||
receivedEvent = true;
|
||||
@@ -748,8 +753,19 @@ export function PipelinePage(props: {
|
||||
setChatText("");
|
||||
setChatAttachments([]);
|
||||
setPendingTagEdits([]);
|
||||
// 已有脚本 → 追问走「改稿」模式(基于当前脚本增强,保留原意);否则全自动出稿
|
||||
void runScriptGeneration(prompt, label || undefined, undefined, currentScript ? "revise" : "auto");
|
||||
// 检测「第N镜 / 场N / 第N个 / 第N段」→ 精准只改那一镜(有当前脚本时);否则整版改稿/出稿
|
||||
let targetIndex: number | undefined;
|
||||
if (currentScript && shots.length) {
|
||||
// 只认「第N镜 / 第N场」;「个/段」太宽(会撞「第3个卖点」「第2段文案」等整体改稿诉求)故不触发
|
||||
const m = text.match(/第\s*([0-9]{1,2}|[一二两三四五六七八九十])\s*(?:镜|场)/);
|
||||
if (m) {
|
||||
const cn: Record<string, number> = { 一: 1, 二: 2, 两: 2, 三: 3, 四: 4, 五: 5, 六: 6, 七: 7, 八: 8, 九: 9, 十: 10 };
|
||||
const n = /^[0-9]+$/.test(m[1]) ? parseInt(m[1], 10) : (cn[m[1]] ?? 0);
|
||||
if (n >= 1 && n <= shots.length) targetIndex = n - 1;
|
||||
}
|
||||
}
|
||||
// 已有脚本 → 追问走「改稿」(指定镜号则只改那镜);否则全自动出稿
|
||||
void runScriptGeneration(prompt, label || undefined, undefined, currentScript ? "revise" : "auto", targetIndex);
|
||||
}
|
||||
function clearChat() {
|
||||
setPendingTagEdits([]);
|
||||
@@ -1846,6 +1862,17 @@ export function PipelinePage(props: {
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
{/* 角色对白(剧情向):speaker→角色名 / 旁白;只读展示 */}
|
||||
{shot.dialogue && shot.dialogue.length > 0 ? (
|
||||
<div className="shot-row">
|
||||
<span className="shot-k">对白</span>
|
||||
<div className="shot-v" style={{ opacity: 0.9 }}>
|
||||
{shot.dialogue.map((d, di) => (
|
||||
<div key={di}><b>{entityName(d.speaker)}:</b>{d.line}</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
{/* 行34 · 该卡之后挂着的本地草稿分镜(点「添加分镜」插入,可编辑,有内容失焦才落库) */}
|
||||
@@ -1878,19 +1905,7 @@ export function PipelinePage(props: {
|
||||
<div className="pane-h">
|
||||
<div className="ai-avatar">AI</div>
|
||||
<strong>脚本助手</strong>
|
||||
{textModels && textModels.length > 0 ? (
|
||||
<select
|
||||
className="setup-select"
|
||||
style={{ height: 24, fontSize: 12, padding: "0 6px", maxWidth: 150, marginLeft: 6 }}
|
||||
value={activeScriptModelId}
|
||||
onChange={(event) => setScriptModelId(event.target.value)}
|
||||
title="选择脚本生成模型(豆包 / GPT / Gemini)"
|
||||
>
|
||||
{textModels.map((m) => <option key={m.id} value={m.id}>{m.display_name || m.name}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
)}
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
<span className="spacer"></span>
|
||||
<button className="btn btn-ghost btn-sm" type="button" id="chat-clear-btn" disabled={!chatText && chatAttachments.length === 0 && chatMsgs.length === 0} onClick={clearChat}>清空对话</button>
|
||||
</div>
|
||||
@@ -1987,6 +2002,18 @@ export function PipelinePage(props: {
|
||||
<button className="chat-icon-btn" id="chat-upload-btn" type="button" title="上传脚本附件" aria-label="上传脚本附件" onClick={() => chatFileRef.current?.click()}>
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M12 5v14M5 12h14" /></svg>
|
||||
</button>
|
||||
{/* 模型选择小按钮(放输入框下方,对齐 ChatGPT/Lovart) */}
|
||||
{textModels && textModels.length > 0 ? (
|
||||
<select
|
||||
className="setup-select"
|
||||
style={{ height: 26, fontSize: 12, padding: "0 6px", maxWidth: 150, marginLeft: 6 }}
|
||||
value={activeScriptModelId}
|
||||
onChange={(event) => setScriptModelId(event.target.value)}
|
||||
title="选择脚本生成模型(豆包 / GPT / Gemini)"
|
||||
>
|
||||
{textModels.map((m) => <option key={m.id} value={m.id}>{m.display_name || m.name}</option>)}
|
||||
</select>
|
||||
) : null}
|
||||
<span className="spacer"></span>
|
||||
<button className="chat-send-btn" id="chat-send-btn" type="button" title="发送" aria-label="发送" disabled={loading || (!setupOpen && !chatText.trim() && pendingTagEdits.length === 0)} onClick={submitChat}>
|
||||
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.2" strokeLinecap="round" strokeLinejoin="round"><path d="M5 12h14M13 6l6 6-6 6" /></svg>
|
||||
|
||||
@@ -94,6 +94,7 @@ export type ScriptVersion = {
|
||||
speaker?: string;
|
||||
product_exposure?: string;
|
||||
entity_refs?: string[];
|
||||
dialogue?: Array<{ speaker: string | null; line: string }>;
|
||||
}>;
|
||||
// metadata 携带 hook/tone/entities(脚本 agent 产出),供结构化渲染与下游故事板 @图N
|
||||
metadata?: Record<string, unknown>;
|
||||
|
||||
Reference in New Issue
Block a user