feat(core): AI 生成 Agent 化 — 多模型流式脚本 agent + 可插拔 Provider + gpt-image-2 参考图 + 模特库
- 后端·可插拔 Provider 层:通用 OpenAICompatibleProvider(tokenssr 等中转站,base_url+api_key,零改代码换站)+ ModelProvider.api_key - 后端·脚本 agent:结构化 ScriptDraft 契约 + 加载电商 skill + 出稿/改稿一体对话 agent(3模式/多模型)+ 流式 SSE 端点(DRF SSE renderer) - 后端·图像:gpt-image-2 参考图出图 + 故事板 @图1@图2@图3 多锚点合成(锁脸锁商品);Seedance 打开 generate_audio - 后端·模特库:gpt-image-2 生成器(9:16氛围图→16:9白底三视图)+ seed_demo_models 管理命令 - DB·迁移:tokenssr 中转站 + 多模型 seed(豆包/GPT-5.5/Gemini + gpt-image-2);ScriptSegment 结构化字段 - 前端·脚本趴:接真 SSE(工具卡 + 思考流)+ 模型下拉 + 3模式 + 改稿;agentScriptStream - skills/ecommerce-video-script 电商脚本技能(运行时依赖) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
seaislee1209
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
a0ffb6fc8e
commit
6464001f84
@@ -669,6 +669,7 @@ export function App() {
|
||||
key={pipelineProject.id}
|
||||
project={pipelineProject}
|
||||
scriptModelName={textModel?.display_name || textModel?.name || "AI"}
|
||||
textModels={modelConfigs.filter((m) => m.capability === "text" && m.status === "active")}
|
||||
loading={loading}
|
||||
navigate={navigate}
|
||||
user={currentUser}
|
||||
|
||||
@@ -223,6 +223,55 @@ export const api = {
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
},
|
||||
// 对话式脚本 agent · 流式(SSE)。逐帧回调 onEvent:tool(工具卡)/delta(思考前言)/draft/saved/done/error。
|
||||
// 用 fetch + ReadableStream 消费 text/event-stream(EventSource 只支持 GET,这里要 POST 带 body)。
|
||||
async agentScriptStream(
|
||||
projectId: string,
|
||||
payload: {
|
||||
mode?: "auto" | "theme" | "revise";
|
||||
prompt?: string;
|
||||
model_config_id?: string;
|
||||
selling_point_ids?: string[];
|
||||
base_version_id?: string;
|
||||
aspect_ratio?: string;
|
||||
total_duration?: number;
|
||||
},
|
||||
onEvent: (evt: { type: string; [k: string]: unknown }) => void
|
||||
): Promise<void> {
|
||||
const token = getToken();
|
||||
const headers = new Headers({ "Content-Type": "application/json", Accept: "text/event-stream" });
|
||||
if (token) headers.set("Authorization", `Token ${token}`);
|
||||
const response = await fetch(`${API_BASE}/api/projects/${projectId}/script-agent-stream/`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new ApiError(response.status, text || "脚本流式生成失败");
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
let buffer = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let sep: number;
|
||||
// SSE 帧以空行分隔(\n\n);每帧取 data: 行解析
|
||||
while ((sep = buffer.indexOf("\n\n")) !== -1) {
|
||||
const frame = buffer.slice(0, sep);
|
||||
buffer = buffer.slice(sep + 2);
|
||||
const dataLine = frame.split("\n").find((l) => l.startsWith("data:"));
|
||||
if (!dataLine) continue;
|
||||
try {
|
||||
onEvent(JSON.parse(dataLine.slice(5).trim()));
|
||||
} catch {
|
||||
/* 跳过解析失败的帧 */
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
adoptScript(projectId: string, script_version_id: string) {
|
||||
return request<ScriptVersion>(`/api/projects/${projectId}/adopt-script/`, {
|
||||
method: "POST",
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Fragment, memo, useCallback, useDeferredValue, useEffect, useMemo, useR
|
||||
import type { ChangeEvent, CSSProperties, PointerEvent as ReactPointerEvent } from "react";
|
||||
import { Play } from "lucide-react";
|
||||
import { api } from "../api";
|
||||
import type { Asset, BillingSummary, ExportPoll, Product, Project, Team, TimelineSavePayload, User } from "../types";
|
||||
import type { Asset, BillingSummary, ExportPoll, ModelConfig, Product, Project, Team, TimelineSavePayload, User } from "../types";
|
||||
import type { Notice, Page } from "./route-config";
|
||||
import { money, stageOrder, statusPill } from "./stage-config";
|
||||
import { CornerMarks, Decorations, Sidebar, ToastLike } from "../components/app-shell";
|
||||
@@ -362,6 +362,7 @@ export function PipelinePage(props: {
|
||||
avatarChar: string;
|
||||
logout: () => void;
|
||||
scriptModelName: string;
|
||||
textModels?: ModelConfig[];
|
||||
onGenerateScript: (prompt: string, source?: string) => Promise<unknown>;
|
||||
onAdoptScript: (scriptId: string) => void | Promise<unknown>;
|
||||
onUpdateShot: (payload: { segment_id: string; narration?: string; visual_prompt?: string; duration_seconds?: number }) => Promise<unknown>;
|
||||
@@ -389,7 +390,7 @@ export function PipelinePage(props: {
|
||||
}) {
|
||||
const {
|
||||
project, loading, navigate, user, team, products, projects, assets, billing, notice, unreadCount, avatarChar, logout,
|
||||
scriptModelName, onGenerateScript, onAdoptScript, onUpdateShot, onAddShot, onDeleteShot, onRerunShot, onSaveProjectMeta, onAdoptVideoVersion, onGenerateVoiceover,
|
||||
scriptModelName, textModels, onGenerateScript, onAdoptScript, onUpdateShot, onAddShot, onDeleteShot, onRerunShot, onSaveProjectMeta, onAdoptVideoVersion, onGenerateVoiceover,
|
||||
onGenerateBaseAsset, onAdoptBaseAsset, onGenerateStoryboard, onSkipStoryboard,
|
||||
onSubmitVideo, onSubmitAllVideos, onPollVideosQuiet, exportResult, onRefreshExport, onRefreshProject,
|
||||
onUploadVideoSegment, onUploadBgm, onSaveTimeline, onSubmitExport
|
||||
@@ -573,7 +574,7 @@ export function PipelinePage(props: {
|
||||
const chatBodyRef = useRef<HTMLDivElement | null>(null);
|
||||
// 对话记录(本地会话态):生成动作可追溯,不再是「点了按钮、对话区永远空着」
|
||||
// kind=progress:进度提示流(行33),steps 逐条滚动出现,done 后折叠成一行结果
|
||||
type ChatMsg = { id: number; role: "ai" | "user"; text: string; time: string; kind?: "progress"; steps?: string[]; done?: boolean; auto?: boolean };
|
||||
type ChatMsg = { id: number; role: "ai" | "user"; text: string; time: string; kind?: "progress"; steps?: string[]; stream?: string; done?: boolean; auto?: boolean };
|
||||
const nowHm = () => new Date().toTimeString().slice(0, 5);
|
||||
const msgIdRef = useRef(1);
|
||||
const nextMsgId = () => msgIdRef.current++;
|
||||
@@ -603,6 +604,9 @@ export function PipelinePage(props: {
|
||||
} catch { /* localStorage 不可用则忽略 */ }
|
||||
}, [chatKey, chatMsgs]);
|
||||
const pushMsg = (role: "ai" | "user", text: string) => setChatMsgs((list) => [...list, { id: nextMsgId(), role, text, time: nowHm() }]);
|
||||
// 脚本模型下拉:用户可选 豆包/GPT-5.5/Gemini(空 = 用后端默认文本模型)
|
||||
const [scriptModelId, setScriptModelId] = useState<string>("");
|
||||
const activeScriptModelId = scriptModelId || textModels?.[0]?.id || "";
|
||||
// 删除分镜的两步确认(行内变红,3 秒不二次点击自动复位;不用原生 confirm)
|
||||
const [armedDelete, setArmedDelete] = useState<string | null>(null);
|
||||
// 行35 · 单条分镜重跑 / 删除的即时反馈:正在处理的 shot id(按钮转「处理中」并禁用)
|
||||
@@ -637,33 +641,59 @@ export function PipelinePage(props: {
|
||||
const el = chatBodyRef.current;
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}, [chatMsgs]);
|
||||
// 行33 · 进度提示流:前端模拟 AI 分步思考(逐条滚动),不需要真后端分步。
|
||||
// 生成期间逐条往同一条 progress 消息追加 step;onGenerateScript 返回后置 done 折叠收起。
|
||||
const PROGRESS_STEPS = [
|
||||
"收到脚本,正在解析商品卖点与创作方向…",
|
||||
"提取关键卖点 · 锁定目标人群画像…",
|
||||
"匹配创作风格与镜头节奏…",
|
||||
"编排分镜 · 旁白与画面逐镜成稿…",
|
||||
"校对时长与转化点,整理输出…"
|
||||
];
|
||||
// 统一的脚本生成对话回合:用户消息 → 进度流 → 成功/失败回执(onGenerateScript 失败被 App 兜住返回 null)
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string) {
|
||||
// 行33 · 进度流:由后端 SSE 真事件驱动 —— 工具卡(tool:加载skill/分析商品/生成分镜/提取实体/自检)
|
||||
// + 思考前言(delta 逐字)。真 agent 体感,不再前端假模拟。流式不可用时兜底退回旧同步端点。
|
||||
function mapSourceToMode(src?: string): "auto" | "theme" | "revise" {
|
||||
if (src === "theme") return "theme";
|
||||
if (src === "revise") return "revise";
|
||||
return "auto"; // ai / manual / 默认
|
||||
}
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string, mode?: "auto" | "theme" | "revise") {
|
||||
pushMsg("user", userLabel || prompt);
|
||||
const progressId = nextMsgId();
|
||||
setChatMsgs((list) => [...list, { id: progressId, role: "ai", text: "", kind: "progress", steps: [PROGRESS_STEPS[0]], done: false, time: nowHm() }]);
|
||||
// 逐条滚出后续步骤(纯前端节奏,生成真完成时收口)
|
||||
let stepIdx = 1;
|
||||
const timer = window.setInterval(() => {
|
||||
if (stepIdx >= PROGRESS_STEPS.length) { window.clearInterval(timer); return; }
|
||||
const step = PROGRESS_STEPS[stepIdx];
|
||||
stepIdx += 1;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId && !m.done ? { ...m, steps: [...(m.steps ?? []), step] } : m)));
|
||||
}, 900);
|
||||
const res = await onGenerateScript(prompt, source ?? chatMode);
|
||||
window.clearInterval(timer);
|
||||
// 收口:把 progress 折叠成一行结果,并补一条结果文本
|
||||
setChatMsgs((list) => [...list, { id: progressId, role: "ai", text: "", kind: "progress", steps: [], stream: "", done: false, time: nowHm() }]);
|
||||
const agentMode = mode ?? mapSourceToMode(source ?? chatMode);
|
||||
const baseVersionId = agentMode === "revise" ? currentScript?.id : undefined;
|
||||
let ok = false;
|
||||
try {
|
||||
await api.agentScriptStream(
|
||||
project.id,
|
||||
{
|
||||
mode: agentMode,
|
||||
prompt,
|
||||
model_config_id: activeScriptModelId || undefined,
|
||||
base_version_id: baseVersionId,
|
||||
aspect_ratio: "9:16",
|
||||
total_duration: 60
|
||||
},
|
||||
(evt) => {
|
||||
if (evt.type === "tool") {
|
||||
// 工具卡:running 时把 label 滚进进度流(done/error 暂只用于结束态)
|
||||
if (evt.status === "running" && typeof evt.label === "string") {
|
||||
const label = evt.label;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, steps: [...(m.steps ?? []), label] } : m)));
|
||||
}
|
||||
} else if (evt.type === "delta" && typeof evt.text === "string") {
|
||||
const piece = evt.text;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, stream: (m.stream ?? "") + piece } : m)));
|
||||
} else if (evt.type === "saved") {
|
||||
ok = true;
|
||||
} else if (evt.type === "error") {
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, done: true } : m)));
|
||||
pushMsg("ai", `生成失败:${typeof evt.detail === "string" ? evt.detail : "请稍后重试"}`);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch {
|
||||
// 流式不可用(网关不支持 SSE 等)→ 退回旧同步端点,保证可用性
|
||||
const res = await onGenerateScript(prompt, source ?? chatMode).catch(() => null);
|
||||
ok = !!res;
|
||||
}
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, done: true } : m)));
|
||||
pushMsg("ai", res ? "镜头脚本已生成,左侧已刷新。可继续输入修改意见整体重写,或点底部「确认脚本」进入下一步。" : "生成没有成功,请查看提示后重试。");
|
||||
if (ok) {
|
||||
await onRefreshProject();
|
||||
pushMsg("ai", "镜头脚本已生成,左侧已刷新。可继续输入修改意见(会基于当前脚本改稿),或点「确认脚本」进入下一步。");
|
||||
}
|
||||
}
|
||||
// 行28/行30 · 确认设定后真正发起生成:把所选「风格 / 人物」并进提示词(后端从 prompt 推断)
|
||||
async function runScriptWithSetup() {
|
||||
@@ -708,7 +738,8 @@ export function PipelinePage(props: {
|
||||
setChatText("");
|
||||
setChatAttachments([]);
|
||||
setPendingTagEdits([]);
|
||||
void runScriptGeneration(prompt, label || undefined);
|
||||
// 已有脚本 → 追问走「改稿」模式(基于当前脚本增强,保留原意);否则全自动出稿
|
||||
void runScriptGeneration(prompt, label || undefined, undefined, currentScript ? "revise" : "auto");
|
||||
}
|
||||
function clearChat() {
|
||||
setPendingTagEdits([]);
|
||||
@@ -1826,7 +1857,19 @@ export function PipelinePage(props: {
|
||||
<div className="pane-h">
|
||||
<div className="ai-avatar">AI</div>
|
||||
<strong>脚本助手</strong>
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
{textModels && textModels.length > 0 ? (
|
||||
<select
|
||||
className="setup-select"
|
||||
style={{ height: 24, fontSize: 12, padding: "0 6px", maxWidth: 150, marginLeft: 6 }}
|
||||
value={activeScriptModelId}
|
||||
onChange={(event) => setScriptModelId(event.target.value)}
|
||||
title="选择脚本生成模型(豆包 / GPT / Gemini)"
|
||||
>
|
||||
{textModels.map((m) => <option key={m.id} value={m.id}>{m.display_name || m.name}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
)}
|
||||
<span className="spacer"></span>
|
||||
<button className="btn btn-ghost btn-sm" type="button" id="chat-clear-btn" disabled={!chatText && chatAttachments.length === 0 && chatMsgs.length === 0} onClick={clearChat}>清空对话</button>
|
||||
</div>
|
||||
@@ -1837,7 +1880,10 @@ export function PipelinePage(props: {
|
||||
<div className={`msg ${msg.role}`} key={msg.id}>
|
||||
<div className="bubble">
|
||||
{msg.kind === "progress"
|
||||
? <ProgressStream steps={msg.steps} done={msg.done} />
|
||||
? <>
|
||||
<ProgressStream steps={msg.steps} done={msg.done} />
|
||||
{msg.stream ? <div style={{ marginTop: 6, opacity: 0.85, whiteSpace: "pre-wrap" }}>{msg.stream}</div> : null}
|
||||
</>
|
||||
: <CollapsibleText text={msg.text} maxLines={10} />}
|
||||
</div>
|
||||
<div className="time">{msg.time}</div>
|
||||
@@ -1933,7 +1979,7 @@ export function PipelinePage(props: {
|
||||
<div className="stage-foot">
|
||||
<div className="info"><span className="mono">[ LLM 用量 ~2.4k tokens · ¥0.04 · 失败不扣 · 通过后扣 ]</span></div>
|
||||
<div className="hstack">
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => void runScriptGeneration("整体重新生成 · 突出商品卖点,节奏紧凑", "重新生成全部")}><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M4 12a8 8 0 0 1 14-5.5L21 9" /><path d="M21 4v5h-5" /><path d="M20 12a8 8 0 0 1-14 5.5L3 15" /><path d="M3 20v-5h5" /></svg> 重新生成全部</button>
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => void runScriptGeneration("整体重新生成 · 突出商品卖点,节奏紧凑", "重新生成全部", undefined, "auto")}><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M4 12a8 8 0 0 1 14-5.5L21 9" /><path d="M21 4v5h-5" /><path d="M20 12a8 8 0 0 1-14 5.5L3 15" /><path d="M3 20v-5h5" /></svg> 重新生成全部</button>
|
||||
<button className="btn btn-primary btn-lg" type="button" disabled={loading || !currentScript} onClick={confirmScript}>{scriptAdopted ? "进入下一步" : "确认脚本,进入下一步"} <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M5 12h14M12 5l7 7-7 7" /></svg></button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -83,7 +83,20 @@ export type ScriptVersion = {
|
||||
content: string;
|
||||
source?: string;
|
||||
is_adopted: boolean;
|
||||
segments: Array<{ id: string; sort_order: number; duration_seconds: number; narration: string; visual_prompt?: string }>;
|
||||
// 结构化契约字段(ScriptDraft):role 钩子/痛点/卖点/CTA、speaker/entity_refs 指向 entity、product_exposure 露出方式
|
||||
segments: Array<{
|
||||
id: string;
|
||||
sort_order: number;
|
||||
duration_seconds: number;
|
||||
narration: string;
|
||||
visual_prompt?: string;
|
||||
role?: string;
|
||||
speaker?: string;
|
||||
product_exposure?: string;
|
||||
entity_refs?: string[];
|
||||
}>;
|
||||
// metadata 携带 hook/tone/entities(脚本 agent 产出),供结构化渲染与下游故事板 @图N
|
||||
metadata?: Record<string, unknown>;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user