feat(core): AI 生成 Agent 化 — 多模型流式脚本 agent + 可插拔 Provider + gpt-image-2 参考图 + 模特库
- 后端·可插拔 Provider 层:通用 OpenAICompatibleProvider(tokenssr 等中转站,base_url+api_key,零改代码换站)+ ModelProvider.api_key - 后端·脚本 agent:结构化 ScriptDraft 契约 + 加载电商 skill + 出稿/改稿一体对话 agent(3模式/多模型)+ 流式 SSE 端点(DRF SSE renderer) - 后端·图像:gpt-image-2 参考图出图 + 故事板 @图1@图2@图3 多锚点合成(锁脸锁商品);Seedance 打开 generate_audio - 后端·模特库:gpt-image-2 生成器(9:16氛围图→16:9白底三视图)+ seed_demo_models 管理命令 - DB·迁移:tokenssr 中转站 + 多模型 seed(豆包/GPT-5.5/Gemini + gpt-image-2);ScriptSegment 结构化字段 - 前端·脚本趴:接真 SSE(工具卡 + 思考流)+ 模型下拉 + 3模式 + 改稿;agentScriptStream - skills/ecommerce-video-script 电商脚本技能(运行时依赖) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
seaislee1209
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
a0ffb6fc8e
commit
6464001f84
@@ -2,7 +2,7 @@ import { Fragment, memo, useCallback, useDeferredValue, useEffect, useMemo, useR
|
||||
import type { ChangeEvent, CSSProperties, PointerEvent as ReactPointerEvent } from "react";
|
||||
import { Play } from "lucide-react";
|
||||
import { api } from "../api";
|
||||
import type { Asset, BillingSummary, ExportPoll, Product, Project, Team, TimelineSavePayload, User } from "../types";
|
||||
import type { Asset, BillingSummary, ExportPoll, ModelConfig, Product, Project, Team, TimelineSavePayload, User } from "../types";
|
||||
import type { Notice, Page } from "./route-config";
|
||||
import { money, stageOrder, statusPill } from "./stage-config";
|
||||
import { CornerMarks, Decorations, Sidebar, ToastLike } from "../components/app-shell";
|
||||
@@ -362,6 +362,7 @@ export function PipelinePage(props: {
|
||||
avatarChar: string;
|
||||
logout: () => void;
|
||||
scriptModelName: string;
|
||||
textModels?: ModelConfig[];
|
||||
onGenerateScript: (prompt: string, source?: string) => Promise<unknown>;
|
||||
onAdoptScript: (scriptId: string) => void | Promise<unknown>;
|
||||
onUpdateShot: (payload: { segment_id: string; narration?: string; visual_prompt?: string; duration_seconds?: number }) => Promise<unknown>;
|
||||
@@ -389,7 +390,7 @@ export function PipelinePage(props: {
|
||||
}) {
|
||||
const {
|
||||
project, loading, navigate, user, team, products, projects, assets, billing, notice, unreadCount, avatarChar, logout,
|
||||
scriptModelName, onGenerateScript, onAdoptScript, onUpdateShot, onAddShot, onDeleteShot, onRerunShot, onSaveProjectMeta, onAdoptVideoVersion, onGenerateVoiceover,
|
||||
scriptModelName, textModels, onGenerateScript, onAdoptScript, onUpdateShot, onAddShot, onDeleteShot, onRerunShot, onSaveProjectMeta, onAdoptVideoVersion, onGenerateVoiceover,
|
||||
onGenerateBaseAsset, onAdoptBaseAsset, onGenerateStoryboard, onSkipStoryboard,
|
||||
onSubmitVideo, onSubmitAllVideos, onPollVideosQuiet, exportResult, onRefreshExport, onRefreshProject,
|
||||
onUploadVideoSegment, onUploadBgm, onSaveTimeline, onSubmitExport
|
||||
@@ -573,7 +574,7 @@ export function PipelinePage(props: {
|
||||
const chatBodyRef = useRef<HTMLDivElement | null>(null);
|
||||
// 对话记录(本地会话态):生成动作可追溯,不再是「点了按钮、对话区永远空着」
|
||||
// kind=progress:进度提示流(行33),steps 逐条滚动出现,done 后折叠成一行结果
|
||||
type ChatMsg = { id: number; role: "ai" | "user"; text: string; time: string; kind?: "progress"; steps?: string[]; done?: boolean; auto?: boolean };
|
||||
type ChatMsg = { id: number; role: "ai" | "user"; text: string; time: string; kind?: "progress"; steps?: string[]; stream?: string; done?: boolean; auto?: boolean };
|
||||
const nowHm = () => new Date().toTimeString().slice(0, 5);
|
||||
const msgIdRef = useRef(1);
|
||||
const nextMsgId = () => msgIdRef.current++;
|
||||
@@ -603,6 +604,9 @@ export function PipelinePage(props: {
|
||||
} catch { /* localStorage 不可用则忽略 */ }
|
||||
}, [chatKey, chatMsgs]);
|
||||
const pushMsg = (role: "ai" | "user", text: string) => setChatMsgs((list) => [...list, { id: nextMsgId(), role, text, time: nowHm() }]);
|
||||
// 脚本模型下拉:用户可选 豆包/GPT-5.5/Gemini(空 = 用后端默认文本模型)
|
||||
const [scriptModelId, setScriptModelId] = useState<string>("");
|
||||
const activeScriptModelId = scriptModelId || textModels?.[0]?.id || "";
|
||||
// 删除分镜的两步确认(行内变红,3 秒不二次点击自动复位;不用原生 confirm)
|
||||
const [armedDelete, setArmedDelete] = useState<string | null>(null);
|
||||
// 行35 · 单条分镜重跑 / 删除的即时反馈:正在处理的 shot id(按钮转「处理中」并禁用)
|
||||
@@ -637,33 +641,59 @@ export function PipelinePage(props: {
|
||||
const el = chatBodyRef.current;
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}, [chatMsgs]);
|
||||
// 行33 · 进度提示流:前端模拟 AI 分步思考(逐条滚动),不需要真后端分步。
|
||||
// 生成期间逐条往同一条 progress 消息追加 step;onGenerateScript 返回后置 done 折叠收起。
|
||||
const PROGRESS_STEPS = [
|
||||
"收到脚本,正在解析商品卖点与创作方向…",
|
||||
"提取关键卖点 · 锁定目标人群画像…",
|
||||
"匹配创作风格与镜头节奏…",
|
||||
"编排分镜 · 旁白与画面逐镜成稿…",
|
||||
"校对时长与转化点,整理输出…"
|
||||
];
|
||||
// 统一的脚本生成对话回合:用户消息 → 进度流 → 成功/失败回执(onGenerateScript 失败被 App 兜住返回 null)
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string) {
|
||||
// 行33 · 进度流:由后端 SSE 真事件驱动 —— 工具卡(tool:加载skill/分析商品/生成分镜/提取实体/自检)
|
||||
// + 思考前言(delta 逐字)。真 agent 体感,不再前端假模拟。流式不可用时兜底退回旧同步端点。
|
||||
function mapSourceToMode(src?: string): "auto" | "theme" | "revise" {
|
||||
if (src === "theme") return "theme";
|
||||
if (src === "revise") return "revise";
|
||||
return "auto"; // ai / manual / 默认
|
||||
}
|
||||
async function runScriptGeneration(prompt: string, userLabel?: string, source?: string, mode?: "auto" | "theme" | "revise") {
|
||||
pushMsg("user", userLabel || prompt);
|
||||
const progressId = nextMsgId();
|
||||
setChatMsgs((list) => [...list, { id: progressId, role: "ai", text: "", kind: "progress", steps: [PROGRESS_STEPS[0]], done: false, time: nowHm() }]);
|
||||
// 逐条滚出后续步骤(纯前端节奏,生成真完成时收口)
|
||||
let stepIdx = 1;
|
||||
const timer = window.setInterval(() => {
|
||||
if (stepIdx >= PROGRESS_STEPS.length) { window.clearInterval(timer); return; }
|
||||
const step = PROGRESS_STEPS[stepIdx];
|
||||
stepIdx += 1;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId && !m.done ? { ...m, steps: [...(m.steps ?? []), step] } : m)));
|
||||
}, 900);
|
||||
const res = await onGenerateScript(prompt, source ?? chatMode);
|
||||
window.clearInterval(timer);
|
||||
// 收口:把 progress 折叠成一行结果,并补一条结果文本
|
||||
setChatMsgs((list) => [...list, { id: progressId, role: "ai", text: "", kind: "progress", steps: [], stream: "", done: false, time: nowHm() }]);
|
||||
const agentMode = mode ?? mapSourceToMode(source ?? chatMode);
|
||||
const baseVersionId = agentMode === "revise" ? currentScript?.id : undefined;
|
||||
let ok = false;
|
||||
try {
|
||||
await api.agentScriptStream(
|
||||
project.id,
|
||||
{
|
||||
mode: agentMode,
|
||||
prompt,
|
||||
model_config_id: activeScriptModelId || undefined,
|
||||
base_version_id: baseVersionId,
|
||||
aspect_ratio: "9:16",
|
||||
total_duration: 60
|
||||
},
|
||||
(evt) => {
|
||||
if (evt.type === "tool") {
|
||||
// 工具卡:running 时把 label 滚进进度流(done/error 暂只用于结束态)
|
||||
if (evt.status === "running" && typeof evt.label === "string") {
|
||||
const label = evt.label;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, steps: [...(m.steps ?? []), label] } : m)));
|
||||
}
|
||||
} else if (evt.type === "delta" && typeof evt.text === "string") {
|
||||
const piece = evt.text;
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, stream: (m.stream ?? "") + piece } : m)));
|
||||
} else if (evt.type === "saved") {
|
||||
ok = true;
|
||||
} else if (evt.type === "error") {
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, done: true } : m)));
|
||||
pushMsg("ai", `生成失败:${typeof evt.detail === "string" ? evt.detail : "请稍后重试"}`);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch {
|
||||
// 流式不可用(网关不支持 SSE 等)→ 退回旧同步端点,保证可用性
|
||||
const res = await onGenerateScript(prompt, source ?? chatMode).catch(() => null);
|
||||
ok = !!res;
|
||||
}
|
||||
setChatMsgs((list) => list.map((m) => (m.id === progressId ? { ...m, done: true } : m)));
|
||||
pushMsg("ai", res ? "镜头脚本已生成,左侧已刷新。可继续输入修改意见整体重写,或点底部「确认脚本」进入下一步。" : "生成没有成功,请查看提示后重试。");
|
||||
if (ok) {
|
||||
await onRefreshProject();
|
||||
pushMsg("ai", "镜头脚本已生成,左侧已刷新。可继续输入修改意见(会基于当前脚本改稿),或点「确认脚本」进入下一步。");
|
||||
}
|
||||
}
|
||||
// 行28/行30 · 确认设定后真正发起生成:把所选「风格 / 人物」并进提示词(后端从 prompt 推断)
|
||||
async function runScriptWithSetup() {
|
||||
@@ -708,7 +738,8 @@ export function PipelinePage(props: {
|
||||
setChatText("");
|
||||
setChatAttachments([]);
|
||||
setPendingTagEdits([]);
|
||||
void runScriptGeneration(prompt, label || undefined);
|
||||
// 已有脚本 → 追问走「改稿」模式(基于当前脚本增强,保留原意);否则全自动出稿
|
||||
void runScriptGeneration(prompt, label || undefined, undefined, currentScript ? "revise" : "auto");
|
||||
}
|
||||
function clearChat() {
|
||||
setPendingTagEdits([]);
|
||||
@@ -1826,7 +1857,19 @@ export function PipelinePage(props: {
|
||||
<div className="pane-h">
|
||||
<div className="ai-avatar">AI</div>
|
||||
<strong>脚本助手</strong>
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
{textModels && textModels.length > 0 ? (
|
||||
<select
|
||||
className="setup-select"
|
||||
style={{ height: 24, fontSize: 12, padding: "0 6px", maxWidth: 150, marginLeft: 6 }}
|
||||
value={activeScriptModelId}
|
||||
onChange={(event) => setScriptModelId(event.target.value)}
|
||||
title="选择脚本生成模型(豆包 / GPT / Gemini)"
|
||||
>
|
||||
{textModels.map((m) => <option key={m.id} value={m.id}>{m.display_name || m.name}</option>)}
|
||||
</select>
|
||||
) : (
|
||||
<span className="muted-2 mono" style={{ fontSize: "12px" }}>· {scriptModelName}</span>
|
||||
)}
|
||||
<span className="spacer"></span>
|
||||
<button className="btn btn-ghost btn-sm" type="button" id="chat-clear-btn" disabled={!chatText && chatAttachments.length === 0 && chatMsgs.length === 0} onClick={clearChat}>清空对话</button>
|
||||
</div>
|
||||
@@ -1837,7 +1880,10 @@ export function PipelinePage(props: {
|
||||
<div className={`msg ${msg.role}`} key={msg.id}>
|
||||
<div className="bubble">
|
||||
{msg.kind === "progress"
|
||||
? <ProgressStream steps={msg.steps} done={msg.done} />
|
||||
? <>
|
||||
<ProgressStream steps={msg.steps} done={msg.done} />
|
||||
{msg.stream ? <div style={{ marginTop: 6, opacity: 0.85, whiteSpace: "pre-wrap" }}>{msg.stream}</div> : null}
|
||||
</>
|
||||
: <CollapsibleText text={msg.text} maxLines={10} />}
|
||||
</div>
|
||||
<div className="time">{msg.time}</div>
|
||||
@@ -1933,7 +1979,7 @@ export function PipelinePage(props: {
|
||||
<div className="stage-foot">
|
||||
<div className="info"><span className="mono">[ LLM 用量 ~2.4k tokens · ¥0.04 · 失败不扣 · 通过后扣 ]</span></div>
|
||||
<div className="hstack">
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => void runScriptGeneration("整体重新生成 · 突出商品卖点,节奏紧凑", "重新生成全部")}><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M4 12a8 8 0 0 1 14-5.5L21 9" /><path d="M21 4v5h-5" /><path d="M20 12a8 8 0 0 1-14 5.5L3 15" /><path d="M3 20v-5h5" /></svg> 重新生成全部</button>
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => void runScriptGeneration("整体重新生成 · 突出商品卖点,节奏紧凑", "重新生成全部", undefined, "auto")}><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M4 12a8 8 0 0 1 14-5.5L21 9" /><path d="M21 4v5h-5" /><path d="M20 12a8 8 0 0 1-14 5.5L3 15" /><path d="M3 20v-5h5" /></svg> 重新生成全部</button>
|
||||
<button className="btn btn-primary btn-lg" type="button" disabled={loading || !currentScript} onClick={confirmScript}>{scriptAdopted ? "进入下一步" : "确认脚本,进入下一步"} <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"><path d="M5 12h14M12 5l7 7-7 7" /></svg></button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user