From 7f5307d97c38b7a5919279bb2dd9b4fc5b40a77a Mon Sep 17 00:00:00 2001 From: "Azmat@qq.com" Date: Wed, 16 Sep 2026 16:07:56 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=A7=E9=87=8F=E4=BC=98=E5=8C=96=E5=85=A8?= =?UTF-8?q?=E8=83=BD=E5=88=9B=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/backend/apps/ai/creation.py | 193 ++++++++++ core/backend/apps/ai/creation_agent.py | 338 +++++++++++++++++- core/backend/apps/ai/free_video.py | 4 +- core/backend/apps/ai/tasks.py | 9 + core/backend/apps/ai/test_creation_agent.py | 60 ++++ core/backend/apps/ai/views.py | 102 ++++++ core/frontend/src/api.ts | 7 + core/frontend/src/omni-session-page.css | 139 +++++++- core/frontend/src/routes/omni-session.tsx | 368 +++++++++++++++++--- 9 files changed, 1155 insertions(+), 65 deletions(-) diff --git a/core/backend/apps/ai/creation.py b/core/backend/apps/ai/creation.py index 1c093de..1870816 100644 --- a/core/backend/apps/ai/creation.py +++ b/core/backend/apps/ai/creation.py @@ -6,6 +6,7 @@ from __future__ import annotations from datetime import timedelta +import uuid from django.core.cache import cache from django.db import transaction @@ -123,9 +124,16 @@ def sync_generating_message(message: CreationMessage) -> bool: if message.kind != CreationMessage.Kind.GENERATING: return False + payload = message.payload or {} + if payload.get("kind") == "video_segments": + return _sync_segmented_video_message(message) task = _message_task(message) if task is None: return False + if payload.get("kind") == "video_merge" and task.status in (AITask.Status.SUBMITTED, AITask.Status.POLLING): + # Celery 不在线的本地环境同样可完成用户已确认的合并；此前从不在普通出片完成时调用。 + run_segmented_video_merge(str(task.id)) + task.refresh_from_db() if task.status == AITask.Status.SUCCEEDED: assets = _assets_from_task(task) if not assets: @@ -138,6 +146,191 @@ def sync_generating_message(message: CreationMessage) -> bool: return False +def _sync_segmented_video_message(message: CreationMessage) -> bool: + """同步全能创作的多段出片。 + + 每段仍是普通 FREE_VIDEO 任务，故可沿用既有计费、轮询和资产落库；这里仅把它们聚合成 + 一张结果卡。所有分段完成后停在结果卡，绝不在此处触发 ffmpeg。 + """ + from .free_video import finalize_free_video + from .models import AITask + + payload = message.payload or {} + ids = [str(value) for value in payload.get("task_ids") or [] if value] + if not ids: + return False + by_id = { + str(task.id): task + for task in AITask.objects.filter(id__in=ids).select_related("model_config") + } + tasks = [by_id.get(task_id) for task_id in ids] + if any(task is None for task in tasks): + fail_generating_message(message, "分段视频任务不完整，请重新生成。") + return True + + # 本地没有 worker 时，用户的会话轮询本身即可推进每个片段；已在 worker 中的任务则幂等返回。 + refreshed = [] + for task in tasks: + assert task is not None + if task.status in (AITask.Status.SUBMITTED, AITask.Status.POLLING): + try: + task = finalize_free_video(task=task) + except Exception: # noqa: BLE001 - 单段网络抖动不能使整组直接失败 + task.refresh_from_db() + else: + task.refresh_from_db() + refreshed.append(task) + + failed = next( + (task for task in refreshed if task.status in (AITask.Status.FAILED, AITask.Status.CANCELLED)), + None, + ) + if failed is not None: + number = next((index + 1 for index, task in enumerate(refreshed) if task.id == failed.id), 1) + fail_generating_message(message, f"第 {number} 段生成失败：{failed.error_message or '请重试'}") + return True + if not all(task.status == AITask.Status.SUCCEEDED for task in refreshed): + return False + + assets: list[dict] = [] + for index, task in enumerate(refreshed, start=1): + task_assets = _assets_from_task(task) + if not task_assets: + return False + for asset in task_assets: + assets.append({**asset, "label": f"第 {index} 段"}) + first = refreshed[0] + finish_generating_message( + message, + assets=assets, + meta={ + **_meta_from_task(first, message), + "kind": "video_segments", + "segment_count": len(refreshed), + "total_duration": payload.get("total_duration") or "", + "needs_merge": True, + "segments": payload.get("segments") or [], + }, + ) + return True + + +def start_segmented_video_merge(*, conversation: CreationConversation, message: CreationMessage, user): + """用户明确点击后才建立合并任务；这之前绝不下载片段或调用 ffmpeg。""" + from .models import AITask + + payload = dict(message.payload or {}) + if message.kind != CreationMessage.Kind.RESULT or not payload.get("needs_merge"): + raise ValueError("这条结果不需要合并") + state = str(payload.get("merge_state") or "") + if state in {"queued", "processing", "completed"}: + raise ValueError("这组片段已经在合并中或已合并") + task_ids = [str(value) for value in payload.get("task_ids") or [] if value] + source_tasks = list(AITask.objects.filter(id__in=task_ids).select_related("model_config")) + if len(source_tasks) != len(task_ids): + raise ValueError("分段视频不完整，无法合并") + model_config = source_tasks[0].model_config + merge_task = AITask.objects.create( + team=conversation.team, + created_by=user, + project=None, + task_type=AITask.Type.EXPORT, + status=AITask.Status.SUBMITTED, + model_config=model_config, + idempotency_key=f"omni-video-merge:{conversation.id}:{message.id}:{uuid.uuid4()}", + request_payload={ + "feature": "omni_video_merge", + "source_message_id": str(message.id), + "source_task_ids": task_ids, + "prompt": payload.get("prompt") or "", + }, + ) + payload["merge_state"] = "queued" + payload["merge_task_id"] = str(merge_task.id) + message.payload = payload + message.save(update_fields=["payload", "updated_at"]) + generating = append_message( + conversation, + role="assistant", + kind=CreationMessage.Kind.GENERATING, + payload={"task_id": str(merge_task.id), "kind": "video_merge", "prompt": payload.get("prompt") or ""}, + task=merge_task, + ) + return merge_task, generating + + +def run_segmented_video_merge(task_id: str) -> None: + """后台执行用户已确认的合并。任务创建不等于执行；仅此函数调用 ffmpeg。""" + from .free_video import _store_free_video_media + from .models import AITask + from .services import asset_stable_url + from .video_replace import _concat_shot_media + from apps.assets.models import Asset + + task = AITask.objects.select_related("team", "created_by").filter(id=task_id).first() + if task is None or task.task_type != AITask.Type.EXPORT: + return + source_message_id = str((task.request_payload or {}).get("source_message_id") or "") + source_message = CreationMessage.objects.filter(id=source_message_id).first() + if source_message is None: + return + try: + with transaction.atomic(): + locked = AITask.objects.select_for_update().get(id=task.id) + if locked.status not in (AITask.Status.SUBMITTED, AITask.Status.POLLING): + return + locked.status = AITask.Status.POSTPROCESSING + locked.save(update_fields=["status", "updated_at"]) + urls: list[str] = [] + for source_id in (locked.request_payload or {}).get("source_task_ids") or []: + source_asset = ( + Asset.objects.filter(origin_task_id=source_id, is_deleted=False, purged_at__isnull=True) + .prefetch_related("files") + .first() + ) + url, _cover = asset_stable_url(source_asset) + if not url: + raise ValueError("有片段尚未准备好，暂时不能合并") + urls.append(url) + merged_bytes = _concat_shot_media(urls) + _store_free_video_media(task=locked, video_bytes=merged_bytes) + with transaction.atomic(): + locked = AITask.objects.select_for_update().get(id=task.id) + if locked.status != AITask.Status.POSTPROCESSING: + return + locked.status = AITask.Status.SUCCEEDED + locked.completed_at = timezone.now() + locked.save(update_fields=["status", "completed_at", "updated_at"]) + source = CreationMessage.objects.select_for_update().get(id=source_message.id) + source_payload = dict(source.payload or {}) + source_payload["merge_state"] = "completed" + source.payload = source_payload + source.save(update_fields=["payload", "updated_at"]) + except Exception as exc: # noqa: BLE001 - 合并失败不影响已完成的分段视频 + import logging + + logging.getLogger(__name__).exception("omni video merge failed for %s", task_id) + with transaction.atomic(): + locked = AITask.objects.select_for_update().get(id=task.id) + if locked.status == AITask.Status.POSTPROCESSING: + locked.status = AITask.Status.FAILED + locked.error_code = "MergeError" + locked.error_message = str(exc)[:500] + locked.completed_at = timezone.now() + locked.save(update_fields=["status", "error_code", "error_message", "completed_at", "updated_at"]) + source = CreationMessage.objects.select_for_update().filter(id=source_message.id).first() + if source is not None: + source_payload = dict(source.payload or {}) + source_payload["merge_state"] = "failed" + source.payload = source_payload + source.save(update_fields=["payload", "updated_at"]) + finally: + # 复用既有「生成中 → 结果/错误」转换，生成的合成片会成为独立结果卡。 + fresh = AITask.objects.filter(id=task_id).first() + if fresh is not None: + sync_generating_for_task(fresh) + + def sync_generating_messages(conversation: CreationConversation) -> int: """把一条会话里所有已结束的 GENERATING 回填。返回改了几条。""" pending = list( diff --git a/core/backend/apps/ai/creation_agent.py b/core/backend/apps/ai/creation_agent.py index f532462..aa73d73 100644 --- a/core/backend/apps/ai/creation_agent.py +++ b/core/backend/apps/ai/creation_agent.py @@ -232,6 +232,34 @@ def get_pending_video_prompt(conversation: CreationConversation) -> str: return str(memory.get("pending_video_prompt") or "").strip() +def append_selling_point_gate(conversation: CreationConversation) -> CreationMessage: + """在视频方案生成前确认卖点来源。 + + 商家可以直接给真实卖点；若交给系统，则后续只从商品资料和参考素材中选择可证实的表达， + 不把“系统推荐”误做成无依据的夸大文案。 + """ + return append_message( + conversation, + role="assistant", + kind=CreationMessage.Kind.ELICIT, + text="这条视频准备先讲哪个卖点？你可以直接写真实卖点，也可以让我从商品和素材里推荐一个。", + payload={ + "interaction": "selling_point_gate", + "fields": [ + { + "key": "selling_point", + "label": "商品卖点", + "type": "text", + "required": False, + "placeholder": "例如：油污一喷一擦就干净，适合厨房重油污…", + } + ], + "submitted": False, + "answers": {}, + }, + ) + + def _step_confirm_payload(step: str) -> dict: return { "interaction": "step_confirm", @@ -438,8 +466,16 @@ IMAGE_MODEL_BY_LABEL = { "YQ image2": "gpt-image", "影擎-Image2": "gpt-image", } -# 「智能时长」= 交给我们定,取一个口播讲得完又不烧钱的中间值 +# 「智能时长」没有可读到的脚本时才用的保守兜底。实际方案生成后必须从时间轴推导， +# 不能把每条片都悄悄压成 15 秒。 SMART_DURATION = 15 +_SMART_DURATION_RE = re.compile( + r"(? int: memory["stage"] = "clarify" memory.pop("pending_video_prompt", None) memory.pop("strategy_confirmed", None) + memory.pop("selling_point_ready", None) + memory.pop("selling_point_mode", None) + memory.pop("selling_point", None) conversation.memory = memory conversation.save(update_fields=["memory", "updated_at"]) @@ -945,6 +984,17 @@ def requested_asset_card_from_context( ) if not wants_card: return None + # 用户明确要某类素材列表时，不能依赖上一条是否碰巧留下追问卡。 + # 否则模型会把检索结果念成名称段落，而不会返回可点击的卡片。 + direct_types = ( + ("product", ("商品", "产品")), + ("character", ("角色", "人物")), + ("model", ("模特",)), + ("scene", ("场景",)), + ) + for type_, keywords in direct_types: + if any(keyword in text for keyword in keywords): + return type_ recent = conversation.messages.filter( kind=CreationMessage.Kind.ELICIT ).order_by("-seq")[:8] @@ -1457,15 +1507,175 @@ def video_model_name(params: dict) -> str: return hit.name if hit else DEFAULT_VIDEO_MODEL -def video_duration(params: dict) -> int: - """「15 秒」→ 15;「智能时长」/ 解析不出 → SMART_DURATION。""" +def _is_smart_duration(params: dict) -> bool: + raw = str((params or {}).get("duration") or "").strip().lower() + return not raw or "智能" in raw or raw in {"smart", "auto"} + + +def infer_script_duration(*, timeline: list[dict] | None = None, prompt: str = "") -> int | None: + """从已写好的方案推算实际片长：优先方案时间轴，其次 Prompt 的秒级分段。""" + ends: list[float] = [] + for item in timeline or []: + if not isinstance(item, dict): + continue + try: + end = float(item.get("end")) + except (TypeError, ValueError): + continue + if end > 0: + ends.append(end) + for match in _SMART_DURATION_RE.finditer(prompt or ""): + try: + ends.append(float(match.group(2))) + except (TypeError, ValueError): + continue + for match in _TOTAL_DURATION_RE.finditer(prompt or ""): + try: + ends.append(float(match.group(1))) + except (TypeError, ValueError): + continue + if not ends: + return None + # 方案的最后一个时间点就是成片总长；向上取整避免 19.2 秒被截成 19 秒。 + return max(4, min(60, int(max(ends) + 0.999))) + + +def _raw_script_duration(*, timeline: list[dict] | None = None, prompt: str = "") -> int | None: + """返回方案真实的最大时间点，不在这里截断，供 60 秒硬上限校验使用。""" + ends: list[float] = [] + for item in timeline or []: + if not isinstance(item, dict): + continue + try: + end = float(item.get("end")) + except (TypeError, ValueError): + continue + if end > 0: + ends.append(end) + for match in _SMART_DURATION_RE.finditer(prompt or ""): + try: + ends.append(float(match.group(2))) + except (TypeError, ValueError): + continue + for match in _TOTAL_DURATION_RE.finditer(prompt or ""): + try: + ends.append(float(match.group(1))) + except (TypeError, ValueError): + continue + return int(max(ends) + 0.999) if ends else None + + +def plan_video_segments(duration: int, timeline: list[dict] | None = None) -> list[dict]: + """按方案节奏把 31–60 秒视频拆成可由 Seedance 2.5 单独完成的片段。 + + 每段不超过 30 秒；优先落在时间轴的自然转场处，找不到合适转场再均分。 + 当前模型上限下 31–60 秒稳定拆为两段，避免无意义地把同一支片拆得过碎。 + """ + total = max(4, min(int(duration or 0), 60)) + if total <= 30: + return [{"index": 1, "start": 0, "end": total, "duration": total}] + + candidates: list[int] = [] + for item in timeline or []: + if not isinstance(item, dict): + continue + try: + end = int(round(float(item.get("end")))) + except (TypeError, ValueError): + continue + if 4 <= end <= total - 4: + candidates.append(end) + # 两段都必须 <= 30 秒，因此 60 秒时唯一合法分点就是 30 秒。 + lower, upper = max(4, total - 30), min(30, total - 4) + target = total / 2 + legal = [point for point in candidates if lower <= point <= upper] + split = min(legal, key=lambda point: abs(point - target)) if legal else int(round(target)) + split = max(lower, min(upper, split)) + return [ + {"index": 1, "start": 0, "end": split, "duration": split}, + {"index": 2, "start": split, "end": total, "duration": total - split}, + ] + + +def segment_video_prompt(prompt: str, segment: dict, total_duration: int) -> str: + """让单段模型只拍本段，不把整条长脚本压回每一个片段里。""" + index = int(segment.get("index") or 1) + start = int(segment.get("start") or 0) + end = int(segment.get("end") or 0) + return ( + f"{prompt.strip()}\n\n" + f"【分段出片约束】这是整支 {total_duration} 秒视频的第 {index} 段，只生成 {start}–{end} 秒的内容。" + "仅呈现这一时间段对应的情节与镜头，承接上一段的角色、服装、商品、场景和光线，" + "为下一段留出自然动作衔接；不要重演完整故事，不要添加字幕、文字、角标或水印。" + ) + + +def video_duration(params: dict, *, prompt: str = "", timeline: list[dict] | None = None) -> int: + """显式时长优先；智能时长从方案/Prompt 推导，完全缺失才回退 15 秒。""" raw = str(params.get("duration") or "") digits = "".join(ch for ch in raw if ch.isdigit()) if not digits: + inferred = infer_script_duration(timeline=timeline, prompt=prompt) + if inferred is not None: + return inferred return SMART_DURATION return max(4, min(int(digits), 60)) +def _model_supports_duration(model_name: str, duration: int) -> bool: + """仅用于智能时长的自动路由；配置缺失时不武断拦截，交由提交校验给出明确错误。""" + model = ModelConfig.objects.filter( + name=model_name, + capability=ModelConfig.Capability.VIDEO, + status=ModelConfig.Status.ACTIVE, + ).first() + if model is None: + return True + meta = model.metadata if isinstance(model.metadata, dict) else {} + listed = (meta.get("capabilities") or {}).get("durations") or meta.get("durations") or [] + values = [int(value) for value in listed if str(value).isdigit()] + return not values or min(values) <= duration <= max(values) + + +def resolve_smart_video_duration( + conversation: CreationConversation, + *, + prompt: str = "", + timeline: list[dict] | None = None, +) -> int: + """把智能时长固化为方案真实时长，并在需要时切换到能承载它的模型。 + + 这一步发生在方案写完、用户看到最终确认卡之前，所以页面显示、计费和实际 API 参数 + 始终是同一个秒数。 + """ + params = dict(conversation.params or {}) + smart_duration = _is_smart_duration(params) + duration = video_duration(params, prompt=prompt, timeline=timeline) + if smart_duration: + params["duration"] = f"{duration} 秒" + selected_model = video_model_name(params) + switched = False + # Seedance 2.5 是当前唯一可稳定承载 16–30 秒单段、以及 31–60 秒分段的模型。 + # 即使总时长 45 秒不在单段能力表里，后续也会拆成两条 <=30 秒的 2.5 任务。 + if duration > 15 and selected_model != DEFAULT_VIDEO_MODEL: + params["model"] = "Seedance 2.5" + switched = True + elif not _model_supports_duration(selected_model, duration): + # 智能模式可自动选能完成完整脚本的模型；确认卡会清楚展示变更，用户仍能手动调整。 + if _model_supports_duration(DEFAULT_VIDEO_MODEL, duration): + params["model"] = "Seedance 2.5" + switched = True + memory = dict(conversation.memory or {}) + memory["smart_duration_resolved"] = duration + if switched: + memory["smart_duration_model_switched"] = True + if smart_duration or switched: + conversation.params = params + conversation.memory = memory + conversation.save(update_fields=["params", "memory", "updated_at"]) + return duration + + def _image_count(params: dict, raw) -> int: """出图张数:首页选过「N 张」就用它,否则用模型传的 count,默认 1,上限 8。""" label = str((params or {}).get("count") or (params or {}).get("duration") or "") @@ -1534,7 +1744,6 @@ def _run_generate_image(context: AgentContext, args: dict) -> tuple[dict, list]: def _video_submit_params(context: AgentContext, prompt: str) -> tuple[dict, list]: """拼 submit_free_video 的入参。references 直接用 resolve_refs 的产物 —— 它已经排好 角色 → 场景 → 商品 的顺序,那正是出片模型 @图N 的语义依据。""" - params = context.conversation.params or {} resolved = resolve_refs(context.team, context.conversation.pinned_refs or []) prompt = apply_video_preset_prompt(context.conversation.preset, prompt) prompt = apply_product_voice_visual_guard(context.conversation, prompt) @@ -1544,6 +1753,9 @@ def _video_submit_params(context: AgentContext, prompt: str) -> tuple[dict, list active_plot_twist_story_depth(context.conversation), prompt, ) + duration = resolve_smart_video_duration(context.conversation, prompt=prompt) + # resolve_smart_video_duration 可能为了完整脚本切到支持长时长的模型，必须重新取参数。 + params = context.conversation.params or {} submit = { "prompt": prompt, "feature": "omni_create", @@ -1551,7 +1763,7 @@ def _video_submit_params(context: AgentContext, prompt: str) -> tuple[dict, list "model": video_model_name(params), "aspect_ratio": params.get("ratio") or "9:16", "resolution": params.get("resolution") or "720p", - "duration": video_duration(params), + "duration": duration, "generate_audio": True, "references": resolved.references, } @@ -1612,7 +1824,10 @@ def submit_confirmed_video(*, conversation: CreationConversation, user, confirm_ **这里不再跑一轮模型**:方案已经确认过了,再让模型决定一次既费钱又可能它不调工具。 返回 (生成中消息, 错误文案),两者必有其一。 """ - from .free_video import submit_free_video + from django.conf import settings + + from .free_video import IN_FLIGHT_STATUSES, submit_free_video + from .models import AITask payload = confirm_message.payload or {} prompt = str(payload.get("video_prompt") or "").strip() @@ -1621,14 +1836,54 @@ def submit_confirmed_video(*, conversation: CreationConversation, user, confirm_ context = AgentContext(conversation=conversation, user=user, model_config=None) submit, _references = _video_submit_params(context, prompt) + total_duration = int(submit["duration"]) + segments = plan_video_segments(total_duration) + # 先整体检查并发余量，再提交任何一段；否则第一段已扣费、第二段才因额度满失败会留下孤儿片段。 + in_flight = AITask.objects.filter( + team=conversation.team, + task_type=AITask.Type.FREE_VIDEO, + status__in=IN_FLIGHT_STATUSES, + ).count() + max_concurrent = int(getattr(settings, "FREE_VIDEO_MAX_CONCURRENT", 3)) + if in_flight + len(segments) > max_concurrent: + return None, f"当前视频任务余量不足，需要同时生成 {len(segments)} 段；请等待现有任务完成后再试。" + tasks = [] try: - task = submit_free_video(team=conversation.team, user=user, params=submit) + for segment in segments: + segment_submit = { + **submit, + "duration": int(segment["duration"]), + "prompt": segment_video_prompt(prompt, segment, total_duration) if len(segments) > 1 else submit["prompt"], + "extra_payload": { + "omni_segment": { + "index": segment["index"], + "start": segment["start"], + "end": segment["end"], + "total_duration": total_duration, + } + }, + } + tasks.append(submit_free_video(team=conversation.team, user=user, params=segment_submit)) except ValueError as exc: # 校验类错误(时长/比例/额度),给用户看原文 return None, str(exc) + if len(tasks) == 1: + message_payload = {"task_id": str(tasks[0].id), "kind": "video", "prompt": prompt} + else: + message_payload = { + "task_id": str(tasks[0].id), + "task_ids": [str(task.id) for task in tasks], + "kind": "video_segments", + "prompt": prompt, + "total_duration": total_duration, + "segments": [ + {**segment, "task_id": str(task.id)} + for segment, task in zip(segments, tasks, strict=True) + ], + } message = append_message( conversation, role="assistant", kind=CreationMessage.Kind.GENERATING, - payload={"task_id": str(task.id), "kind": "video", "prompt": prompt}, task=task, + payload=message_payload, task=tasks[0], ) _remember_artifact(conversation, prompt, "video") set_video_gate_stage(conversation, "done", clear_pending_prompt=True) @@ -1849,6 +2104,18 @@ def build_system_prompt(context: AgentContext, *, allow_plan: bool = True, has_c ) stage = get_video_gate_stage(conversation) memory = conversation.memory if isinstance(conversation.memory, dict) else {} + selling_point = str(memory.get("selling_point") or "").strip() + selling_mode = str(memory.get("selling_point_mode") or "").strip() + if selling_mode == "manual" and selling_point: + lines.append( + f"- 商家已确认核心卖点：【{selling_point}】。策略、方案、脚本和出片指令必须围绕它展开；" + "只补充可从素材或正常使用中证明的支撑，不得替换或夸大。" + ) + elif selling_mode == "auto": + lines.append( + "- 商家已授权系统推荐卖点。你必须从商品资料、可见素材和正常使用动作中选择一个最易证明的核心卖点；" + "不要虚构功效、价格或规格。" + ) strategy_confirmed = bool(memory.get("strategy_confirmed")) stage_hint = { "clarify": "当前阶段=澄清:缺关键信息就 ask_user;信息够了只调 write_strategy。", @@ -2315,6 +2582,29 @@ def iter_creation_agent_events( yield {"type": "done"} return + # 明确要商品/角色/场景列表时，直接生成真实选择卡，绝不先让模型念出素材名称。 + requested_card = requested_asset_card_from_context(conversation, text) + if requested_card: + result, _stop = _dispatch_tool( + context, + "ask_user", + {"fields": [{ + "key": requested_card, + "label": _ASSET_PICK_LABEL.get( + requested_card, + _ASSET_CARD_LABELS.get(requested_card, "请选择素材"), + ), + "type": "asset", + "required": True, + "asset_types": [requested_card], + }]}, + allow_pick=True, + ) + for event in result.get("_events", []): + yield event + yield {"type": "done"} + return + model_config = _prefer_vision_text_model(model_config, conversation.team, conversation.pinned_refs or []) context.model_config = model_config @@ -2796,6 +3086,14 @@ def _dispatch_tool( return {"payload": _run_search_library(context, args)}, False if name == "write_strategy": + memory = context.conversation.memory if isinstance(context.conversation.memory, dict) else {} + if context.is_video and not memory.get("selling_point_ready"): + gate = append_selling_point_gate(context.conversation) + set_video_gate_stage(context.conversation, "clarify") + return { + "payload": {"asked": True, "field": "selling_point"}, + "_events": [{"type": "message", "message": _message_payload(gate)}], + }, True strategy_payload = _coerce_strategy_args(args if isinstance(args, dict) else {}) # 空卡会落成「只有标签没有正文」——拒绝,让模型把四字段写满再调 if not all(strategy_payload.values()): @@ -2849,11 +3147,25 @@ def _dispatch_tool( ) } }, False - duration = 15 - try: - duration = int(float(str((context.conversation.params or {}).get("duration") or "15").replace("秒", "").strip() or "15")) - except (TypeError, ValueError): - duration = 15 + selling_memory = context.conversation.memory if isinstance(context.conversation.memory, dict) else {} + chosen_selling_point = str(selling_memory.get("selling_point") or "").strip() + if chosen_selling_point and str(selling_memory.get("selling_point_mode") or "") == "manual": + # 用户亲自给出的真实卖点是本轮的唯一 USP，模型只能围绕它补充画面证据，不能擅自替换。 + card["usp"] = chosen_selling_point + video_prompt = ( + f"{video_prompt}\n\n【商家确认的核心卖点】{chosen_selling_point}\n" + "整条视频必须围绕这个卖点展开，并用真实使用动作或素材可见细节证明；不得替换、夸大或新增未经确认的功效。" + ) + raw_duration = _raw_script_duration(timeline=card["timeline"], prompt=video_prompt) + if raw_duration is not None and raw_duration > 60: + return { + "payload": {"error": "脚本时长不能超过 60 秒。请把方案收束到 60 秒以内，再重新给出完整时间轴和出片指令。"} + }, False + duration = resolve_smart_video_duration( + context.conversation, + prompt=video_prompt, + timeline=card["timeline"], + ) lo = max(20, round(duration * 3.4)) hi = max(lo + 1, round(duration * 4)) plan_payload = { diff --git a/core/backend/apps/ai/free_video.py b/core/backend/apps/ai/free_video.py index 815be89..f4dce1b 100644 --- a/core/backend/apps/ai/free_video.py +++ b/core/backend/apps/ai/free_video.py @@ -867,7 +867,7 @@ def _store_free_video_media(*, task: AITask, media: str = "", video_bytes: bytes if feature == "video_replace": mode_label = "角色复刻" if payload.get("replace_mode") == "character" else "商品复刻" asset_name = f"{subject}{mode_label}" if subject else mode_label - elif feature == "omni_create": + elif feature in {"omni_create", "omni_video_merge"}: asset_name = prompt[:255] or "全能创作视频" else: asset_name = prompt[:255] or "自由创作视频" @@ -883,7 +883,7 @@ def _store_free_video_media(*, task: AITask, media: str = "", video_bytes: bytes category=Asset.Category.FREE_CREATE, origin_task=task, # 全能创作成品只挂在会话结果卡上,不进资产库/专业创作用的视频列表。 - in_library=feature != "omni_create", + in_library=feature not in {"omni_create", "omni_video_merge"}, metadata={"feature": feature}, ) AssetFile.objects.create( diff --git a/core/backend/apps/ai/tasks.py b/core/backend/apps/ai/tasks.py index 2ce911b..68b2f11 100644 --- a/core/backend/apps/ai/tasks.py +++ b/core/backend/apps/ai/tasks.py @@ -121,6 +121,15 @@ def poll_free_video_task(self, task_id: str, attempt: int = 0) -> str: return task_id +@app.task(bind=True, max_retries=0) +def merge_omni_video_segments_task(self, task_id: str) -> str: + """全能创作的分段视频仅在用户点「合并成片」后进入此任务。""" + from apps.ai.creation import run_segmented_video_merge + + run_segmented_video_merge(task_id) + return task_id + + @app.task(bind=True, max_retries=0, queue=QUICK_QUEUE) def drain_asset_reviews_task(self) -> str: """全平台人脸审核兜底:未送审自动送火山、审核中自动拉绿/红盾。 diff --git a/core/backend/apps/ai/test_creation_agent.py b/core/backend/apps/ai/test_creation_agent.py index cb9fef0..bbae5e0 100644 --- a/core/backend/apps/ai/test_creation_agent.py +++ b/core/backend/apps/ai/test_creation_agent.py @@ -34,6 +34,8 @@ from .creation_agent import ( get_creation_chat_model, get_video_gate_stage, has_creative_intent, + infer_script_duration, + plan_video_segments, is_continue_intent, is_greeting, is_pure_chitchat, @@ -943,6 +945,8 @@ class VideoPlanAndConfirmTests(CreationAgentBaseTests): self.assertNotIn("generate_image", names) def test_strategy_card_stops_with_step_confirm(self): + self.conversation.memory = {"selling_point_ready": True, "selling_point_mode": "auto"} + self.conversation.save(update_fields=["memory", "updated_at"]) fake = FakeProvider([ _tool_chunks("write_strategy", {"target": "油皮通勤人群", "trust": "真实使用反馈", "belief": "值得一试", "direction": "达人 UGC 口播"}), @@ -964,6 +968,21 @@ class VideoPlanAndConfirmTests(CreationAgentBaseTests): # 策略闸门必须停下等人确认,不能同轮连写方案 self.assertEqual(len(fake.calls), 1) + def test_selling_point_is_confirmed_before_strategy(self): + fake = FakeProvider([ + _tool_chunks("write_strategy", {"target": "油皮通勤人群", "trust": "真实使用反馈", + "belief": "值得一试", "direction": "达人 UGC 口播"}), + ]) + with patch("apps.ai.creation_agent.build_provider", return_value=fake): + events = _events(stream_creation_agent(conversation=self.conversation, user=self.user, + text="做条视频", model_config=self.model)) + cards = [ + event["message"] for event in events + if event.get("type") == "message" and event["message"]["kind"] == "elicit" + ] + self.assertEqual(cards[-1]["payload"].get("interaction"), "selling_point_gate") + self.assertFalse(any(event.get("message", {}).get("kind") == "strategy" for event in events)) + def test_plan_emits_plan_and_step_confirm_only(self): fake = FakeProvider([ _tool_chunks("write_plan", self._plan_args()), @@ -991,6 +1010,23 @@ class VideoPlanAndConfirmTests(CreationAgentBaseTests): # 方案闸门停下,不能同轮出 Prompt/积分卡 self.assertEqual(len(fake.calls), 1) + def test_manual_selling_point_is_locked_as_plan_usp(self): + self.conversation.memory = { + "selling_point_ready": True, + "selling_point_mode": "manual", + "selling_point": "一喷一擦去除厨房重油污", + } + self.conversation.save(update_fields=["memory", "updated_at"]) + fake = FakeProvider([_tool_chunks("write_plan", self._plan_args())]) + with patch("apps.ai.creation_agent.build_provider", return_value=fake): + events = _events(stream_creation_agent(conversation=self.conversation, user=self.user, + text="继续", model_config=self.model)) + plan = next( + event["message"] for event in events + if event.get("type") == "message" and event["message"]["kind"] == "plan" + ) + self.assertEqual(plan["payload"]["usp"], "一喷一擦去除厨房重油污") + def test_write_prompt_emits_document_and_requires_prompt_confirmation(self): fake = FakeProvider([ _tool_chunks("write_prompt", {"video_prompt": "0-3秒 近景手持商品…"}), @@ -1015,6 +1051,9 @@ class VideoPlanAndConfirmTests(CreationAgentBaseTests): """模型若同轮连调 write_strategy+write_plan,只落策略闸门。""" from apps.ai.creation_agent import _parse_arguments # noqa: F401 + self.conversation.memory = {"selling_point_ready": True, "selling_point_mode": "auto"} + self.conversation.save(update_fields=["memory", "updated_at"]) + # FakeProvider 一轮里多个 tool_call:模拟两个独立 rounds 不行,需单轮多 call。 # 用手动合并:第一帧带两个 tool call deltas。 def _multi_tool_chunks(*pairs): @@ -1174,6 +1213,27 @@ class VideoParamParsingTests(TestCase): # 当前视频工作流支持完整短剧，超出预设上限时仍需夹住。 self.assertEqual(video_duration({"duration": "99 秒"}), 60) + def test_smart_duration_uses_final_plan_timestamp(self): + timeline = [ + {"start": 0, "end": 2, "stage": "Hook"}, + {"start": 2, "end": 14, "stage": "正文"}, + {"start": 14, "end": 18, "stage": "证据"}, + {"start": 18, "end": 20, "stage": "CTA"}, + ] + self.assertEqual(infer_script_duration(timeline=timeline), 20) + self.assertEqual(video_duration({"duration": "智能时长"}, timeline=timeline), 20) + + def test_long_video_is_split_into_seedance_sized_segments(self): + self.assertEqual( + plan_video_segments(45, [{"end": 14}, {"end": 25}, {"end": 45}]), + [ + {"index": 1, "start": 0, "end": 25, "duration": 25}, + {"index": 2, "start": 25, "end": 45, "duration": 20}, + ], + ) + self.assertEqual(plan_video_segments(60)[0]["duration"], 30) + self.assertEqual(plan_video_segments(60)[1]["duration"], 30) + def test_model_label_maps_to_volcano_name(self): self.assertEqual(video_model_name({"model": "Seedance 2.0 Fast"}), "doubao-seedance-2-0-fast-260128") self.assertEqual(video_model_name({"model": "没见过的模型"}), DEFAULT_VIDEO_MODEL) diff --git a/core/backend/apps/ai/views.py b/core/backend/apps/ai/views.py index c433594..0cfb3fc 100644 --- a/core/backend/apps/ai/views.py +++ b/core/backend/apps/ai/views.py @@ -30,6 +30,7 @@ from .creation import ( cleanup_stale_agent_planning, finish_agent_planning, request_agent_cancel, + start_segmented_video_merge, sync_generating_messages, team_agent_busy, ) @@ -139,6 +140,11 @@ _WANTS_CARD_RE = re.compile( re.IGNORECASE, ) +_WANTS_PRODUCT_PICKER_RE = re.compile( + r"(?:商品|产品).{0,8}(?:列表|库)|(?:列表|库).{0,8}(?:商品|产品)", + re.IGNORECASE, +) + # 步骤确认卡下，用户往往不会去点「按这个继续」，而是自然地回一句 # 「这个还行」「就这样」「没问题」。这些是确认，不应被误判成修改意见。 _STEP_TEXT_CONFIRM_RE = re.compile( @@ -169,6 +175,50 @@ def _pending_chat_question(conversation: CreationConversation) -> CreationMessag return None +def _open_product_picker( + conversation: CreationConversation, + user_text: str, + pending: CreationMessage | None = None, +): + """用户要求商品列表时，直接返回可点击的商品卡。""" + if pending is not None and not bool((pending.payload or {}).get("submitted")): + payload = dict(pending.payload or {}) + payload.update({"submitted": True, "superseded_by": "product_picker"}) + pending.payload = payload + pending.save(update_fields=["payload", "updated_at"]) + user_message = append_message(conversation, role="user", text=user_text) + field = { + "key": "product", + "label": _ASSET_CARD_LABELS.get("product", "选择要推广的商品"), + "type": "asset", + "required": True, + "asset_types": ["product"], + } + picker = append_message( + conversation, + role="assistant", + kind=CreationMessage.Kind.ELICIT, + text=field["label"], + payload={ + "interaction": "asset_picker", + "phase": "pick", + "fields": [field], + "submitted": False, + "answers": {}, + }, + ) + conversation.agent_status = CreationConversation.AgentStatus.AWAITING_USER + conversation.save(update_fields=["agent_status", "updated_at"]) + return JsonResponse({ + "conversation_id": str(conversation.id), + "agent_status": conversation.agent_status, + "messages": [ + CreationMessageSerializer(user_message).data, + CreationMessageSerializer(picker).data, + ], + }, status=200) + + def _plot_twist_depth_continuation(depth: dict) -> str: """让故事深度卡的回答直接进入对应的创作分支。""" if depth.get("value") == "smart": @@ -1694,6 +1744,28 @@ class CreationConversationViewSet(TeamScopedViewSetMixin, ModelViewSet): raise ValidationError({"detail": "after_seq 必须是整数"}) from exc return Response(CreationMessageSerializer(queryset, many=True).data) + @action(detail=True, methods=["post"], url_path="merge-video-segments") + def merge_video_segments(self, request, pk=None): + """用户确认后才合并多段成片；未点击前不下载视频、更不调用 ffmpeg。""" + conversation = self.get_object() + message_id = str(request.data.get("message_id") or "").strip() + message = conversation.messages.filter(id=message_id).first() if message_id else None + if message is None: + return JsonResponse({"detail": "找不到要合并的视频片段"}, status=404) + try: + merge_task, generating = start_segmented_video_merge( + conversation=conversation, message=message, user=request.user + ) + except ValueError as exc: + return JsonResponse({"detail": str(exc)}, status=400) + try: + from .tasks import merge_omni_video_segments_task + + merge_omni_video_segments_task.apply_async(args=[str(merge_task.id)]) + except Exception: # noqa: BLE001 - 没有 worker 时前端轮询仍可看到任务，避免重复执行 ffmpeg + logger.warning("omni video merge enqueue failed for %s", merge_task.id, exc_info=True) + return JsonResponse({"message": CreationMessageSerializer(generating).data}, status=202) + @action(detail=True, methods=["post"], url_path="cancel") def cancel(self, request, pk=None): """终止进行中的整理方案(agent planning)。 @@ -1765,6 +1837,10 @@ class CreationConversationViewSet(TeamScopedViewSetMixin, ModelViewSet): # 打招呼不应被误当成上一条素材追问的答案；让 agent 立刻简短回应即可。 elif kind == "text" and text and not is_greeting(text): pending = _pending_chat_question(conversation) + # 无论旧会话遗留了什么追问，只要用户要商品列表，就直接出可点击卡片。 + # 不能再交给模型 search_library 后用文字念出商品名。 + if _WANTS_PRODUCT_PICKER_RE.search(text): + return _open_product_picker(conversation, text, pending) if pending is not None: payload = dict(pending.payload or {}) if payload.get("interaction") == "step_confirm": @@ -2049,6 +2125,13 @@ class CreationConversationViewSet(TeamScopedViewSetMixin, ModelViewSet): record_user_message = False # force_creative / continuation 已设;跳过后面普通 elicit 逻辑 else: + if payload.get("interaction") == "selling_point_gate": + mode = str(answers.get("selling_point_mode") or "").strip().lower() + selling_point = str(answers.get("selling_point") or "").strip() + if mode not in {"manual", "auto"}: + return JsonResponse({"detail": "请选择自己填写卖点或系统推荐"}, status=400) + if mode == "manual" and not selling_point: + return JsonResponse({"detail": "请先填写一个真实卖点，或选择系统推荐"}, status=400) payload["answers"] = answers payload["submitted"] = True card.payload = payload @@ -2078,6 +2161,25 @@ class CreationConversationViewSet(TeamScopedViewSetMixin, ModelViewSet): continuation_instruction = _plot_twist_direction_continuation( conversation, payload, choice ) + elif payload.get("interaction") == "selling_point_gate": + mode = str(answers.get("selling_point_mode") or "").strip().lower() + selling_point = str(answers.get("selling_point") or "").strip() + memory = dict(conversation.memory or {}) + memory["selling_point_ready"] = True + memory["selling_point_mode"] = mode + memory["selling_point"] = selling_point if mode == "manual" else "" + conversation.memory = memory + conversation.save(update_fields=["memory", "updated_at"]) + text = "" + record_user_message = False + force_creative_turn = True + continuation_instruction = ( + f"商家已确认核心卖点：【{selling_point}】。现在只调用 write_strategy 写创作策略，" + "再等待商家确认；策略、后续方案和视频都必须围绕这个真实卖点，不要替换或夸大。" + if mode == "manual" + else "商家选择系统推荐卖点。现在只调用 write_strategy 写创作策略；" + "从商品资料和现有素材中挑一个最容易被画面证明的真实核心卖点，不要虚构功效、价格或规格。" + ) elif payload.get("phase") == "gate": choice = str(answers.get("_asset_gate") or "").strip() if choice == "send": diff --git a/core/frontend/src/api.ts b/core/frontend/src/api.ts index 3bab9df..de85532 100644 --- a/core/frontend/src/api.ts +++ b/core/frontend/src/api.ts @@ -601,6 +601,13 @@ export const api = { } ); }, + /** 分段视频全部完成后，用户明确确认才触发服务端拼接。 */ + mergeCreationVideoSegments(id: string, messageId: string) { + return request<{ message: CreationMessage }>(`/api/ai/creations/${id}/merge-video-segments/`, { + method: "POST", + body: JSON.stringify({ message_id: messageId }), + }); + }, /** * 发一条消息 → 异步整理方案。成功 **202** `{conversation_id, agent_status}`， * 闸门短路径可能 **200** 带 `messages`；冲突 **409**（团队已有对话在整理方案）。 diff --git a/core/frontend/src/omni-session-page.css b/core/frontend/src/omni-session-page.css index d983d3c..2ec9aa2 100644 --- a/core/frontend/src/omni-session-page.css +++ b/core/frontend/src/omni-session-page.css @@ -281,34 +281,81 @@ .omni-reply-custom-input { display: flex; - flex: 1 1 220px; + flex: 1 1 260px; gap: 6px; + width: min(300px, 100%); max-width: 300px; } .omni-reply-custom-input .input { min-width: 0; - height: 30px; - padding: 0 10px; + height: 36px; + padding: 0 12px; font-size: 12px; } .omni-reply-custom-input button { - flex: 0 0 30px; - width: 30px; + flex: 0 0 36px; + width: 36px; padding: 0; display: inline-flex; align-items: center; justify-content: center; } +/* 小云雀式跟随追问：答案留在问题附近，尺寸不抢占对话空间。 */ +.omni-chat-question-input { + display: flex; + align-items: center; + gap: 6px; + width: min(300px, 100%); +} + +.omni-chat-question-input .input { + min-width: 0; + height: 36px; + padding: 0 12px; + font-size: 12px; + border-color: var(--border-faint); + background: var(--surface); +} + +.omni-chat-question-input button { + flex: 0 0 36px; + width: 36px; + height: 36px; + padding: 0; + display: inline-flex; + align-items: center; + justify-content: center; + border: 1px solid var(--border-faint); + border-radius: var(--r-md); + color: var(--text-secondary); + background: var(--surface); + cursor: pointer; +} + +.omni-chat-question-input button:hover:not(:disabled) { + border-color: var(--heat-40); + color: var(--heat); + background: var(--heat-12); +} + +.omni-chat-question-input button:disabled { + color: var(--black-alpha-48); + background: var(--black-alpha-4); + cursor: not-allowed; +} + .omni-chat-row.is-user .omni-chat-bubble { align-items: flex-end; - border-color: var(--black); + border-color: var(--black-alpha-12); border-bottom-right-radius: 4px; - color: #fff; - background: var(--black); + color: var(--accent-black); + background: color-mix(in srgb, var(--surface) 72%, transparent); + -webkit-backdrop-filter: blur(14px) saturate(115%); + backdrop-filter: blur(14px) saturate(115%); } .omni-result-card, @@ -1359,6 +1406,64 @@ padding: 14px 18px 16px; } +/* 卖点确认在策略和脚本之前出现：输入真实卖点，或把筛选权交给系统。 */ +.omni-selling-point-card { + box-sizing: border-box; + width: min(760px, calc(100% - 44px)); + margin: 0 0 24px 44px; + overflow: hidden; + border-radius: var(--r-md); + background: var(--surface); + box-shadow: inset 0 0 0 1px var(--border-faint); +} + +.omni-selling-point-card > header { + display: flex; + align-items: center; + min-height: 58px; + padding: 12px 16px; + box-shadow: inset 0 -1px 0 var(--border-faint); +} + +.omni-selling-point-card header > div { + display: flex; + flex-direction: column; + gap: 3px; +} + +.omni-selling-point-card header strong { + color: var(--accent-black); + font-size: 14px; + font-weight: 600; +} + +.omni-selling-point-card header span, +.omni-selling-point-selected { + color: var(--black-alpha-56); + font-size: 12px; +} + +.omni-selling-point-card form { + display: flex; + flex-direction: column; + gap: 10px; + padding: 14px 16px 16px; +} + +.omni-selling-point-card .input { + width: 100%; +} + +.omni-selling-point-actions { + display: flex; + justify-content: flex-end; + gap: 8px; +} + +.omni-selling-point-selected { + padding: 14px 16px 16px; +} + /* 剧情反转预设的方向选择：三个完整方向直接平铺，避免只留一句「我准备了三个方向」。 */ .omni-direction-card { box-sizing: border-box; @@ -1497,6 +1602,20 @@ cursor: not-allowed; } +.omni-direction-confirm { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; + padding: 0 16px 16px; +} + +.omni-direction-confirm > span { + margin-right: auto; + color: var(--black-alpha-56); + font-size: 12px; +} + .omni-elicit-field > label { display: block; margin-bottom: 9px; @@ -2104,8 +2223,8 @@ } .omni-chat-row.is-user .omni-chat-stack.is-pending .omni-chat-bubble { - background: #3a424e; - border-color: #3a424e; + background: color-mix(in srgb, var(--surface) 84%, transparent); + border-color: var(--black-alpha-16); } .omni-send-spinner { diff --git a/core/frontend/src/routes/omni-session.tsx b/core/frontend/src/routes/omni-session.tsx index adc7668..0e2c335 100644 --- a/core/frontend/src/routes/omni-session.tsx +++ b/core/frontend/src/routes/omni-session.tsx @@ -41,7 +41,7 @@ type PromptBlock = | { type: "shot"; heading: string; fields: Array<{ label: string; value: string }>; text: string } | { type: "text"; heading: string; text: string }; -type ReplyOption = { label: string; text: string }; +type ReplyOption = { label: string; text: string; action?: "upload" }; function renderInlineMarkdown(text: string): ReactNode[] { return text.split(/(\*\*[^*]+\*\*|`[^`]+`|\*[^*]+\*)/g).filter(Boolean).map((part, index) => { @@ -143,6 +143,10 @@ function stripNumericReplyInstruction(text: string): string { function numberedReplyOptions(text: string): ReplyOption[] { const matches = [...(text || "").matchAll(/(?:^|\n)\s*([1-3])[.、．]\s*(?:\*\*)?([^\n*]{1,80})/g)]; if (matches.length < 2) return []; + // 分镜、步骤、卖点清单也经常使用 1./2./3.；只有助手明确要求用户在这些项中选择时， + // 才把编号转成操作按钮，避免把分镜 1–6 错当成三个可选方案。 + const hasChoiceInstruction = /(?:请|你)?(?:选择|选)(?:其中|一个|一项|下面|以下|第[一二三123]|\s*1\s*[/、,]\s*2)|(?:回复|输入)(?:数字|编号)\s*1\s*[/、,]\s*2|(?:下面|以下)(?:有|为)?(?:三|3)个?(?:方向|方案|选项)/.test(text || ""); + if (!hasChoiceInstruction) return []; return matches.slice(0, 3).map((match) => ({ label: match[1], text: `选择第${match[1]}个方向`, @@ -150,10 +154,26 @@ function numberedReplyOptions(text: string): ReplyOption[] { } function contextualReplyOptions(text: string): ReplyOption[] { + if (/(?:两位|两个|多位|多个).{0,24}(?:人物|角色|模特|男士|女生).{0,80}(?:哪位|哪个|选择|用哪|出镜)/i.test(text)) { + return [ + { label: "1", text: "选择第 1 位人物出镜" }, + { label: "2", text: "选择第 2 位人物出镜" }, + { label: "你来决定", text: "请根据当前脚本帮我选择更合适的人物" }, + ]; + } + // 人物参考和商品参考是两件事。已锁定人物后，助手若提到实物图，不能又给出「上传人物图」—— + // 这会让用户误以为刚上传的参考没有生效。 + if (/实物图|商品图|产品图|商品外观|产品外观|产品素材|耳机(?:图|外观)?|IP设计/i.test(text)) { + return [ + { label: "上传商品实物图", text: "我上传商品实物图", action: "upload" }, + { label: "按当前描述继续", text: "先按当前商品描述继续，不再补图" }, + { label: "换一个商品", text: "我想换一个商品来创作" }, + ]; + } if (/颜色|色号|色彩|SKU|款式|几种/i.test(text)) { return [ { label: "补充颜色和顺序", text: "我来补充每个颜色和展示顺序" }, - { label: "上传各款实物图", text: "我补充各颜色/款式的实物图" }, + { label: "上传各款实物图", text: "我补充各颜色/款式的实物图", action: "upload" }, { label: "先按当前主款做", text: "先按当前主款做，其他颜色后面再补" }, ]; } @@ -166,14 +186,14 @@ function contextualReplyOptions(text: string): ReplyOption[] { } if (/人物|角色|模特|出镜/i.test(text)) { return [ - { label: "上传人物图", text: "我上传人物参考图" }, + { label: "上传人物图", text: "我上传人物参考图", action: "upload" }, { label: "由你设定角色", text: "你先帮我设定一个合适的角色" }, { label: "不需要人物", text: "这条先不需要人物出镜" }, ]; } if (/场景|地点|背景|在哪/i.test(text)) { return [ - { label: "上传场景图", text: "我上传场景参考图" }, + { label: "上传场景图", text: "我上传场景参考图", action: "upload" }, { label: "你来推荐场景", text: "你按商品和预设推荐场景" }, { label: "用干净日常场景", text: "先用干净自然的日常场景" }, ]; @@ -188,30 +208,58 @@ function contextualReplyOptions(text: string): ReplyOption[] { return []; } +function uploadReplyText(option: ReplyOption): string { + if (/人物|角色|模特/.test(option.label)) { + return "我上传人物参考图，请使用刚上传的这张图作为本次视频唯一的出镜人物参考，不需要再让我选择其他人物。"; + } + if (/商品|实物|款/.test(option.label)) { + return "我上传商品实物图，请使用刚上传的图片作为本次视频的商品外观参考。"; + } + if (/场景/.test(option.label)) { + return "我上传场景参考图，请使用刚上传的图片作为本次视频的主要场景参考。"; + } + return option.text; +} + function replyOptionsForMessage(text: string, payload: Record): ReplyOption[] { const numbered = numberedReplyOptions(text); if (numbered.length) return numbered; + // 文本正在明确索要的内容永远优先于旧的 reply_options。 + // Agent 的历史 payload 可能残留「上传人物图」，但当前气泡已改成要商品实物图； + // 此时沿用旧按钮会让用户上传错素材。 + const contextual = contextualReplyOptions(text); + if (contextual.length) return contextual; const stored = payload.reply_options; if (Array.isArray(stored)) { const options = stored .filter((item): item is Record => Boolean(item) && typeof item === "object") - .map((item) => ({ label: String(item.label || "").trim(), text: String(item.text || "").trim() })) + .map((item) => { + const label = String(item.label || "").trim(); + return { + label, + text: String(item.text || "").trim(), + // 任何引导里的“上传图片/实物图/素材图”都走本地文件选择器，避免同一套对话里有的能传、有的只是文字回复。 + action: /上传.*(?:图|图片|素材)|(?:图|图片|素材).*上传/.test(label) ? "upload" as const : undefined, + }; + }) .filter((item) => item.label && item.text) .slice(0, 3); const legacyGeneric = ["继续完善方案", "换个场景", "改卖点", "按这个方向出图", "改人物状态"]; if (options.length && !options.every((item) => legacyGeneric.includes(item.label))) return options; } - return contextualReplyOptions(text); + return []; } function ReplyActions({ options, disabled, onSubmit, + onUpload, }: { options: ReplyOption[]; disabled?: boolean; onSubmit: (text: string) => void; + onUpload: (option: ReplyOption) => void; }) { const [customIdea, setCustomIdea] = useState(""); const submitCustomIdea = (event: FormEvent) => { @@ -230,7 +278,10 @@ function ReplyActions({ className={index === 0 && options.length < 3 ? "primary" : ""} key={option.text} disabled={disabled} - onClick={() => onSubmit(option.text)} + onClick={() => { + if (option.action === "upload") onUpload(option); + else onSubmit(option.text); + }} > {option.label} @@ -522,8 +573,50 @@ type SessionAsset = { src?: string; promptTitle?: string; promptBody?: string; + documentDescription?: string; }; +function formatDocumentTime(value: unknown): string { + const number = Number(value); + return Number.isFinite(number) ? (Number.isInteger(number) ? String(number) : number.toFixed(1)) : "—"; +} + +function strategyDocumentBody(payload: Record): string { + const sections: Array<[string, string]> = [ + ["这条视频给谁看", strategyField(payload, "target", "audience", "who", "给谁看", "目标人群")], + ["用户为什么相信", strategyField(payload, "trust", "credibility", "为什么相信", "信任")], + ["希望用户相信什么", strategyField(payload, "belief", "希望相信", "想让他信什么", "认知")], + ["创作方向", strategyField(payload, "direction", "创作方向", "方向", "style")], + ]; + return ["# 创作策略摘要", ...sections.map(([label, value]) => `## ${label}\n${value || "—"}`)].join("\n\n"); +} + +function planDocumentBody(payload: Record): string { + const usp = strategyField(payload, "usp", "主打卖点", "卖点"); + const points = coercePlanPoints(payload.points); + const timeline = (payload.timeline as PlanTimelineItem[] | undefined) || []; + const voice = coerceVoiceChars(payload.voice_chars); + const pointText = points.length ? points.map((point) => `- ${point}`).join("\n") : "—"; + const timelineText = timeline.length + ? timeline.map((item) => `- ${formatDocumentTime(item.start)}–${formatDocumentTime(item.end)} 秒 · ${item.stage}${item.desc ? `：${item.desc}` : ""}`).join("\n") + : "—"; + const voiceText = voice.length === 2 ? `${voice[0]}–${voice[1]} 字，预计覆盖约 90% 时长` : "—"; + return [ + "# 视频最终方案", + `## 主打卖点 USP\n${usp || "—"}`, + `## 核心支撑\n${pointText}`, + `## 时间轴\n${timelineText}`, + `## 口播目标\n${voiceText}`, + `## 出片准备\n已整合生成参数与 ${String(payload.ref_count ?? 0)} 组参考素材。`, + ].join("\n\n"); +} + +function documentDescription(title: string): string { + if (title.includes("创作策略")) return "创作策略 · 点击查看"; + if (title.includes("视频最终方案")) return "视频方案 · 点击查看"; + return "出片指令 · 点击查看"; +} + /** 从本会话已加载 messages(+pinned_refs / 会话上传)聚合聊天资产，不另打 BE。 */ function collectSessionAssets( messages: CreationMessage[], @@ -579,8 +672,43 @@ function collectSessionAssets( } - // 视频确认卡保存的是最终用于出片的 Prompt。它不是聊天正文，而是本会话 - // 的一份可复查资源；右侧资源栏始终只保留最新的一版，避免多次改稿堆出一串副本。 + // 策略、方案、Prompt 都是会话的正式产物。资源栏只保留各自最新的一版， + // 避免多次修改后堆出一串相同文档。 + const latestDocument = (kind: CreationMessage["kind"]) => { + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message.kind !== kind) continue; + return { message, payload: message.payload || {}, index }; + } + return null; + }; + + const strategy = latestDocument("strategy"); + if (strategy) { + const title = "创作策略摘要.md"; + push({ + key: `strategy:${strategy.message.id || strategy.index}`, + kind: "document", + title, + promptTitle: title, + promptBody: strategyDocumentBody(strategy.payload), + documentDescription: documentDescription(title), + }); + } + + const plan = latestDocument("plan"); + if (plan) { + const title = "视频最终方案.md"; + push({ + key: `plan:${plan.message.id || plan.index}`, + kind: "document", + title, + promptTitle: title, + promptBody: planDocumentBody(plan.payload), + documentDescription: documentDescription(title), + }); + } + for (let index = messages.length - 1; index >= 0; index -= 1) { const message = messages[index]; const payload = message.payload || {}; @@ -594,6 +722,7 @@ function collectSessionAssets( title: String(payload.title || "视频生成Prompt.md"), promptTitle: String(payload.title || "视频生成Prompt.md"), promptBody: body, + documentDescription: documentDescription(String(payload.title || "视频生成Prompt.md")), }); break; } @@ -913,12 +1042,15 @@ function ElicitCard({ message, disabled, onSubmit, + onChatAnswer, }: { message: CreationMessage; disabled: boolean; /** answers 给模型读(人话),refs 给后端取卖点和参考图。**选素材必须两样都回**, 只回名字的话同名素材会配错货。 */ onSubmit: (answers: Record, refs: CreationRef[]) => void; + /** 普通追问直接在气泡下回答，仍走 text 链路以保留用户消息与上下文。 */ + onChatAnswer: (text: string) => void; }) { const fields = ((message.payload.fields as CreationField[] | undefined) || []).filter(Boolean); const submitted = Boolean(message.payload.submitted); @@ -929,6 +1061,7 @@ function ElicitCard({ const [assetOptions, setAssetOptions] = useState>({}); const [assetPicked, setAssetPicked] = useState>({}); const [customDirection, setCustomDirection] = useState(""); + const [chatAnswer, setChatAnswer] = useState(""); useEffect(() => { if (submitted || interaction === "chat" || interaction === "step_confirm") return; @@ -972,6 +1105,58 @@ function ElicitCard({ ); } + if (interaction === "selling_point_gate") { + const sellingPoint = typeof answers.selling_point === "string" ? answers.selling_point : ""; + const savedMode = String(saved.selling_point_mode || ""); + const submitManual = (event: FormEvent) => { + event.preventDefault(); + const value = sellingPoint.trim(); + if (!value || disabled) return; + onSubmit({ selling_point_mode: "manual", selling_point: value }, []); + }; + return ( +
+
+
+
+ 先确定核心卖点 + 它会贯穿后面的策略、脚本和视频画面 +
+
+ {submitted ? ( +

+ {savedMode === "auto" ? "已交给系统从商品和素材中推荐卖点" : `已使用：${String(saved.selling_point || "")}`} +

+ ) : ( +
+ setAnswers((prev) => ({ ...prev, selling_point: event.target.value }))} + /> +
+ + +
+
+ )} +
+
+ ); + } + if (interaction === "plot_twist_directions") { const directions = Array.isArray(message.payload.directions) ? message.payload.directions @@ -986,12 +1171,16 @@ function ElicitCard({ })) .filter((item) => item.id) : []; - const selected = String(saved.story_direction || ""); + // 选卡片只是本地暂存；明确点「确认选择」后才真正提交并开始整理下一步。 + const selected = String(answers.story_direction || saved.story_direction || ""); + const selectedDirection = directions.find( + (direction) => selected === direction.id || selected === direction.title, + ); const submitCustomDirection = (event: FormEvent) => { event.preventDefault(); const idea = customDirection.trim(); if (!idea || disabled) return; - onSubmit({ story_direction: idea }, []); + setAnswers((prev) => ({ ...prev, story_direction: idea })); setCustomDirection(""); }; return ( @@ -1000,7 +1189,7 @@ function ElicitCard({
三个剧情反转方向 - 直接选一个，我会继续展开成完整方案 + 选择方向后确认，再继续展开完整方案
@@ -1012,7 +1201,7 @@ function ElicitCard({ key={direction.id} className={isSelected ? "is-selected" : ""} disabled={disabled || submitted} - onClick={() => onSubmit({ story_direction: direction.id }, [])} + onClick={() => setAnswers((prev) => ({ ...prev, story_direction: direction.id }))} > 方向 {index + 1} {direction.title} @@ -1025,17 +1214,40 @@ function ElicitCard({ })}
{!submitted ? ( -
- setCustomDirection(event.target.value)} - placeholder="或者写下你自己的剧情想法…" - aria-label="输入自己的剧情想法" - /> - -
+ <> +
+ setCustomDirection(event.target.value)} + placeholder="或者写下你自己的剧情想法…" + aria-label="输入自己的剧情想法" + /> + +
+
+ {selectedDirection ? `已选：${selectedDirection.title}` : selected ? "已选自定义剧情方向" : "请先选择一个方向"} + {selected ? ( + + ) : null} + +
+ ) : null} @@ -1102,6 +1314,13 @@ function ElicitCard({ if (interaction === "chat") { const question = message.text || fields[0]?.label || "这项你想怎么定？"; + const submitChatAnswer = (event: FormEvent) => { + event.preventDefault(); + const text = chatAnswer.trim(); + if (!text || disabled) return; + onChatAnswer(text); + setChatAnswer(""); + }; return (
@@ -1113,11 +1332,19 @@ function ElicitCard({
{!submitted ? (
-
- -
+
) : null} @@ -1324,8 +1551,12 @@ function ConfirmCard({ function ResultCard({ payload, + onMerge, + merging, }: { payload: Record; + onMerge?: () => void; + merging?: boolean; }) { const assets = (payload.assets as Array> | undefined) || []; const first = assets[0] || {}; @@ -1340,7 +1571,7 @@ function ResultCard({ const cover = asset.cover || asset.url || ""; const url = asset.url || cover; const video = asset.type === "video"; - const label = assets.length > 1 ? `生成结果 ${index + 1}` : "生成结果"; + const label = String(asset.label || (assets.length > 1 ? `第 ${index + 1} 段` : "生成结果")); return (
+ ) : null} }) { - const isVideo = payload.kind === "video"; + const isSegmentedVideo = payload.kind === "video_segments"; + const isMerge = payload.kind === "video_merge"; + const isVideo = payload.kind === "video" || isSegmentedVideo || isMerge; + const segmentCount = Array.isArray(payload.segments) ? payload.segments.length : 0; return (
@@ -1687,8 +1934,8 @@ function ProcessCard({ payload }: { payload: Record }) {
- {isVideo ? "正在生成视频" : "正在出图"} - {isVideo ? "方案编译中，请稍候" : "画面生成中，请稍候"} + {isMerge ? "正在合并成片" : isSegmentedVideo ? `正在生成 ${segmentCount || 2} 段视频` : isVideo ? "正在生成视频" : "正在出图"} + {isMerge ? "正在拼接已确认的片段" : isSegmentedVideo ? "片段完成后会先展示给你确认" : isVideo ? "方案编译中，请稍候" : "画面生成中，请稍候"}
@@ -1755,6 +2002,8 @@ export function OmniSessionPage({ const [sessionUploads, setSessionUploads] = useState(firstUploads || []); const [uploading, setUploading] = useState(false); const fileInputRef = useRef(null); + // 对话引导里的「上传人物图」要直达系统文件选择器，而不是绕回通用素材菜单。 + const quickUploadReplyRef = useRef(null); const composerRef = useRef(null); const [streaming, setStreaming] = useState(false); const [liveText, setLiveText] = useState(""); @@ -1768,6 +2017,7 @@ export function OmniSessionPage({ const mentionWrapRef = useRef(null); const [uploadMenuOpen, setUploadMenuOpen] = useState(false); const [confirming, setConfirming] = useState(false); + const [mergingMessageIds, setMergingMessageIds] = useState([]); const [stopping, setStopping] = useState(false); const [promptView, setPromptView] = useState<{ title: string; body: string } | null>(null); const [assetsOpen, setAssetsOpen] = useState(false); @@ -2354,6 +2604,24 @@ export function OmniSessionPage({ } }; + const handleMergeSegments = async (message: CreationMessage) => { + if (mergingMessageIds.includes(message.id)) return; + setMergingMessageIds((prev) => [...prev, message.id]); + try { + const result = await api.mergeCreationVideoSegments(conversationId, message.id); + setMessages((prev) => [ + ...prev.map((item) => item.id === message.id + ? { ...item, payload: { ...item.payload, merge_state: "queued" } } + : item), + result.message, + ]); + notify("info", "正在合并成片"); + } catch (error) { + notify("error", (error as Error).message || "合并失败，请重试"); + setMergingMessageIds((prev) => prev.filter((id) => id !== message.id)); + } + }; + const handleSend = () => { // 顺序要紧:先判 streaming 再清输入框。反过来的话,流式期间敲一次回车 // 会把已经打好的内容清空,消息却没发出去。 @@ -2462,6 +2730,7 @@ export function OmniSessionPage({ ); void send({ kind: "elicit_answer", reply_to: message.id, answers, refs }); }} + onChatAnswer={(text) => void send({ kind: "text", text })} /> ); case "strategy": @@ -2509,6 +2778,8 @@ export function OmniSessionPage({ void handleMergeSegments(message) : undefined} /> ); default: { @@ -2545,6 +2816,11 @@ export function OmniSessionPage({ options={replyOptions} disabled={streaming} onSubmit={(text) => void send({ kind: "text", text })} + onUpload={(option) => { + if (streaming || uploading) return; + quickUploadReplyRef.current = option; + fileInputRef.current?.click(); + }} /> ) : null} @@ -2696,6 +2972,7 @@ export function OmniSessionPage({ disabled={uploading || streaming} onClick={() => { if (uploading || streaming) return; + quickUploadReplyRef.current = null; setUploadMenuOpen((open) => !open); setMentionMenuOpen(false); }} @@ -2720,6 +2997,7 @@ export function OmniSessionPage({ );