修改全能创作发现问题

2026-09-03 19:00:59 +08:00
parent e7122b78fb
commit 3e904479d9
23 changed files with 1325 additions and 433 deletions
@@ -29,7 +29,7 @@ from .creation import append_message, pin_refs
 from .creation_presets import preset_guidance
 from .mentions import TYPE_LABELS, infer_field_types, resolve_refs, search_mentions
 from .models import CreationConversation, CreationMessage, ModelConfig
-from .services import build_provider, get_default_model
+from .services import build_provider, get_default_model, resolve_text_model

 logger = logging.getLogger(__name__)

@@ -209,6 +209,52 @@ def apply_session_params(conversation, fields, answers: dict) -> bool:
    return changed


+def snapshot_session_params(conversation) -> dict:
+    params = conversation.params or {}
+    return {
+        "model": str(params.get("model") or ""),
+        "resolution": str(params.get("resolution") or ""),
+        "ratio": str(params.get("ratio") or ""),
+        "duration": str(params.get("duration") or ""),
+        "count": str(params.get("count") or params.get("duration") or ""),
+    }
+
+
+def confirm_param_options(is_video: bool) -> dict:
+    return {
+        "model": VIDEO_MODELS if is_video else IMAGE_MODELS,
+        "resolution": RESOLUTIONS if is_video else [],
+        "ratio": RATIOS,
+        "duration": VIDEO_DURATIONS if is_video else [],
+        "count": IMAGE_COUNTS if not is_video else [],
+    }
+
+
+def apply_confirm_params(conversation, incoming: dict | None) -> tuple[dict, bool]:
+    """确认卡上改的参数写回会话。返回 (最新 params, 视频时长是否变了)。"""
+    current = dict(conversation.params or {})
+    old_duration = str(current.get("duration") or "")
+    changed = False
+    for key, raw in (incoming or {}).items():
+        if key not in {"model", "ratio", "resolution", "duration", "count"}:
+            continue
+        value = str(raw or "").strip()
+        if not value or current.get(key) == value:
+            continue
+        current[key] = value
+        changed = True
+    duration_changed = (
+        conversation.mode == CreationConversation.Mode.VIDEO
+        and str(current.get("duration") or "") != old_duration
+        and bool(str(current.get("duration") or ""))
+        and bool(old_duration)
+    )
+    if changed:
+        conversation.params = current
+        conversation.save(update_fields=["params", "updated_at"])
+    return snapshot_session_params(conversation), duration_changed
+
+
 # ---------------------------------------------------------------- 工具 schema

 def tool_schemas(context: AgentContext) -> list[dict]:
@@ -330,24 +376,6 @@ def tool_schemas(context: AgentContext) -> list[dict]:
                                "required": ["start", "end", "stage"],
                            },
                        },
-                        "matrix": {
-                            "type": "object",
-                            "description": "卖点覆盖矩阵:哪个卖点落在第几个镜头",
-                            "properties": {
-                                "shots": {"type": "integer"},
-                                "rows": {
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "point": {"type": "string"},
-                                            "hits": {"type": "array", "items": {"type": "integer"}},
-                                        },
-                                        "required": ["point", "hits"],
-                                    },
-                                },
-                            },
-                        },
                        "voice_chars": {
                            "type": "array", "items": {"type": "integer"},
                            "description": "口播字数区间 [下限, 上限]",
@@ -587,9 +615,134 @@ def submit_confirmed_video(*, conversation: CreationConversation, user, confirm_
    return message, ""


+def submit_confirmed_image(*, conversation: CreationConversation, user, confirm_message: CreationMessage):
+    """用户点了确认 → 按确认卡里存的画面描述出图。同样不跑一轮模型。"""
+    payload = confirm_message.payload or {}
+    prompt = str(payload.get("prompt") or payload.get("image_prompt") or "").strip()
+    if not prompt:
+        return None, "这条方案没有存下出图指令，请让我重新写一次。"
+
+    context = AgentContext(conversation=conversation, user=user, model_config=None)
+    try:
+        _result, tasks = _run_generate_image(context, {"prompt": prompt})
+    except AgentError as exc:
+        return None, str(exc)
+    except ValueError as exc:
+        return None, str(exc)
+
+    message = None
+    for task in tasks:
+        message = append_message(
+            conversation, role="assistant",
+            kind=CreationMessage.Kind.GENERATING,
+            payload={"task_id": str(task.id), "kind": "image", "prompt": prompt},
+            task=task,
+        )
+    if message is None:
+        return None, "出图没有提交成功，请再试一次。"
+    _remember_artifact(conversation, prompt, "image")
+    return message, ""
+
+
 # ---------------------------------------------------------------- 提示词


+
+def get_creation_chat_model(requested: ModelConfig | None = None) -> ModelConfig | None:
+    """全能创作对话模型:走平台文本解析,DeepSeek 一律换成 Seed 2.1 Pro。"""
+    return resolve_text_model(requested)
+
+
+def _creation_model_sees_images(model_config: ModelConfig | None) -> bool:
+    """对话模型能不能收图。参考图只在能看图时才塞进 chat messages,避免纯文本模型整轮失败。"""
+    if model_config is None:
+        return False
+    if getattr(model_config, "capability", "") == ModelConfig.Capability.VISION:
+        return True
+    name = str(getattr(model_config, "name", "") or "").lower()
+    # 豆包 Seed 2.x / 1.6 文本档都支持图文;vl / vision 后缀同理。
+    if name.startswith("doubao-seed-") or "vision" in name or name.endswith("-vl") or "-vl-" in name:
+        return True
+    metadata = model_config.metadata if isinstance(getattr(model_config, "metadata", None), dict) else {}
+    capabilities = metadata.get("capabilities") if isinstance(metadata.get("capabilities"), dict) else {}
+    features = {str(item) for item in capabilities.get("features") or []}
+    return bool({"vision", "image_input", "multimodal"} & features)
+
+
+def _prefer_vision_text_model(current: ModelConfig | None, team, refs: list | None) -> ModelConfig | None:
+    """有参考图时,尽量换成能看图的文本模型(豆包 Seed 等),否则聊天侧完全看不见男女。"""
+    if current is not None and _creation_model_sees_images(current):
+        return current
+    if not _ref_image_urls(team, refs):
+        return current
+    qs = (
+        ModelConfig.objects.select_related("provider")
+        .filter(
+            capability=ModelConfig.Capability.TEXT,
+            status=ModelConfig.Status.ACTIVE,
+            provider__status="active",
+        )
+        .order_by("created_at")
+    )
+    for candidate in qs:
+        if _creation_model_sees_images(candidate):
+            return candidate
+    return get_default_model(ModelConfig.Capability.VISION) or current
+
+
+def _ref_image_urls(team, refs: list | None) -> list[str]:
+    resolved = resolve_refs(team, refs or [])
+    urls: list[str] = []
+    seen: set[str] = set()
+    for item in resolved.references:
+        url = str(item.get("url") or "").strip()
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        urls.append(url)
+    return urls[:6]
+
+
+def _attach_ref_images(messages: list[dict], image_urls: list[str]) -> list[dict]:
+    """把锁定素材图挂到最近一条 user 消息上(OpenAI image_url 格式)。"""
+    if not image_urls or not messages:
+        return messages
+    note = (
+        f"【参考图·请亲眼看】下面 {len(image_urls)} 张是用户锁定的素材。"
+        "人物的性别、年龄段、发型、服装必须以图为准;看不清再问用户,禁止凭文件名猜测性别。"
+    )
+    out = [dict(message) for message in messages]
+    index = next((i for i in range(len(out) - 1, -1, -1) if out[i].get("role") == "user"), None)
+    if index is None:
+        content = [{"type": "text", "text": note}]
+        content.extend({"type": "image_url", "image_url": {"url": url}} for url in image_urls)
+        out.append({"role": "user", "content": content})
+        return out
+    last = dict(out[index])
+    raw = last.get("content")
+    if isinstance(raw, list):
+        content = list(raw)
+        text_bits = [str(item.get("text") or "") for item in content if isinstance(item, dict) and item.get("type") == "text"]
+        if not any(note[:8] in bit for bit in text_bits):
+            content.append({"type": "text", "text": note})
+        existing = {
+            (item.get("image_url") or {}).get("url")
+            for item in content
+            if isinstance(item, dict) and item.get("type") == "image_url"
+        }
+        content.extend(
+            {"type": "image_url", "image_url": {"url": url}}
+            for url in image_urls
+            if url not in existing
+        )
+    else:
+        content = [{"type": "text", "text": f"{raw or ''}\n\n{note}".strip()}]
+        content.extend({"type": "image_url", "image_url": {"url": url}} for url in image_urls)
+    last["content"] = content
+    out[index] = last
+    return out
+
+
 def build_system_prompt(context: AgentContext) -> str:
    conversation = context.conversation
    params = conversation.params or {}
@@ -635,7 +788,9 @@ def build_system_prompt(context: AgentContext) -> str:
        lines.append("\n【本次会话已锁定的素材事实】")
        lines.append(resolved.facts_text)
        lines.append(
-            "以上素材的参考图会自动附给生成模型锁人锁物,你在 prompt 里不需要重复描述它们的外观。"
+            "以上素材的参考图会附给你看,出片时也会自动锁人锁物。"
+            "人物性别、年龄段、发型、服装、商品颜色外形必须以图为准;"
+            "图上看不清或没附图时,必须问用户,禁止凭文件名猜测男女。"
        )
    memory = conversation.memory or {}
    if memory.get("summary"):
@@ -682,6 +837,11 @@ def build_messages(context: AgentContext) -> list[dict]:
            messages.append({"role": "assistant", "content": f"(我生成了一版,prompt:{prompt[:200]})"})
        elif message.kind == CreationMessage.Kind.ERROR:
            messages.append({"role": "assistant", "content": f"(上一次生成失败:{message.text})"})
+    if _creation_model_sees_images(context.model_config):
+        messages = _attach_ref_images(
+            messages,
+            _ref_image_urls(context.team, context.conversation.pinned_refs or []),
+        )
    return messages


@@ -727,7 +887,7 @@ def stream_creation_agent(
 ) -> Iterator[str]:
    """一条用户消息 → SSE 流。生成器,由 StreamingHttpResponse 逐帧下发。"""
    refs = refs or []
-    model_config = model_config or get_default_model(ModelConfig.Capability.TEXT)
+    model_config = get_creation_chat_model(model_config)
    if model_config is None:
        yield _sse({"type": "error", "detail": "没有可用的文本模型,请先在模型库配置"})
        return
@@ -738,6 +898,8 @@ def stream_creation_agent(
            pin_refs(conversation, refs)
            user_message = append_message(conversation, role="user", text=text, refs=refs)
        yield _sse({"type": "message", "message": _message_payload(user_message)})
+        model_config = _prefer_vision_text_model(model_config, conversation.team, conversation.pinned_refs or [])
+        context.model_config = model_config

        resolved = resolve_refs(context.team, refs)
        if resolved.missing:
@@ -898,7 +1060,6 @@ def _dispatch_tool(context: AgentContext, name: str, args: dict) -> tuple[dict,
            "usp": str(args.get("usp") or ""),
            "points": [str(p) for p in (args.get("points") or [])][:3],
            "timeline": args.get("timeline") or [],
-            "matrix": args.get("matrix") or {},
            "voice_chars": args.get("voice_chars") or [],
            "ref_count": len(context.conversation.pinned_refs or []),
        }
@@ -922,8 +1083,15 @@ def _dispatch_tool(context: AgentContext, name: str, args: dict) -> tuple[dict,
        confirm = append_message(
            context.conversation, role="assistant",
            kind=CreationMessage.Kind.CONFIRM,
-            payload={"label": "开始生成", "estimated_credits": credits,
-                     "video_prompt": video_prompt, "submitted": False},
+            payload={
+                "kind": "video",
+                "label": "开始生成",
+                "estimated_credits": credits,
+                "video_prompt": video_prompt,
+                "submitted": False,
+                "params": snapshot_session_params(context.conversation),
+                "param_options": confirm_param_options(True),
+            },
        )
        events.append({"type": "message", "message": _message_payload(confirm)})
        events.append({"type": "credits", "estimated": credits})
@@ -934,19 +1102,25 @@ def _dispatch_tool(context: AgentContext, name: str, args: dict) -> tuple[dict,
        if context.generations_used >= MAX_BILLED_GENERATIONS:
            # 一条用户消息只计费一次。模型想连出好几版时在这里挡住。
            return {"payload": {"error": "本轮已经生成过一次了,请让用户看过再决定要不要改"}}, True
-        payload, tasks = _run_generate_image(context, args)
-        events = []
-        for task in tasks:
-            message = append_message(
-                context.conversation, role="assistant",
-                kind=CreationMessage.Kind.GENERATING,
-                payload={"task_id": str(task.id), "kind": "image", "prompt": args.get("prompt", "")},
-                task=task,
-            )
-            events.append({"type": "message", "message": _message_payload(message)})
-            events.append({"type": "task", "task_id": str(task.id), "kind": "image"})
-        _remember_artifact(context.conversation, args.get("prompt", ""), "image")
-        return {"payload": payload, "_events": events}, True
+        prompt = str(args.get("prompt") or "").strip()
+        if not prompt:
+            return {"payload": {"error": "生成失败:模型没有给出画面描述"}}, False
+        # 出图也走确认卡:用户先看当前模型/比例/张数,点了才提交。
+        confirm = append_message(
+            context.conversation, role="assistant",
+            kind=CreationMessage.Kind.CONFIRM,
+            payload={
+                "kind": "image",
+                "label": "开始生成",
+                "estimated_credits": 0,
+                "prompt": prompt,
+                "submitted": False,
+                "params": snapshot_session_params(context.conversation),
+                "param_options": confirm_param_options(False),
+            },
+        )
+        events = [{"type": "message", "message": _message_payload(confirm)}]
+        return {"payload": {"awaiting_confirmation": True}, "_events": events}, True

    return {"payload": {"error": f"未知工具 {name}"}}, False