diff --git a/core/backend/apps/ai/services.py b/core/backend/apps/ai/services.py index 9722b57..e8c3326 100644 --- a/core/backend/apps/ai/services.py +++ b/core/backend/apps/ai/services.py @@ -62,6 +62,23 @@ def get_default_model(capability: str) -> ModelConfig: return qs.filter(is_default=True).order_by("created_at").first() or qs.order_by("created_at").first() +def get_storyboard_image_model() -> ModelConfig: + """故事板出图钉 YunQi gpt-image-2 多图 edits(与手工测通的 curl 同一条链路)。 + 找不到再回落默认图像模型,避免测试/未 seed 环境直接挂。""" + pinned = ( + ModelConfig.objects.select_related("provider") + .filter( + capability=ModelConfig.Capability.IMAGE, + status=ModelConfig.Status.ACTIVE, + provider__status="active", + provider__name="yunqi", + name="gpt-image-2", + ) + .first() + ) + return pinned or get_default_model(ModelConfig.Capability.IMAGE) + + def resolve_image_model(key: str | None) -> "ModelConfig | None": """前端「生图模型选择」→ ModelConfig。用户显式选的可以是 disabled 模型(故不按 status 过滤)。 · "volcano" → 火山官方 Seedream(取最新一版) @@ -2625,7 +2642,7 @@ def submit_storyboard(*, project, user, prompt: str = "", shot_ids: list | None adopted_script = project.script_versions.filter(is_adopted=True).first() if adopted_script is None: raise ValueError("script must be adopted before generating storyboard") - if get_default_model(ModelConfig.Capability.IMAGE) is None: + if get_storyboard_image_model() is None: raise ValueError("no active image model configured") if prompt: meta = dict(project.metadata or {}) @@ -3046,7 +3063,7 @@ def poll_storyboard(*, project, user) -> dict: ).values_list("request_payload__storyboard_shot", flat=True) if v } - model_config = get_default_model(ModelConfig.Capability.IMAGE) + model_config = get_storyboard_image_model() extra_prompt = (project.metadata or {}).get("storyboard_prompt", "") or "" spawnable = [s for s in active if str(s.id) not in inflight_shot_ids] slots = max(0, STORYBOARD_MAX_PARALLEL - len(inflight_shot_ids)) diff --git a/core/backend/apps/ai/test_video_digest.py b/core/backend/apps/ai/test_video_digest.py index 8eb9099..49cc321 100644 --- a/core/backend/apps/ai/test_video_digest.py +++ b/core/backend/apps/ai/test_video_digest.py @@ -8,9 +8,11 @@ from __future__ import annotations import subprocess from django.core.files.uploadedfile import SimpleUploadedFile -from django.test import SimpleTestCase +from django.test import SimpleTestCase, TestCase +from apps.ai.models import ModelConfig, ModelProvider from apps.ai.video_digest import ( + DIGEST_VISION_MODEL_NAME, MAX_DURATION_SECONDS, MAX_FRAMES, MAX_UPLOAD_BYTES, @@ -22,6 +24,7 @@ from apps.ai.video_digest import ( frames_from_upload, load_digest_skill, plan_frame_times, + resolve_digest_model_config, validate_digest_text, ) @@ -121,6 +124,57 @@ class DigestValidationTests(SimpleTestCase): self.assertEqual(validate_digest_text(f" {good} "), good.strip()) +class DigestModelPinTests(TestCase): + """视频提炼必须钉会看图的 Gemini,不能跟默认语言模型走。""" + + def setUp(self): + ModelConfig.objects.filter(capability=ModelConfig.Capability.TEXT).update( + status=ModelConfig.Status.DISABLED + ) + + def _provider(self, name, display=""): + provider, _ = ModelProvider.objects.get_or_create( + name=name, + defaults={"display_name": display or name, "status": ModelProvider.Status.ACTIVE}, + ) + provider.display_name = display or name + provider.status = ModelProvider.Status.ACTIVE + provider.save(update_fields=["display_name", "status"]) + return provider + + def _model(self, provider, name, display_name, *, is_default=False): + model, _ = ModelConfig.objects.update_or_create( + provider=provider, + name=name, + capability=ModelConfig.Capability.TEXT, + defaults={ + "display_name": display_name, + "status": ModelConfig.Status.ACTIVE, + "is_default": is_default, + }, + ) + return model + + def test_prefers_official_relay_gemini_over_default_text_model(self): + doubao = self._provider("digest-pin-doubao", "火山") + relay = self._provider("digest-pin-relay", "Gemini 官转") + self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True) + gemini = self._model(relay, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro 官转") + picked = resolve_digest_model_config() + self.assertEqual(picked.id, gemini.id) + + def test_falls_back_to_gemini_name_when_label_has_no_relay_tag(self): + yunqi = self._provider("digest-pin-yunqi", "YunQi · Gemini") + gemini = self._model(yunqi, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro") + picked = resolve_digest_model_config() + self.assertEqual(picked.id, gemini.id) + + def test_does_not_use_default_language_model_if_gemini_missing(self): + doubao = self._provider("digest-pin-text-only", "火山") + self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True) + self.assertIsNone(resolve_digest_model_config()) + + # --------------------------------------------------------------------------- # # helpers # --------------------------------------------------------------------------- # diff --git a/core/backend/apps/ai/video_digest.py b/core/backend/apps/ai/video_digest.py index bd6c8a9..d2692b0 100644 --- a/core/backend/apps/ai/video_digest.py +++ b/core/backend/apps/ai/video_digest.py @@ -3,8 +3,9 @@ 链路:ffmpeg 抽帧(均匀采样) → 帧内联进多模态 messages → 走现有文本模型路由 → 纯文本分镜稿。 两个「本来以为要新建、其实已经有」的前提: - 1. **读图能力**:默认脚本模型(YunQi gemini-3.1-pro)本身是多模态的,OpenAI 兼容的 - ``content: [{type:"text"},{type:"image_url"}]`` 直接透传即可,不需要新 provider、新模型、新 key。 + 1. **读图能力**:拆视频是多模态(帧图 + 文本)。默认语言模型现在可能是纯文本豆包, + **不能再用 get_default_model(TEXT)**。固定钉 Gemini 3.1 Pro 官转 + (``gemini-3.1-pro-preview``,展示名带「官转」优先)。 2. **ffmpeg**:第 5 阶段导出早就依赖它,已装进后端镜像(见 Dockerfile)。 **没有音轨**:帧里读不到口播,只能读画面上的字幕。这是刻意取舍——接语音转写要另开火山 ASR 服务、 @@ -198,6 +199,59 @@ def validate_digest_text(text: str) -> str: return cleaned +# 拆视频必须会看图。默认文本模型现在常是纯文本豆包,传帧图会直接失败。 +# 钉 Gemini 3.1 Pro;展示名带「官转」的优先(中转站官方 Gemini 通道)。 +DIGEST_VISION_MODEL_NAME = "gemini-3.1-pro-preview" + + +def resolve_digest_model_config(): + """视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。""" + from apps.ai.models import ModelConfig + + qs = ( + ModelConfig.objects.select_related("provider") + .filter( + capability=ModelConfig.Capability.TEXT, + status=ModelConfig.Status.ACTIVE, + provider__status="active", + ) + ) + + def _blob(model) -> str: + return " ".join( + filter( + None, + [ + model.name, + model.display_name, + getattr(model.provider, "name", ""), + getattr(model.provider, "display_name", ""), + ], + ) + ) + + ranked = [] + for model in qs: + blob = _blob(model) + name_hit = "gemini-3.1" in model.name.lower() or "gemini-3.1" in (model.display_name or "").lower() + label_hit = "gemini 3.1" in (model.display_name or "").lower() + if not (name_hit or label_hit or model.name == DIGEST_VISION_MODEL_NAME): + continue + # 官转 > 精确模型名 > 其它 Gemini 3.1 + score = 0 + if "官转" in blob: + score += 100 + if model.name == DIGEST_VISION_MODEL_NAME: + score += 20 + if getattr(model.provider, "name", "") == "yunqi_gemini": + score += 5 + ranked.append((score, model.created_at, model)) + if not ranked: + return None + ranked.sort(key=lambda item: (-item[0], item[1])) + return ranked[0][2] + + # --------------------------------------------------------------------------- # # 入口:一次真实的计费调用 # --------------------------------------------------------------------------- # @@ -206,15 +260,15 @@ def digest_project_video(*, project, user, upload) -> dict: from django.db import transaction from django.utils import timezone - from apps.ai.models import AITask, ModelConfig - from apps.ai.services import create_ai_task, execute_routed_text_request, get_default_model + from apps.ai.models import AITask + from apps.ai.services import create_ai_task, execute_routed_text_request from apps.billing.services.ledger import charge_reserved_credit frames, duration = frames_from_upload(upload) - model_config = get_default_model(ModelConfig.Capability.TEXT) + model_config = resolve_digest_model_config() if model_config is None: - raise VideoDigestError("暂时没有可用的模型,请联系管理员") + raise VideoDigestError("视频提炼需要 Gemini 3.1 Pro(会看图),当前没有启用,请联系管理员") product = getattr(project, "product", None) messages = build_digest_messages( diff --git a/core/frontend/src/pipeline-page.css b/core/frontend/src/pipeline-page.css index 8f62a11..7401fc5 100644 --- a/core/frontend/src/pipeline-page.css +++ b/core/frontend/src/pipeline-page.css @@ -352,7 +352,7 @@ .del { text-decoration: line-through; color: var(--black-alpha-48); } .ins { background: var(--forest-bg); color: var(--accent-forest); padding: 0 3px; } .chat-input { padding: 14px 18px 18px; border-top: 1px solid var(--border-faint); } - .chat-input-card { background: var(--background-base); border: 1px solid var(--border-faint); border-radius: var(--r-md); padding: 12px 14px 10px; transition: border-color var(--t-base), box-shadow var(--t-base); } + .chat-input-card { background: var(--background-base); border: 1px solid var(--border-faint); border-radius: var(--r-md); padding: 12px 14px 10px; min-width: 0; overflow: hidden; transition: border-color var(--t-base), box-shadow var(--t-base); } .chat-input-card:focus-within { border-color: rgba(0, 47, 167, 0.38); box-shadow: 0 0 0 3px rgba(0, 47, 167, 0.07); } .chat-input-area { width: 100%; border: none; outline: none; background: transparent; font-family: var(--font-sans); font-size: 13px; color: var(--accent-black); line-height: 1.55; resize: none; padding: 0; min-height: 42px; } .chat-input-area::placeholder { color: var(--black-alpha-40); } @@ -367,7 +367,45 @@ .chat-send-btn:hover { background: var(--klein-hover); border-color: var(--klein-hover); } .chat-send-btn:active { transform: scale(.95); } .chat-send-btn:disabled { background: var(--black-alpha-12); border-color: var(--black-alpha-12); color: var(--black-alpha-40); cursor: not-allowed; transform: none; } - .chat-attach-row { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; } + .chat-attach-row { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; min-width: 0; max-width: 100%; } + .chat-file { + display: flex; + align-items: center; + gap: 8px; + max-width: 100%; + min-width: 0; + padding: 8px 10px; + background: var(--surface); + border: 1px solid var(--border-faint); + border-radius: var(--r-md); + } + .chat-file-ico { width: 16px; height: 16px; flex-shrink: 0; color: var(--black-alpha-48); } + .chat-file-meta { min-width: 0; flex: 1; display: flex; flex-direction: column; gap: 1px; } + .chat-file-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 13px; + color: var(--accent-black); + line-height: 1.3; + } + .chat-file-sub { font-size: 12px; color: var(--black-alpha-48); line-height: 1.3; } + .chat-file-x { + flex-shrink: 0; + width: 18px; + height: 18px; + display: grid; + place-items: center; + background: transparent; + border: 0; + border-radius: var(--r-sm); + color: var(--black-alpha-48); + cursor: pointer; + font-size: 14px; + line-height: 1; + padding: 0; + } + .chat-file-x:hover { background: var(--black-alpha-4); color: var(--accent-black); } .shot-list { display: flex; flex-direction: column; } .shots-body { padding: 16px 18px; flex: 1; overflow-y: auto; max-height: 540px; display: flex; flex-direction: column; gap: 12px; position: relative; align-items: stretch; } diff --git a/core/frontend/src/routes/pipeline.tsx b/core/frontend/src/routes/pipeline.tsx index ada8f69..c9d4307 100644 --- a/core/frontend/src/routes/pipeline.tsx +++ b/core/frontend/src/routes/pipeline.tsx @@ -3157,10 +3157,13 @@ export function PipelinePage(props: {