优化故事版生成

This commit is contained in:
Azmat@qq.com
2026-08-21 18:22:50 +08:00
parent 158c66ac25
commit 409d1c2969
5 changed files with 181 additions and 15 deletions
+19 -2
View File
@@ -62,6 +62,23 @@ def get_default_model(capability: str) -> ModelConfig:
return qs.filter(is_default=True).order_by("created_at").first() or qs.order_by("created_at").first() return qs.filter(is_default=True).order_by("created_at").first() or qs.order_by("created_at").first()
def get_storyboard_image_model() -> ModelConfig:
"""故事板出图钉 YunQi gpt-image-2 多图 edits(与手工测通的 curl 同一条链路)。
找不到再回落默认图像模型,避免测试/未 seed 环境直接挂。"""
pinned = (
ModelConfig.objects.select_related("provider")
.filter(
capability=ModelConfig.Capability.IMAGE,
status=ModelConfig.Status.ACTIVE,
provider__status="active",
provider__name="yunqi",
name="gpt-image-2",
)
.first()
)
return pinned or get_default_model(ModelConfig.Capability.IMAGE)
def resolve_image_model(key: str | None) -> "ModelConfig | None": def resolve_image_model(key: str | None) -> "ModelConfig | None":
"""前端「生图模型选择」→ ModelConfig。用户显式选的可以是 disabled 模型(故不按 status 过滤)。 """前端「生图模型选择」→ ModelConfig。用户显式选的可以是 disabled 模型(故不按 status 过滤)。
· "volcano" → 火山官方 Seedream(取最新一版) · "volcano" → 火山官方 Seedream(取最新一版)
@@ -2625,7 +2642,7 @@ def submit_storyboard(*, project, user, prompt: str = "", shot_ids: list | None
adopted_script = project.script_versions.filter(is_adopted=True).first() adopted_script = project.script_versions.filter(is_adopted=True).first()
if adopted_script is None: if adopted_script is None:
raise ValueError("script must be adopted before generating storyboard") raise ValueError("script must be adopted before generating storyboard")
if get_default_model(ModelConfig.Capability.IMAGE) is None: if get_storyboard_image_model() is None:
raise ValueError("no active image model configured") raise ValueError("no active image model configured")
if prompt: if prompt:
meta = dict(project.metadata or {}) meta = dict(project.metadata or {})
@@ -3046,7 +3063,7 @@ def poll_storyboard(*, project, user) -> dict:
).values_list("request_payload__storyboard_shot", flat=True) ).values_list("request_payload__storyboard_shot", flat=True)
if v if v
} }
model_config = get_default_model(ModelConfig.Capability.IMAGE) model_config = get_storyboard_image_model()
extra_prompt = (project.metadata or {}).get("storyboard_prompt", "") or "" extra_prompt = (project.metadata or {}).get("storyboard_prompt", "") or ""
spawnable = [s for s in active if str(s.id) not in inflight_shot_ids] spawnable = [s for s in active if str(s.id) not in inflight_shot_ids]
slots = max(0, STORYBOARD_MAX_PARALLEL - len(inflight_shot_ids)) slots = max(0, STORYBOARD_MAX_PARALLEL - len(inflight_shot_ids))
+55 -1
View File
@@ -8,9 +8,11 @@ from __future__ import annotations
import subprocess import subprocess
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import SimpleTestCase from django.test import SimpleTestCase, TestCase
from apps.ai.models import ModelConfig, ModelProvider
from apps.ai.video_digest import ( from apps.ai.video_digest import (
DIGEST_VISION_MODEL_NAME,
MAX_DURATION_SECONDS, MAX_DURATION_SECONDS,
MAX_FRAMES, MAX_FRAMES,
MAX_UPLOAD_BYTES, MAX_UPLOAD_BYTES,
@@ -22,6 +24,7 @@ from apps.ai.video_digest import (
frames_from_upload, frames_from_upload,
load_digest_skill, load_digest_skill,
plan_frame_times, plan_frame_times,
resolve_digest_model_config,
validate_digest_text, validate_digest_text,
) )
@@ -121,6 +124,57 @@ class DigestValidationTests(SimpleTestCase):
self.assertEqual(validate_digest_text(f" {good} "), good.strip()) self.assertEqual(validate_digest_text(f" {good} "), good.strip())
class DigestModelPinTests(TestCase):
"""视频提炼必须钉会看图的 Gemini,不能跟默认语言模型走。"""
def setUp(self):
ModelConfig.objects.filter(capability=ModelConfig.Capability.TEXT).update(
status=ModelConfig.Status.DISABLED
)
def _provider(self, name, display=""):
provider, _ = ModelProvider.objects.get_or_create(
name=name,
defaults={"display_name": display or name, "status": ModelProvider.Status.ACTIVE},
)
provider.display_name = display or name
provider.status = ModelProvider.Status.ACTIVE
provider.save(update_fields=["display_name", "status"])
return provider
def _model(self, provider, name, display_name, *, is_default=False):
model, _ = ModelConfig.objects.update_or_create(
provider=provider,
name=name,
capability=ModelConfig.Capability.TEXT,
defaults={
"display_name": display_name,
"status": ModelConfig.Status.ACTIVE,
"is_default": is_default,
},
)
return model
def test_prefers_official_relay_gemini_over_default_text_model(self):
doubao = self._provider("digest-pin-doubao", "火山")
relay = self._provider("digest-pin-relay", "Gemini 官转")
self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True)
gemini = self._model(relay, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro 官转")
picked = resolve_digest_model_config()
self.assertEqual(picked.id, gemini.id)
def test_falls_back_to_gemini_name_when_label_has_no_relay_tag(self):
yunqi = self._provider("digest-pin-yunqi", "YunQi · Gemini")
gemini = self._model(yunqi, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro")
picked = resolve_digest_model_config()
self.assertEqual(picked.id, gemini.id)
def test_does_not_use_default_language_model_if_gemini_missing(self):
doubao = self._provider("digest-pin-text-only", "火山")
self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True)
self.assertIsNone(resolve_digest_model_config())
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# helpers # helpers
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
+60 -6
View File
@@ -3,8 +3,9 @@
链路:ffmpeg 抽帧(均匀采样) → 帧内联进多模态 messages → 走现有文本模型路由 → 纯文本分镜稿。 链路:ffmpeg 抽帧(均匀采样) → 帧内联进多模态 messages → 走现有文本模型路由 → 纯文本分镜稿。
两个「本来以为要新建、其实已经有」的前提: 两个「本来以为要新建、其实已经有」的前提:
1. **读图能力**:默认脚本模型(YunQi gemini-3.1-pro)本身是多模态的,OpenAI 兼容的 1. **读图能力**:拆视频是多模态(帧图 + 文本)。默认语言模型现在可能是纯文本豆包,
``content: [{type:"text"},{type:"image_url"}]`` 直接透传即可,不需要新 provider、新模型、新 key。 **不能再用 get_default_model(TEXT)**。固定钉 Gemini 3.1 Pro 官转
(``gemini-3.1-pro-preview``,展示名带「官转」优先)。
2. **ffmpeg**:第 5 阶段导出早就依赖它,已装进后端镜像(见 Dockerfile)。 2. **ffmpeg**:第 5 阶段导出早就依赖它,已装进后端镜像(见 Dockerfile)。
**没有音轨**:帧里读不到口播,只能读画面上的字幕。这是刻意取舍——接语音转写要另开火山 ASR 服务、 **没有音轨**:帧里读不到口播,只能读画面上的字幕。这是刻意取舍——接语音转写要另开火山 ASR 服务、
@@ -198,6 +199,59 @@ def validate_digest_text(text: str) -> str:
return cleaned return cleaned
# 拆视频必须会看图。默认文本模型现在常是纯文本豆包,传帧图会直接失败。
# 钉 Gemini 3.1 Pro;展示名带「官转」的优先(中转站官方 Gemini 通道)。
DIGEST_VISION_MODEL_NAME = "gemini-3.1-pro-preview"
def resolve_digest_model_config():
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。"""
from apps.ai.models import ModelConfig
qs = (
ModelConfig.objects.select_related("provider")
.filter(
capability=ModelConfig.Capability.TEXT,
status=ModelConfig.Status.ACTIVE,
provider__status="active",
)
)
def _blob(model) -> str:
return " ".join(
filter(
None,
[
model.name,
model.display_name,
getattr(model.provider, "name", ""),
getattr(model.provider, "display_name", ""),
],
)
)
ranked = []
for model in qs:
blob = _blob(model)
name_hit = "gemini-3.1" in model.name.lower() or "gemini-3.1" in (model.display_name or "").lower()
label_hit = "gemini 3.1" in (model.display_name or "").lower()
if not (name_hit or label_hit or model.name == DIGEST_VISION_MODEL_NAME):
continue
# 官转 > 精确模型名 > 其它 Gemini 3.1
score = 0
if "官转" in blob:
score += 100
if model.name == DIGEST_VISION_MODEL_NAME:
score += 20
if getattr(model.provider, "name", "") == "yunqi_gemini":
score += 5
ranked.append((score, model.created_at, model))
if not ranked:
return None
ranked.sort(key=lambda item: (-item[0], item[1]))
return ranked[0][2]
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# 入口:一次真实的计费调用 # 入口:一次真实的计费调用
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
@@ -206,15 +260,15 @@ def digest_project_video(*, project, user, upload) -> dict:
from django.db import transaction from django.db import transaction
from django.utils import timezone from django.utils import timezone
from apps.ai.models import AITask, ModelConfig from apps.ai.models import AITask
from apps.ai.services import create_ai_task, execute_routed_text_request, get_default_model from apps.ai.services import create_ai_task, execute_routed_text_request
from apps.billing.services.ledger import charge_reserved_credit from apps.billing.services.ledger import charge_reserved_credit
frames, duration = frames_from_upload(upload) frames, duration = frames_from_upload(upload)
model_config = get_default_model(ModelConfig.Capability.TEXT) model_config = resolve_digest_model_config()
if model_config is None: if model_config is None:
raise VideoDigestError("暂时没有可用的模型,请联系管理员") raise VideoDigestError("视频提炼需要 Gemini 3.1 Pro(会看图),当前没有启用,请联系管理员")
product = getattr(project, "product", None) product = getattr(project, "product", None)
messages = build_digest_messages( messages = build_digest_messages(
+40 -2
View File
@@ -352,7 +352,7 @@
.del { text-decoration: line-through; color: var(--black-alpha-48); } .del { text-decoration: line-through; color: var(--black-alpha-48); }
.ins { background: var(--forest-bg); color: var(--accent-forest); padding: 0 3px; } .ins { background: var(--forest-bg); color: var(--accent-forest); padding: 0 3px; }
.chat-input { padding: 14px 18px 18px; border-top: 1px solid var(--border-faint); } .chat-input { padding: 14px 18px 18px; border-top: 1px solid var(--border-faint); }
.chat-input-card { background: var(--background-base); border: 1px solid var(--border-faint); border-radius: var(--r-md); padding: 12px 14px 10px; transition: border-color var(--t-base), box-shadow var(--t-base); } .chat-input-card { background: var(--background-base); border: 1px solid var(--border-faint); border-radius: var(--r-md); padding: 12px 14px 10px; min-width: 0; overflow: hidden; transition: border-color var(--t-base), box-shadow var(--t-base); }
.chat-input-card:focus-within { border-color: rgba(0, 47, 167, 0.38); box-shadow: 0 0 0 3px rgba(0, 47, 167, 0.07); } .chat-input-card:focus-within { border-color: rgba(0, 47, 167, 0.38); box-shadow: 0 0 0 3px rgba(0, 47, 167, 0.07); }
.chat-input-area { width: 100%; border: none; outline: none; background: transparent; font-family: var(--font-sans); font-size: 13px; color: var(--accent-black); line-height: 1.55; resize: none; padding: 0; min-height: 42px; } .chat-input-area { width: 100%; border: none; outline: none; background: transparent; font-family: var(--font-sans); font-size: 13px; color: var(--accent-black); line-height: 1.55; resize: none; padding: 0; min-height: 42px; }
.chat-input-area::placeholder { color: var(--black-alpha-40); } .chat-input-area::placeholder { color: var(--black-alpha-40); }
@@ -367,7 +367,45 @@
.chat-send-btn:hover { background: var(--klein-hover); border-color: var(--klein-hover); } .chat-send-btn:hover { background: var(--klein-hover); border-color: var(--klein-hover); }
.chat-send-btn:active { transform: scale(.95); } .chat-send-btn:active { transform: scale(.95); }
.chat-send-btn:disabled { background: var(--black-alpha-12); border-color: var(--black-alpha-12); color: var(--black-alpha-40); cursor: not-allowed; transform: none; } .chat-send-btn:disabled { background: var(--black-alpha-12); border-color: var(--black-alpha-12); color: var(--black-alpha-40); cursor: not-allowed; transform: none; }
.chat-attach-row { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; } .chat-attach-row { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; min-width: 0; max-width: 100%; }
.chat-file {
display: flex;
align-items: center;
gap: 8px;
max-width: 100%;
min-width: 0;
padding: 8px 10px;
background: var(--surface);
border: 1px solid var(--border-faint);
border-radius: var(--r-md);
}
.chat-file-ico { width: 16px; height: 16px; flex-shrink: 0; color: var(--black-alpha-48); }
.chat-file-meta { min-width: 0; flex: 1; display: flex; flex-direction: column; gap: 1px; }
.chat-file-name {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-size: 13px;
color: var(--accent-black);
line-height: 1.3;
}
.chat-file-sub { font-size: 12px; color: var(--black-alpha-48); line-height: 1.3; }
.chat-file-x {
flex-shrink: 0;
width: 18px;
height: 18px;
display: grid;
place-items: center;
background: transparent;
border: 0;
border-radius: var(--r-sm);
color: var(--black-alpha-48);
cursor: pointer;
font-size: 14px;
line-height: 1;
padding: 0;
}
.chat-file-x:hover { background: var(--black-alpha-4); color: var(--accent-black); }
.shot-list { display: flex; flex-direction: column; } .shot-list { display: flex; flex-direction: column; }
.shots-body { padding: 16px 18px; flex: 1; overflow-y: auto; max-height: 540px; display: flex; flex-direction: column; gap: 12px; position: relative; align-items: stretch; } .shots-body { padding: 16px 18px; flex: 1; overflow-y: auto; max-height: 540px; display: flex; flex-direction: column; gap: 12px; position: relative; align-items: stretch; }
+7 -4
View File
@@ -3157,10 +3157,13 @@ export function PipelinePage(props: {
</div> </div>
<div className="chat-attach-row" id="chat-attach-row" hidden={chatAttachments.length === 0}> <div className="chat-attach-row" id="chat-attach-row" hidden={chatAttachments.length === 0}>
{chatAttachments.map((att, index) => ( {chatAttachments.map((att, index) => (
<span className="chip" key={`${att.name}-${index}`} style={{ marginRight: 6 }}> <span className="chat-file" key={`${att.name}-${index}`}>
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round" style={{ marginRight: 4 }}><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z" /><path d="M14 2v6h6" /></svg> <svg className="chat-file-ico" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z" /><path d="M14 2v6h6" /></svg>
{att.name} · {att.chars} 字 <span className="chat-file-meta">
<button type="button" aria-label="移除附件" style={{ marginLeft: 4, background: "none", border: 0, cursor: "pointer", color: "inherit" }} onClick={() => setChatAttachments((list) => list.filter((_, i) => i !== index))}>×</button> <span className="chat-file-name" title={att.name}>{att.name}</span>
<span className="chat-file-sub">{att.chars} 字</span>
</span>
<button className="chat-file-x" type="button" aria-label="移除附件" onClick={() => setChatAttachments((list) => list.filter((_, i) => i !== index))}>×</button>
</span> </span>
))} ))}
</div> </div>