azmat1127/yingqing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加视频复刻和优化脚本旁白

Browse Source
This commit is contained in:
Azmat@qq.com
2026-08-24 10:48:01 +08:00
parent 368586d33e
commit 2f5b7304b0
33 changed files with 1172 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/backend/apps/ai/video_digest.py
+116 -34
View File
@@ -17,14 +17,18 @@ from __future__ import annotations
import base64
import json
import logging
import math
import shutil
import subprocess
import tempfile
import uuid
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
logger = logging.getLogger(__name__)
from django.conf import settings
# 上传限制:超了直接 400,不进 ffmpeg,也不花模型钱
@@ -199,13 +203,31 @@ def validate_digest_text(text: str) -> str:
return cleaned
# 拆视频必须会看图。默认文本模型现在常是纯文本豆包,传帧图会直接失败。
# 钉 Gemini 3.1 Pro;展示名带「官转」的优先(中转站官方 Gemini 通道)。
# 拆视频必须会看图。火山豆包直连读不了这组帧图,会在 ARK 上挂满 120s。
# 只认中转站的 Gemini 3.1 Pro;展示名带「官转」的优先。
DIGEST_VISION_MODEL_NAME = "gemini-3.1-pro-preview"
def resolve_digest_model_config():
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。"""
def _is_digest_vision_model(model) -> bool:
from apps.ai.services import OFFICIAL_DIRECT_PROVIDERS
provider_name = getattr(getattr(model, "provider", None), "name", "") or ""
if provider_name in OFFICIAL_DIRECT_PROVIDERS:
return False
blob = f"{model.name} {model.display_name}".lower()
return (
model.name == DIGEST_VISION_MODEL_NAME
or "gemini-3.1" in blob
or "gemini 3.1" in blob
)
def resolve_digest_model_config(preferred_id=None):
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。
前端可传 model_config_id(跟生成脚本同一套下拉)。只有会看图的 Gemini 3.1 才认,
选了豆包等纯文本模型仍钉回官转,避免拆帧直接失败。
"""
from apps.ai.models import ModelConfig
qs = (
@@ -216,6 +238,10 @@ def resolve_digest_model_config():
provider__status="active",
)
)
if preferred_id:
chosen = qs.filter(pk=preferred_id).first()
if chosen is not None and _is_digest_vision_model(chosen):
return chosen
def _blob(model) -> str:
return " ".join(
@@ -232,11 +258,9 @@ def resolve_digest_model_config():
ranked = []
for model in qs:
blob = _blob(model)
name_hit = "gemini-3.1" in model.name.lower() or "gemini-3.1" in (model.display_name or "").lower()
label_hit = "gemini 3.1" in (model.display_name or "").lower()
if not (name_hit or label_hit or model.name == DIGEST_VISION_MODEL_NAME):
if not _is_digest_vision_model(model):
continue
blob = _blob(model)
# 官转 > 精确模型名 > 其它 Gemini 3.1
score = 0
if "官转" in blob:
@@ -255,44 +279,99 @@ def resolve_digest_model_config():
# --------------------------------------------------------------------------- #
# 入口:一次真实的计费调用
# --------------------------------------------------------------------------- #
def digest_project_video(*, project, user, upload) -> dict:
def digest_project_video(*, project, user, upload, model_config_id=None) -> dict:
"""上传视频 → 分镜稿。抽帧在建任务之前做,文件不合格不占积分。"""
product = getattr(project, "product", None)
return _digest_video(
team=project.team,
user=user,
upload=upload,
project=project,
product_hint=" · ".join(
filter(None, [getattr(product, "title", ""), getattr(product, "category", "")])
),
model_config_id=model_config_id,
)
def digest_team_video(*, team, user, upload, model_config_id=None) -> dict:
"""视频复刻页:不绑项目,计费挂当前团队。"""
return _digest_video(
team=team,
user=user,
upload=upload,
project=None,
product_hint="",
model_config_id=model_config_id,
)
def _digest_video(*, team, user, upload, project=None, product_hint="", model_config_id=None) -> dict:
"""上传视频 → 分镜稿。抽帧在建任务之前做,文件不合格不占积分。"""
from django.db import transaction
from django.utils import timezone
from apps.ai.models import AITask
from apps.ai.services import create_ai_task, execute_routed_text_request
from apps.billing.services.ledger import charge_reserved_credit
from apps.billing.pricing import quote_flat
from apps.billing.services.ledger import charge_reserved_credit, reserve_credit
frames, duration = frames_from_upload(upload)
model_config = resolve_digest_model_config()
model_config = resolve_digest_model_config(preferred_id=model_config_id)
if model_config is None:
raise VideoDigestError("视频提炼需要 Gemini 3.1 Pro(会看图),当前没有启用,请联系管理员")
product = getattr(project, "product", None)
messages = build_digest_messages(
frames,
duration,
product_hint=" · ".join(
filter(None, [getattr(product, "title", ""), getattr(product, "category", "")])
),
logger.info(
"video digest using %s:%s (%s)",
model_config.provider.name,
model_config.name,
model_config.display_name,
)
task = create_ai_task(
project=project,
user=user,
task_type=AITask.Type.VIDEO_DIGEST,
model_config=model_config,
# 帧是几百 KB base64,绝不进 request_payload(会把 AITask 表撑爆),只记形状
request_payload={
"model": model_config.name,
"endpoint": model_config.endpoint,
"duration_seconds": round(duration, 2),
"frame_count": len(frames),
"frame_times": [f.at_seconds for f in frames],
},
)
messages = build_digest_messages(frames, duration, product_hint=product_hint)
request_payload = {
"model": model_config.name,
"endpoint": model_config.endpoint,
"feature": "video_remix" if project is None else "video_digest",
"duration_seconds": round(duration, 2),
"frame_count": len(frames),
"frame_times": [f.at_seconds for f in frames],
}
if project is not None:
task = create_ai_task(
project=project,
user=user,
task_type=AITask.Type.VIDEO_DIGEST,
model_config=model_config,
# 帧是几百 KB base64,绝不进 request_payload(会把 AITask 表撑爆),只记形状
request_payload=request_payload,
)
else:
quote = quote_flat(model_config, team=team)
if quote.meta.get("rate"):
request_payload = {**request_payload, "points_per_yuan_snapshot": quote.meta["rate"]}
try:
with transaction.atomic():
task = AITask.objects.create(
team=team,
created_by=user,
project=None,
task_type=AITask.Type.VIDEO_DIGEST,
status=AITask.Status.CREATED,
model_config=model_config,
idempotency_key=f"video_digest:{team.id}:{uuid.uuid4()}",
request_payload=request_payload,
estimated_cost=quote.points,
base_cost=quote.base_cost_yuan,
)
reserve_credit(team=team, user=user, task=task, amount=quote.points)
task.status = AITask.Status.RESERVED
task.save(update_fields=["status", "updated_at"])
except ValueError as exc:
if "insufficient credit" in str(exc).lower():
raise VideoDigestError("团队余额不足，请充值后重试") from exc
raise VideoDigestError(str(exc)) from exc
reservation = task.credit_reservation
try:
task.status = AITask.Status.SUBMITTED
@@ -302,12 +381,14 @@ def digest_project_video(*, project, user, upload) -> dict:
task=task,
primary_model=model_config,
messages=messages,
streaming=False,
streaming=True,
structured_output=False,
business_operation="video_digest",
temperature=0.4,
validate_text=validate_digest_text,
request_summary={"duration_seconds": round(duration, 2), "frame_count": len(frames)},
allow_retry=False,
allow_fallback=False,
)
_text, _response, digest = routed.value
except Exception as exc: # noqa: BLE001
@@ -328,6 +409,7 @@ def digest_project_video(*, project, user, upload) -> dict:
"frames": len(frames),
"duration": round(duration, 1),
"task_id": str(task.id),
"estimated_cost": str(task.estimated_cost),
}