添加视频复刻和优化脚本旁白

This commit is contained in:
Azmat@qq.com
2026-08-24 10:48:01 +08:00
parent 368586d33e
commit 2f5b7304b0
33 changed files with 1172 additions and 109 deletions
@@ -53,6 +53,7 @@ OPERATION_LABELS = {
"video_generate": "视频",
"voiceover_generate": "配音",
"entity_extract": "脚本信息",
"video_digest": "视频提炼",
}
TASK_OPERATIONS = {
@@ -67,6 +68,7 @@ TASK_OPERATIONS = {
"video_segment": "video_generate",
"voiceover": "voiceover_generate",
"free_video": "video_generate",
"video_digest": "video_digest",
}
+6
View File
@@ -316,6 +316,8 @@ def execute_model_call(
error_metadata: Callable[[Exception, ModelConfig], AttemptMetadata | Mapping[str, Any]] | None = None,
error_classifier: Callable[[Exception, ModelConfig], PublicGenerationError] | None = None,
candidate_resolver: Callable[..., list[ModelConfig]] | None = None,
allow_retry: bool = True,
allow_fallback: bool = True,
sleep: Callable[[float], None] = time.sleep,
monotonic: Callable[[], float] = time.monotonic,
uniform: Callable[[float, float], float] = random.uniform,
@@ -328,7 +330,11 @@ def execute_model_call(
raise ValueError("主模型 capability 与本次 ModelRequirements 不一致")
policy, retry_delays, request_timeout, total_timeout, retry_after_cap = _ability_policy(requirements)
if not allow_retry:
retry_delays = ()
resolver = candidate_resolver or resolve_fallback_candidates
if not allow_fallback:
resolver = lambda **_kwargs: []
result_meta = result_metadata or _default_result_metadata
error_meta = error_metadata or _default_error_metadata
classify = error_classifier or (
+6 -5
View File
@@ -83,10 +83,11 @@ PERSONA_BRIEFS: dict[str, str] = {
}
_PERSONA_KEY_BY_LABEL = {label: key for key, label in PERSONA_LABELS.items()}
# 可懂语速上限 3.5 字/秒;15 秒口播还要有下限,否则模型写一句 20 字就收,撑不满
NARRATION_CHARS_PER_SECOND = 3.5
NARRATION_CHARS_PER_SECOND_MIN = 2.4
NARRATION_CHARS_HARD_CAP = 55
# 成片有同期画面与自然停顿,3.5 字/秒会让 15 秒口播在约 10 秒时说完
# 电商真人口播以 3.6–4.5 字/秒为可用区间:既留出换气,也不会让画面空转。
NARRATION_CHARS_PER_SECOND = 4.5
NARRATION_CHARS_PER_SECOND_MIN = 3.6
NARRATION_CHARS_HARD_CAP = 68
VISUAL_CHARS_MIN = 72
SHOT_BEATS_MIN = 3
BEAT_SPAN_RE = re.compile(
@@ -160,7 +161,7 @@ def coerce_combo(fmt: str | None, structure: str | None) -> tuple[str, str]:
def narration_limit(duration: int) -> int:
"""这一镜旁白的字数上限:秒数 × 3.5,且不超过硬上限 55"""
"""这一镜旁白的字数上限:秒数 × 4.5,且不超过硬上限 68"""
return max(1, min(NARRATION_CHARS_HARD_CAP, int(duration * NARRATION_CHARS_PER_SECOND)))
+12 -2
View File
@@ -122,8 +122,14 @@ def public_model_name(model_config: ModelConfig) -> str:
def resolve_provider_credentials(provider) -> tuple[str | None, str | None]:
"""解析中转站凭证。可插拔顺序:DB(ModelProvider.base_url/api_key)优先 → settings(.env)回退。
两者都不写死;换站只改 DB 这一行,或改 .env 对应项。"""
base_url = (provider.base_url or "").strip() or settings.PROVIDER_BASE_URLS.get(provider.name)
api_key = (getattr(provider, "api_key", "") or "").strip() or settings.PROVIDER_KEYS.get(provider.name)
name = provider.name
base_url = (provider.base_url or "").strip() or settings.PROVIDER_BASE_URLS.get(name)
api_key = (getattr(provider, "api_key", "") or "").strip() or settings.PROVIDER_KEYS.get(name)
# 官转等 yunqi_gemini_* 变体共用同一把 YunQi Gemini key,避免新 provider 没写进 .env 就落到火山。
if not api_key and name.startswith("yunqi_gemini"):
api_key = (settings.PROVIDER_KEYS.get("yunqi_gemini") or "").strip()
if not base_url and name.startswith("yunqi_gemini"):
base_url = (settings.PROVIDER_BASE_URLS.get("yunqi_gemini") or "").strip()
return (base_url or None), (api_key or None)
@@ -616,6 +622,8 @@ def execute_routed_text_request(
request_summary: dict | None = None,
stream_event_callback=None,
abort_check=None,
allow_retry: bool = True,
allow_fallback: bool = True,
):
"""文本入口共用的模型调用薄层:能力声明、Provider 调用、输出校验和尝试成本审计。
@@ -721,6 +729,8 @@ def execute_routed_text_request(
internal_kind="processing_failed" if isinstance(exc, _RoutedTextStreamCancelled) else "",
reference_id=str(task.id),
),
allow_retry=allow_retry,
allow_fallback=allow_fallback,
)
@@ -121,6 +121,31 @@ class RoutingExecutorTests(TestCase):
self.assertEqual([item.is_retry for item in attempts], [False, True])
self.assertEqual(attempts[1].previous_attempt_id, attempts[0].id)
def test_allow_retry_false_fails_once_without_fallback(self):
primary = self.model(self.provider("exec-no-retry", 100), "primary")
candidate = self.model(self.provider("exec-no-retry-unused", 10), "unused")
task = self.task(primary, "exec-no-retry")
resolver = Mock(return_value=[candidate])
calls = []
def invoke(model, timeout):
calls.append(model.id)
raise requests.ConnectionError("temporary network failure")
with self.assertRaises(requests.ConnectionError):
self.execute(
task=task,
primary=primary,
invoke=invoke,
candidate_resolver=resolver,
allow_retry=False,
allow_fallback=False,
)
self.assertEqual(calls, [primary.id])
resolver.assert_not_called()
self.assertEqual(task.model_attempts.count(), 1)
def test_outcome_unknown_never_retries_or_resolves_fallback_candidates(self):
primary = self.model(self.provider("exec-unknown-primary", 100), "primary")
candidate = self.model(self.provider("exec-unknown-candidate", 10), "candidate")
@@ -284,15 +284,15 @@ class PromptAssemblyTests(SimpleTestCase):
class NarrationLimitTests(SimpleTestCase):
def test_limit_scales_with_shot_length(self):
self.assertEqual(narration_limit(4), 14)
self.assertEqual(narration_limit(8), 28)
self.assertEqual(narration_limit(15), 52)
self.assertEqual(narration_limit(4), 18)
self.assertEqual(narration_limit(8), 36)
self.assertEqual(narration_limit(15), 67)
def test_never_exceeds_hard_cap(self):
self.assertLessEqual(narration_limit(60), 55)
self.assertLessEqual(narration_limit(60), 68)
def test_fifteen_second_floor_is_about_thirty_six(self):
self.assertEqual(narration_floor(15), 36)
def test_fifteen_second_floor_is_about_fifty_four(self):
self.assertEqual(narration_floor(15), 54)
self.assertLess(narration_floor(15), narration_limit(15))
@@ -321,7 +321,7 @@ class ShotDensityTests(SimpleTestCase):
self.assertIn("旁白太短", str(ctx.exception))
def test_oral_meeting_floor_passes(self):
narration = "下午三点工位犯困,键盘都敲不利索。我倒了杯茶,第一口是回甘不是苦,整个人才慢慢醒过来。"
narration = "下午三点工位犯困,键盘都敲不利索。我以前总是硬扛,越扛脑子越乱。后来会先倒杯热茶,第一口是回甘不是苦,桌上的文件终于能看进去,整个人才慢慢醒过来。"
self.assertGreaterEqual(len(narration.replace(" ", "")), narration_floor(15))
self.assertGreaterEqual(len(self._VISUAL.replace(" ", "")), VISUAL_CHARS_MIN)
assert_shot_density(
+90
View File
@@ -7,8 +7,13 @@ from __future__ import annotations
import subprocess
from unittest.mock import patch
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import SimpleTestCase, TestCase
from rest_framework.test import APIClient
from apps.accounts.models import Team, TeamMember, User
from apps.ai.models import ModelConfig, ModelProvider
from apps.ai.video_digest import (
@@ -174,6 +179,30 @@ class DigestModelPinTests(TestCase):
self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True)
self.assertIsNone(resolve_digest_model_config())
def test_honors_preferred_gemini_id(self):
relay = self._provider("digest-pin-two", "Gemini")
first = self._model(relay, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro")
official = self._model(
self._provider("digest-pin-official", "官转"),
"gemini-3.1-pro-preview-official",
"Gemini 3.1 Pro 官转",
)
self.assertEqual(resolve_digest_model_config(preferred_id=first.id).id, first.id)
self.assertEqual(resolve_digest_model_config(preferred_id=official.id).id, official.id)
def test_ignores_preferred_text_only_model(self):
doubao = self._provider("digest-pin-ignore", "火山")
text_only = self._model(doubao, "doubao-seed-2-0-pro-260215", "豆包 2.0 Pro", is_default=True)
gemini = self._model(self._provider("digest-pin-keep", "官转"), DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro 官转")
self.assertEqual(resolve_digest_model_config(preferred_id=text_only.id).id, gemini.id)
def test_ignores_volcano_hosted_gemini_label(self):
volcano = self._provider("volcengine", "火山")
fake = self._model(volcano, DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro 官转", is_default=True)
relay = self._model(self._provider("digest-pin-relay-real", "官转"), DIGEST_VISION_MODEL_NAME, "Gemini 3.1 Pro 官转")
self.assertEqual(resolve_digest_model_config().id, relay.id)
self.assertEqual(resolve_digest_model_config(preferred_id=fake.id).id, relay.id)
# --------------------------------------------------------------------------- #
# helpers
@@ -194,3 +223,64 @@ def _digest_with_duration(clip, duration: float):
with patch("apps.ai.video_digest.probe_duration", return_value=duration):
return frames_from_upload(clip)
class VideoDigestApiTests(TestCase):
"""视频复刻页独立入口:不绑项目,缺文件直接 400。"""
def setUp(self):
self.user = User.objects.create_user(username="remix-api", password="p")
self.team = Team.objects.create(name="Remix", owner=self.user)
TeamMember.objects.create(team=self.team, user=self.user, role=TeamMember.Role.OWNER)
self.client = APIClient()
self.client.force_authenticate(self.user)
def test_rejects_missing_file(self):
response = self.client.post("/api/ai/video-digest/")
self.assertEqual(response.status_code, 400)
self.assertIn("视频", response.data["detail"])
def test_returns_digest_without_project(self):
payload = {
"text": "【整体结构】\n形式:口播\n" + "【第 1 镜】0-4 秒 · 钩子\n主体:一位女生\n" * 3,
"chars": 120,
"frames": 4,
"duration": 15.0,
"task_id": "00000000-0000-0000-0000-000000000001",
"estimated_cost": "30",
}
with patch("apps.ai.video_digest.digest_team_video", return_value=payload):
upload = SimpleUploadedFile("ref.mp4", b"x", content_type="video/mp4")
response = self.client.post("/api/ai/video-digest/", {"file": upload}, format="multipart")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["name"], "ref.mp4")
self.assertIn("【第 1 镜】", response.data["text"])
self.assertEqual(response.data["duration"], 15.0)
def test_forwards_model_config_id(self):
payload = {
"text": "【整体结构】\n形式:口播\n" + "【第 1 镜】0-4 秒 · 钩子\n主体:一位女生\n" * 3,
"chars": 120,
"frames": 4,
"duration": 15.0,
"task_id": "00000000-0000-0000-0000-000000000002",
"estimated_cost": "30",
}
with patch("apps.ai.video_digest.digest_team_video", return_value=payload) as mocked:
upload = SimpleUploadedFile("ref.mp4", b"x", content_type="video/mp4")
response = self.client.post(
"/api/ai/video-digest/",
{"file": upload, "model_config_id": "abc-123"},
format="multipart",
)
self.assertEqual(response.status_code, 200)
self.assertEqual(mocked.call_args.kwargs["model_config_id"], "abc-123")
def test_provider_failure_returns_json_not_500(self):
with patch("apps.ai.video_digest.digest_team_video", side_effect=RuntimeError("upstream 402")):
upload = SimpleUploadedFile("ref.mp4", b"x", content_type="video/mp4")
response = self.client.post("/api/ai/video-digest/", {"file": upload}, format="multipart")
self.assertEqual(response.status_code, 502)
self.assertIsInstance(response.data["detail"], str)
self.assertTrue(response.data["detail"])
self.assertEqual(response.data["error"]["operation"], "video_digest")
+2
View File
@@ -3,6 +3,7 @@ from rest_framework.routers import DefaultRouter
from .views import (
AITaskViewSet,
VideoDigestView,
FreeVideoDetailView,
FreeVideoFavoriteView,
FreeVideoPurgeView,
@@ -23,6 +24,7 @@ router.register("image-conversations", ImageConversationViewSet, basename="image
urlpatterns = [
path("generate-image/", GenerateImageView.as_view(), name="ai-generate-image"),
path("video-digest/", VideoDigestView.as_view(), name="ai-video-digest"),
path("free-video/", FreeVideoView.as_view(), name="ai-free-video"),
path("free-video/trash/", FreeVideoTrashView.as_view(), name="ai-free-video-trash"),
path("free-video/upload/", FreeVideoUploadView.as_view(), name="ai-free-video-upload"),
+116 -34
View File
@@ -17,14 +17,18 @@ from __future__ import annotations
import base64
import json
import logging
import math
import shutil
import subprocess
import tempfile
import uuid
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
logger = logging.getLogger(__name__)
from django.conf import settings
# 上传限制:超了直接 400,不进 ffmpeg,也不花模型钱
@@ -199,13 +203,31 @@ def validate_digest_text(text: str) -> str:
return cleaned
# 拆视频必须会看图。默认文本模型现在常是纯文本豆包,传帧图会直接失败
# Gemini 3.1 Pro;展示名带「官转」的优先(中转站官方 Gemini 通道)
# 拆视频必须会看图。火山豆包直连读不了这组帧图,会在 ARK 上挂满 120s
# 只认中转站的 Gemini 3.1 Pro;展示名带「官转」的优先。
DIGEST_VISION_MODEL_NAME = "gemini-3.1-pro-preview"
def resolve_digest_model_config():
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。"""
def _is_digest_vision_model(model) -> bool:
from apps.ai.services import OFFICIAL_DIRECT_PROVIDERS
provider_name = getattr(getattr(model, "provider", None), "name", "") or ""
if provider_name in OFFICIAL_DIRECT_PROVIDERS:
return False
blob = f"{model.name} {model.display_name}".lower()
return (
model.name == DIGEST_VISION_MODEL_NAME
or "gemini-3.1" in blob
or "gemini 3.1" in blob
)
def resolve_digest_model_config(preferred_id=None):
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。
前端可传 model_config_id(跟生成脚本同一套下拉)只有会看图的 Gemini 3.1 才认,
选了豆包等纯文本模型仍钉回官转,避免拆帧直接失败
"""
from apps.ai.models import ModelConfig
qs = (
@@ -216,6 +238,10 @@ def resolve_digest_model_config():
provider__status="active",
)
)
if preferred_id:
chosen = qs.filter(pk=preferred_id).first()
if chosen is not None and _is_digest_vision_model(chosen):
return chosen
def _blob(model) -> str:
return " ".join(
@@ -232,11 +258,9 @@ def resolve_digest_model_config():
ranked = []
for model in qs:
blob = _blob(model)
name_hit = "gemini-3.1" in model.name.lower() or "gemini-3.1" in (model.display_name or "").lower()
label_hit = "gemini 3.1" in (model.display_name or "").lower()
if not (name_hit or label_hit or model.name == DIGEST_VISION_MODEL_NAME):
if not _is_digest_vision_model(model):
continue
blob = _blob(model)
# 官转 > 精确模型名 > 其它 Gemini 3.1
score = 0
if "官转" in blob:
@@ -255,44 +279,99 @@ def resolve_digest_model_config():
# --------------------------------------------------------------------------- #
# 入口:一次真实的计费调用
# --------------------------------------------------------------------------- #
def digest_project_video(*, project, user, upload) -> dict:
def digest_project_video(*, project, user, upload, model_config_id=None) -> dict:
"""上传视频 → 分镜稿。抽帧在建任务之前做,文件不合格不占积分。"""
product = getattr(project, "product", None)
return _digest_video(
team=project.team,
user=user,
upload=upload,
project=project,
product_hint=" · ".join(
filter(None, [getattr(product, "title", ""), getattr(product, "category", "")])
),
model_config_id=model_config_id,
)
def digest_team_video(*, team, user, upload, model_config_id=None) -> dict:
"""视频复刻页:不绑项目,计费挂当前团队。"""
return _digest_video(
team=team,
user=user,
upload=upload,
project=None,
product_hint="",
model_config_id=model_config_id,
)
def _digest_video(*, team, user, upload, project=None, product_hint="", model_config_id=None) -> dict:
"""上传视频 → 分镜稿。抽帧在建任务之前做,文件不合格不占积分。"""
from django.db import transaction
from django.utils import timezone
from apps.ai.models import AITask
from apps.ai.services import create_ai_task, execute_routed_text_request
from apps.billing.services.ledger import charge_reserved_credit
from apps.billing.pricing import quote_flat
from apps.billing.services.ledger import charge_reserved_credit, reserve_credit
frames, duration = frames_from_upload(upload)
model_config = resolve_digest_model_config()
model_config = resolve_digest_model_config(preferred_id=model_config_id)
if model_config is None:
raise VideoDigestError("视频提炼需要 Gemini 3.1 Pro(会看图),当前没有启用,请联系管理员")
product = getattr(project, "product", None)
messages = build_digest_messages(
frames,
duration,
product_hint=" · ".join(
filter(None, [getattr(product, "title", ""), getattr(product, "category", "")])
),
logger.info(
"video digest using %s:%s (%s)",
model_config.provider.name,
model_config.name,
model_config.display_name,
)
task = create_ai_task(
project=project,
user=user,
task_type=AITask.Type.VIDEO_DIGEST,
model_config=model_config,
# 帧是几百 KB base64,绝不进 request_payload(会把 AITask 表撑爆),只记形状
request_payload={
"model": model_config.name,
"endpoint": model_config.endpoint,
"duration_seconds": round(duration, 2),
"frame_count": len(frames),
"frame_times": [f.at_seconds for f in frames],
},
)
messages = build_digest_messages(frames, duration, product_hint=product_hint)
request_payload = {
"model": model_config.name,
"endpoint": model_config.endpoint,
"feature": "video_remix" if project is None else "video_digest",
"duration_seconds": round(duration, 2),
"frame_count": len(frames),
"frame_times": [f.at_seconds for f in frames],
}
if project is not None:
task = create_ai_task(
project=project,
user=user,
task_type=AITask.Type.VIDEO_DIGEST,
model_config=model_config,
# 帧是几百 KB base64,绝不进 request_payload(会把 AITask 表撑爆),只记形状
request_payload=request_payload,
)
else:
quote = quote_flat(model_config, team=team)
if quote.meta.get("rate"):
request_payload = {**request_payload, "points_per_yuan_snapshot": quote.meta["rate"]}
try:
with transaction.atomic():
task = AITask.objects.create(
team=team,
created_by=user,
project=None,
task_type=AITask.Type.VIDEO_DIGEST,
status=AITask.Status.CREATED,
model_config=model_config,
idempotency_key=f"video_digest:{team.id}:{uuid.uuid4()}",
request_payload=request_payload,
estimated_cost=quote.points,
base_cost=quote.base_cost_yuan,
)
reserve_credit(team=team, user=user, task=task, amount=quote.points)
task.status = AITask.Status.RESERVED
task.save(update_fields=["status", "updated_at"])
except ValueError as exc:
if "insufficient credit" in str(exc).lower():
raise VideoDigestError("团队余额不足,请充值后重试") from exc
raise VideoDigestError(str(exc)) from exc
reservation = task.credit_reservation
try:
task.status = AITask.Status.SUBMITTED
@@ -302,12 +381,14 @@ def digest_project_video(*, project, user, upload) -> dict:
task=task,
primary_model=model_config,
messages=messages,
streaming=False,
streaming=True,
structured_output=False,
business_operation="video_digest",
temperature=0.4,
validate_text=validate_digest_text,
request_summary={"duration_seconds": round(duration, 2), "frame_count": len(frames)},
allow_retry=False,
allow_fallback=False,
)
_text, _response, digest = routed.value
except Exception as exc: # noqa: BLE001
@@ -328,6 +409,7 @@ def digest_project_video(*, project, user, upload) -> dict:
"frames": len(frames),
"duration": round(duration, 1),
"task_id": str(task.id),
"estimated_cost": str(task.estimated_cost),
}
+37
View File
@@ -1,3 +1,4 @@
import logging
import uuid
from django.db import transaction
@@ -27,6 +28,8 @@ from .serializers import (
)
from .services import enqueue_standalone_images
logger = logging.getLogger(__name__)
class GenerateImageView(APIView):
"""独立生图(不绑项目)· 图片创作/模特图/平台套图共用 —— **异步**。
@@ -693,6 +696,40 @@ def _set_free_video_generated_assets_deleted(task, deleted):
Asset.objects.filter(team=task.team, origin_task=task, purged_at__isnull=True).update(is_deleted=deleted)
class VideoDigestView(APIView):
"""视频复刻 · 上传参考视频提炼分镜稿(不绑项目)。
POST /api/ai/video-digest/ multipart file 中文分镜稿(30~60 ),计一次文本费
"""
parser_classes = [MultiPartParser, FormParser]
def post(self, request):
from .video_digest import VideoDigestError, digest_team_video
upload = request.FILES.get("file") or request.data.get("file")
if upload is None:
return Response({"detail": "请先上传参考视频"}, status=status.HTTP_400_BAD_REQUEST)
team = get_current_team(request.user)
try:
result = digest_team_video(
team=team,
user=request.user,
upload=upload,
model_config_id=request.data.get("model_config_id") or None,
)
except VideoDigestError as exc:
return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST)
except Exception as exc: # noqa: BLE001 — 模型/网络失败:走统一安全文案
logger.exception("video digest failed")
public_error = classify_generation_error(exc, operation="video_digest")
return Response(
{"detail": public_error.fallback_message, "error": public_error.as_dict()},
status=status.HTTP_502_BAD_GATEWAY,
)
return Response({"name": getattr(upload, "name", "") or "参考视频", **result})
class FreeVideoView(APIView):
"""自由创作·视频生成(不绑项目,universal 全能参考 / keyframe 首尾帧)。
+10 -2
View File
@@ -1372,12 +1372,20 @@ class ProjectViewSet(TeamScopedViewSetMixin, ModelViewSet):
if upload is None:
return Response({"detail": "no file uploaded"}, status=status.HTTP_400_BAD_REQUEST)
try:
result = digest_project_video(project=project, user=request.user, upload=upload)
result = digest_project_video(
project=project,
user=request.user,
upload=upload,
model_config_id=request.data.get("model_config_id") or None,
)
except VideoDigestError as exc:
return Response({"detail": str(exc)}, status=status.HTTP_400_BAD_REQUEST)
except Exception as exc: # noqa: BLE001 — 模型/网络失败:走统一安全文案,不回传原始异常
public_error = classify_generation_error(exc, operation="video_digest")
return Response(public_error, status=status.HTTP_502_BAD_GATEWAY)
return Response(
{"detail": public_error.fallback_message, "error": public_error.as_dict()},
status=status.HTTP_502_BAD_GATEWAY,
)
return Response({"name": getattr(upload, "name", "") or "参考视频", **result})
@action(detail=True, methods=["post"], url_path="upload-bgm", parser_classes=[MultiPartParser, FormParser])