350 lines
14 KiB
Python
350 lines
14 KiB
Python
"""上传视频提炼 —— 参考视频 → 可人工逐镜编辑的中文分镜稿。
|
||
|
||
链路:ffmpeg 抽帧(均匀采样) → 帧内联进多模态 messages → 走现有文本模型路由 → 纯文本分镜稿。
|
||
|
||
两个「本来以为要新建、其实已经有」的前提:
|
||
1. **读图能力**:拆视频是多模态(帧图 + 文本)。默认语言模型现在可能是纯文本豆包,
|
||
**不能再用 get_default_model(TEXT)**。固定钉 Gemini 3.1 Pro 官转
|
||
(``gemini-3.1-pro-preview``,展示名带「官转」优先)。
|
||
2. **ffmpeg**:第 5 阶段导出早就依赖它,已装进后端镜像(见 Dockerfile)。
|
||
|
||
**没有音轨**:帧里读不到口播,只能读画面上的字幕。这是刻意取舍——接语音转写要另开火山 ASR 服务、
|
||
另加一套凭证与计价,而带货参考片绝大多数带硬字幕,且口播词下游本来就要按用户自己的商品重写。
|
||
skill 里已要求模型「无字幕就如实写缺失,不许编口播词」。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import json
|
||
import math
|
||
import shutil
|
||
import subprocess
|
||
import tempfile
|
||
from dataclasses import dataclass
|
||
from functools import lru_cache
|
||
from pathlib import Path
|
||
|
||
from django.conf import settings
|
||
|
||
# 上传限制:超了直接 400,不进 ffmpeg,也不花模型钱
|
||
ALLOWED_SUFFIXES = (".mp4", ".mov", ".m4v", ".webm")
|
||
MAX_UPLOAD_BYTES = 200 * 1024 * 1024 # 200 MB
|
||
MAX_DURATION_SECONDS = 180 # 3 分钟。带货参考片远短于此;更长的帧采样密度不够,拆出来也是错的
|
||
|
||
# 抽帧:每 5 秒一帧,夹在 4~12 帧之间。12 帧 × 512px JPEG ≈ 0.5 MB base64,单次请求扛得住
|
||
SECONDS_PER_FRAME = 5
|
||
MIN_FRAMES = 4
|
||
MAX_FRAMES = 12
|
||
FRAME_WIDTH = 512 # 帧宽上限;分镜拆解看的是构图与景别,不需要原分辨率
|
||
FRAME_QUALITY = 5 # ffmpeg -q:v,2(最好)~31(最差)
|
||
|
||
_FFMPEG_TIMEOUT = 60
|
||
|
||
|
||
class VideoDigestError(ValueError):
|
||
"""用户可见的失败(文件不合格 / ffmpeg 读不动),一律 400。"""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VideoFrame:
|
||
at_seconds: int
|
||
jpeg: bytes
|
||
|
||
def as_data_url(self) -> str:
|
||
return "data:image/jpeg;base64," + base64.b64encode(self.jpeg).decode("ascii")
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# skill 加载
|
||
# --------------------------------------------------------------------------- #
|
||
def _skill_dir() -> Path:
|
||
"""与 script_agent._skill_dir 同源:优先 BASE_DIR/skills(镜像内),回落仓库根(本地旧布局)。"""
|
||
base = Path(settings.BASE_DIR)
|
||
for cand in (base / "skills", base.parent.parent / "skills"):
|
||
if (cand / "video-shot-digest").is_dir():
|
||
return cand / "video-shot-digest"
|
||
return base / "skills" / "video-shot-digest"
|
||
|
||
|
||
@lru_cache(maxsize=1)
|
||
def load_digest_skill() -> str:
|
||
main = _skill_dir() / "SKILL.md"
|
||
if main.exists():
|
||
return main.read_text(encoding="utf-8")
|
||
# 兜底:skill 丢了也别整条链路挂掉,退化成一句话提示词(产出会明显变差,交接文档已注明须带 skills 目录)
|
||
return (
|
||
"你是分镜拆解 agent。输入是一条电商短视频按时间均匀抽出的截帧。"
|
||
"逐镜还原分镜,每镜写清主体/动作/场景/景别/运镜/光线氛围/商品露出七要素,"
|
||
"台词只抄画面上的字幕,看不见的不要编。输出中文纯文本。"
|
||
)
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# ffmpeg:探时长 + 抽帧
|
||
# --------------------------------------------------------------------------- #
|
||
def _binary(name: str) -> str:
|
||
found = shutil.which(name)
|
||
if not found:
|
||
raise VideoDigestError("服务器暂时无法解析视频,请稍后再试")
|
||
return found
|
||
|
||
|
||
def probe_duration(path: str | Path) -> float:
|
||
"""ffprobe 读时长(秒)。读不到 = 不是能解的视频。"""
|
||
try:
|
||
out = subprocess.run(
|
||
[
|
||
_binary("ffprobe"), "-v", "error",
|
||
"-print_format", "json", "-show_format",
|
||
str(path),
|
||
],
|
||
capture_output=True, timeout=_FFMPEG_TIMEOUT, check=True,
|
||
).stdout
|
||
duration = float(json.loads(out)["format"]["duration"])
|
||
except VideoDigestError:
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 — ffprobe 各种失败对用户是同一件事
|
||
raise VideoDigestError("这个视频读不出来,请换一个 mp4 / mov 文件") from exc
|
||
if duration <= 0:
|
||
raise VideoDigestError("这个视频读不出来,请换一个 mp4 / mov 文件")
|
||
return duration
|
||
|
||
|
||
def plan_frame_times(duration: float) -> list[int]:
|
||
"""均匀采样时间点。取每段的**中点**,避开首尾黑场与片尾卡片。"""
|
||
count = max(MIN_FRAMES, min(MAX_FRAMES, math.ceil(duration / SECONDS_PER_FRAME)))
|
||
step = duration / count
|
||
return [int(step * (i + 0.5)) for i in range(count)]
|
||
|
||
|
||
def extract_frames(path: str | Path, times: list[int]) -> list[VideoFrame]:
|
||
"""逐时间点抽一帧。``-ss`` 放在 ``-i`` 前走关键帧快速定位,每帧约几十毫秒。"""
|
||
ffmpeg = _binary("ffmpeg")
|
||
frames: list[VideoFrame] = []
|
||
for at in times:
|
||
try:
|
||
done = subprocess.run(
|
||
[
|
||
ffmpeg, "-v", "error", "-ss", str(at), "-i", str(path),
|
||
"-frames:v", "1", "-vf", f"scale={FRAME_WIDTH}:-2",
|
||
"-q:v", str(FRAME_QUALITY), "-f", "image2", "-",
|
||
],
|
||
capture_output=True, timeout=_FFMPEG_TIMEOUT, check=True,
|
||
)
|
||
except Exception: # noqa: BLE001 — 单帧抽失败(定位越界等)跳过,别拖垮整次提炼
|
||
continue
|
||
if done.stdout:
|
||
frames.append(VideoFrame(at_seconds=at, jpeg=done.stdout))
|
||
if not frames:
|
||
raise VideoDigestError("没能从这个视频里取到画面,请换一个文件")
|
||
return frames
|
||
|
||
|
||
def frames_from_upload(upload) -> tuple[list[VideoFrame], float]:
|
||
"""校验上传文件 → 落临时盘 → 探时长 → 抽帧。临时文件退出即删。"""
|
||
name = (getattr(upload, "name", "") or "").lower()
|
||
if not name.endswith(ALLOWED_SUFFIXES):
|
||
raise VideoDigestError("只支持 mp4 / mov / m4v / webm 四种视频格式")
|
||
size = getattr(upload, "size", 0) or 0
|
||
if size > MAX_UPLOAD_BYTES:
|
||
raise VideoDigestError(f"视频不能超过 {MAX_UPLOAD_BYTES // 1024 // 1024} MB,请压缩后再传")
|
||
|
||
suffix = Path(name).suffix or ".mp4"
|
||
with tempfile.NamedTemporaryFile(suffix=suffix) as tmp:
|
||
for chunk in upload.chunks():
|
||
tmp.write(chunk)
|
||
tmp.flush()
|
||
duration = probe_duration(tmp.name)
|
||
if duration > MAX_DURATION_SECONDS:
|
||
raise VideoDigestError(
|
||
f"视频不能超过 {MAX_DURATION_SECONDS // 60} 分钟,请剪出要参考的那一段再传"
|
||
)
|
||
return extract_frames(tmp.name, plan_frame_times(duration)), duration
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# 组装多模态消息
|
||
# --------------------------------------------------------------------------- #
|
||
def build_digest_messages(
|
||
frames: list[VideoFrame],
|
||
duration: float,
|
||
*,
|
||
product_hint: str = "",
|
||
) -> list[dict]:
|
||
"""system = 拆解 skill;user = 时间戳 + 帧图交替,让模型知道每张图在原片的第几秒。"""
|
||
head = [
|
||
f"这是一条时长约 {round(duration)} 秒的电商带货短视频,",
|
||
f"按时间顺序均匀抽了 {len(frames)} 帧。每帧图前面标了它在原片中的时间点。",
|
||
]
|
||
if product_hint:
|
||
head.append(f"用户接下来想用这条片子的结构去拍自己的商品:{product_hint}。")
|
||
head.append("请按技能里的输出格式还原它的分镜。")
|
||
|
||
content: list[dict] = [{"type": "text", "text": "".join(head)}]
|
||
for frame in frames:
|
||
content.append({"type": "text", "text": f"[第 {frame.at_seconds} 秒]"})
|
||
content.append({"type": "image_url", "image_url": {"url": frame.as_data_url()}})
|
||
return [
|
||
{"role": "system", "content": load_digest_skill()},
|
||
{"role": "user", "content": content},
|
||
]
|
||
|
||
|
||
def validate_digest_text(text: str) -> str:
|
||
"""模型偶尔吐空 / 吐一句道歉。判空后交给路由层重试或切模型,别把废稿塞给用户。"""
|
||
cleaned = (text or "").strip()
|
||
if len(cleaned) < 80 or "【" not in cleaned:
|
||
raise ValueError("视频拆解结果不完整")
|
||
return cleaned
|
||
|
||
|
||
# 拆视频必须会看图。默认文本模型现在常是纯文本豆包,传帧图会直接失败。
|
||
# 钉 Gemini 3.1 Pro;展示名带「官转」的优先(中转站官方 Gemini 通道)。
|
||
DIGEST_VISION_MODEL_NAME = "gemini-3.1-pro-preview"
|
||
|
||
|
||
def resolve_digest_model_config():
|
||
"""视频提炼用的多模态文本模型:Gemini 3.1 Pro 官转。找不到不回落默认语言模型。"""
|
||
from apps.ai.models import ModelConfig
|
||
|
||
qs = (
|
||
ModelConfig.objects.select_related("provider")
|
||
.filter(
|
||
capability=ModelConfig.Capability.TEXT,
|
||
status=ModelConfig.Status.ACTIVE,
|
||
provider__status="active",
|
||
)
|
||
)
|
||
|
||
def _blob(model) -> str:
|
||
return " ".join(
|
||
filter(
|
||
None,
|
||
[
|
||
model.name,
|
||
model.display_name,
|
||
getattr(model.provider, "name", ""),
|
||
getattr(model.provider, "display_name", ""),
|
||
],
|
||
)
|
||
)
|
||
|
||
ranked = []
|
||
for model in qs:
|
||
blob = _blob(model)
|
||
name_hit = "gemini-3.1" in model.name.lower() or "gemini-3.1" in (model.display_name or "").lower()
|
||
label_hit = "gemini 3.1" in (model.display_name or "").lower()
|
||
if not (name_hit or label_hit or model.name == DIGEST_VISION_MODEL_NAME):
|
||
continue
|
||
# 官转 > 精确模型名 > 其它 Gemini 3.1
|
||
score = 0
|
||
if "官转" in blob:
|
||
score += 100
|
||
if model.name == DIGEST_VISION_MODEL_NAME:
|
||
score += 20
|
||
if getattr(model.provider, "name", "") == "yunqi_gemini":
|
||
score += 5
|
||
ranked.append((score, model.created_at, model))
|
||
if not ranked:
|
||
return None
|
||
ranked.sort(key=lambda item: (-item[0], item[1]))
|
||
return ranked[0][2]
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# 入口:一次真实的计费调用
|
||
# --------------------------------------------------------------------------- #
|
||
def digest_project_video(*, project, user, upload) -> dict:
|
||
"""上传视频 → 分镜稿。抽帧在建任务之前做,文件不合格不占积分。"""
|
||
from django.db import transaction
|
||
from django.utils import timezone
|
||
|
||
from apps.ai.models import AITask
|
||
from apps.ai.services import create_ai_task, execute_routed_text_request
|
||
from apps.billing.services.ledger import charge_reserved_credit
|
||
|
||
frames, duration = frames_from_upload(upload)
|
||
|
||
model_config = resolve_digest_model_config()
|
||
if model_config is None:
|
||
raise VideoDigestError("视频提炼需要 Gemini 3.1 Pro(会看图),当前没有启用,请联系管理员")
|
||
|
||
product = getattr(project, "product", None)
|
||
messages = build_digest_messages(
|
||
frames,
|
||
duration,
|
||
product_hint=" · ".join(
|
||
filter(None, [getattr(product, "title", ""), getattr(product, "category", "")])
|
||
),
|
||
)
|
||
|
||
task = create_ai_task(
|
||
project=project,
|
||
user=user,
|
||
task_type=AITask.Type.VIDEO_DIGEST,
|
||
model_config=model_config,
|
||
# 帧是几百 KB base64,绝不进 request_payload(会把 AITask 表撑爆),只记形状
|
||
request_payload={
|
||
"model": model_config.name,
|
||
"endpoint": model_config.endpoint,
|
||
"duration_seconds": round(duration, 2),
|
||
"frame_count": len(frames),
|
||
"frame_times": [f.at_seconds for f in frames],
|
||
},
|
||
)
|
||
reservation = task.credit_reservation
|
||
try:
|
||
task.status = AITask.Status.SUBMITTED
|
||
task.submitted_at = timezone.now()
|
||
task.save(update_fields=["status", "submitted_at", "updated_at"])
|
||
routed = execute_routed_text_request(
|
||
task=task,
|
||
primary_model=model_config,
|
||
messages=messages,
|
||
streaming=False,
|
||
structured_output=False,
|
||
business_operation="video_digest",
|
||
temperature=0.4,
|
||
validate_text=validate_digest_text,
|
||
request_summary={"duration_seconds": round(duration, 2), "frame_count": len(frames)},
|
||
)
|
||
_text, _response, digest = routed.value
|
||
except Exception as exc: # noqa: BLE001
|
||
_fail_digest_task(task, reservation, str(exc))
|
||
raise
|
||
|
||
with transaction.atomic():
|
||
task.status = AITask.Status.SUCCEEDED
|
||
task.response_payload = {"digest": digest[:8000]}
|
||
task.actual_cost = task.estimated_cost
|
||
task.completed_at = timezone.now()
|
||
task.save(update_fields=["status", "response_payload", "actual_cost", "completed_at", "updated_at"])
|
||
charge_reserved_credit(reservation=reservation, actual_amount=task.actual_cost)
|
||
|
||
return {
|
||
"text": digest,
|
||
"chars": len(digest),
|
||
"frames": len(frames),
|
||
"duration": round(duration, 1),
|
||
"task_id": str(task.id),
|
||
}
|
||
|
||
|
||
def _fail_digest_task(task, reservation, message: str) -> None:
|
||
from django.utils import timezone
|
||
|
||
from apps.ai.models import AITask
|
||
from apps.billing.services.ledger import release_credit
|
||
|
||
try:
|
||
task.status = AITask.Status.FAILED
|
||
task.error_message = message[:2000]
|
||
task.completed_at = timezone.now()
|
||
task.save(update_fields=["status", "error_message", "completed_at", "updated_at"])
|
||
finally:
|
||
try:
|
||
release_credit(reservation=reservation, reason=message[:200])
|
||
except Exception: # noqa: BLE001
|
||
pass
|