feat(core/backend): pipeline continuity + threaded ffmpeg burn-in export + upload/save-timeline
Video pipeline (script→assets→storyboard→video→stitch): - robust split_script_into_segments (4 non-empty scenes), scene-aware storyboard/video prompts - link VideoSegment→ScriptSegment + storyboard-frame reference image (graceful text fallback) - idempotent poll_video_segment (no double-charge on repeated polling) - threaded export (no Celery worker needed) + poll-export endpoint - run_export_job rewritten to filter_complex: per-clip trim, xfade transitions, subtitle burn-in (Pillow PNG overlay; this ffmpeg lacks libass), BGM mix - upload-video-segment / upload-bgm / save-timeline endpoints - serializers embed asset preview URLs (beat assets pagination); Pillow added to requirements Also includes prior uncommitted backend work: account preferences/sessions, billing trend, product/asset endpoints, accounts 0002 migration. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,15 +1,39 @@
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
import requests
|
||||
from django.db import transaction
|
||||
from django.db import connections, transaction
|
||||
|
||||
from apps.assets.models import Asset, AssetFile
|
||||
from apps.assets.storage import TosStorage
|
||||
from apps.projects.models import ExportJob
|
||||
|
||||
|
||||
# 字幕样式(对齐 stage5 四个 swatch)。本机 ffmpeg 无 libass/drawtext,改用 Pillow 渲染 PNG 再 overlay 烧入。
|
||||
# RGBA 颜色;box 为半透明黑底(影视),stroke 为描边色。
|
||||
SUBTITLE_STYLES: dict[str, dict] = {
|
||||
"plain": {"size": 58, "fill": (255, 255, 255, 255), "stroke": (0, 0, 0, 255), "stroke_w": 4, "box": None}, # 朴素白底
|
||||
"cinema": {"size": 56, "fill": (255, 255, 255, 255), "stroke": (0, 0, 0, 0), "stroke_w": 0, "box": (0, 0, 0, 165)}, # 影视黑底
|
||||
"handwrite": {"size": 60, "fill": (255, 255, 255, 255), "stroke": (250, 93, 25, 255), "stroke_w": 7, "box": None}, # 手写描边(主橙 #fa5d19)
|
||||
"variety": {"size": 60, "fill": (255, 220, 60, 255), "stroke": (0, 0, 0, 255), "stroke_w": 6, "box": None}, # 综艺暖黄
|
||||
}
|
||||
# 候选 CJK 字体(mac 优先,Linux 兜底)
|
||||
_FONT_CANDIDATES = [
|
||||
"/System/Library/Fonts/STHeiti Medium.ttc",
|
||||
"/System/Library/Fonts/Hiragino Sans GB.ttc",
|
||||
"/System/Library/Fonts/PingFang.ttc",
|
||||
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
||||
"/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
|
||||
]
|
||||
# 转场(UI 选项)→ ffmpeg xfade transition 名。"none" 表示纯拼接。
|
||||
XFADE_MAP: dict[str, str] = {
|
||||
"fade": "fade", "dissolve": "dissolve", "slide": "slideleft", "slideleft": "slideleft",
|
||||
"slideright": "slideright", "wipe": "wiperight", "wiperight": "wiperight", "circle": "circleopen", "smooth": "smoothleft",
|
||||
}
|
||||
|
||||
|
||||
def _download_asset_primary_file(asset, target_path: Path) -> None:
|
||||
primary = asset.files.filter(is_primary=True).first() or asset.files.first()
|
||||
if primary is None:
|
||||
@@ -20,6 +44,174 @@ def _download_asset_primary_file(asset, target_path: Path) -> None:
|
||||
target_path.write_bytes(response.content)
|
||||
|
||||
|
||||
def _load_font(size: int):
|
||||
from PIL import ImageFont
|
||||
|
||||
for path in _FONT_CANDIDATES:
|
||||
try:
|
||||
return ImageFont.truetype(path, size, index=0)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _wrap_cjk(draw, text: str, font, max_width: int) -> list[str]:
|
||||
"""按像素宽折行(中文逐字、英文整体不强拆)。"""
|
||||
lines: list[str] = []
|
||||
line = ""
|
||||
for ch in text:
|
||||
trial = line + ch
|
||||
if draw.textlength(trial, font=font) <= max_width or not line:
|
||||
line = trial
|
||||
else:
|
||||
lines.append(line)
|
||||
line = ch
|
||||
if line:
|
||||
lines.append(line)
|
||||
return lines[:3] # 最多 3 行,够长截断
|
||||
|
||||
|
||||
def _render_subtitle_png(text: str, style_key: str, path: Path) -> tuple[int, int]:
|
||||
"""把一条字幕渲染成 1080 宽的透明 PNG(居中,带描边/底框),返回 (w,h)。"""
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
st = SUBTITLE_STYLES.get(style_key) or SUBTITLE_STYLES["plain"]
|
||||
canvas_w = 1080
|
||||
margin_x = 90
|
||||
font = _load_font(st["size"])
|
||||
probe = ImageDraw.Draw(Image.new("RGBA", (10, 10)))
|
||||
lines = _wrap_cjk(probe, (text or "").strip().replace("\n", " "), font, canvas_w - 2 * margin_x)
|
||||
line_h = st["size"] + 16
|
||||
pad = 22
|
||||
text_h = line_h * len(lines)
|
||||
canvas_h = text_h + 2 * pad
|
||||
img = Image.new("RGBA", (canvas_w, canvas_h), (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(img)
|
||||
if st["box"]:
|
||||
widest = max((draw.textlength(ln, font=font) for ln in lines), default=0)
|
||||
box_w = int(widest) + 2 * pad + 24
|
||||
x0 = (canvas_w - box_w) // 2
|
||||
draw.rounded_rectangle([x0, 0, x0 + box_w, canvas_h], radius=16, fill=st["box"])
|
||||
y = pad
|
||||
for ln in lines:
|
||||
w = draw.textlength(ln, font=font)
|
||||
x = (canvas_w - w) / 2
|
||||
draw.text((x, y), ln, font=font, fill=st["fill"],
|
||||
stroke_width=st["stroke_w"], stroke_fill=st["stroke"])
|
||||
y += line_h
|
||||
img.save(path)
|
||||
return canvas_w, canvas_h
|
||||
|
||||
|
||||
def _clip_specs(clips) -> list[dict]:
|
||||
"""每个 clip 的入点/出点/时长(秒),考虑 trim。"""
|
||||
specs = []
|
||||
for clip in clips:
|
||||
ts = (clip.trim_start_ms or 0) / 1000.0
|
||||
te = (clip.trim_end_ms / 1000.0) if clip.trim_end_ms else ts + (clip.duration_ms or 15000) / 1000.0
|
||||
specs.append({"ts": ts, "te": te, "dur": max(0.1, te - ts)})
|
||||
return specs
|
||||
|
||||
|
||||
def _output_starts(specs: list[dict], xfade: float) -> tuple[list[float], float]:
|
||||
"""每个 clip 在输出时间轴上的起点 + 输出总时长(xfade 会压缩总长)。"""
|
||||
starts, cum = [], 0.0
|
||||
for i, s in enumerate(specs):
|
||||
starts.append(0.0 if i == 0 else max(0.0, cum - i * xfade))
|
||||
cum += s["dur"]
|
||||
total = sum(s["dur"] for s in specs) - (len(specs) - 1) * xfade if xfade > 0 else sum(s["dur"] for s in specs)
|
||||
return starts, max(0.1, total)
|
||||
|
||||
|
||||
def _build_export_command(*, n: int, specs: list[dict], starts: list[float], total: float,
|
||||
transition: str, sub_overlays: list[tuple[str, float, float]],
|
||||
bgm_name: str | None, bgm_volume: float) -> list[str]:
|
||||
parts: list[str] = []
|
||||
for i, s in enumerate(specs):
|
||||
parts.append(
|
||||
f"[{i}:v]trim=start={s['ts']:.3f}:end={s['te']:.3f},setpts=PTS-STARTPTS,"
|
||||
"scale=1080:1920:force_original_aspect_ratio=decrease,"
|
||||
"pad=1080:1920:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=30,format=yuv420p[v" + str(i) + "]"
|
||||
)
|
||||
xname = XFADE_MAP.get(transition or "none")
|
||||
if xname and n > 1:
|
||||
prev = "v0"
|
||||
for i in range(1, n):
|
||||
out = "vbase" if i == n - 1 else f"x{i}"
|
||||
parts.append(f"[{prev}][v{i}]xfade=transition={xname}:duration=0.5:offset={starts[i]:.3f}[{out}]")
|
||||
prev = out
|
||||
else:
|
||||
parts.append("".join(f"[v{i}]" for i in range(n)) + f"concat=n={n}:v=1:a=0[vbase]")
|
||||
|
||||
# 字幕:每条一张 PNG,按时间窗 overlay 到底部居中(本机 ffmpeg 无 libass,用图片烧入)
|
||||
sub_base = n + (1 if bgm_name else 0)
|
||||
vlabel = "vbase"
|
||||
for j, (_png, start, end) in enumerate(sub_overlays):
|
||||
idx = sub_base + j
|
||||
out = "vout" if j == len(sub_overlays) - 1 else f"ov{j}"
|
||||
parts.append(
|
||||
f"[{vlabel}][{idx}:v]overlay=x=(W-w)/2:y=H-h-150:enable='between(t,{start:.3f},{end:.3f})'[{out}]"
|
||||
)
|
||||
vlabel = out
|
||||
if bgm_name:
|
||||
parts.append(f"[{n}:a]volume={bgm_volume:.3f},atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]")
|
||||
|
||||
cmd = ["ffmpeg", "-y"]
|
||||
for i in range(n):
|
||||
cmd += ["-i", f"clip{i}.mp4"]
|
||||
if bgm_name:
|
||||
cmd += ["-stream_loop", "-1", "-i", bgm_name]
|
||||
for png, _s, _e in sub_overlays:
|
||||
cmd += ["-loop", "1", "-i", png]
|
||||
cmd += ["-filter_complex", ";".join(parts), "-map", f"[{vlabel}]"]
|
||||
if bgm_name:
|
||||
cmd += ["-map", "[aout]"]
|
||||
cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", "-preset", "veryfast"]
|
||||
if bgm_name:
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"]
|
||||
cmd += ["-t", f"{total:.3f}", "-movflags", "+faststart", "output.mp4"]
|
||||
return cmd
|
||||
|
||||
|
||||
def run_export_job_in_thread(export_job_id: str) -> None:
|
||||
"""后台线程跑拼接导出。本机无 Celery worker(dev),故事板/视频已用线程模式,导出沿用同一打法:
|
||||
HTTP 秒回,真实 ffmpeg 拼接在线程里跑,前端轮询 poll-export 看进度 / 取成片。失败落库供轮询上报。"""
|
||||
|
||||
def _worker() -> None:
|
||||
try:
|
||||
run_export_job(export_job_id)
|
||||
except Exception as exc: # noqa: BLE001 — 失败落库,poll-export 据此上报
|
||||
job = ExportJob.objects.filter(id=export_job_id).first()
|
||||
if job is not None:
|
||||
job.status = ExportJob.Status.FAILED
|
||||
job.error_message = str(exc)
|
||||
job.save(update_fields=["status", "error_message", "updated_at"])
|
||||
finally:
|
||||
connections.close_all()
|
||||
|
||||
threading.Thread(target=_worker, daemon=True).start()
|
||||
|
||||
|
||||
def _subtitle_cues(timeline, project, specs, starts, total) -> list[tuple[float, float, str]]:
|
||||
"""字幕条目:文本取 SubtitleTrack.content,空则回退脚本旁白;时间按输出布局(对 xfade 也对齐)。"""
|
||||
track = timeline.subtitle_tracks.filter(enabled=True).first() or timeline.subtitle_tracks.first()
|
||||
if track is None or track.enabled is False:
|
||||
return []
|
||||
texts: list[str] = [str((c or {}).get("text", "")) for c in (track.content or [])]
|
||||
if not any(t.strip() for t in texts):
|
||||
script = project.script_versions.filter(is_adopted=True).prefetch_related("segments").first()
|
||||
if script is not None:
|
||||
texts = [seg.narration for seg in script.segments.all().order_by("sort_order")]
|
||||
cues: list[tuple[float, float, str]] = []
|
||||
for i in range(len(specs)):
|
||||
text = texts[i] if i < len(texts) else ""
|
||||
start = starts[i]
|
||||
end = starts[i + 1] if i + 1 < len(starts) else total
|
||||
if text and text.strip():
|
||||
cues.append((start, max(start + 0.5, end), text))
|
||||
return cues
|
||||
|
||||
|
||||
def run_export_job(export_job_id: str) -> ExportJob:
|
||||
export_job = ExportJob.objects.select_related("timeline", "timeline__project").get(id=export_job_id)
|
||||
timeline = export_job.timeline
|
||||
@@ -32,43 +224,45 @@ def run_export_job(export_job_id: str) -> ExportJob:
|
||||
export_job.progress = 10
|
||||
export_job.save(update_fields=["status", "progress", "updated_at"])
|
||||
|
||||
transition = str((timeline.metadata or {}).get("transition", {}).get("type", "none"))
|
||||
bgm_track = timeline.bgm_tracks.select_related("asset").first()
|
||||
subtitle_track = timeline.subtitle_tracks.filter(enabled=True).first()
|
||||
style_key = str((subtitle_track.style or {}).get("key", "plain")) if subtitle_track else "plain"
|
||||
|
||||
specs = _clip_specs(clips)
|
||||
xfade = 0.5 if XFADE_MAP.get(transition) and len(clips) > 1 else 0.0
|
||||
starts, total = _output_starts(specs, xfade)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="airshelf-export-") as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
concat_file = tmp / "concat.txt"
|
||||
downloaded_files: list[Path] = []
|
||||
for index, clip in enumerate(clips):
|
||||
clip_path = tmp / f"clip-{index}.mp4"
|
||||
_download_asset_primary_file(clip.asset, clip_path)
|
||||
downloaded_files.append(clip_path)
|
||||
concat_file.write_text(
|
||||
"\n".join(f"file '{path.as_posix()}'" for path in downloaded_files),
|
||||
encoding="utf-8",
|
||||
_download_asset_primary_file(clip.asset, tmp / f"clip{index}.mp4")
|
||||
|
||||
bgm_name = None
|
||||
if bgm_track is not None and bgm_track.asset_id:
|
||||
primary = bgm_track.asset.files.filter(is_primary=True).first() or bgm_track.asset.files.first()
|
||||
suffix = Path(primary.object_key).suffix or ".mp3" if primary else ".mp3"
|
||||
bgm_name = f"bgm{suffix}"
|
||||
_download_asset_primary_file(bgm_track.asset, tmp / bgm_name)
|
||||
|
||||
cues = _subtitle_cues(timeline, project, specs, starts, total)
|
||||
sub_overlays: list[tuple[str, float, float]] = []
|
||||
for i, (start, end, text) in enumerate(cues):
|
||||
png = f"sub{i}.png"
|
||||
_render_subtitle_png(text, style_key, tmp / png)
|
||||
sub_overlays.append((png, start, end))
|
||||
|
||||
export_job.progress = 35
|
||||
export_job.save(update_fields=["progress", "updated_at"])
|
||||
|
||||
command = _build_export_command(
|
||||
n=len(clips), specs=specs, starts=starts, total=total, transition=transition,
|
||||
sub_overlays=sub_overlays, bgm_name=bgm_name, bgm_volume=(bgm_track.volume / 100.0) if bgm_track else 1.0,
|
||||
)
|
||||
proc = subprocess.run(command, cwd=str(tmp), capture_output=True)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"ffmpeg export failed: {proc.stderr.decode('utf-8', 'ignore')[-1200:]}")
|
||||
output_path = tmp / "output.mp4"
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-f",
|
||||
"concat",
|
||||
"-safe",
|
||||
"0",
|
||||
"-i",
|
||||
str(concat_file),
|
||||
"-vf",
|
||||
"scale=1080:1920:force_original_aspect_ratio=decrease,pad=1080:1920:(ow-iw)/2:(oh-ih)/2",
|
||||
"-r",
|
||||
"30",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(output_path),
|
||||
]
|
||||
subprocess.run(command, check=True, capture_output=True)
|
||||
export_job.progress = 85
|
||||
export_job.save(update_fields=["progress", "updated_at"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user