feat(core/backend): pipeline continuity + threaded ffmpeg burn-in export + upload/save-timeline

Video pipeline (script→assets→storyboard→video→stitch):
- robust split_script_into_segments (4 non-empty scenes), scene-aware storyboard/video prompts
- link VideoSegment→ScriptSegment + storyboard-frame reference image (graceful text fallback)
- idempotent poll_video_segment (no double-charge on repeated polling)
- threaded export (no Celery worker needed) + poll-export endpoint
- run_export_job rewritten to filter_complex: per-clip trim, xfade transitions,
  subtitle burn-in (Pillow PNG overlay; this ffmpeg lacks libass), BGM mix
- upload-video-segment / upload-bgm / save-timeline endpoints
- serializers embed asset preview URLs (beat assets pagination); Pillow added to requirements

Also includes prior uncommitted backend work: account preferences/sessions,
billing trend, product/asset endpoints, accounts 0002 migration.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
zyc
2026-06-09 14:46:16 +08:00
co-authored by Claude Opus 4.8
parent 8959946241
commit 92826dec14
13 changed files with 1229 additions and 97 deletions
+227 -33
View File
@@ -1,15 +1,39 @@
from pathlib import Path
import subprocess
import tempfile
import threading
import requests
from django.db import transaction
from django.db import connections, transaction
from apps.assets.models import Asset, AssetFile
from apps.assets.storage import TosStorage
from apps.projects.models import ExportJob
# 字幕样式(对齐 stage5 四个 swatch)。本机 ffmpeg 无 libass/drawtext,改用 Pillow 渲染 PNG 再 overlay 烧入。
# RGBA 颜色;box 为半透明黑底(影视),stroke 为描边色。
SUBTITLE_STYLES: dict[str, dict] = {
"plain": {"size": 58, "fill": (255, 255, 255, 255), "stroke": (0, 0, 0, 255), "stroke_w": 4, "box": None}, # 朴素白底
"cinema": {"size": 56, "fill": (255, 255, 255, 255), "stroke": (0, 0, 0, 0), "stroke_w": 0, "box": (0, 0, 0, 165)}, # 影视黑底
"handwrite": {"size": 60, "fill": (255, 255, 255, 255), "stroke": (250, 93, 25, 255), "stroke_w": 7, "box": None}, # 手写描边(主橙 #fa5d19)
"variety": {"size": 60, "fill": (255, 220, 60, 255), "stroke": (0, 0, 0, 255), "stroke_w": 6, "box": None}, # 综艺暖黄
}
# 候选 CJK 字体(mac 优先,Linux 兜底)
_FONT_CANDIDATES = [
"/System/Library/Fonts/STHeiti Medium.ttc",
"/System/Library/Fonts/Hiragino Sans GB.ttc",
"/System/Library/Fonts/PingFang.ttc",
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
]
# 转场(UI 选项)→ ffmpeg xfade transition 名。"none" 表示纯拼接。
XFADE_MAP: dict[str, str] = {
"fade": "fade", "dissolve": "dissolve", "slide": "slideleft", "slideleft": "slideleft",
"slideright": "slideright", "wipe": "wiperight", "wiperight": "wiperight", "circle": "circleopen", "smooth": "smoothleft",
}
def _download_asset_primary_file(asset, target_path: Path) -> None:
primary = asset.files.filter(is_primary=True).first() or asset.files.first()
if primary is None:
@@ -20,6 +44,174 @@ def _download_asset_primary_file(asset, target_path: Path) -> None:
target_path.write_bytes(response.content)
def _load_font(size: int):
from PIL import ImageFont
for path in _FONT_CANDIDATES:
try:
return ImageFont.truetype(path, size, index=0)
except Exception: # noqa: BLE001
continue
return ImageFont.load_default()
def _wrap_cjk(draw, text: str, font, max_width: int) -> list[str]:
"""按像素宽折行(中文逐字、英文整体不强拆)。"""
lines: list[str] = []
line = ""
for ch in text:
trial = line + ch
if draw.textlength(trial, font=font) <= max_width or not line:
line = trial
else:
lines.append(line)
line = ch
if line:
lines.append(line)
return lines[:3] # 最多 3 行,够长截断
def _render_subtitle_png(text: str, style_key: str, path: Path) -> tuple[int, int]:
"""把一条字幕渲染成 1080 宽的透明 PNG(居中,带描边/底框),返回 (w,h)。"""
from PIL import Image, ImageDraw
st = SUBTITLE_STYLES.get(style_key) or SUBTITLE_STYLES["plain"]
canvas_w = 1080
margin_x = 90
font = _load_font(st["size"])
probe = ImageDraw.Draw(Image.new("RGBA", (10, 10)))
lines = _wrap_cjk(probe, (text or "").strip().replace("\n", " "), font, canvas_w - 2 * margin_x)
line_h = st["size"] + 16
pad = 22
text_h = line_h * len(lines)
canvas_h = text_h + 2 * pad
img = Image.new("RGBA", (canvas_w, canvas_h), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
if st["box"]:
widest = max((draw.textlength(ln, font=font) for ln in lines), default=0)
box_w = int(widest) + 2 * pad + 24
x0 = (canvas_w - box_w) // 2
draw.rounded_rectangle([x0, 0, x0 + box_w, canvas_h], radius=16, fill=st["box"])
y = pad
for ln in lines:
w = draw.textlength(ln, font=font)
x = (canvas_w - w) / 2
draw.text((x, y), ln, font=font, fill=st["fill"],
stroke_width=st["stroke_w"], stroke_fill=st["stroke"])
y += line_h
img.save(path)
return canvas_w, canvas_h
def _clip_specs(clips) -> list[dict]:
"""每个 clip 的入点/出点/时长(秒),考虑 trim。"""
specs = []
for clip in clips:
ts = (clip.trim_start_ms or 0) / 1000.0
te = (clip.trim_end_ms / 1000.0) if clip.trim_end_ms else ts + (clip.duration_ms or 15000) / 1000.0
specs.append({"ts": ts, "te": te, "dur": max(0.1, te - ts)})
return specs
def _output_starts(specs: list[dict], xfade: float) -> tuple[list[float], float]:
"""每个 clip 在输出时间轴上的起点 + 输出总时长(xfade 会压缩总长)。"""
starts, cum = [], 0.0
for i, s in enumerate(specs):
starts.append(0.0 if i == 0 else max(0.0, cum - i * xfade))
cum += s["dur"]
total = sum(s["dur"] for s in specs) - (len(specs) - 1) * xfade if xfade > 0 else sum(s["dur"] for s in specs)
return starts, max(0.1, total)
def _build_export_command(*, n: int, specs: list[dict], starts: list[float], total: float,
transition: str, sub_overlays: list[tuple[str, float, float]],
bgm_name: str | None, bgm_volume: float) -> list[str]:
parts: list[str] = []
for i, s in enumerate(specs):
parts.append(
f"[{i}:v]trim=start={s['ts']:.3f}:end={s['te']:.3f},setpts=PTS-STARTPTS,"
"scale=1080:1920:force_original_aspect_ratio=decrease,"
"pad=1080:1920:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=30,format=yuv420p[v" + str(i) + "]"
)
xname = XFADE_MAP.get(transition or "none")
if xname and n > 1:
prev = "v0"
for i in range(1, n):
out = "vbase" if i == n - 1 else f"x{i}"
parts.append(f"[{prev}][v{i}]xfade=transition={xname}:duration=0.5:offset={starts[i]:.3f}[{out}]")
prev = out
else:
parts.append("".join(f"[v{i}]" for i in range(n)) + f"concat=n={n}:v=1:a=0[vbase]")
# 字幕:每条一张 PNG,按时间窗 overlay 到底部居中(本机 ffmpeg 无 libass,用图片烧入)
sub_base = n + (1 if bgm_name else 0)
vlabel = "vbase"
for j, (_png, start, end) in enumerate(sub_overlays):
idx = sub_base + j
out = "vout" if j == len(sub_overlays) - 1 else f"ov{j}"
parts.append(
f"[{vlabel}][{idx}:v]overlay=x=(W-w)/2:y=H-h-150:enable='between(t,{start:.3f},{end:.3f})'[{out}]"
)
vlabel = out
if bgm_name:
parts.append(f"[{n}:a]volume={bgm_volume:.3f},atrim=0:{total:.3f},asetpts=PTS-STARTPTS[aout]")
cmd = ["ffmpeg", "-y"]
for i in range(n):
cmd += ["-i", f"clip{i}.mp4"]
if bgm_name:
cmd += ["-stream_loop", "-1", "-i", bgm_name]
for png, _s, _e in sub_overlays:
cmd += ["-loop", "1", "-i", png]
cmd += ["-filter_complex", ";".join(parts), "-map", f"[{vlabel}]"]
if bgm_name:
cmd += ["-map", "[aout]"]
cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", "-preset", "veryfast"]
if bgm_name:
cmd += ["-c:a", "aac", "-b:a", "192k"]
cmd += ["-t", f"{total:.3f}", "-movflags", "+faststart", "output.mp4"]
return cmd
def run_export_job_in_thread(export_job_id: str) -> None:
"""后台线程跑拼接导出。本机无 Celery worker(dev),故事板/视频已用线程模式,导出沿用同一打法:
HTTP 秒回,真实 ffmpeg 拼接在线程里跑,前端轮询 poll-export 看进度 / 取成片。失败落库供轮询上报。"""
def _worker() -> None:
try:
run_export_job(export_job_id)
except Exception as exc: # noqa: BLE001 — 失败落库,poll-export 据此上报
job = ExportJob.objects.filter(id=export_job_id).first()
if job is not None:
job.status = ExportJob.Status.FAILED
job.error_message = str(exc)
job.save(update_fields=["status", "error_message", "updated_at"])
finally:
connections.close_all()
threading.Thread(target=_worker, daemon=True).start()
def _subtitle_cues(timeline, project, specs, starts, total) -> list[tuple[float, float, str]]:
"""字幕条目:文本取 SubtitleTrack.content,空则回退脚本旁白;时间按输出布局(对 xfade 也对齐)。"""
track = timeline.subtitle_tracks.filter(enabled=True).first() or timeline.subtitle_tracks.first()
if track is None or track.enabled is False:
return []
texts: list[str] = [str((c or {}).get("text", "")) for c in (track.content or [])]
if not any(t.strip() for t in texts):
script = project.script_versions.filter(is_adopted=True).prefetch_related("segments").first()
if script is not None:
texts = [seg.narration for seg in script.segments.all().order_by("sort_order")]
cues: list[tuple[float, float, str]] = []
for i in range(len(specs)):
text = texts[i] if i < len(texts) else ""
start = starts[i]
end = starts[i + 1] if i + 1 < len(starts) else total
if text and text.strip():
cues.append((start, max(start + 0.5, end), text))
return cues
def run_export_job(export_job_id: str) -> ExportJob:
export_job = ExportJob.objects.select_related("timeline", "timeline__project").get(id=export_job_id)
timeline = export_job.timeline
@@ -32,43 +224,45 @@ def run_export_job(export_job_id: str) -> ExportJob:
export_job.progress = 10
export_job.save(update_fields=["status", "progress", "updated_at"])
transition = str((timeline.metadata or {}).get("transition", {}).get("type", "none"))
bgm_track = timeline.bgm_tracks.select_related("asset").first()
subtitle_track = timeline.subtitle_tracks.filter(enabled=True).first()
style_key = str((subtitle_track.style or {}).get("key", "plain")) if subtitle_track else "plain"
specs = _clip_specs(clips)
xfade = 0.5 if XFADE_MAP.get(transition) and len(clips) > 1 else 0.0
starts, total = _output_starts(specs, xfade)
with tempfile.TemporaryDirectory(prefix="airshelf-export-") as tmp_dir:
tmp = Path(tmp_dir)
concat_file = tmp / "concat.txt"
downloaded_files: list[Path] = []
for index, clip in enumerate(clips):
clip_path = tmp / f"clip-{index}.mp4"
_download_asset_primary_file(clip.asset, clip_path)
downloaded_files.append(clip_path)
concat_file.write_text(
"\n".join(f"file '{path.as_posix()}'" for path in downloaded_files),
encoding="utf-8",
_download_asset_primary_file(clip.asset, tmp / f"clip{index}.mp4")
bgm_name = None
if bgm_track is not None and bgm_track.asset_id:
primary = bgm_track.asset.files.filter(is_primary=True).first() or bgm_track.asset.files.first()
suffix = Path(primary.object_key).suffix or ".mp3" if primary else ".mp3"
bgm_name = f"bgm{suffix}"
_download_asset_primary_file(bgm_track.asset, tmp / bgm_name)
cues = _subtitle_cues(timeline, project, specs, starts, total)
sub_overlays: list[tuple[str, float, float]] = []
for i, (start, end, text) in enumerate(cues):
png = f"sub{i}.png"
_render_subtitle_png(text, style_key, tmp / png)
sub_overlays.append((png, start, end))
export_job.progress = 35
export_job.save(update_fields=["progress", "updated_at"])
command = _build_export_command(
n=len(clips), specs=specs, starts=starts, total=total, transition=transition,
sub_overlays=sub_overlays, bgm_name=bgm_name, bgm_volume=(bgm_track.volume / 100.0) if bgm_track else 1.0,
)
proc = subprocess.run(command, cwd=str(tmp), capture_output=True)
if proc.returncode != 0:
raise RuntimeError(f"ffmpeg export failed: {proc.stderr.decode('utf-8', 'ignore')[-1200:]}")
output_path = tmp / "output.mp4"
command = [
"ffmpeg",
"-y",
"-f",
"concat",
"-safe",
"0",
"-i",
str(concat_file),
"-vf",
"scale=1080:1920:force_original_aspect_ratio=decrease,pad=1080:1920:(ow-iw)/2:(oh-ih)/2",
"-r",
"30",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-movflags",
"+faststart",
str(output_path),
]
subprocess.run(command, check=True, capture_output=True)
export_job.progress = 85
export_job.save(update_fields=["progress", "updated_at"])