修复线上视频导出失败 + 配音预览音画同步
- Dockerfile: 补装 ffmpeg 与中文字体 fonts-noto-cjk,修线上导出报 "No such file or directory: 'ffmpeg'" 及字幕中文方块;换阿里云 apt 源防构建超时 - export.py: 字幕字体加载加 Noto CJK glob 兜底,适配不同版本 fonts-noto-cjk - pipeline.tsx: 配音预览改 Web Audio 精确排程(借鉴 WebAV)——预解码 + source.start(when,offset) + 增量重排,正在发声的句绝不掐断,消除字幕音错位与卡顿 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,15 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 视频导出(stage5 拼接)靠 ffmpeg 二进制;字幕用 Pillow 渲染 PNG 需中文字体(fonts-noto-cjk),
|
||||
# 否则线上导出报 "No such file or directory: 'ffmpeg'" / 中文字幕变方块。slim 镜像默认两者都无。
|
||||
# 换阿里云 apt 源避免国内构建超时(兼容 bookworm 的 deb822 与旧 sources.list 两种格式)。
|
||||
RUN (sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null \
|
||||
|| sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true) \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg fonts-noto-cjk \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# PyMySQL is pure-python (install_as_MySQLdb), boto3/gunicorn need no C deps,
|
||||
# so the slim image is enough — no build-essential required.
|
||||
COPY requirements.txt .
|
||||
|
||||
@@ -65,6 +65,20 @@ def _load_font(size: int):
|
||||
return ImageFont.truetype(path, size, index=0)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
# 兜底:扫常见字体目录里任意 Noto CJK 文件(fonts-noto-cjk 不同版本文件名可能不同),
|
||||
# 找不到才退英文位图字体(中文会变方块)——保证线上装了 noto-cjk 就一定能用上
|
||||
import glob
|
||||
|
||||
for pattern in (
|
||||
"/usr/share/fonts/**/NotoSansCJK*.ttc",
|
||||
"/usr/share/fonts/**/NotoSansCJK*.otf",
|
||||
"/usr/share/fonts/**/NotoSerifCJK*.ttc",
|
||||
):
|
||||
for path in glob.glob(pattern, recursive=True):
|
||||
try:
|
||||
return ImageFont.truetype(path, size, index=0)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
|
||||
@@ -676,17 +676,22 @@ export function PipelinePage(props: {
|
||||
const voCuesRef = useRef<Record<number, VoCue[]>>({});
|
||||
const edPlayingRef = useRef(false);
|
||||
const edMutedRef = useRef(false);
|
||||
const voActiveRef = useRef(false); // 上次是否在「配音发声态」(边沿检测,防依赖变化误停)
|
||||
const voGainRef = useRef<GainNode | null>(null);
|
||||
const voSourcesRef = useRef<AudioBufferSourceNode[]>([]);
|
||||
// 已排程的句:asset → { src, ctxStart(该句在 AudioContext 时间轴上的起点秒) }。
|
||||
// 用 Map 做增量排程——重排时位置基本没变的「正在发声句」直接保留,不掐断重启(否则每次微漂移都爆音)
|
||||
const voSourcesRef = useRef<Map<string, { src: AudioBufferSourceNode; ctxStart: number }>>(new Map());
|
||||
const voClockRef = useRef<{ baseGlobalMs: number; ctxTime: number } | null>(null);
|
||||
const stopVoices = useCallback(() => {
|
||||
for (const s of voSourcesRef.current) { try { s.stop(); } catch { /* 已停 */ } }
|
||||
voSourcesRef.current = [];
|
||||
for (const { src } of voSourcesRef.current.values()) { try { src.stop(); } catch { /* 已停 */ } }
|
||||
voSourcesRef.current.clear();
|
||||
voClockRef.current = null;
|
||||
}, []);
|
||||
const scheduleVoices = useCallback((globalMs: number) => {
|
||||
stopVoices();
|
||||
if (!edPlayingRef.current) return;
|
||||
// 增量排程(借鉴 WebAV:AudioContext 时钟权威 + 排了就交给时钟、绝不回头掐断正在发声的句)。
|
||||
// hard=false(温和:解码晚到/拖句/换段)→ 正在发声的句无条件保留,只动还没开始的未来句;
|
||||
// hard=true(真 seek/大漂移)→ 正在发声且位置变了的句才停掉重排(用户跳转,预期内)。
|
||||
const scheduleVoices = useCallback((globalMs: number, hard = false) => {
|
||||
if (!edPlayingRef.current) { stopVoices(); return; }
|
||||
const ctx = getVoiceCtx();
|
||||
if (ctx.state === "suspended") void ctx.resume();
|
||||
if (!voGainRef.current) {
|
||||
@@ -694,21 +699,46 @@ export function PipelinePage(props: {
|
||||
voGainRef.current.connect(ctx.destination);
|
||||
}
|
||||
voGainRef.current.gain.value = edMutedRef.current ? 0 : 1;
|
||||
const gain = voGainRef.current;
|
||||
const idx = edIdxRef.current;
|
||||
const within = globalMs - (edOffsetsRef.current[idx] ?? 0);
|
||||
const cues = voCuesRef.current[idx] || [];
|
||||
const now = ctx.currentTime + 0.03; // 30ms 起播余量
|
||||
const live = voSourcesRef.current;
|
||||
const wanted = new Set<string>();
|
||||
for (const q of cues) {
|
||||
const buf = voiceBufferReady.get(q.asset);
|
||||
if (!buf) continue; // 未解码完的句子先跳过(漂移校验会再补排)
|
||||
if (!buf) continue; // 未解码完的句子先跳过(解码完 voCuesPerClip 变 → 意图 effect 补排)
|
||||
const startIn = (q.offsetMs - within) / 1000;
|
||||
if (startIn + buf.duration <= 0) continue; // 这句已经过去
|
||||
wanted.add(q.asset);
|
||||
const desiredCtxStart = now + startIn; // 这句应当开始的绝对 ctx 时间(可能为过去=正在发声)
|
||||
const existing = live.get(q.asset);
|
||||
if (existing) {
|
||||
const playing = existing.ctxStart <= ctx.currentTime; // 这句已经在发声
|
||||
// 温和重排:正在发声的句一律保留(WebAV 式不掐断);未来句漂移 <120ms 也保留
|
||||
if (!hard && playing) continue;
|
||||
if (Math.abs(existing.ctxStart - desiredCtxStart) < 0.12) continue;
|
||||
try { existing.src.stop(); } catch { /* 已停 */ }
|
||||
}
|
||||
const src = ctx.createBufferSource();
|
||||
src.buffer = buf;
|
||||
src.connect(voGainRef.current);
|
||||
src.connect(gain);
|
||||
if (startIn >= 0) src.start(now + startIn);
|
||||
else src.start(now, -startIn); // 播放头在句中:从句内偏移处续播
|
||||
voSourcesRef.current.push(src);
|
||||
const asset = q.asset;
|
||||
src.onended = () => { if (live.get(asset)?.src === src) live.delete(asset); };
|
||||
live.set(asset, { src, ctxStart: desiredCtxStart });
|
||||
}
|
||||
// 不再属于当前片段/已挪走的句:停掉。但正在发声的句在温和重排里不掐——
|
||||
// 换段边界有一帧 edIdx 与播放头不同步,会把刚起声的下一句误判「不属于本片段」,
|
||||
// 放它播完(onended 自清)即可,杜绝边界顿挫(WebAV:从不掐断正在发声的句)
|
||||
for (const [asset, entry] of live) {
|
||||
if (wanted.has(asset)) continue;
|
||||
const playing = entry.ctxStart <= ctx.currentTime;
|
||||
if (playing && !hard) continue;
|
||||
try { entry.src.stop(); } catch { /* 已停 */ }
|
||||
live.delete(asset);
|
||||
}
|
||||
voClockRef.current = { baseGlobalMs: globalMs, ctxTime: now };
|
||||
}, [stopVoices]);
|
||||
@@ -716,9 +746,10 @@ export function PipelinePage(props: {
|
||||
if (!edPlayingRef.current) return;
|
||||
const clock = voClockRef.current;
|
||||
if (!clock) { scheduleVoices(globalMs); return; }
|
||||
// 视频缓冲卡顿时播放头停、AudioContext 时钟不停 → 漂移超阈值整体重排(声画重新对齐)
|
||||
// syncVoice 的重排永远是「卡顿/累积漂移对齐」,不是用户 seek(seek 走 gotoClip/scrub 落点)→
|
||||
// 一律温和(hard=false):正在发声的句绝不掐断,只对齐还没起声的未来句(WebAV 铁律)
|
||||
const engineGlobal = clock.baseGlobalMs + (getVoiceCtx().currentTime - clock.ctxTime) * 1000;
|
||||
if (Math.abs(engineGlobal - globalMs) > 250) scheduleVoices(globalMs);
|
||||
if (Math.abs(engineGlobal - globalMs) > 600) scheduleVoices(globalMs);
|
||||
}, [scheduleVoices]);
|
||||
useEffect(() => { edIdxRef.current = edIdx; }, [edIdx]);
|
||||
useEffect(() => { edOffsetsRef.current = edOffsets; }, [edOffsets]);
|
||||
@@ -887,13 +918,17 @@ export function PipelinePage(props: {
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [edPlaying, edIdx, showingFinal, bgmPreviewUrl]);
|
||||
// 旁白配音跟随播放意图:播放/换段/拖动句子(voCuesPerClip 变)都整体重排;暂停即停
|
||||
// 旁白配音跟随播放意图:播放/换段/拖动句子(voCuesPerClip 变)都温和重排;暂停才停。
|
||||
// 边沿检测:只在「发声态 → 停」真翻转时 stopVoices——否则解码晚到让 voCuesPerClip 变、
|
||||
// effect 重跑时会误走 else 把正在念的句掐掉(0:17 边界顿挫的真凶)
|
||||
useEffect(() => {
|
||||
edPlayingRef.current = edPlaying && !showingFinal;
|
||||
edMutedRef.current = edMuted;
|
||||
if (voGainRef.current) voGainRef.current.gain.value = edMuted ? 0 : 1;
|
||||
if (edPlayingRef.current && voEnabled) scheduleVoices(edOffsetMs(edIdx) + edClipMsRef.current);
|
||||
else stopVoices();
|
||||
const active = edPlayingRef.current && voEnabled;
|
||||
if (active) scheduleVoices(edOffsetMs(edIdx) + edClipMsRef.current);
|
||||
else if (voActiveRef.current) stopVoices(); // 仅播放→暂停/切成片/关配音 时停一次
|
||||
voActiveRef.current = active;
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [edPlaying, edIdx, showingFinal, edMuted, voEnabled, voCuesPerClip]);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user