From 5e2bc0fc5ac512af4d5239c5082569f88546ebfc Mon Sep 17 00:00:00 2001 From: zyc <1439655764@qq.com> Date: Mon, 15 Jun 2026 13:20:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BA=BF=E4=B8=8A=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E5=AF=BC=E5=87=BA=E5=A4=B1=E8=B4=A5=20+=20=E9=85=8D?= =?UTF-8?q?=E9=9F=B3=E9=A2=84=E8=A7=88=E9=9F=B3=E7=94=BB=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dockerfile: 补装 ffmpeg 与中文字体 fonts-noto-cjk,修线上导出报 "No such file or directory: 'ffmpeg'" 及字幕中文方块;换阿里云 apt 源防构建超时 - export.py: 字幕字体加载加 Noto CJK glob 兜底,适配不同版本 fonts-noto-cjk - pipeline.tsx: 配音预览改 Web Audio 精确排程(借鉴 WebAV)——预解码 + source.start(when,offset) + 增量重排,正在发声的句绝不掐断,消除字幕音错位与卡顿 Co-Authored-By: Claude Opus 4.8 --- core/backend/Dockerfile | 9 +++ core/backend/apps/projects/services/export.py | 14 +++++ core/frontend/src/routes/pipeline.tsx | 63 ++++++++++++++----- 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/core/backend/Dockerfile b/core/backend/Dockerfile index 27d17db..daa424e 100644 --- a/core/backend/Dockerfile +++ b/core/backend/Dockerfile @@ -8,6 +8,15 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app +# 视频导出(stage5 拼接)靠 ffmpeg 二进制;字幕用 Pillow 渲染 PNG 需中文字体(fonts-noto-cjk), +# 否则线上导出报 "No such file or directory: 'ffmpeg'" / 中文字幕变方块。slim 镜像默认两者都无。 +# 换阿里云 apt 源避免国内构建超时(兼容 bookworm 的 deb822 与旧 sources.list 两种格式)。 +RUN (sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null \ + || sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true) \ + && apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg fonts-noto-cjk \ + && rm -rf /var/lib/apt/lists/* + # PyMySQL is pure-python (install_as_MySQLdb), boto3/gunicorn need no C deps, # so the slim image is enough — no build-essential required. COPY requirements.txt . diff --git a/core/backend/apps/projects/services/export.py b/core/backend/apps/projects/services/export.py index 299c1e5..22565bc 100644 --- a/core/backend/apps/projects/services/export.py +++ b/core/backend/apps/projects/services/export.py @@ -65,6 +65,20 @@ def _load_font(size: int): return ImageFont.truetype(path, size, index=0) except Exception: # noqa: BLE001 continue + # 兜底:扫常见字体目录里任意 Noto CJK 文件(fonts-noto-cjk 不同版本文件名可能不同), + # 找不到才退英文位图字体(中文会变方块)——保证线上装了 noto-cjk 就一定能用上 + import glob + + for pattern in ( + "/usr/share/fonts/**/NotoSansCJK*.ttc", + "/usr/share/fonts/**/NotoSansCJK*.otf", + "/usr/share/fonts/**/NotoSerifCJK*.ttc", + ): + for path in glob.glob(pattern, recursive=True): + try: + return ImageFont.truetype(path, size, index=0) + except Exception: # noqa: BLE001 + continue return ImageFont.load_default() diff --git a/core/frontend/src/routes/pipeline.tsx b/core/frontend/src/routes/pipeline.tsx index 92d9958..195f497 100644 --- a/core/frontend/src/routes/pipeline.tsx +++ b/core/frontend/src/routes/pipeline.tsx @@ -676,17 +676,22 @@ export function PipelinePage(props: { const voCuesRef = useRef>({}); const edPlayingRef = useRef(false); const edMutedRef = useRef(false); + const voActiveRef = useRef(false); // 上次是否在「配音发声态」(边沿检测,防依赖变化误停) const voGainRef = useRef(null); - const voSourcesRef = useRef([]); + // 已排程的句:asset → { src, ctxStart(该句在 AudioContext 时间轴上的起点秒) }。 + // 用 Map 做增量排程——重排时位置基本没变的「正在发声句」直接保留,不掐断重启(否则每次微漂移都爆音) + const voSourcesRef = useRef>(new Map()); const voClockRef = useRef<{ baseGlobalMs: number; ctxTime: number } | null>(null); const stopVoices = useCallback(() => { - for (const s of voSourcesRef.current) { try { s.stop(); } catch { /* 已停 */ } } - voSourcesRef.current = []; + for (const { src } of voSourcesRef.current.values()) { try { src.stop(); } catch { /* 已停 */ } } + voSourcesRef.current.clear(); voClockRef.current = null; }, []); - const scheduleVoices = useCallback((globalMs: number) => { - stopVoices(); - if (!edPlayingRef.current) return; + // 增量排程(借鉴 WebAV:AudioContext 时钟权威 + 排了就交给时钟、绝不回头掐断正在发声的句)。 + // hard=false(温和:解码晚到/拖句/换段)→ 正在发声的句无条件保留,只动还没开始的未来句; + // hard=true(真 seek/大漂移)→ 正在发声且位置变了的句才停掉重排(用户跳转,预期内)。 + const scheduleVoices = useCallback((globalMs: number, hard = false) => { + if (!edPlayingRef.current) { stopVoices(); return; } const ctx = getVoiceCtx(); if (ctx.state === "suspended") void ctx.resume(); if (!voGainRef.current) { @@ -694,21 +699,46 @@ export function PipelinePage(props: { voGainRef.current.connect(ctx.destination); } voGainRef.current.gain.value = edMutedRef.current ? 0 : 1; + const gain = voGainRef.current; const idx = edIdxRef.current; const within = globalMs - (edOffsetsRef.current[idx] ?? 0); const cues = voCuesRef.current[idx] || []; const now = ctx.currentTime + 0.03; // 30ms 起播余量 + const live = voSourcesRef.current; + const wanted = new Set(); for (const q of cues) { const buf = voiceBufferReady.get(q.asset); - if (!buf) continue; // 未解码完的句子先跳过(漂移校验会再补排) + if (!buf) continue; // 未解码完的句子先跳过(解码完 voCuesPerClip 变 → 意图 effect 补排) const startIn = (q.offsetMs - within) / 1000; if (startIn + buf.duration <= 0) continue; // 这句已经过去 + wanted.add(q.asset); + const desiredCtxStart = now + startIn; // 这句应当开始的绝对 ctx 时间(可能为过去=正在发声) + const existing = live.get(q.asset); + if (existing) { + const playing = existing.ctxStart <= ctx.currentTime; // 这句已经在发声 + // 温和重排:正在发声的句一律保留(WebAV 式不掐断);未来句漂移 <120ms 也保留 + if (!hard && playing) continue; + if (Math.abs(existing.ctxStart - desiredCtxStart) < 0.12) continue; + try { existing.src.stop(); } catch { /* 已停 */ } + } const src = ctx.createBufferSource(); src.buffer = buf; - src.connect(voGainRef.current); + src.connect(gain); if (startIn >= 0) src.start(now + startIn); else src.start(now, -startIn); // 播放头在句中:从句内偏移处续播 - voSourcesRef.current.push(src); + const asset = q.asset; + src.onended = () => { if (live.get(asset)?.src === src) live.delete(asset); }; + live.set(asset, { src, ctxStart: desiredCtxStart }); + } + // 不再属于当前片段/已挪走的句:停掉。但正在发声的句在温和重排里不掐—— + // 换段边界有一帧 edIdx 与播放头不同步,会把刚起声的下一句误判「不属于本片段」, + // 放它播完(onended 自清)即可,杜绝边界顿挫(WebAV:从不掐断正在发声的句) + for (const [asset, entry] of live) { + if (wanted.has(asset)) continue; + const playing = entry.ctxStart <= ctx.currentTime; + if (playing && !hard) continue; + try { entry.src.stop(); } catch { /* 已停 */ } + live.delete(asset); } voClockRef.current = { baseGlobalMs: globalMs, ctxTime: now }; }, [stopVoices]); @@ -716,9 +746,10 @@ export function PipelinePage(props: { if (!edPlayingRef.current) return; const clock = voClockRef.current; if (!clock) { scheduleVoices(globalMs); return; } - // 视频缓冲卡顿时播放头停、AudioContext 时钟不停 → 漂移超阈值整体重排(声画重新对齐) + // syncVoice 的重排永远是「卡顿/累积漂移对齐」,不是用户 seek(seek 走 gotoClip/scrub 落点)→ + // 一律温和(hard=false):正在发声的句绝不掐断,只对齐还没起声的未来句(WebAV 铁律) const engineGlobal = clock.baseGlobalMs + (getVoiceCtx().currentTime - clock.ctxTime) * 1000; - if (Math.abs(engineGlobal - globalMs) > 250) scheduleVoices(globalMs); + if (Math.abs(engineGlobal - globalMs) > 600) scheduleVoices(globalMs); }, [scheduleVoices]); useEffect(() => { edIdxRef.current = edIdx; }, [edIdx]); useEffect(() => { edOffsetsRef.current = edOffsets; }, [edOffsets]); @@ -887,13 +918,17 @@ export function PipelinePage(props: { } // eslint-disable-next-line react-hooks/exhaustive-deps }, [edPlaying, edIdx, showingFinal, bgmPreviewUrl]); - // 旁白配音跟随播放意图:播放/换段/拖动句子(voCuesPerClip 变)都整体重排;暂停即停 + // 旁白配音跟随播放意图:播放/换段/拖动句子(voCuesPerClip 变)都温和重排;暂停才停。 + // 边沿检测:只在「发声态 → 停」真翻转时 stopVoices——否则解码晚到让 voCuesPerClip 变、 + // effect 重跑时会误走 else 把正在念的句掐掉(0:17 边界顿挫的真凶) useEffect(() => { edPlayingRef.current = edPlaying && !showingFinal; edMutedRef.current = edMuted; if (voGainRef.current) voGainRef.current.gain.value = edMuted ? 0 : 1; - if (edPlayingRef.current && voEnabled) scheduleVoices(edOffsetMs(edIdx) + edClipMsRef.current); - else stopVoices(); + const active = edPlayingRef.current && voEnabled; + if (active) scheduleVoices(edOffsetMs(edIdx) + edClipMsRef.current); + else if (voActiveRef.current) stopVoices(); // 仅播放→暂停/切成片/关配音 时停一次 + voActiveRef.current = active; // eslint-disable-next-line react-hooks/exhaustive-deps }, [edPlaying, edIdx, showingFinal, edMuted, voEnabled, voCuesPerClip]);