修复线上视频导出卡死(OOM)+ 导出健壮性加固
根因:后端容器内存 limit 仅 768Mi,ffmpeg 编码 1080P 竖屏时内存峰值超限被 K8s OOMKill,导出后台线程随 Pod 重启而死,任务永远停在 35%(本地无 limit 29s 出片)。 - k8s/core/api-deployment.yaml: 后端内存 limit 768Mi→2Gi、requests 256→512Mi, 给 ffmpeg 足够编码空间(根治) - export.py: ffmpeg subprocess 加 900s 超时(卡死不再永久 RUNNING)+ -threads 2 降内存峰值 - views.py: submit-export 清理超 5 分钟未结束的僵尸任务,前端不再无限转圈 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -268,7 +268,9 @@ def _build_export_command(*, n: int, specs: list[dict], starts: list[float], tot
|
|||||||
cmd += ["-filter_complex", ";".join(parts), "-map", f"[{vlabel}]"]
|
cmd += ["-filter_complex", ";".join(parts), "-map", f"[{vlabel}]"]
|
||||||
if audio_label:
|
if audio_label:
|
||||||
cmd += ["-map", f"[{audio_label}]"]
|
cmd += ["-map", f"[{audio_label}]"]
|
||||||
cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", fps_expr, "-preset", "veryfast"]
|
# -threads 2:限制 x264 线程数,降低小容器(K8s 内存 limit 小)里的内存峰值,缓解 OOM 被杀;
|
||||||
|
# 同时避免在受限 CPU 上 fork 过多线程争抢。veryfast 保持编码速度。
|
||||||
|
cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", fps_expr, "-preset", "veryfast", "-threads", "2"]
|
||||||
if audio_label:
|
if audio_label:
|
||||||
cmd += ["-c:a", "aac", "-b:a", "192k"]
|
cmd += ["-c:a", "aac", "-b:a", "192k"]
|
||||||
cmd += ["-t", f"{total:.3f}", "-movflags", "+faststart", "output.mp4"]
|
cmd += ["-t", f"{total:.3f}", "-movflags", "+faststart", "output.mp4"]
|
||||||
@@ -479,7 +481,12 @@ def run_export_job(export_job_id: str) -> ExportJob:
|
|||||||
sub_overlays=sub_overlays, bgm_name=bgm_name, bgm_volume=(bgm_track.volume / 100.0) if bgm_track else 1.0,
|
sub_overlays=sub_overlays, bgm_name=bgm_name, bgm_volume=(bgm_track.volume / 100.0) if bgm_track else 1.0,
|
||||||
has_audio=has_audio, fps=output_fps, voice_overlays=voice_overlays,
|
has_audio=has_audio, fps=output_fps, voice_overlays=voice_overlays,
|
||||||
)
|
)
|
||||||
proc = subprocess.run(command, cwd=str(tmp), capture_output=True)
|
# 加超时:ffmpeg 卡死(资源不足/被 OOM 杀)时不能让任务永久停在 RUNNING、前端无限转圈;
|
||||||
|
# 超时即按失败收尾,前端能看到「导出失败」而非一直卡。15 分钟足够正常 60s 成片。
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(command, cwd=str(tmp), capture_output=True, timeout=900)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
raise RuntimeError("ffmpeg 导出超时(15 分钟未完成),通常是服务器资源不足或视频过大,请重试或联系运维扩容")
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise RuntimeError(f"ffmpeg export failed: {proc.stderr.decode('utf-8', 'ignore')[-1200:]}")
|
raise RuntimeError(f"ffmpeg export failed: {proc.stderr.decode('utf-8', 'ignore')[-1200:]}")
|
||||||
output_path = tmp / "output.mp4"
|
output_path = tmp / "output.mp4"
|
||||||
|
|||||||
@@ -371,6 +371,15 @@ class ProjectViewSet(TeamScopedViewSetMixin, ModelViewSet):
|
|||||||
duration_ms=segment.target_duration_seconds * 1000,
|
duration_ms=segment.target_duration_seconds * 1000,
|
||||||
)
|
)
|
||||||
start_ms += segment.target_duration_seconds * 1000
|
start_ms += segment.target_duration_seconds * 1000
|
||||||
|
# 清理僵尸任务:后台线程跑导出时若 Pod 被重启/OOM 杀掉,旧任务会永远停在 RUNNING、
|
||||||
|
# 前端无限轮询。新建前把本时间线超过 5 分钟仍未结束的旧任务标失败,避免界面一直转圈。
|
||||||
|
from django.utils import timezone
|
||||||
|
from datetime import timedelta
|
||||||
|
ExportJob.objects.filter(
|
||||||
|
timeline=timeline,
|
||||||
|
status__in=[ExportJob.Status.QUEUED, ExportJob.Status.RUNNING],
|
||||||
|
updated_at__lt=timezone.now() - timedelta(minutes=5),
|
||||||
|
).update(status=ExportJob.Status.FAILED, error_message="任务中断(服务重启或超时),已自动失败,请重新导出")
|
||||||
export_job = create_export_job(timeline=timeline, user=request.user)
|
export_job = create_export_job(timeline=timeline, user=request.user)
|
||||||
stage, _ = ProjectStage.objects.get_or_create(project=project, stage=ProjectStage.Stage.EXPORT)
|
stage, _ = ProjectStage.objects.get_or_create(project=project, stage=ProjectStage.Stage.EXPORT)
|
||||||
stage.status = ProjectStage.Status.RUNNING
|
stage.status = ProjectStage.Status.RUNNING
|
||||||
|
|||||||
@@ -51,11 +51,13 @@ spec:
|
|||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
memory: "256Mi"
|
memory: "512Mi"
|
||||||
cpu: "100m"
|
cpu: "250m"
|
||||||
limits:
|
limits:
|
||||||
memory: "768Mi"
|
# 视频导出在本容器后台线程跑 ffmpeg,编码 1080P 竖屏 + 多路输入 + 字幕/配音混音
|
||||||
cpu: "1000m"
|
# 内存峰值可达 1.5GB,原 768Mi 必被 OOMKill(导出卡 35%、Pod 重启)。提到 2Gi 给足空间。
|
||||||
|
memory: "2Gi"
|
||||||
|
cpu: "1500m"
|
||||||
---
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Service
|
kind: Service
|
||||||
|
|||||||
Reference in New Issue
Block a user