优化脚本生成卡住问题
This commit is contained in:
@@ -27,6 +27,7 @@ from apps.ai.script_agent import (
|
||||
)
|
||||
from apps.ai.services import (
|
||||
collect_video_review_blockers,
|
||||
reap_stale_script_tasks,
|
||||
create_export_job,
|
||||
generate_base_asset,
|
||||
generate_person_triview,
|
||||
@@ -550,9 +551,17 @@ def _advance_script(job: QuickCreateJob) -> int | None:
|
||||
_enqueue_script(str(job.id))
|
||||
return SCRIPT_POLL_SECONDS
|
||||
|
||||
# 先回收被进程重启丢下的僵尸脚本任务(部署滚动更新最常见),否则下面会一直命中「在途」分支:
|
||||
# 任务行永远停在 SUBMITTED、预扣积分冻着,而本函数底部的 SCRIPT_TIMEOUT 判定根本走不到。
|
||||
reap_stale_script_tasks(project=job.project)
|
||||
latest = _latest_script_task(job.project)
|
||||
if latest is not None and latest.status in _ACTIVE_TASK_STATUSES:
|
||||
started_at = _parse_iso(metadata.get("script_started_at"))
|
||||
# ★ 超时判定必须在这里也做一遍。原来它只写在函数末尾,而末尾只有「查不到在途任务」时才够得着 ——
|
||||
# 任务卡在 SUBMITTED 时永远从这个分支 return,30 分钟上限形同虚设(实测挂了 2.5 小时还在转)。
|
||||
if started_at and timezone.now() - started_at > SCRIPT_TIMEOUT:
|
||||
fail_quick_create(job, "脚本生成超时,请稍后重试或进入专业模式查看")
|
||||
return None
|
||||
elapsed = int((timezone.now() - started_at).total_seconds()) if started_at else 0
|
||||
_save_job(job, progress=min(44, 28 + elapsed // 8), message="正在生成分镜脚本…")
|
||||
return SCRIPT_POLL_SECONDS
|
||||
|
||||
@@ -584,6 +584,30 @@ class QuickCreateCoordinatorTests(TestCase):
|
||||
self.assertIsNone(delay)
|
||||
self.assertEqual(self.job.status, QuickCreateJob.Status.CANCELLED)
|
||||
|
||||
def test_script_stuck_in_flight_does_not_spin_forever(self):
|
||||
"""线上故障复现:11:41 起的脚本任务被部署打断,行永远停在 SUBMITTED,
|
||||
到 14:18 任务监控还是「进行中」。原因是「在途」分支提前 return,
|
||||
函数末尾那个 30 分钟 SCRIPT_TIMEOUT 判定根本走不到。"""
|
||||
task = self._script_task(AITask.Status.SUBMITTED, key="stuck-by-deploy")
|
||||
stale = timezone.now() - timedelta(hours=3)
|
||||
AITask.objects.filter(id=task.id).update(created_at=stale, updated_at=stale)
|
||||
self.job.status = QuickCreateJob.Status.RUNNING
|
||||
self.job.phase = QuickCreateJob.Phase.SCRIPT
|
||||
self.job.metadata = {
|
||||
"script_started": True,
|
||||
"script_started_at": stale.isoformat(),
|
||||
}
|
||||
self.job.save(update_fields=["status", "phase", "metadata", "updated_at"])
|
||||
|
||||
advance_quick_create(str(self.job.id))
|
||||
self.job.refresh_from_db()
|
||||
task.refresh_from_db()
|
||||
|
||||
# 僵尸行必须被收掉(积分解冻),整单落到可恢复的失败态,而不是永远转圈
|
||||
self.assertEqual(task.status, AITask.Status.FAILED)
|
||||
self.assertNotEqual(self.job.status, QuickCreateJob.Status.RUNNING)
|
||||
self.assertNotEqual(self.job.message, "正在生成分镜脚本…")
|
||||
|
||||
@patch("apps.projects.services.quick_create._enqueue_script")
|
||||
def test_recover_requeues_script_when_queue_drops_it(self, enqueue_script):
|
||||
self.job.status = QuickCreateJob.Status.RUNNING
|
||||
|
||||
Reference in New Issue
Block a user