fix(core): 故事板单帧网络超时自动重试 + 在途锁放宽防重复扣费
中转站(tokenssr)偶发 Read timeout 会掐掉单帧,4 镜出 3 张卡住不收尾。 出图调用加瞬时错误有界重试(超时/502/连接重置才重试,审核拦截等确定性失败立即抛)。 配套把 poll 在途锁窗口 3min→12min(可配 STORYBOARD_INFLIGHT_STALE_MINUTES), 覆盖 worker 最坏重试时长,避免重试期间任务被判僵尸 → 同镜重复起线程 + 重复扣费。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -830,9 +830,11 @@ def _is_transient_error(exc: Exception) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _call_image_with_retry(fn, *, attempts: int = 3, base_delay: float = 2.0):
|
||||
def _call_image_with_retry(fn, *, attempts: int = 2, base_delay: float = 2.0):
|
||||
"""对一次出图网络调用做有界重试:仅瞬时错误重试(指数退避),确定性失败立即抛出。
|
||||
出图无副作用(失败=没拿到图),重试安全;成功一次即返回。"""
|
||||
出图无副作用(失败=没拿到图),重试安全;成功一次即返回。
|
||||
attempts 默认 2(一次重试):单次 HTTP 超时上限 300s,2 次≈10min,须 < poll 的「在途锁过期窗口」
|
||||
(STORYBOARD_INFLIGHT_STALE_MINUTES)否则 worker 重试期间任务被判僵尸 → 重复起线程 + 重复扣费。"""
|
||||
import time
|
||||
|
||||
last: Exception | None = None
|
||||
@@ -872,18 +874,23 @@ def _storyboard_frame_worker(task_id, version_id, segment_id, user_id) -> None:
|
||||
# gpt-image-2 多图参考:必须用 refs 版提示词(点名「参考图N=角色/场景/商品」+锁脸锁商品),
|
||||
# 不能复用 request_payload['prompt'](那是建任务时写死的基础提示词,恒为真值会架空一致性约束)。
|
||||
frame_prompt = build_storyboard_frame_prompt_refs(project, version, segment, refs)
|
||||
response = provider.image_edit(
|
||||
model=model_config.name,
|
||||
prompt=frame_prompt,
|
||||
images=ref_urls,
|
||||
size="1024x1536",
|
||||
# 中转站偶发 Read timeout 会掐掉单帧(4 镜出 3 张的根因)→ 瞬时错误有界重试
|
||||
response = _call_image_with_retry(
|
||||
lambda: provider.image_edit(
|
||||
model=model_config.name,
|
||||
prompt=frame_prompt,
|
||||
images=ref_urls,
|
||||
size="1024x1536",
|
||||
)
|
||||
)
|
||||
else:
|
||||
frame_prompt = task.request_payload.get("prompt") or build_storyboard_frame_prompt(project, version, segment)
|
||||
response = provider.image_generation(
|
||||
model=model_config.name,
|
||||
endpoint=model_config.endpoint,
|
||||
prompt=frame_prompt,
|
||||
response = _call_image_with_retry(
|
||||
lambda: provider.image_generation(
|
||||
model=model_config.name,
|
||||
endpoint=model_config.endpoint,
|
||||
prompt=frame_prompt,
|
||||
)
|
||||
)
|
||||
media = provider.extract_first_media_url(response)
|
||||
# 注意顺序:task 是 poll 端的「占位锁」,必须等帧真正落库后才置 SUCCEEDED。
|
||||
@@ -951,10 +958,13 @@ def generate_storyboard_frame(*, project, user) -> dict:
|
||||
# ★ 实测注记(2026-06-10):4 线程并发时当前 Seedream 端点在服务侧排队,单帧 25s→83-105s,
|
||||
# 整版总时长 ≈ 串行(117s)。瓶颈是 ARK 端点并发配额而非本机;并行无额外成本,
|
||||
# 配额提升后自动受益。可用 settings.STORYBOARD_MAX_PARALLEL 调并发(1=回到串行)。
|
||||
# 仅算「近 3 分钟内」的任务:线程意外中断留下的僵尸任务超时后不再占锁,允许重新发起。
|
||||
# 仅算「近 N 分钟内」的任务:线程意外中断留下的僵尸任务超时后不再占锁,允许重新发起。
|
||||
# 窗口须 > worker 最坏运行时长(单帧 HTTP 超时 300s × 重试 2 次 ≈ 10min),否则 worker 正在
|
||||
# 重试时其任务被误判为僵尸 → 同一镜重复起线程 + 重复扣费。故默认 12min(覆盖一次瞬时重试)。
|
||||
from django.conf import settings as dj_settings
|
||||
STORYBOARD_MAX_PARALLEL = int(getattr(dj_settings, "STORYBOARD_MAX_PARALLEL", 4))
|
||||
stale_cutoff = timezone.now() - timedelta(minutes=3)
|
||||
stale_minutes = int(getattr(dj_settings, "STORYBOARD_INFLIGHT_STALE_MINUTES", 12))
|
||||
stale_cutoff = timezone.now() - timedelta(minutes=stale_minutes)
|
||||
inflight_segment_ids = {
|
||||
str(v)
|
||||
for v in AITask.objects.filter(
|
||||
|
||||
Reference in New Issue
Block a user