完成极速成品和脚本优化
This commit is contained in:
@@ -27,13 +27,39 @@ class WorkerUnavailable(APIException):
|
||||
_OK_TTL = 30.0
|
||||
_FAIL_TTL = 5.0
|
||||
_cache = {"ok": False, "expires": 0.0}
|
||||
# True / False / None(inspect 没答上来,不当成「未部署」)
|
||||
_registered_task_cache: dict[str, tuple[bool | None, float]] = {}
|
||||
|
||||
|
||||
def _task_name_listed(task_name: str, task_names) -> bool:
|
||||
"""Celery inspect.registered() 可能给全名、短名,或带 [rate] 后缀。"""
|
||||
short = task_name.rsplit(".", 1)[-1]
|
||||
for raw in task_names or []:
|
||||
name = str(raw).split("[", 1)[0].strip()
|
||||
if name in {task_name, short} or name.endswith("." + short):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _inspect_task_registered(task_name: str) -> bool | None:
|
||||
"""True 已注册 / False 明确没有 / None 问不到(超时或空应答)。"""
|
||||
try:
|
||||
from airshelf.celery import app as celery_app
|
||||
|
||||
registered = celery_app.control.inspect(timeout=2.5).registered()
|
||||
except Exception: # noqa: BLE001 — inspect 失败不等于任务没部署
|
||||
return None
|
||||
if not registered:
|
||||
return None
|
||||
return any(_task_name_listed(task_name, names) for names in registered.values())
|
||||
|
||||
|
||||
def _ping_workers() -> bool:
|
||||
from airshelf.celery import app as celery_app
|
||||
|
||||
# limit=1:收到第一个 worker 回包立即返回,不傻等满 timeout
|
||||
return bool(celery_app.control.ping(timeout=1.0, limit=1))
|
||||
# limit=1:收到第一个 worker 回包立即返回。本机连火山 Redis 往返常超过 1s,
|
||||
# 超时会被误判成 worker 未运行,生成入口整页锁死。
|
||||
return bool(celery_app.control.ping(timeout=3.0, limit=1))
|
||||
|
||||
|
||||
def celery_worker_available() -> bool:
|
||||
@@ -55,3 +81,29 @@ def require_worker() -> None:
|
||||
"""生成类提交入口的前置检查:无 worker 直接 503,不让任务出门。"""
|
||||
if not celery_worker_available():
|
||||
raise WorkerUnavailable()
|
||||
|
||||
|
||||
def require_worker_task(task_name: str) -> None:
|
||||
"""确认在线 Worker 已加载指定任务,避免新功能被旧进程静默丢弃。
|
||||
|
||||
Worker 在线不代表它已经加载了最新 tasks.py。Celery 收到未注册任务会直接丢弃消息,
|
||||
页面只能看到永久「生成中」。提交极速成片等新编排前,额外确认任务名已注册。
|
||||
|
||||
inspect 超时/空应答不能当成「未部署」——本机常见 ping 通、inspect 1s 问不到,
|
||||
会误报「正在升级」。只有明确看到已注册列表且没有该任务时才拦截。
|
||||
"""
|
||||
if getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False):
|
||||
return
|
||||
require_worker()
|
||||
now = time.monotonic()
|
||||
cached = _registered_task_cache.get(task_name)
|
||||
if cached and now < cached[1]:
|
||||
available = cached[0]
|
||||
else:
|
||||
available = _inspect_task_registered(task_name)
|
||||
ttl = _OK_TTL if available else _FAIL_TTL
|
||||
_registered_task_cache[task_name] = (available, now + ttl)
|
||||
if available is False:
|
||||
exc = WorkerUnavailable()
|
||||
exc.detail = "极速成片后台任务尚未加载,请重启 Celery worker 后再试。"
|
||||
raise exc
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from django.test import SimpleTestCase, override_settings
|
||||
from rest_framework.exceptions import APIException
|
||||
|
||||
from apps.common import celery_health
|
||||
from apps.common.celery_health import require_worker_task
|
||||
|
||||
|
||||
class RequireWorkerTaskTests(SimpleTestCase):
|
||||
def setUp(self):
|
||||
celery_health._registered_task_cache.clear()
|
||||
celery_health._cache["expires"] = 0.0
|
||||
|
||||
def tearDown(self):
|
||||
celery_health._registered_task_cache.clear()
|
||||
celery_health._cache["expires"] = 0.0
|
||||
|
||||
def test_registered_name_accepts_short_name_and_rate_suffix(self):
|
||||
listed = ["apps.projects.tasks.advance_quick_create_task[1/s]"]
|
||||
self.assertTrue(
|
||||
celery_health._task_name_listed("apps.projects.tasks.advance_quick_create_task", listed)
|
||||
)
|
||||
self.assertTrue(celery_health._task_name_listed("advance_quick_create_task", listed))
|
||||
self.assertFalse(celery_health._task_name_listed("apps.projects.tasks.missing_task", listed))
|
||||
|
||||
@override_settings(CELERY_TASK_ALWAYS_EAGER=False)
|
||||
@patch("apps.common.celery_health.celery_worker_available", return_value=True)
|
||||
@patch("apps.common.celery_health._inspect_task_registered", return_value=None)
|
||||
def test_inspect_unknown_does_not_block(self, _inspect, _available):
|
||||
require_worker_task("apps.projects.tasks.advance_quick_create_task")
|
||||
|
||||
@override_settings(CELERY_TASK_ALWAYS_EAGER=False)
|
||||
@patch("apps.common.celery_health.celery_worker_available", return_value=True)
|
||||
@patch("apps.common.celery_health._inspect_task_registered", return_value=False)
|
||||
def test_missing_task_blocks_with_restart_hint(self, _inspect, _available):
|
||||
with self.assertRaises(APIException) as raised:
|
||||
require_worker_task("apps.projects.tasks.advance_quick_create_task")
|
||||
self.assertEqual(raised.exception.status_code, 503)
|
||||
self.assertIn("重启", str(raised.exception.detail))
|
||||
self.assertNotIn("正在升级", str(raised.exception.detail))
|
||||
Reference in New Issue
Block a user