fix(deploy): migrate 跨 pod 串行化 + rescale 迁移幂等守卫(测试环境×10⁶事故复盘)

事故:连推 3 commit 触发 3 轮 rolling 部署,多个新 pod 并发跑 migrate 且崩溃重跑,
积分 ×10 rescale 被交错重放——ai.0025 执行 6 次(单价/任务计价 ×10⁶),accounts.0008
执行 2 次(限额 ×100),billing.0003/0004 从未完成(django_migrations 漏记录)。
测试库数据已按精确倍率手工修复并补记迁移记录(备份于本机)。

两层防复发:
1. docker-entrypoint 用 MySQL GET_LOCK('airshelf_migrate') 串行化 migrate,
   后到 pod 等锁,拿到时迁移已被记录 → 自然 no-op;
2. 三个 rescale 迁移加 airshelf_rescale_marker 幂等标记(与数据变更同事务提交):
   记录丢失/崩溃重跑时,标记在 → 跳过,不会重复 ×10。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
zyc
2026-07-07 11:25:40 +08:00
co-authored by Claude Fable 5
parent d466bd60c6
commit 2fcd6d996c
4 changed files with 133 additions and 11 deletions
@@ -20,10 +20,41 @@ REVERSE_STATEMENTS = [
]
# 幂等守卫(2026-07-07 测试环境事故复盘):并发/崩溃后的 migrate 重跑会把 ×10 重复应用
# (事故中本迁移被交错重放,单价被乘到 ×10⁶)。标记行与数据变更同一事务提交:
# 上次成功 → 标记在 → 跳过;上次崩溃回滚 → 标记不在 → 安全重放。
MARKER = "ai.0025_points_rescale"
_MARKER_DDL = "CREATE TABLE IF NOT EXISTS airshelf_rescale_marker (name varchar(80) NOT NULL PRIMARY KEY)"
def _run(statements):
def apply(apps, schema_editor):
with transaction.atomic(using=schema_editor.connection.alias):
with schema_editor.connection.cursor() as cursor:
conn = schema_editor.connection
with conn.cursor() as cursor: # DDL 幂等,MySQL 隐式提交故放事务外
cursor.execute(_MARKER_DDL)
with transaction.atomic(using=conn.alias):
with conn.cursor() as cursor:
cursor.execute("SELECT COUNT(*) FROM airshelf_rescale_marker WHERE name = %s", [MARKER])
if cursor.fetchone()[0]:
return # 已应用过(django_migrations 记录丢失/并发重跑),幂等跳过
cursor.execute("INSERT INTO airshelf_rescale_marker (name) VALUES (%s)", [MARKER])
for sql in statements:
cursor.execute(sql)
return apply
def _run_reverse(statements):
def apply(apps, schema_editor):
conn = schema_editor.connection
with conn.cursor() as cursor:
cursor.execute(_MARKER_DDL)
with transaction.atomic(using=conn.alias):
with conn.cursor() as cursor:
cursor.execute("SELECT COUNT(*) FROM airshelf_rescale_marker WHERE name = %s", [MARKER])
if not cursor.fetchone()[0]:
return # 未应用过或已回滚,无需反向
cursor.execute("DELETE FROM airshelf_rescale_marker WHERE name = %s", [MARKER])
for sql in statements:
cursor.execute(sql)
@@ -32,4 +63,4 @@ def _run(statements):
class Migration(migrations.Migration):
dependencies = [("ai", "0024_aitask_base_cost")]
operations = [migrations.RunPython(_run(RESCALE_STATEMENTS), _run(REVERSE_STATEMENTS))]
operations = [migrations.RunPython(_run(RESCALE_STATEMENTS), _run_reverse(REVERSE_STATEMENTS))]