feat(core): AI 生成 Agent 化 — 多模型流式脚本 agent + 可插拔 Provider + gpt-image-2 参考图 + 模特库
- 后端·可插拔 Provider 层:通用 OpenAICompatibleProvider(tokenssr 等中转站,base_url+api_key,零改代码换站)+ ModelProvider.api_key - 后端·脚本 agent:结构化 ScriptDraft 契约 + 加载电商 skill + 出稿/改稿一体对话 agent(3模式/多模型)+ 流式 SSE 端点(DRF SSE renderer) - 后端·图像:gpt-image-2 参考图出图 + 故事板 @图1@图2@图3 多锚点合成(锁脸锁商品);Seedance 打开 generate_audio - 后端·模特库:gpt-image-2 生成器(9:16氛围图→16:9白底三视图)+ seed_demo_models 管理命令 - DB·迁移:tokenssr 中转站 + 多模型 seed(豆包/GPT-5.5/Gemini + gpt-image-2);ScriptSegment 结构化字段 - 前端·脚本趴:接真 SSE(工具卡 + 思考流)+ 模型下拉 + 3模式 + 改稿;agentScriptStream - skills/ecommerce-video-script 电商脚本技能(运行时依赖) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
seaislee1209
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
a0ffb6fc8e
commit
6464001f84
@@ -7,12 +7,18 @@ from datetime import timedelta
|
||||
from decimal import Decimal
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.ai.models import AITask, ModelConfig
|
||||
from apps.ai.providers import TtsNotConfigured, VolcanoArkProvider, VolcanoTtsProvider, YunqiProvider
|
||||
from apps.ai.providers import (
|
||||
OpenAICompatibleProvider,
|
||||
TtsNotConfigured,
|
||||
VolcanoArkProvider,
|
||||
VolcanoTtsProvider,
|
||||
)
|
||||
from apps.assets.models import Asset, AssetFile
|
||||
from apps.assets.storage import TosStorage
|
||||
from apps.billing.services.ledger import charge_reserved_credit, release_credit, reserve_credit
|
||||
@@ -39,11 +45,39 @@ def get_default_model(capability: str) -> ModelConfig:
|
||||
)
|
||||
|
||||
|
||||
# 火山官方直连(SeeDream 生图 / Seedance 视频 / 豆包文本)走 ARK SDK;其余 provider 一律
|
||||
# 视为「OpenAI 兼容中转站」走通用适配器。加/换中转站 = DB 加一行 ModelProvider,零改代码。
|
||||
# 注意:DB 里火山 provider 实际命名为 "volcengine"(豆包),必须包含,否则会被错路由到中转站。
|
||||
OFFICIAL_DIRECT_PROVIDERS = {"volcengine", "volcano", "ark", "volcano_ark"}
|
||||
|
||||
|
||||
def resolve_provider_credentials(provider) -> tuple[str | None, str | None]:
|
||||
"""解析中转站凭证。可插拔顺序:DB(ModelProvider.base_url/api_key)优先 → settings(.env)回退。
|
||||
两者都不写死;换站只改 DB 这一行,或改 .env 对应项。"""
|
||||
base_url = (provider.base_url or "").strip() or settings.PROVIDER_BASE_URLS.get(provider.name)
|
||||
api_key = (getattr(provider, "api_key", "") or "").strip() or settings.PROVIDER_KEYS.get(provider.name)
|
||||
return (base_url or None), (api_key or None)
|
||||
|
||||
|
||||
def build_provider(model_config: ModelConfig):
|
||||
"""按 provider.name 分流:火山官方直连 → VolcanoArkProvider;其余 → 通用 OpenAICompatibleProvider。"""
|
||||
provider = model_config.provider
|
||||
if provider.name in OFFICIAL_DIRECT_PROVIDERS:
|
||||
return VolcanoArkProvider(base_url=provider.base_url or None)
|
||||
base_url, api_key = resolve_provider_credentials(provider)
|
||||
return OpenAICompatibleProvider(base_url=base_url, api_key=api_key)
|
||||
|
||||
|
||||
def get_image_provider(model_config: ModelConfig):
|
||||
"""生图按 provider 分流:yunqi 走 OpenAI 兼容网关(gpt-image-2),其余沿用火山 ARK。"""
|
||||
if model_config.provider.name == "yunqi":
|
||||
return YunqiProvider(base_url=model_config.provider.base_url or None)
|
||||
return VolcanoArkProvider(base_url=model_config.provider.base_url or None)
|
||||
return build_provider(model_config)
|
||||
|
||||
|
||||
def get_text_provider(model_config: ModelConfig):
|
||||
return build_provider(model_config)
|
||||
|
||||
|
||||
def get_video_provider(model_config: ModelConfig):
|
||||
return build_provider(model_config)
|
||||
|
||||
|
||||
def estimate_cost(model_config: ModelConfig) -> Decimal:
|
||||
@@ -179,7 +213,7 @@ def extract_cast_and_scenes(*, project, user, content: str) -> dict:
|
||||
task.submitted_at = timezone.now()
|
||||
task.save(update_fields=["status", "submitted_at", "updated_at"])
|
||||
|
||||
provider = VolcanoArkProvider(base_url=model_config.provider.base_url or None)
|
||||
provider = build_provider(model_config)
|
||||
response = provider.chat_completion(model=model_config.name, endpoint=model_config.endpoint, messages=messages)
|
||||
text = provider.extract_text(response)
|
||||
|
||||
@@ -289,7 +323,7 @@ def generate_project_script(*, project, user, user_prompt: str, selling_point_id
|
||||
task.submitted_at = timezone.now()
|
||||
task.save(update_fields=["status", "submitted_at", "updated_at"])
|
||||
|
||||
provider = VolcanoArkProvider(base_url=model_config.provider.base_url or None)
|
||||
provider = build_provider(model_config)
|
||||
response = provider.chat_completion(model=model_config.name, endpoint=model_config.endpoint, messages=messages)
|
||||
content = provider.extract_text(response)
|
||||
|
||||
@@ -422,7 +456,7 @@ def regenerate_script_segment(*, project, user, segment, instruction: str = "")
|
||||
task.submitted_at = timezone.now()
|
||||
task.save(update_fields=["status", "submitted_at", "updated_at"])
|
||||
|
||||
provider = VolcanoArkProvider(base_url=model_config.provider.base_url or None)
|
||||
provider = build_provider(model_config)
|
||||
response = provider.chat_completion(model=model_config.name, endpoint=model_config.endpoint, messages=messages)
|
||||
content = provider.extract_text(response)
|
||||
narration, visual = parse_segment_fields(content)
|
||||
@@ -653,6 +687,65 @@ def submit_storyboard(*, project, user, prompt: str = "") -> StoryboardVersion:
|
||||
return version
|
||||
|
||||
|
||||
_ENTITY_TYPE_CN = {"character": "角色", "scene": "场景", "product": "商品"}
|
||||
|
||||
|
||||
def _storyboard_reference_images(project, segment) -> list[dict]:
|
||||
"""按本镜 entity_refs 取参考图(角色/场景/商品的已采用基础资产),供 gpt-image-2 多图合成 @图N。
|
||||
返回 [{url,label,type}],最多 4 张;无匹配时兜底商品组。依赖脚本 agent 落进 metadata 的 script_entities。"""
|
||||
entities = {
|
||||
e.get("id"): e
|
||||
for e in (project.metadata or {}).get("script_entities", [])
|
||||
if isinstance(e, dict)
|
||||
}
|
||||
kind_by_type = {
|
||||
"character": BaseAssetGroup.Kind.PERSON,
|
||||
"scene": BaseAssetGroup.Kind.SCENE,
|
||||
"product": BaseAssetGroup.Kind.PRODUCT,
|
||||
}
|
||||
groups = list(project.base_asset_groups.filter(adopted_asset__isnull=False).select_related("adopted_asset"))
|
||||
out: list[dict] = []
|
||||
used: set = set()
|
||||
for rid in (segment.entity_refs or []):
|
||||
ent = entities.get(rid)
|
||||
if not ent:
|
||||
continue
|
||||
kind = kind_by_type.get(ent.get("type"))
|
||||
name = (ent.get("name") or "").strip()
|
||||
match = next(
|
||||
(g for g in groups if g.kind == kind and (g.metadata or {}).get("label", "").strip() == name and g.id not in used),
|
||||
None,
|
||||
) or next((g for g in groups if g.kind == kind and g.id not in used), None)
|
||||
if match:
|
||||
used.add(match.id)
|
||||
url = _asset_preview_url(match.adopted_asset)
|
||||
if url:
|
||||
out.append({"url": url, "label": name or _ENTITY_TYPE_CN.get(ent.get("type"), "参考"), "type": ent.get("type")})
|
||||
if len(out) >= 4:
|
||||
break
|
||||
if not out:
|
||||
pg = next((g for g in groups if g.kind == BaseAssetGroup.Kind.PRODUCT), None)
|
||||
if pg:
|
||||
url = _asset_preview_url(pg.adopted_asset)
|
||||
if url:
|
||||
out.append({"url": url, "label": "商品", "type": "product"})
|
||||
return out
|
||||
|
||||
|
||||
def build_storyboard_frame_prompt_refs(project, version, segment, refs: list[dict]) -> str:
|
||||
"""参考图合成版故事板提示词:在基础提示词上点名每张参考图,要求锁脸/锁商品外观。"""
|
||||
base = build_storyboard_frame_prompt(project, version, segment)
|
||||
if not refs:
|
||||
return base
|
||||
ref_lines = "；".join(
|
||||
f"参考图{i + 1}={r['label']}({_ENTITY_TYPE_CN.get(r.get('type'), '参考')})" for i, r in enumerate(refs)
|
||||
)
|
||||
return (
|
||||
f"{base}\n参考图对应:{ref_lines}。"
|
||||
"请严格保持各参考图中角色的同一张脸、同一商品的外观与配色,按本镜画面重新构图合成为一张电商竖屏分镜图。"
|
||||
)
|
||||
|
||||
|
||||
def _storyboard_frame_worker(task_id, version_id, segment_id, user_id) -> None:
|
||||
"""后台线程:真正调 ARK 生成一帧故事板图并落库。每次 poll 不阻塞在此——HTTP 永远秒回。"""
|
||||
import threading # noqa: F401 — 仅标注此函数运行在独立线程
|
||||
@@ -672,12 +765,26 @@ def _storyboard_frame_worker(task_id, version_id, segment_id, user_id) -> None:
|
||||
task.save(update_fields=["status", "updated_at"])
|
||||
try:
|
||||
provider = get_image_provider(model_config)
|
||||
frame_prompt = task.request_payload.get("prompt") or build_storyboard_frame_prompt(project, version, segment)
|
||||
response = provider.image_generation(
|
||||
model=model_config.name,
|
||||
endpoint=model_config.endpoint,
|
||||
prompt=frame_prompt,
|
||||
)
|
||||
refs = _storyboard_reference_images(project, segment)
|
||||
ref_urls = [r["url"] for r in refs]
|
||||
if ref_urls and hasattr(provider, "image_edit"):
|
||||
# gpt-image-2 多图参考:把角色/场景/商品合成进本镜(@图1@图2@图3),锁脸锁外观保一致
|
||||
frame_prompt = task.request_payload.get("prompt") or build_storyboard_frame_prompt_refs(
|
||||
project, version, segment, refs
|
||||
)
|
||||
response = provider.image_edit(
|
||||
model=model_config.name,
|
||||
prompt=frame_prompt,
|
||||
images=ref_urls,
|
||||
size="1024x1536",
|
||||
)
|
||||
else:
|
||||
frame_prompt = task.request_payload.get("prompt") or build_storyboard_frame_prompt(project, version, segment)
|
||||
response = provider.image_generation(
|
||||
model=model_config.name,
|
||||
endpoint=model_config.endpoint,
|
||||
prompt=frame_prompt,
|
||||
)
|
||||
media = provider.extract_first_media_url(response)
|
||||
# 注意顺序:task 是 poll 端的「占位锁」,必须等帧真正落库后才置 SUCCEEDED。
|
||||
# 旧实现先置 SUCCEEDED 再上传 TOS(数秒)最后建帧,中间窗口 poll 会判「无在途且帧缺失」
|
||||
@@ -885,7 +992,7 @@ def submit_video_segment(*, video_segment: VideoSegment, user, prompt: str) -> V
|
||||
},
|
||||
)
|
||||
try:
|
||||
provider = VolcanoArkProvider(base_url=model_config.provider.base_url or None)
|
||||
provider = build_provider(model_config)
|
||||
try:
|
||||
response = provider.create_video_task(
|
||||
model=model_config.name,
|
||||
@@ -956,7 +1063,7 @@ def poll_video_segment(*, video_segment: VideoSegment, user) -> VideoSegmentVers
|
||||
if ai_task.status in (AITask.Status.FAILED, AITask.Status.CANCELLED):
|
||||
return None
|
||||
|
||||
provider = VolcanoArkProvider(base_url=ai_task.model_config.provider.base_url or None)
|
||||
provider = build_provider(ai_task.model_config)
|
||||
response = provider.poll_video_task(endpoint=ai_task.model_config.endpoint, provider_task_id=ai_task.provider_task_id)
|
||||
remote_status = response.get("status")
|
||||
if remote_status in {"queued", "running", "processing"}:
|
||||
|
||||
Reference in New Issue
Block a user