video-workflow/src/app/services/reference-video-prompt-planner.service.ts

import { Injectable } from '@angular/core';
import { Observable, from, of } from 'rxjs';
import { catchError, map, switchMap } from 'rxjs/operators';
import { JimengAspectRatio } from './jimeng.service';
import { LlmService } from './llm.service';

export interface ReferenceImageBrief {
  mainSubject: string;
  identityFeatures: string[];
  visualStyle: string;
  colorsAndLighting: string;
  composition: string;
  objects: string[];
  doNotChange: string[];
}

export interface ReferenceVideoPlanQualityHints {
  composition: string;
  style: string;
  lighting: string;
  aspectRatioSuggestion: JimengAspectRatio;
}

export interface ReferenceVideoPromptPlan {
  intentSummary: string;
  referenceKeep: string[];
  imagePrompt: string;
  videoPrompt: string;
  negativePrompt: string;
  qualityHints: ReferenceVideoPlanQualityHints;
  riskWarnings: string[];
  imageBrief: ReferenceImageBrief;
  source: 'llm' | 'fallback';
}

export interface ReferenceVideoPromptPlanInput {
  referenceImageUrl: string;
  userPrompt: string;
  aspect: JimengAspectRatio;
}

const ASPECTS: JimengAspectRatio[] = ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9'];

@Injectable({ providedIn: 'root' })
export class ReferenceVideoPromptPlannerService {
  constructor(private llm: LlmService) {}

  plan(input: ReferenceVideoPromptPlanInput): Observable<ReferenceVideoPromptPlan> {
    const referenceImageUrl = String(input.referenceImageUrl || '').trim();
    const userPrompt = String(input.userPrompt || '').trim();
    if (!referenceImageUrl) {
      throw new Error('缺少参考图，无法规划参考图生成视频');
    }
    if (!userPrompt) {
      throw new Error('缺少提示词，无法规划参考图生成视频');
    }

    return from(this.llm.urlToBase64(referenceImageUrl)).pipe(
      switchMap(({ base64, mimeType }) => this.llm.analyzeImage(
        base64,
        mimeType,
        this.imageAnalysisPrompt(),
        {
          model: 'gemini-2.5-flash',
          generationConfig: { temperature: 0.25, maxOutputTokens: 1600 },
        },
      )),
      map((text) => this.parseImageBrief(text)),
      switchMap((brief) => this.llm.askWithSystemDetailed(
        this.plannerSystemPrompt(),
        this.plannerUserPrompt(brief, userPrompt, input.aspect),
        { model: 'gpt-4o-mini', temperature: 0.35, max_tokens: 1800 },
      ).pipe(
        map((result) => this.parsePlan(result.content, brief, input.aspect)),
        catchError((error) => of(this.fallbackPlan(brief, userPrompt, input.aspect, error))),
      )),
      catchError((error) => {
        const emptyBrief = this.emptyBrief();
        return of(this.fallbackPlan(emptyBrief, userPrompt, input.aspect, error));
      }),
    );
  }

  private imageAnalysisPrompt(): string {
    return `请分析这张参考图，输出严格的 JSON，不要 markdown，不要解释。
{
  "mainSubject": "主体是什么，30字内",
  "identityFeatures": ["必须保留的主体特征，每条20字内"],
  "visualStyle": "画风/摄影风格/质感，30字内",
  "colorsAndLighting": "色彩和光线，40字内",
  "composition": "构图、景别、视角，40字内",
  "objects": ["重要物体"],
  "doNotChange": ["不应改变的内容"]
}
只描述图中可见内容，不要推测品牌、身份或不可见信息。`;
  }

  private plannerSystemPrompt(): string {
    return `你是“参考图生视频”的提示词规划器。
任务：根据参考图摘要和用户需求，输出严格 JSON，用于两段式生成。
第一段 imagePrompt 给图片生成模型：参考图只是视觉参考，生成一张新的、适合作为视频首帧的图片。
第二段 videoPrompt 给图生视频模型：基于新首帧生成自然动态视频。
规则：
1. imagePrompt 必须包含“以参考图为视觉参考”和“不要直接复制原图构图”。
2. imagePrompt 必须说明保留参考图核心主体、关键外观、材质、风格或构图关系。
3. videoPrompt 只描述主体动作、环境变化、镜头运动和一致性约束。
4. videoPrompt 不要引入 imagePrompt 没有建立的主体、场景或风格。
5. negativePrompt 写成一句中文，包含身份/主体/产品形态/水印文字限制。
6. 不要输出 markdown，不要解释，只输出 JSON。
7. 输出字段必须完整。
JSON 结构：
{
  "intentSummary": "",
  "referenceKeep": [],
  "imagePrompt": "",
  "videoPrompt": "",
  "negativePrompt": "",
  "qualityHints": {
    "composition": "",
    "style": "",
    "lighting": "",
    "aspectRatioSuggestion": "16:9"
  },
  "riskWarnings": []
}`;
  }

  private plannerUserPrompt(brief: ReferenceImageBrief, userPrompt: string, aspect: JimengAspectRatio): string {
    return [
      `【参考图摘要】${JSON.stringify(brief)}`,
      `【用户需求】${userPrompt}`,
      `【当前画面比例】${aspect}`,
      '请输出严格 JSON。',
    ].join('\n');
  }

  private parseImageBrief(text: string): ReferenceImageBrief {
    const parsed = this.parseJsonObject(text);
    return {
      mainSubject: this.clean(parsed['mainSubject']),
      identityFeatures: this.cleanList(parsed['identityFeatures']).slice(0, 8),
      visualStyle: this.clean(parsed['visualStyle']),
      colorsAndLighting: this.clean(parsed['colorsAndLighting']),
      composition: this.clean(parsed['composition']),
      objects: this.cleanList(parsed['objects']).slice(0, 8),
      doNotChange: this.cleanList(parsed['doNotChange']).slice(0, 8),
    };
  }

  private parsePlan(text: string, brief: ReferenceImageBrief, fallbackAspect: JimengAspectRatio): ReferenceVideoPromptPlan {
    const parsed = this.parseJsonObject(text);
    const hints = parsed['qualityHints'] && typeof parsed['qualityHints'] === 'object'
      ? parsed['qualityHints'] as Record<string, unknown>
      : {};
    const plan: ReferenceVideoPromptPlan = {
      intentSummary: this.clean(parsed['intentSummary']) || '参考图生成视频',
      referenceKeep: this.cleanList(parsed['referenceKeep']).slice(0, 8),
      imagePrompt: this.clean(parsed['imagePrompt']),
      videoPrompt: this.clean(parsed['videoPrompt']),
      negativePrompt: this.clean(parsed['negativePrompt']),
      qualityHints: {
        composition: this.clean(hints['composition']),
        style: this.clean(hints['style']),
        lighting: this.clean(hints['lighting']),
        aspectRatioSuggestion: this.normalizeAspect(hints['aspectRatioSuggestion'], fallbackAspect),
      },
      riskWarnings: this.cleanList(parsed['riskWarnings']).slice(0, 6),
      imageBrief: brief,
      source: 'llm',
    };

    if (!plan.imagePrompt || !plan.videoPrompt) {
      throw new Error('LLM 规划缺少 imagePrompt 或 videoPrompt');
    }
    if (!plan.imagePrompt.includes('参考图')) {
      plan.imagePrompt = `以参考图为视觉参考，${plan.imagePrompt}`;
    }
    if (!plan.imagePrompt.includes('不要直接复制原图构图')) {
      plan.imagePrompt = `${plan.imagePrompt}，但不要直接复制原图构图。`;
    }
    return plan;
  }

  private fallbackPlan(
    brief: ReferenceImageBrief,
    userPrompt: string,
    aspect: JimengAspectRatio,
    error: unknown,
  ): ReferenceVideoPromptPlan {
    const keep = [
      brief.mainSubject ? `保留${brief.mainSubject}` : '',
      ...brief.identityFeatures.map((item) => `保留${item}`),
      ...brief.doNotChange.map((item) => `不要改变${item}`),
    ].filter(Boolean).slice(0, 8);
    const keepText = keep.length
      ? keep.join('，')
      : '保留参考图中的核心主体、主要外观、色彩关系和风格质感';
    const styleText = [brief.visualStyle, brief.colorsAndLighting, brief.composition].filter(Boolean).join('，');
    const message = error instanceof Error ? error.message : String(error || 'LLM 规划失败');

    return {
      intentSummary: userPrompt.slice(0, 40) || '参考图生成视频',
      referenceKeep: keep,
      imagePrompt: [
        '以参考图为视觉参考',
        keepText,
        '但不要直接复制原图构图',
        `根据用户需求生成一张清晰、主体完整、适合作为视频首帧的高清图片：${userPrompt}`,
        styleText ? `参考视觉风格：${styleText}` : '',
        '画面完整，无文字水印，无畸形结构。',
      ].filter(Boolean).join('，'),
      videoPrompt: [
        '基于首帧画面生成自然动态视频',
        userPrompt,
        '保持主体外观、场景风格和光线一致，动作自然流畅，不要改变主体身份，不要新增无关物体。',
      ].filter(Boolean).join('。'),
      negativePrompt: '不要改变主体身份、产品形态、关键结构，不要出现水印、字幕、乱码文字或无关物体。',
      qualityHints: {
        composition: brief.composition || '主体完整，适合作为视频首帧',
        style: brief.visualStyle || '高质量视觉风格',
        lighting: brief.colorsAndLighting || '自然统一的光线',
        aspectRatioSuggestion: aspect,
      },
      riskWarnings: [`已使用保守兜底规划：${message}`],
      imageBrief: brief,
      source: 'fallback',
    };
  }

  private parseJsonObject(text: string): Record<string, unknown> {
    const raw = String(text || '').trim();
    const cleaned = raw
      .replace(/```(?:json)?/gi, '')
      .replace(/```/g, '')
      .trim();
    const firstBrace = cleaned.indexOf('{');
    const lastBrace = cleaned.lastIndexOf('}');
    if (firstBrace < 0 || lastBrace <= firstBrace) {
      throw new Error(`LLM 未返回 JSON 对象：${raw.slice(0, 200)}`);
    }
    const parsed = JSON.parse(cleaned.slice(firstBrace, lastBrace + 1));
    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
      throw new Error('LLM 返回的 JSON 不是对象');
    }
    return parsed as Record<string, unknown>;
  }

  private emptyBrief(): ReferenceImageBrief {
    return {
      mainSubject: '',
      identityFeatures: [],
      visualStyle: '',
      colorsAndLighting: '',
      composition: '',
      objects: [],
      doNotChange: [],
    };
  }

  private clean(value: unknown): string {
    return String(value ?? '').replace(/\s+/g, ' ').trim();
  }

  private cleanList(value: unknown): string[] {
    if (!Array.isArray(value)) return [];
    return value.map((item) => this.clean(item)).filter(Boolean);
  }

  private normalizeAspect(value: unknown, fallback: JimengAspectRatio): JimengAspectRatio {
    const raw = this.clean(value) as JimengAspectRatio;
    return ASPECTS.includes(raw) ? raw : fallback;
  }
}