import { Injectable } from '@angular/core'; import { Observable, from, of } from 'rxjs'; import { catchError, map, switchMap } from 'rxjs/operators'; import { JimengAspectRatio } from './jimeng.service'; import { LlmService } from './llm.service'; export interface ReferenceImageBrief { mainSubject: string; identityFeatures: string[]; visualStyle: string; colorsAndLighting: string; composition: string; objects: string[]; doNotChange: string[]; } export interface ReferenceVideoPlanQualityHints { composition: string; style: string; lighting: string; aspectRatioSuggestion: JimengAspectRatio; } export interface ReferenceVideoPromptPlan { intentSummary: string; referenceKeep: string[]; imagePrompt: string; videoPrompt: string; negativePrompt: string; qualityHints: ReferenceVideoPlanQualityHints; riskWarnings: string[]; imageBrief: ReferenceImageBrief; source: 'llm' | 'fallback'; } export interface ReferenceVideoPromptPlanInput { referenceImageUrl: string; userPrompt: string; aspect: JimengAspectRatio; } const ASPECTS: JimengAspectRatio[] = ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9']; @Injectable({ providedIn: 'root' }) export class ReferenceVideoPromptPlannerService { constructor(private llm: LlmService) {} plan(input: ReferenceVideoPromptPlanInput): Observable { const referenceImageUrl = String(input.referenceImageUrl || '').trim(); const userPrompt = String(input.userPrompt || '').trim(); if (!referenceImageUrl) { throw new Error('缺少参考图,无法规划参考图生成视频'); } if (!userPrompt) { throw new Error('缺少提示词,无法规划参考图生成视频'); } return from(this.llm.urlToBase64(referenceImageUrl)).pipe( switchMap(({ base64, mimeType }) => this.llm.analyzeImage( base64, mimeType, this.imageAnalysisPrompt(), { model: 'gemini-2.5-flash', generationConfig: { temperature: 0.25, maxOutputTokens: 1600 }, }, )), map((text) => this.parseImageBrief(text)), switchMap((brief) => this.llm.askWithSystemDetailed( this.plannerSystemPrompt(), this.plannerUserPrompt(brief, userPrompt, input.aspect), { model: 'gpt-4o-mini', temperature: 0.35, max_tokens: 1800 }, ).pipe( map((result) => this.parsePlan(result.content, brief, input.aspect)), catchError((error) => of(this.fallbackPlan(brief, userPrompt, input.aspect, error))), )), catchError((error) => { const emptyBrief = this.emptyBrief(); return of(this.fallbackPlan(emptyBrief, userPrompt, input.aspect, error)); }), ); } private imageAnalysisPrompt(): string { return `请分析这张参考图,输出严格的 JSON,不要 markdown,不要解释。 { "mainSubject": "主体是什么,30字内", "identityFeatures": ["必须保留的主体特征,每条20字内"], "visualStyle": "画风/摄影风格/质感,30字内", "colorsAndLighting": "色彩和光线,40字内", "composition": "构图、景别、视角,40字内", "objects": ["重要物体"], "doNotChange": ["不应改变的内容"] } 只描述图中可见内容,不要推测品牌、身份或不可见信息。`; } private plannerSystemPrompt(): string { return `你是“参考图生视频”的提示词规划器。 任务:根据参考图摘要和用户需求,输出严格 JSON,用于两段式生成。 第一段 imagePrompt 给图片生成模型:参考图只是视觉参考,生成一张新的、适合作为视频首帧的图片。 第二段 videoPrompt 给图生视频模型:基于新首帧生成自然动态视频。 规则: 1. imagePrompt 必须包含“以参考图为视觉参考”和“不要直接复制原图构图”。 2. imagePrompt 必须说明保留参考图核心主体、关键外观、材质、风格或构图关系。 3. videoPrompt 只描述主体动作、环境变化、镜头运动和一致性约束。 4. videoPrompt 不要引入 imagePrompt 没有建立的主体、场景或风格。 5. negativePrompt 写成一句中文,包含身份/主体/产品形态/水印文字限制。 6. 不要输出 markdown,不要解释,只输出 JSON。 7. 输出字段必须完整。 JSON 结构: { "intentSummary": "", "referenceKeep": [], "imagePrompt": "", "videoPrompt": "", "negativePrompt": "", "qualityHints": { "composition": "", "style": "", "lighting": "", "aspectRatioSuggestion": "16:9" }, "riskWarnings": [] }`; } private plannerUserPrompt(brief: ReferenceImageBrief, userPrompt: string, aspect: JimengAspectRatio): string { return [ `【参考图摘要】${JSON.stringify(brief)}`, `【用户需求】${userPrompt}`, `【当前画面比例】${aspect}`, '请输出严格 JSON。', ].join('\n'); } private parseImageBrief(text: string): ReferenceImageBrief { const parsed = this.parseJsonObject(text); return { mainSubject: this.clean(parsed['mainSubject']), identityFeatures: this.cleanList(parsed['identityFeatures']).slice(0, 8), visualStyle: this.clean(parsed['visualStyle']), colorsAndLighting: this.clean(parsed['colorsAndLighting']), composition: this.clean(parsed['composition']), objects: this.cleanList(parsed['objects']).slice(0, 8), doNotChange: this.cleanList(parsed['doNotChange']).slice(0, 8), }; } private parsePlan(text: string, brief: ReferenceImageBrief, fallbackAspect: JimengAspectRatio): ReferenceVideoPromptPlan { const parsed = this.parseJsonObject(text); const hints = parsed['qualityHints'] && typeof parsed['qualityHints'] === 'object' ? parsed['qualityHints'] as Record : {}; const plan: ReferenceVideoPromptPlan = { intentSummary: this.clean(parsed['intentSummary']) || '参考图生成视频', referenceKeep: this.cleanList(parsed['referenceKeep']).slice(0, 8), imagePrompt: this.clean(parsed['imagePrompt']), videoPrompt: this.clean(parsed['videoPrompt']), negativePrompt: this.clean(parsed['negativePrompt']), qualityHints: { composition: this.clean(hints['composition']), style: this.clean(hints['style']), lighting: this.clean(hints['lighting']), aspectRatioSuggestion: this.normalizeAspect(hints['aspectRatioSuggestion'], fallbackAspect), }, riskWarnings: this.cleanList(parsed['riskWarnings']).slice(0, 6), imageBrief: brief, source: 'llm', }; if (!plan.imagePrompt || !plan.videoPrompt) { throw new Error('LLM 规划缺少 imagePrompt 或 videoPrompt'); } if (!plan.imagePrompt.includes('参考图')) { plan.imagePrompt = `以参考图为视觉参考,${plan.imagePrompt}`; } if (!plan.imagePrompt.includes('不要直接复制原图构图')) { plan.imagePrompt = `${plan.imagePrompt},但不要直接复制原图构图。`; } return plan; } private fallbackPlan( brief: ReferenceImageBrief, userPrompt: string, aspect: JimengAspectRatio, error: unknown, ): ReferenceVideoPromptPlan { const keep = [ brief.mainSubject ? `保留${brief.mainSubject}` : '', ...brief.identityFeatures.map((item) => `保留${item}`), ...brief.doNotChange.map((item) => `不要改变${item}`), ].filter(Boolean).slice(0, 8); const keepText = keep.length ? keep.join(',') : '保留参考图中的核心主体、主要外观、色彩关系和风格质感'; const styleText = [brief.visualStyle, brief.colorsAndLighting, brief.composition].filter(Boolean).join(','); const message = error instanceof Error ? error.message : String(error || 'LLM 规划失败'); return { intentSummary: userPrompt.slice(0, 40) || '参考图生成视频', referenceKeep: keep, imagePrompt: [ '以参考图为视觉参考', keepText, '但不要直接复制原图构图', `根据用户需求生成一张清晰、主体完整、适合作为视频首帧的高清图片:${userPrompt}`, styleText ? `参考视觉风格:${styleText}` : '', '画面完整,无文字水印,无畸形结构。', ].filter(Boolean).join(','), videoPrompt: [ '基于首帧画面生成自然动态视频', userPrompt, '保持主体外观、场景风格和光线一致,动作自然流畅,不要改变主体身份,不要新增无关物体。', ].filter(Boolean).join('。'), negativePrompt: '不要改变主体身份、产品形态、关键结构,不要出现水印、字幕、乱码文字或无关物体。', qualityHints: { composition: brief.composition || '主体完整,适合作为视频首帧', style: brief.visualStyle || '高质量视觉风格', lighting: brief.colorsAndLighting || '自然统一的光线', aspectRatioSuggestion: aspect, }, riskWarnings: [`已使用保守兜底规划:${message}`], imageBrief: brief, source: 'fallback', }; } private parseJsonObject(text: string): Record { const raw = String(text || '').trim(); const cleaned = raw .replace(/```(?:json)?/gi, '') .replace(/```/g, '') .trim(); const firstBrace = cleaned.indexOf('{'); const lastBrace = cleaned.lastIndexOf('}'); if (firstBrace < 0 || lastBrace <= firstBrace) { throw new Error(`LLM 未返回 JSON 对象:${raw.slice(0, 200)}`); } const parsed = JSON.parse(cleaned.slice(firstBrace, lastBrace + 1)); if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { throw new Error('LLM 返回的 JSON 不是对象'); } return parsed as Record; } private emptyBrief(): ReferenceImageBrief { return { mainSubject: '', identityFeatures: [], visualStyle: '', colorsAndLighting: '', composition: '', objects: [], doNotChange: [], }; } private clean(value: unknown): string { return String(value ?? '').replace(/\s+/g, ' ').trim(); } private cleanList(value: unknown): string[] { if (!Array.isArray(value)) return []; return value.map((item) => this.clean(item)).filter(Boolean); } private normalizeAspect(value: unknown, fallback: JimengAspectRatio): JimengAspectRatio { const raw = this.clean(value) as JimengAspectRatio; return ASPECTS.includes(raw) ? raw : fallback; } }