// scripts/pipeline-a.mjs — 产线A 编排:AI 单曲 MV 端到端 // // 主题 → 作曲填词 → AI 音乐 → 人声/伴奏分离 → 关键帧 → 图生视频 → 唇形 → 剪辑成片 // // 编排原则: // · 每一步都能单独跑(对应 fmode-mv ),pipeline 只是串起来; // · 通道不可用的步骤「如实跳过并记录原因」,能兜底的就兜底(视频→Ken Burns); // · 最后打印一张「步骤执行表」,一眼看清哪些是真产物、哪些是兜底/跳过。 import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { parseArgs, readJSON, writeJSON, paths, ensureWork, log, humanSize, DEFAULT_ANCHOR, } from '../lib/index.mjs'; import { buildLyrics } from './lyrics.mjs'; import { buildStoryboardFromSong, renderStoryboardMd } from './storyboard.mjs'; import { generateKeyframes } from './keyframe.mjs'; import { generateScenes } from './scene.mjs'; import { generateClips } from './video.mjs'; import { assemble } from './assemble.mjs'; import { getProvider } from '../lib/providers.mjs'; import { probe as ffprobe, listMedia } from '../lib/ffmpeg.mjs'; export async function runPipelineA(theme, opts = {}) { const { title = '', genre = '', mood = '', limit = 0, aspect = '16:9', shots = 8, perShot = 6, outFile = join(paths.out, 'mv.mp4'), music = '', skipMusic = false, skipVideo = false, dryRun = false, // v0.2 新增 characterSheetFile = '', skipCharacter = false, characterSteps = 4, single = false, tier = '1k', videoMode = 'auto', } = opts; ensureWork(); const steps = []; const mark = (name, status, detail) => { steps.push({ name, status, detail }); return status; }; // ---------- 1. 作曲填词 ---------- log.step('[1/8] 作曲填词'); let song; if (existsSync(paths.song) && opts.resume !== false) { song = readJSON(paths.song); mark('作曲填词', 'reused', `沿用 work/song.json《${song.title}》`); log.ok(`沿用已有:《${song.title}》`); } else { song = await buildLyrics(theme, { genre, mood }); writeJSON(paths.song, song); mark('作曲填词', 'ok', `《${song.title}》`); log.ok(`《${song.title}》(${song.genre || '-'} / ${song.bpm || '?'}BPM)`); } // ---------- 2. 分镜 ---------- log.step('[2/8] 分镜脚本'); const sb = await buildStoryboardFromSong(song, { shotCount: shots, aspect, perShot }); writeJSON(paths.storyboard, sb); writeFileSync(join(paths.work, 'storyboard.md'), renderStoryboardMd(sb), 'utf8'); mark('分镜脚本', 'ok', `${sb.shots.length} 镜 / ${sb.totalDuration}s`); log.ok(`${sb.shots.length} 个镜头,总时长 ${sb.totalDuration}s`); // ---------- 3. AI 音乐 ---------- log.step('[3/8] AI 音乐生成'); let musicFile = music && existsSync(music) ? music : null; if (musicFile) { mark('AI 音乐', 'provided', `使用外部提供:${musicFile}`); log.ok(`使用指定音乐:${musicFile}`); } else if (skipMusic) { mark('AI 音乐', 'skipped', '--skip-music'); log.warn('按参数跳过'); } else { const p = getProvider('music'); const pr = await p.probe({ deep: false }); if (pr.available === false) { mark('AI 音乐', 'unavailable', pr.detail.split('\n')[0]); log.warn(`通道未开通,跳过:${pr.detail}`); log.dim('替代:--music <你的音频文件> / 等平台开通后重跑 / 自建兼容端点'); } else { try { const { generateMusic } = await import('./music.mjs'); const r = await generateMusic({ prompt: song.stylePrompt, lyrics: song.lyrics, title: song.title }); musicFile = r.files[0].path; mark('AI 音乐', 'ok', musicFile); log.ok(musicFile); } catch (e) { mark('AI 音乐', 'failed', e.message.split('\n')[0]); log.err(`失败:${e.message.split('\n')[0]}`); } } } // ---------- 4. 音轨分离 ---------- log.step('[4/8] 人声 / 伴奏分离'); if (musicFile) { try { const { separate } = await import('./separate.mjs'); const r = await separate(musicFile, { mode: 'both' }); mark('音轨分离', 'ok', r.method === 'demucs' ? 'demucs(高质量)' : 'ffmpeg 相位兜底(近似)'); log.ok(`方式:${r.method}`); } catch (e) { mark('音轨分离', 'failed', e.message.split('\n')[0]); log.err(`失败:${e.message.split('\n')[0]}`); } } else { mark('音轨分离', 'skipped', '无音乐文件'); log.warn('无音乐文件,跳过'); } // ---------- 4.5 人物资产(④,纯 AI 出片不换脸的关键) ---------- log.step('[4.5/8] 人物资产多视图(即梦)'); const sbLimited = { ...sb, shots: limit ? sb.shots.slice(0, limit) : sb.shots }; let characterSheet = characterSheetFile || ''; if (characterSheet && existsSync(characterSheet)) { mark('人物资产', 'provided', characterSheet); log.ok(`使用指定人物多视图:${characterSheet}`); } else if (skipCharacter) { mark('人物资产', 'skipped', '--skip-character'); log.warn('按参数跳过(跨镜头可能换脸)'); } else { const jp = getProvider('jimeng'); const jr = await jp.probe({}); if (!jr.available) { mark('人物资产', 'unavailable', jr.detail.split('\n')[0]); log.warn(`即梦通道不可用:${jr.detail}`); log.dim('替代:后续关键帧只靠分镜的 appearanceAnchor,跨镜头一致性会变差'); } else { try { const { buildCharacterSheet } = await import('./character-sheet.mjs'); const cs = await buildCharacterSheet({ description: sb.appearanceAnchor || DEFAULT_ANCHOR.song, outDir: paths.out, limit: characterSteps, single: !!single, }); const sel = cs.steps.find((s) => s.step === '4-多视图' && s.file) || cs.steps.filter((s) => s.file).pop(); characterSheet = sel?.file || ''; mark('人物资产', cs.cost.total > 0 ? 'ok' : 'reused', `${cs.steps.length} 步,¥${cs.cost.total.toFixed(2)}${characterSheet ? ` → ${characterSheet}` : ''}`); log.ok(`完成 ${cs.steps.length} 步,花费 ¥${cs.cost.total.toFixed(2)}`); } catch (e) { mark('人物资产', 'failed', e.message.split('\n')[0]); log.err(`失败:${e.message.split('\n')[0]}`); } } } // ---------- 5. 场景图 + 关键帧(⑤) ---------- log.step('[5/8] 场景图(无人)→ 关键帧(有人)'); let kf = { okCount: 0, failCount: 0 }; const jp2 = getProvider('jimeng'); const jr2 = await jp2.probe({}); if (jr2.available) { const sc = await generateScenes({ storyboard: sbLimited, characterSheet, sceneDir: join(paths.work, 'scene'), frameDir: paths.frames, tier, }); kf = { okCount: sc.ok, failCount: sc.failed }; mark('场景图+关键帧', sc.failed ? 'partial' : 'ok', `成功 ${sc.ok} / 失败 ${sc.failed} / 花费 ¥${sc.cost.total.toFixed(2)}`); log.ok(`成功 ${sc.ok},失败 ${sc.failed},花费 ¥${sc.cost.total.toFixed(2)}`); } else { // 即梦不可用 → 退回旧 image 通道出关键帧 log.warn(`即梦通道不可用,退回旧 image 通道出关键帧:${jr2.detail}`); const r = await generateKeyframes({ storyboard: sbLimited, outDir: paths.frames, aspect }); kf = { okCount: r.okCount, failCount: r.failCount }; mark('场景图+关键帧', r.failCount ? 'partial' : 'ok', `(旧通道)成功 ${r.okCount} / 失败 ${r.failCount}`); log.ok(`(旧通道)成功 ${r.okCount} 张,失败 ${r.failCount} 张 → ${paths.frames}`); } // ---------- 6. 图生视频(⑥) ---------- log.step('[6/8] 图生视频(即梦)'); let clipResult = { results: [], mode: 'skipped' }; if (skipVideo) { mark('图生视频', 'skipped', '--skip-video'); log.warn('按参数跳过'); } else { clipResult = await generateClips({ storyboard: sbLimited, mode: videoMode }); const ok = clipResult.results.filter((x) => x.status === 'ok').length; const degraded = clipResult.results.filter((x) => x.degraded).length; mark( '图生视频', clipResult.mode === 'jimeng' ? 'ok' : 'fallback', `${clipResult.mode === 'jimeng' ? '即梦真通道' : 'Ken Burns 兜底'}:${ok} 段${degraded ? `(${degraded} 段首尾帧降级)` : ''},花费 ¥${clipResult.cost.total.toFixed(2)}` ); log.ok(`${clipResult.mode === 'jimeng' ? '即梦真通道' : '兜底'}:${ok} 段,花费 ¥${clipResult.cost.total.toFixed(2)}`); } // ---------- 7. 唇形驱动 ---------- log.step('[7/8] 唇形驱动'); const lipShots = sbLimited.shots.filter((s) => s.lipSync); if (!lipShots.length) { mark('唇形驱动', 'n/a', '分镜中无 lipSync:true 镜头'); log.dim('分镜里没有标记唇形的镜头,跳过'); } else { const p = getProvider('lipsync'); const pr = await p.probe({}); if (!pr.available) { mark('唇形驱动', 'unavailable', `${lipShots.length} 个待处理镜头未处理`); log.warn(`通道不可用,${lipShots.length} 个唇形镜头未处理:${pr.detail}`); log.dim('替代:自建 Wav2Lip/SadTalker 并设 FEME_LIPSYNC_ENDPOINT;或改用表情微动镜头'); } else { try { const { driveLip } = await import('./lipsync.mjs'); let n = 0; for (const s of lipShots) { const face = join(paths.frames, `shot-${String(s.id).padStart(2, '0')}.png`); const audio = (musicFile && join(paths.audio, 'vocals.wav')) || join(paths.audio, 'vocals.wav'); if (!existsSync(face) || !existsSync(audio)) continue; await driveLip(s, { faceImage: face, audio }); n++; } mark('唇形驱动', 'ok', `${n} 个镜头`); log.ok(`${n} 个镜头完成`); } catch (e) { mark('唇形驱动', 'failed', e.message.split('\n')[0]); log.err(`失败:${e.message.split('\n')[0]}`); } } } // ---------- 8. 剪辑合成 ---------- log.step('[8/8] 剪辑合成'); let final = null; try { const r = await assemble({ storyboard: sbLimited, clipsDir: paths.clips, outFile, music: musicFile || undefined, beatSync: true, }); final = r.outFile; mark('剪辑合成', 'ok', `${r.info.duration.toFixed(1)}s / ${humanSize(r.info.size)}`); } catch (e) { mark('剪辑合成', 'failed', e.message.split('\n')[0]); log.err(`失败:${e.message.split('\n')[0]}`); } // ---------- 汇总 ---------- const report = { pipeline: 'A', theme, title: song.title, aspect, shots: sb.shots.length, final, steps, generatedAt: new Date().toISOString(), }; writeJSON(join(paths.work, 'pipeline-a.run.json'), report); printReport('产线A · AI 单曲 MV', report, { song, final }); return report; } export function printReport(name, report, extra = {}) { console.log(''); console.log('═'.repeat(64)); console.log(` ${name} · 执行报告`); console.log('═'.repeat(64)); const icon = { ok: '✅', reused: '♻️ ', provided: '📎', fallback: '◐ ', partial: '◐ ', skipped: '⏭️ ', unavailable: '⚠️ ', failed: '❌', 'n/a': '· ' }; const w = Math.max(...report.steps.map((s) => s.name.length)); for (const s of report.steps) { console.log(` ${icon[s.status] || ' '} ${s.name.padEnd(w)} ${s.detail || ''}`); } console.log('─'.repeat(64)); if (report.final) { console.log(` 成片: ${report.final}`); } else { console.log(' ⚠️ 未产出成片(见上方失败/跳过项)'); } console.log(''); const bad = report.steps.filter((s) => s.status === 'failed' || s.status === 'unavailable'); if (bad.length) { console.log(' 未完成项:'); for (const b of bad) console.log(` · ${b.name}: ${b.detail}`); console.log(' 详见 SKILL.md「能力现状」与 references/capability-matrix.md'); } console.log('═'.repeat(64)); console.log(''); } export async function main(argv = process.argv.slice(2)) { const { _, flags } = parseArgs(argv); const theme = _.join(' ').trim(); if (flags.help || !theme) { console.log(` fmode-mv song — 产线A:AI 单曲 MV 端到端编排 用法: fmode-mv song "主题描述" [选项] 流程: 作曲填词 → 分镜 → AI音乐 → 音轨分离 → 关键帧 → 图生视频 → 唇形 → 剪辑成片 选项: --title "歌名" 指定歌名(否则模型自定) --genre "曲风" --mood "情绪" --shots 镜头数(默认 8) --per-shot 单镜秒数(默认 6) --limit 只处理前 n 镜(试跑省钱) --aspect 16:9|9:16|1:1 --music 用自己已有的音乐(跳过 AI 音乐步骤) --skip-music 跳过音乐步骤 --skip-video 跳过视频生成 --out 成片输出(默认 out/mv.mp4) --fresh 忽略已有 work/song.json,重新作曲填词 示例: fmode-mv song "深夜末班地铁上的城市白领" --genre "city pop" --limit 3 fmode-mv song "..." --music my-song.mp3 fmode-mv song "..." --limit 1 --char-steps 4 # 先跑 1 镜 + 完整人物资产(试水) ⚠️ 通道现状(fmode-mv providers 可查): ✅ chat / image / ffmpeg / 即梦(文生图·图生图·图生视频·数字人)可用; ❌ music / lipsync / tts / 旧 video 通道平台未开通。 未开通的步骤会如实跳过或降级兜底,并在报告中标出,绝不伪造产物。 v0.2 新增选项: --character 指定现成的人物多视图(跳过自动生成人物资产) --skip-character 跳过人物资产(省钱;跨镜头可能换脸) --char-steps <1-4> 人物资产跑到第几步(默认 4=多视图) --single 人物资产每步只出 1 张(跳过择优) --tier 1k|2k 场景/关键帧尺寸档(默认 1k) --video-mode auto|jimeng|still 视频通道(默认 auto;still=零 API 费用的 KB 兜底) `); return 0; } await runPipelineA(theme, { title: flags.title || '', genre: flags.genre || '', mood: flags.mood || '', limit: Number(flags.limit) || 0, aspect: flags.aspect || '16:9', shots: Number(flags.shots) || 8, perShot: Number(flags['per-shot']) || 6, outFile: String(flags.out || join(paths.out, 'mv.mp4')), music: typeof flags.music === 'string' ? flags.music : '', skipMusic: !!flags['skip-music'], skipVideo: !!flags['skip-video'], resume: !flags.fresh, characterSheetFile: typeof flags.character === 'string' ? flags.character : '', skipCharacter: !!flags['skip-character'], characterSteps: Number(flags['char-steps']) || 4, single: !!flags.single, tier: flags.tier || '1k', videoMode: flags['video-mode'] || 'auto', }); return 0; } if (import.meta.url === `file://${process.argv[1]}`) { main().then((c) => process.exit(Number(c) || 0)).catch((e) => { console.error(`✗ ${e.message}`); if (process.env.DEBUG) console.error(e.stack); process.exit(1); }); }