import { Injectable, inject } from '@angular/core'; import { FFmpeg } from '@ffmpeg/ffmpeg'; import { fetchFile, toBlobURL } from '@ffmpeg/util'; import { QiniuUploadService } from './qiniu-upload.service'; // =================================================================== // 合成相关公共类型(原来在 src/app/pipelines/composite-sse.ts 中, // 后端 SSE 客户端已废弃,类型集中到此处) // =================================================================== export type CompositeStage = 'download' | 'compose' | 'concat' | 'cleanup'; /** * 单段输入。按 (image/video, audio/silent) 四态分发: * - imageUrl + audioUrl -> 图片 + 解说(按音频时长) * - videoUrl + audioUrl -> 动态视频 + 解说(按音频时长) * - videoUrl 无 audioUrl -> 纯动态视频(按视频原长) * - imageUrl 无 audioUrl -> 静态画面(按 duration,默认 3s) */ export interface CompositeSegmentInput { id?: string; imageUrl?: string; videoUrl?: string; audioUrl?: string; /** 仅在 imageUrl 无 audioUrl 时生效;缺省 3 秒 */ duration?: number; } export interface CompositeProgressEvent { stage: CompositeStage; current?: number; total?: number; percent?: number; message?: string; } export interface CompositeDoneEvent { success: boolean; videoUrl: string; filename: string; size: number; segments: number; } export interface CompositeCallbacks { onStage?: (data: { stage: CompositeStage; message?: string }) => void; onProgress?: (data: CompositeProgressEvent) => void; onDone?: (data: CompositeDoneEvent) => void; onError?: (data: { error: string }) => void; } export interface CompositeHandle { abort: () => void; promise: Promise; } export interface BrowserAudioExtractionResult { blob: Blob; base64: string; mimeType: string; filename: string; sizeMB: number; } /** * 浏览器端合成参数。与 composeVideoStream 的入参兼容。 */ export interface BrowserComposeParams extends CompositeCallbacks { segments: CompositeSegmentInput[]; title: string; /** 输出短边像素,默认 720(1280x720)。可选 1080。 */ shortSide?: 720 | 1080; /** 浏览器端 fetchFile 失败时是否走后端 /backend/api/video-proxy 兜底。默认 true。 */ useBackendProxyFallback?: boolean; } @Injectable({ providedIn: 'root' }) export class BrowserFfmpegService { private ffmpeg?: FFmpeg; private loading?: Promise; private qiniu = inject(QiniuUploadService); async extractAudioForGemini( input: File | Blob | string, filename = `audio-${Date.now()}.m4a`, ): Promise { const ffmpeg = await this.getFfmpeg(); const inputName = this.inputName(input); const outputName = filename.replace(/\.[^.]+$/, '') + '.m4a'; await ffmpeg.writeFile(inputName, await fetchFile(input)); const exitCode = await ffmpeg.exec([ '-i', inputName, '-vn', '-c:a', 'aac', '-b:a', '64k', '-ar', '16000', '-ac', '1', outputName, ], 300000); if (exitCode !== 0) { await this.safeDelete(ffmpeg, inputName); throw new Error(`浏览器音频提取失败 (exit=${exitCode})`); } const data = await ffmpeg.readFile(outputName); await this.safeDelete(ffmpeg, inputName); await this.safeDelete(ffmpeg, outputName); const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data; const copy = new Uint8Array(bytes.byteLength); copy.set(bytes); const blob = new Blob([copy.buffer], { type: 'audio/mp4' }); const base64 = await this.blobToBase64(blob); return { blob, base64, mimeType: 'audio/mp4', filename: outputName, sizeMB: blob.size / 1024 / 1024, }; } async extractAudioBlob( input: File | Blob | string, filename = `extracted-${Date.now()}.m4a`, ): Promise { const result = await this.extractAudioForGemini(input, filename); return new File([result.blob], result.filename, { type: result.mimeType }); } private async getFfmpeg(): Promise { if (!this.ffmpeg) { this.ffmpeg = new FFmpeg(); } if (!this.ffmpeg.loaded) { this.loading ??= this.load(this.ffmpeg).finally(() => { this.loading = undefined; }); await this.loading; } return this.ffmpeg; } private async load(ffmpeg: FFmpeg): Promise { const baseUrl = new URL('assets/ffmpeg/', document.baseURI).toString(); const workerBaseUrl = new URL('worker/', baseUrl).toString(); await ffmpeg.load({ classWorkerURL: `${workerBaseUrl}worker.js`, coreURL: await toBlobURL(`${baseUrl}ffmpeg-core.js`, 'text/javascript'), wasmURL: await toBlobURL(`${baseUrl}ffmpeg-core.wasm`, 'application/wasm'), }); } private inputName(input: File | Blob | string): string { if (typeof input === 'string') { const clean = input.split('?')[0].split('#')[0]; const tail = clean.split('/').pop() || ''; return this.safeName(tail || `input-${Date.now()}.mp4`); } if (input instanceof File && input.name) { return this.safeName(input.name); } return `input-${Date.now()}.mp4`; } private safeName(name: string): string { const ext = (name.match(/\.[a-z0-9]+$/i)?.[0] || '.mp4').toLowerCase(); const base = name.replace(/\.[^.]+$/, '').replace(/[^a-zA-Z0-9_-]/g, '_') || 'input'; return `${base}${ext}`; } private async safeDelete(ffmpeg: FFmpeg, path: string): Promise { try { await ffmpeg.deleteFile(path); } catch { // best effort cleanup } } private blobToBase64(blob: Blob): Promise { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => { const value = String(reader.result || ''); resolve(value.includes(',') ? value.split(',')[1] : value); }; reader.onerror = () => reject(reader.error || new Error('Blob to base64 failed')); reader.readAsDataURL(blob); }); } // =================================================================== // 视频合成(浏览器端) — 替代 server.js 的 /api/video/composite/stream // // 与后端一致的四态分发: // A: imageUrl + audioUrl -> 图片 + 解说,按音频时长 // B: videoUrl + audioUrl -> 视频 + 解说,按音频时长 // C: videoUrl 无 audioUrl -> 纯视频 + 静音轨,按视频原长 // D: imageUrl 无 audioUrl -> 静态画面 + 静音轨,按 duration // // 统一规格(保证 concat -c copy 可用): // 视频 1280x720(或 1920x1080)/ 30fps / yuv420p / libx264 ultrafast // 音频 AAC 128k / 44100 / stereo // =================================================================== composeVideo(params: BrowserComposeParams): CompositeHandle { let aborted = false; const abortController = new AbortController(); const promise: Promise = (async () => { const { segments, title } = params; const shortSide = params.shortSide ?? 720; const useProxy = params.useBackendProxyFallback !== false; if (!segments?.length) throw new Error('segments empty'); const ffmpeg = await this.getFfmpeg(); const tempFiles = new Set(); // 进度权重:单段编码占 80%(按段均分),拼接 15%,上传 5% const segWeight = 80 / segments.length; let baseProgress = 0; let currentSegProgress = 0; const emitProgress = ( stage: CompositeStage, message?: string, current?: number, total?: number, ) => { const percent = Math.min(100, Math.round(baseProgress + currentSegProgress)); params.onProgress?.({ stage, percent, current, total, message }); }; const emitStage = (stage: CompositeStage, message?: string) => { params.onStage?.({ stage, message }); }; const onEncodeProgress = (ratio: number) => { currentSegProgress = Math.max(0, Math.min(segWeight, segWeight * (Number(ratio) || 0))); params.onProgress?.({ stage: 'compose', percent: Math.min(100, Math.round(baseProgress + currentSegProgress)), }); }; try { emitStage('download', `prepare ${segments.length} segments`); const segmentNames: string[] = []; for (let i = 0; i < segments.length; i += 1) { if (aborted) throw new Error('aborted'); currentSegProgress = 0; emitStage('compose', `compose ${i + 1}/${segments.length}`); emitProgress('compose', `compose ${i + 1}/${segments.length}`, i + 1, segments.length); const segName = await this.composeOneSegment( ffmpeg, segments[i], i, shortSide, tempFiles, useProxy, abortController.signal, onEncodeProgress, ); segmentNames.push(segName); baseProgress += segWeight; currentSegProgress = 0; } if (aborted) throw new Error('aborted'); emitStage('concat', `concat ${segmentNames.length} segments`); emitProgress('concat', `concat ${segmentNames.length} segments`); const listText = segmentNames.map((n) => `file '${n}'`).join('\n'); await ffmpeg.writeFile('list.txt', new TextEncoder().encode(listText)); tempFiles.add('list.txt'); const finalName = `final-${Date.now()}.mp4`; const concatExit = await ffmpeg.exec([ '-f', 'concat', '-safe', '0', '-i', 'list.txt', '-c', 'copy', '-movflags', '+faststart', finalName, ]); if (concatExit !== 0) throw new Error(`concat failed (exit=${concatExit})`); tempFiles.add(finalName); baseProgress = 95; const data = await ffmpeg.readFile(finalName); const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data; const copy = new Uint8Array(bytes.byteLength); copy.set(bytes); const blob = new Blob([copy.buffer], { type: 'video/mp4' }); const size = blob.size; emitStage('cleanup', 'upload final mp4'); const safeTitle = String(title || 'video') .replace(/[^a-zA-Z0-9\u4e00-\u9fff_-]/g, '_') .substring(0, 50) || 'video'; const filename = `${safeTitle}-${Date.now()}.mp4`; const url = await new Promise((resolve, reject) => { this.qiniu .uploadFileWithProgress(blob, filename, 'video/mp4', 'video') .subscribe({ next: (ev) => { if (ev.state === 'progress') { params.onProgress?.({ stage: 'cleanup', percent: Math.min(100, 95 + Math.round((ev.progress || 0) * 0.05)), message: `uploading ${Math.round((ev.progress || 0) * 100)}%`, }); } else if (ev.state === 'done' && ev.url) { resolve(ev.url); } }, error: (err) => reject(err instanceof Error ? err : new Error(String(err))), }); }); await Promise.all([...tempFiles].map((n) => this.safeDelete(ffmpeg, n))); const done: CompositeDoneEvent = { success: true, videoUrl: url, filename, size, segments: segments.length, }; params.onProgress?.({ stage: 'cleanup', percent: 100 }); params.onDone?.(done); return done; } catch (err: any) { await Promise.all([...tempFiles].map((n) => this.safeDelete(ffmpeg, n))).catch(() => {}); const msg = err?.message || String(err); params.onError?.({ error: msg }); throw err instanceof Error ? err : new Error(msg); } })(); return { abort: () => { aborted = true; abortController.abort(); try { this.ffmpeg?.terminate(); } catch { /* noop */ } this.ffmpeg = undefined; this.loading = undefined; }, promise, }; } private async composeOneSegment( ffmpeg: FFmpeg, seg: CompositeSegmentInput, idx: number, shortSide: 720 | 1080, tempFiles: Set, useProxy: boolean, signal: AbortSignal, onEncodeProgress?: (ratio: number) => void, ): Promise { const W = shortSide === 1080 ? 1920 : 1280; const H = shortSide; const baseVf = `scale=${W}:${H}:force_original_aspect_ratio=decrease,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:black`; const audioArgs = ['-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2']; const videoArgs = ['-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-r', '30', '-g', '60']; const hasImage = !!seg.imageUrl; const hasVideo = !!seg.videoUrl; const hasAudio = !!seg.audioUrl; if (!hasImage && !hasVideo) { throw new Error(`segment ${seg.id ?? idx} missing imageUrl/videoUrl`); } const segName = `seg-${idx}.mp4`; let audioName = ''; let audioDuration = 0; // 仅在「真正编码段视频」的 exec 期间挂载 progress 监听,避免 probeDuration 触发段内进度。 const runEncode = async (args: string[]): Promise => { const handler = (e: { progress: number }) => { onEncodeProgress?.(Number(e.progress) || 0); }; ffmpeg.on('progress', handler as any); try { return await ffmpeg.exec(args); } finally { try { ffmpeg.off('progress', handler as any); } catch { /* noop */ } } }; if (hasAudio) { audioName = `audio-${idx}${this.guessExt(seg.audioUrl!, 'm4a')}`; await ffmpeg.writeFile(audioName, await this.fetchAsset(seg.audioUrl!, useProxy, signal)); tempFiles.add(audioName); audioDuration = await this.probeDuration(ffmpeg, audioName); if (!audioDuration) throw new Error(`segment ${seg.id ?? idx} audio duration probe failed`); } let exit = 0; if (hasVideo) { const videoName = `video-${idx}.mp4`; await ffmpeg.writeFile(videoName, await this.fetchAsset(seg.videoUrl!, useProxy, signal)); tempFiles.add(videoName); const videoDuration = await this.probeDuration(ffmpeg, videoName).catch(() => 0); if (hasAudio) { // Case B: video + audio, align to audio duration let vfChain = baseVf; if (videoDuration > 0 && videoDuration < audioDuration - 0.05) { const padSec = (audioDuration - videoDuration).toFixed(3); vfChain += `,tpad=stop_mode=clone:stop_duration=${padSec}`; } exit = await runEncode([ '-i', videoName, '-i', audioName, '-map', '0:v:0', '-map', '1:a:0', ...videoArgs, '-vf', vfChain, ...audioArgs, '-t', String(audioDuration), segName, ]); } else { // Case C: video only + silent audio exit = await runEncode([ '-i', videoName, '-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100', '-map', '0:v:0', '-map', '1:a:0', ...videoArgs, '-vf', baseVf, ...audioArgs, '-shortest', segName, ]); } } else { const imgName = `img-${idx}${this.guessExt(seg.imageUrl!, 'jpg')}`; await ffmpeg.writeFile(imgName, await this.fetchAsset(seg.imageUrl!, useProxy, signal)); tempFiles.add(imgName); if (hasAudio) { // Case A: image + audio exit = await runEncode([ '-loop', '1', '-i', imgName, '-i', audioName, ...videoArgs, '-vf', baseVf, ...audioArgs, '-t', String(audioDuration), '-shortest', segName, ]); } else { // Case D: image only + silent audio (use seg.duration, default 3s) const dur = Math.max(0.5, Number(seg.duration) || 3); exit = await runEncode([ '-loop', '1', '-i', imgName, '-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100', ...videoArgs, '-vf', baseVf, ...audioArgs, '-t', String(dur), '-shortest', segName, ]); } } if (exit !== 0) throw new Error(`segment ${seg.id ?? idx} encode failed (exit=${exit})`); tempFiles.add(segName); return segName; } /** 通过运行 ffmpeg -i -f null - 解析 stderr 拿到时长(秒)。 */ private async probeDuration(ffmpeg: FFmpeg, name: string): Promise { let dur = 0; const handler = ({ message }: { message: string }) => { const m = message?.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/); if (m) { dur = (+m[1]) * 3600 + (+m[2]) * 60 + parseFloat(m[3]); } }; ffmpeg.on('log', handler as any); try { await ffmpeg.exec(['-i', name, '-f', 'null', '-']); } catch { // ffmpeg 用 null muxer 时偶尔非零退出,时长仍能从 stderr 拿到 } finally { try { ffmpeg.off('log', handler as any); } catch { /* noop */ } } return dur; } /** 拉取远程资源到 Uint8Array。直连失败(CORS/network)时回退到 /backend/api/video-proxy。 */ private async fetchAsset( url: string, useProxy: boolean, signal: AbortSignal, ): Promise { if (signal.aborted) throw new Error('aborted'); try { const resp = await fetch(url, { signal }); if (!resp.ok) throw new Error(`HTTP ${resp.status}`); return new Uint8Array(await resp.arrayBuffer()); } catch (err) { if (!useProxy) throw err; const proxied = `/backend/api/video-proxy?url=${encodeURIComponent(url)}`; const resp = await fetch(proxied, { signal }); if (!resp.ok) throw new Error(`proxy fetch failed HTTP ${resp.status}`); return new Uint8Array(await resp.arrayBuffer()); } } private guessExt(url: string, fallback: string): string { const m = url.split('?')[0].match(/\.([a-z0-9]{2,5})$/i); return `.${(m ? m[1] : fallback).toLowerCase()}`; } }