// Copyright (c) 未来飞马 // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. // // Trademark Notice: // The MPL-2.0 license grants copyright permissions for source code only. // It does NOT grant any rights to use trademarks including "未来飞马", // "Harness Loop", "RSI", and associated slogan "让AI进化提前发生,让AI落地快人一步". // Any use of these trademarks requires separate written permission. /** * 视频桥:抽帧(ffmpeg)+ 音频转写(复用兄弟技能 skill-listen) * * 产出: * - frames[] 按时间轴均匀抽出的关键帧(落地到 --frames-out,交给 vision 做理解) * - transcript 音轨转写文本(skill-listen / listen-runner.mjs,走 Fmode 网关,凭据仅服务端) * - durationMs 时长(ffprobe;探测不到则 null) * * 缺 ffmpeg / ffprobe 或 skill-listen 时,返回 available:false + 明确安装提示,绝不伪造结果。 * 绝不写任何 token / 凭据。 */ import fs from 'node:fs'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; import { parseArgs } from 'node:util'; import { ensureDir, resolveSiblingScript, truncate } from './lib.mjs'; const LISTEN_REL = path.join('skill-listen', 'scripts', 'listen-runner.mjs'); function hasBinary(name) { const probe = spawnSync(name, ['-version'], { encoding: 'utf8' }); return probe.status === 0 || Boolean(probe.stdout); } export function resolveFfmpeg() { return process.env.CASE_FFMPEG || 'ffmpeg'; } export function probeDurationMs(videoPath) { const probe = spawnSync(process.env.CASE_FFPROBE || 'ffprobe', [ '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', videoPath, ], { encoding: 'utf8' }); const seconds = Number.parseFloat(String(probe.stdout || '').trim()); return Number.isFinite(seconds) && seconds > 0 ? Math.round(seconds * 1000) : null; } /** * 均匀抽帧。 * @returns {{ok:boolean, frames:object[], error:string|null}} */ export function extractFrames(videoPath, framesOut, options = {}) { const count = Math.max(1, Math.min(24, Number(options.frameCount) || 6)); ensureDir(framesOut); const durationMs = options.durationMs || probeDurationMs(videoPath); const ffmpeg = options.ffmpeg || resolveFfmpeg(); const frames = []; if (durationMs && durationMs > 0) { // 均匀取 count 个时间点 for (let i = 0; i < count; i++) { const tMs = Math.round((durationMs * (i + 0.5)) / count); const target = path.join(framesOut, `frame-${String(i + 1).padStart(3, '0')}.jpg`); const run = spawnSync(ffmpeg, [ '-y', '-loglevel', 'error', '-ss', (tMs / 1000).toFixed(3), '-i', videoPath, '-frames:v', '1', '-q:v', '3', target, ], { encoding: 'utf8' }); if (run.status === 0 && fs.existsSync(target)) { frames.push({ index: i + 1, atMs: tMs, localPath: target }); } } } else { // 拿不到时长:按 fps 抽前 count 帧 const run = spawnSync(ffmpeg, [ '-y', '-loglevel', 'error', '-i', videoPath, '-frames:v', String(count), '-vf', 'fps=1', '-q:v', '3', path.join(framesOut, 'frame-%03d.jpg'), ], { encoding: 'utf8' }); if (run.status === 0) { for (const name of fs.readdirSync(framesOut).sort().slice(0, count)) { frames.push({ index: frames.length + 1, atMs: null, localPath: path.join(framesOut, name) }); } } } if (!frames.length) { return { ok: false, frames: [], error: 'FFMPEG_FRAME_EXTRACT_FAILED(检查 ffmpeg 是否可用及视频是否可解码)' }; } return { ok: true, frames, error: null }; } /** 抽出 16kHz 单声道 wav 音轨(转写前压体积,更稳)。 */ export function extractAudio(videoPath, audioOut, options = {}) { ensureDir(path.dirname(audioOut)); const ffmpeg = options.ffmpeg || resolveFfmpeg(); const run = spawnSync(ffmpeg, [ '-y', '-loglevel', 'error', '-i', videoPath, '-vn', '-ar', '16000', '-ac', '1', '-f', 'wav', audioOut, ], { encoding: 'utf8' }); if (run.status === 0 && fs.existsSync(audioOut) && fs.statSync(audioOut).size > 1024) { return { ok: true, audioPath: audioOut, error: null }; } return { ok: false, audioPath: '', error: 'FFMPEG_AUDIO_EXTRACT_FAILED(视频可能没有音轨)' }; } /** 调 skill-listen 转写音轨。缺技能时返回 available:false,不抛栈。 */ export async function transcribeAudio(audioPath, options = {}) { const listenPath = resolveSiblingScript('CASE_LISTEN_SCRIPT', LISTEN_REL); if (!listenPath) { return { available: false, text: '', segments: [], error: `未找到 skill-listen(期望 ${LISTEN_REL})。请先安装:npx skill-listen@latest install`, }; } try { const runner = await import(new URL(`file://${listenPath.split(path.sep).join('/')}`).href); const durationMs = probeDurationMs(audioPath); const out = await runner.transcribeFile({ filePath: audioPath, language: options.language || 'autodialect', diarize: Boolean(options.diarize), speakers: options.speakers, durationMs: durationMs || undefined, }); return { available: true, text: String(out.text || ''), segments: out.segments || [], error: null, }; } catch (error) { return { available: true, text: '', segments: [], error: `TRANSCRIBE_FAILED: ${error.message}` }; } } /** * 视频处理总入口:抽帧 → 抽音轨 → 转写。 */ export async function processVideo(videoPath, options = {}) { const workDir = ensureDir(options.workDir || path.join(process.cwd(), 'case-get-outputs', 'video')); const durationMs = probeDurationMs(videoPath); const ffmpegOk = hasBinary(options.ffmpeg || resolveFfmpeg()); const report = { video: path.resolve(videoPath), durationMs, frameCount: 0, frames: [], transcription: { available: false, text: '', segments: [], error: null }, warnings: [], }; if (!ffmpegOk) { report.warnings.push('未找到 ffmpeg:无法抽帧/抽音轨。请安装 ffmpeg 后重试。'); return report; } const frameResult = extractFrames(videoPath, path.join(workDir, 'frames'), { frameCount: options.frameCount, durationMs }); report.frames = frameResult.frames; report.frameCount = frameResult.frames.length; if (frameResult.error) report.warnings.push(frameResult.error); const audio = extractAudio(videoPath, path.join(workDir, 'audio.wav')); if (audio.ok) { report.transcription = await transcribeAudio(audio.audioPath, options); if (report.transcription.error) report.warnings.push(report.transcription.error); } else { report.transcription = { available: false, text: '', segments: [], error: audio.error }; report.warnings.push(audio.error); } report.summary = truncate(report.transcription.text || '', 200); return report; } // --------------------------------------------------------------------------- // CLI // --------------------------------------------------------------------------- async function main() { const { values, positionals } = parseArgs({ options: { video: { type: 'string', multiple: true, default: [] }, out: { type: 'string' }, 'work-dir': { type: 'string' }, 'frame-count': { type: 'string' }, diarize: { type: 'boolean', default: false }, speakers: { type: 'string' }, language: { type: 'string' }, help: { type: 'boolean', default: false }, }, allowPositionals: true, }); if (values.help) { process.stdout.write([ 'skill-case-get / listen-bridge — 视频抽帧 + 音轨转写(复用 skill-listen)', '', ' node listen-bridge.mjs --video a.mp4 [--frame-count 6] [--diarize] [--out video.json]', '', '依赖:ffmpeg / ffprobe(抽帧、抽音轨)、skill-listen(转写)。缺任一都会在 warnings 里说明,不伪造结果。', ].join('\n')); return 0; } const inputs = [...values.video, ...positionals]; if (!inputs.length) { process.stderr.write('至少需要一个 --video \n'); return 2; } const reports = []; for (const video of inputs) { reports.push(await processVideo(video, { workDir: values['work-dir'] || path.join(path.dirname(path.resolve(video)), 'case-get-video'), frameCount: Number(values['frame-count']) || 6, diarize: values.diarize, speakers: values.speakers ? Number(values.speakers) : undefined, language: values.language, })); } const payload = { ok: reports.every((r) => !r.warnings.length), count: reports.length, reports }; const text = JSON.stringify(payload, null, 2); if (values.out) { fs.mkdirSync(path.dirname(path.resolve(values.out)), { recursive: true }); fs.writeFileSync(path.resolve(values.out), `${text}\n`, 'utf8'); } else { process.stdout.write(`${text}\n`); } return payload.ok ? 0 : 1; } const invokedDirectly = process.argv[1] && path.resolve(process.argv[1]).endsWith(path.join('scripts', 'listen-bridge.mjs')); if (invokedDirectly) { main().then((code) => process.exit(code)).catch((error) => { process.stderr.write(`listen-bridge 失败:${error.message}\n`); process.exit(1); }); }