#!/usr/bin/env node const fs = require('fs'); const path = require('path'); const { runTihaoSourcing } = require('../mcp/src/features/tihao-sourcing/sourcing-workflow'); const { redactSecrets } = require('../mcp/src/core/result-envelope'); const STATUS_STRONG = '\u5f3a\u63a8\u8350'; const STATUS_BACKUP = '\u5907\u9009'; const STATUS_REVIEW = '\u9700\u590d\u6838'; async function main() { const config = collectRuntimeConfig(process.env); const outputRoot = path.resolve(process.env.TIHAO_AB_OUTPUT || path.join(__dirname, '..', 'outputs', `video-hit-rate-audit-${Date.now()}`)); fs.mkdirSync(outputRoot, { recursive: true }); const runtimePreflight = buildRuntimePreflight(config, outputRoot); if (!runtimePreflight.readyForVideoAb) { const files = writeRuntimePreflight(runtimePreflight, outputRoot); console.error(redactSecrets(JSON.stringify({ readyForVideoAb: false, failureCount: runtimePreflight.failureCount, summaryFile: files.summaryFile, reportFile: files.reportFile, nextActions: runtimePreflight.nextActions }, null, 2))); process.exitCode = 2; return; } const baseInput = { collectionMode: 'live', brief: config.brief, referenceLinks: [{ url: config.referenceUrl, platform: 'xiaohongshu', contentType: 'video' }], keywordLimit: Number(process.env.TIHAO_AB_KEYWORD_LIMIT || 1), pagesPerKeyword: Number(process.env.TIHAO_AB_PAGES_PER_KEYWORD || 1), tihaoToken: config.runtimeCredential, company: config.company }; if (process.env.TIHAO_AB_EVIDENCE_CREATORS_LIMIT) { baseInput.evidenceCreatorsLimit = Number(process.env.TIHAO_AB_EVIDENCE_CREATORS_LIMIT); } console.error('video-hit-rate-audit: running baseline live sourcing'); const baseline = await withoutProviderEnv(() => runTihaoSourcing({ ...baseInput, output: path.join(outputRoot, 'A-no-video-detail') })); console.error('video-hit-rate-audit: running VOC social + Doubao enhanced sourcing'); const enhanced = await runTihaoSourcing({ ...baseInput, output: path.join(outputRoot, 'B-voc-social-doubao'), vocSocialToken: config.vocSocialToken, enableVocSocialReferenceEnrichment: true, resultFirstMode: config.resultFirstMode, optimizeForResults: config.resultFirstMode, videoAnalysisBaseUrl: config.videoBaseUrl, videoAnalysisModel: config.videoModel, videoAnalysisToken: config.videoToken }); const summary = { outputRoot, brief: config.brief, referenceUrl: config.referenceUrl, resultFirstMode: config.resultFirstMode, proofContext: { mode: 'live', collectionMode: 'live', requiresRuntimeCredential: true, requiresVocSocialProvider: true, requiresVideoAnalysisProvider: true, videoAnalysisBaseUrl: config.videoBaseUrl, videoAnalysisModel: config.videoModel, generatedBy: 'acceptance:video-ab' }, baseline: summarizeRun(baseline), enhanced: summarizeRun(enhanced) }; summary.delta = { candidates: summary.enhanced.counts.candidates - summary.baseline.counts.candidates, strong: summary.enhanced.counts.strong - summary.baseline.counts.strong, evidenceCards: summary.enhanced.counts.evidenceCards - summary.baseline.counts.evidenceCards, referenceResources: summary.enhanced.counts.referenceResources - summary.baseline.counts.referenceResources, top10AvgReferenceStyleFitScore: round(summary.enhanced.top10.avgReferenceStyleFitScore - summary.baseline.top10.avgReferenceStyleFitScore), top10AvgScore: round(summary.enhanced.top10.avgScore - summary.baseline.top10.avgScore), top10EvidenceHitCandidates: summary.enhanced.top10.evidenceHitCandidates - summary.baseline.top10.evidenceHitCandidates }; summary.efficiency = { enhancedEvidenceCards: summary.enhanced.counts.evidenceCards, strongLiftPerEvidenceCard: ratio(summary.delta.strong, summary.enhanced.counts.evidenceCards), scoreLiftPerEvidenceCard: ratio(summary.delta.top10AvgScore, summary.enhanced.counts.evidenceCards), referenceScoreLiftPerEvidenceCard: ratio(summary.delta.top10AvgReferenceStyleFitScore, summary.enhanced.counts.evidenceCards) }; summary.acceptance = computeVideoAcceptance(summary); summary.gates = buildAcceptanceGates(summary); const file = path.join(outputRoot, 'video-hit-rate-summary.json'); const reportFile = path.join(outputRoot, 'video-hit-rate-report.md'); fs.writeFileSync(file, JSON.stringify(summary, null, 2)); fs.writeFileSync(reportFile, withBom(renderVideoAbReport(summary)), 'utf8'); scanForLeaks(outputRoot, [config.runtimeCredential, config.vocSocialToken, config.videoToken]); console.log(redactSecrets(JSON.stringify({ summaryFile: file, reportFile, baseline: summary.baseline.counts, enhanced: summary.enhanced.counts, delta: summary.delta, efficiency: summary.efficiency, acceptance: summary.acceptance }, null, 2))); assert(summary.acceptance.passed, `Reference-video A/B did not pass. See ${file}`); } async function withoutProviderEnv(fn) { const keys = [ 'VOC_SOCIAL_TOKEN', 'VOC_SOCIAL_BASE_URL', 'ENABLE_VOC_SOCIAL_REFERENCE_ENRICHMENT', 'TIHAO_ENABLE_VOC_SOCIAL_REFERENCE_ENRICHMENT', 'VIDEO_ANALYSIS_BASE_URL', 'VIDEO_ANALYSIS_MODEL', 'VIDEO_ANALYSIS_TOKEN', 'VIDEO_ANALYSIS_PATH', 'VIDEO_ANALYSIS_MODE', 'DOUBAO_VISION_BASE_URL', 'DOUBAO_VISION_MODEL', 'DOUBAO_VISION_TOKEN', 'DOUBAO_VISION_PATH', 'DOUBAO_VISION_MODE', 'TIHAO_EVIDENCE_BASE_URL', 'TIHAO_EVIDENCE_TOKEN', 'MULTIMODAL_EVIDENCE_TOKEN' ]; const snapshot = Object.fromEntries(keys.map(key => [key, process.env[key]])); for (const key of keys) delete process.env[key]; try { return await fn(); } finally { for (const [key, value] of Object.entries(snapshot)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } } } function collectRuntimeConfig(env = process.env) { const runtimeCredential = env.TIHAO_SESSION_TOKEN || env.VOC_ECOMMERCE_TOKEN || env.VOC_TOKEN || ''; const company = env.TIHAO_COMPANY || env.VOC_ECOMMERCE_COMPANY || env.COMPANY_OBJECT_ID || ''; return { runtimeCredential, company, vocSocialToken: env.VOC_SOCIAL_TOKEN || runtimeCredential, videoToken: env.VIDEO_ANALYSIS_TOKEN || env.DOUBAO_VISION_TOKEN || '', videoBaseUrl: env.VIDEO_ANALYSIS_BASE_URL || env.DOUBAO_VISION_BASE_URL || 'https://api.fmode.cn', videoModel: env.VIDEO_ANALYSIS_MODEL || env.DOUBAO_VISION_MODEL || 'doubao-seed-2-0-pro', brief: env.TIHAO_AB_BRIEF || path.resolve(__dirname, '..', '..', 'dha_brief.xlsx'), referenceUrl: env.TIHAO_AB_REFERENCE_URL || 'http://xhslink.com/o/2fg3MzG2Iau', resultFirstMode: env.TIHAO_AB_RESULT_FIRST === 'true' || env.TIHAO_RESULT_FIRST_MODE === 'true' }; } function buildRuntimePreflight(config, outputRoot = '') { const checks = [ runtimeCheck( 'runtime-credential-present', Boolean(config.runtimeCredential), '配置 TIHAO_SESSION_TOKEN / VOC_ECOMMERCE_TOKEN / VOC_TOKEN 中任一运行时凭证。', 'proofContext.requiresRuntimeCredential=true' ), runtimeCheck( 'company-resolved', Boolean(config.company), '配置 TIHAO_COMPANY / VOC_ECOMMERCE_COMPANY / COMPANY_OBJECT_ID,或先用 live:preflight 解析 sessionToken 上的 company。', 'company resolved for live run' ), runtimeCheck( 'voc-social-provider-token-present', Boolean(config.vocSocialToken), '配置 VOC_SOCIAL_TOKEN,或确认运行时凭证可用于 VOC social provider。', 'proofContext.requiresVocSocialProvider=true' ), runtimeCheck( 'video-analysis-token-present', Boolean(config.videoToken), '配置 VIDEO_ANALYSIS_TOKEN / DOUBAO_VISION_TOKEN。', 'proofContext.requiresVideoAnalysisProvider=true' ), runtimeCheck( 'video-analysis-model-doubao-seed-2-0-pro', config.videoModel === 'doubao-seed-2-0-pro', '配置 VIDEO_ANALYSIS_MODEL / DOUBAO_VISION_MODEL 为 doubao-seed-2-0-pro。', 'videoAnalysisModel=doubao-seed-2-0-pro' ) ]; const failures = checks.filter(item => item.status === 'fail'); return { generatedAt: new Date().toISOString(), materialType: 'video_ab_runtime_preflight', directCustomerProof: false, proofLevel: 'not_business_proof', canCloseProofGap: false, readyForVideoAb: failures.length === 0, failureCount: failures.length, outputRoot, proofContext: { mode: failures.length ? 'not_run' : 'live-ready', collectionMode: failures.length ? 'not_run' : 'live-ready', generatedBy: 'acceptance:video-ab-preflight', requiresRuntimeCredential: true, requiresVocSocialProvider: true, requiresVideoAnalysisProvider: true, videoAnalysisBaseUrl: config.videoBaseUrl, videoAnalysisModel: config.videoModel }, runtime: { runtimeCredentialPresent: Boolean(config.runtimeCredential), companyResolved: Boolean(config.company), vocSocialProviderTokenPresent: Boolean(config.vocSocialToken), videoAnalysisTokenPresent: Boolean(config.videoToken), videoAnalysisBaseUrlPresent: Boolean(config.videoBaseUrl), videoAnalysisModel: config.videoModel, brief: config.brief, referenceUrl: config.referenceUrl, resultFirstMode: config.resultFirstMode }, checks, missingProofRequirements: failures.map(item => item.missingProofRequirement), nextActions: buildRuntimePreflightNextActions(failures) }; } function runtimeCheck(id, pass, action, missingProofRequirement) { return { id, status: pass ? 'pass' : 'fail', action: pass ? '' : action, missingProofRequirement: pass ? '' : missingProofRequirement }; } function buildRuntimePreflightNextActions(failures) { if (!failures.length) return ['运行 npm run acceptance:video-ab,生成带 live proofContext 的 video-hit-rate-summary.json。']; const actions = failures.map(item => item.action); actions.push('补齐后先运行 npm run live:preflight -- --strict,再运行 npm run acceptance:video-ab。'); actions.push('preflight 失败产物不能关闭 video-ab-live-proof,也不能证明提号率提升。'); return [...new Set(actions)]; } function writeRuntimePreflight(summary, outputRoot) { const summaryFile = path.join(outputRoot, 'video-hit-rate-preflight-summary.json'); const reportFile = path.join(outputRoot, 'video-hit-rate-preflight-report.md'); fs.writeFileSync(summaryFile, JSON.stringify(summary, null, 2), 'utf8'); fs.writeFileSync(reportFile, withBom(renderRuntimePreflightReport(summary)), 'utf8'); return { summaryFile, reportFile }; } function renderRuntimePreflightReport(summary) { const lines = [ '# 提号参考视频 A/B 运行前置检查', '', `- readyForVideoAb:${summary.readyForVideoAb}`, `- failureCount:${summary.failureCount}`, `- proofLevel:${summary.proofLevel}`, `- canCloseProofGap:${summary.canCloseProofGap}`, '', '## 检查项', '', '| 状态 | 检查项 | 缺失证明 | 下一步 |', '| --- | --- | --- | --- |', ...summary.checks.map(item => `| ${item.status} | ${escapeCell(item.id)} | ${escapeCell(item.missingProofRequirement)} | ${escapeCell(item.action)} |`), '', '## 下一步', '', ...summary.nextActions.map(item => `- ${item}`), '', '## 边界', '', '- 本文件只是运行前置检查,不是 video-hit-rate-summary.json。', '- 只有 acceptance:video-ab 在真实 provider、真实视频资源和 live proofContext 下通过,才能关闭 video-ab-live-proof。' ]; return lines.join('\n'); } function summarizeRun(result) { const data = result.data || {}; const criteria = data.criteria || {}; const candidates = data.candidates || []; const top10 = candidates.slice(0, 10); return { status: result.status, counts: { candidates: candidates.length, strong: countStatus(candidates, STATUS_STRONG), backup: countStatus(candidates, STATUS_BACKUP), review: countStatus(candidates, STATUS_REVIEW), referenceFingerprints: (criteria.referenceStyleFingerprints || []).length, referenceResources: (criteria.referenceResources || []).length, evidenceCards: (criteria.evidenceCards || []).length, recallRecords: (criteria.recallRecords || []).length }, provider: { reference: criteria.referenceEvidenceStatus || {}, fingerprint: criteria.referenceFingerprintStatus || {}, evidence: criteria.evidenceStatus || {} }, resourceEvidence: summarizeResourceEvidence(criteria), evidenceQuality: summarizeEvidenceQuality(criteria), top10: { avgScore: avg(top10.map(item => item.score)), avgReferenceStyleFitScore: avg(top10.map(item => item.referenceStyleFitScore)), evidenceHitCandidates: top10.filter(item => (item.evidenceSignals || []).length || item.evidenceCard).length, names: top10.map(item => ({ name: item.displayName, status: item.recommendStatus, score: item.score, referenceStyleFitScore: item.referenceStyleFitScore, evidenceSignals: (item.evidenceSignals || []).slice(0, 5), referenceStyleHitPoints: (item.referenceStyleHitPoints || []).slice(0, 5) })) }, files: result.files || {} }; } function summarizeResourceEvidence(criteria) { const resources = criteria.referenceResources || []; return { referenceResources: resources.length, videoUrlCount: resources.filter(item => item.videoUrl || item.coverOrVideoResource || /\.mp4|\.mov|\.m3u8|video/i.test(String(item.url || ''))).length, coverUrlCount: resources.filter(item => item.coverUrl || item.cover || item.coverImage).length, asrTextCount: resources.filter(item => item.asrText || item.transcript || item.subtitleText).length, frameUrlCount: resources.filter(item => Array.isArray(item.frameUrls) && item.frameUrls.length).length, sample: resources.slice(0, 5).map(item => ({ platform: item.platform || '', contentType: item.contentType || '', title: item.title || '', hasVideoUrl: Boolean(item.videoUrl || item.coverOrVideoResource), hasCoverUrl: Boolean(item.coverUrl || item.cover || item.coverImage), hasAsrText: Boolean(item.asrText || item.transcript || item.subtitleText), hasFrameUrls: Boolean(Array.isArray(item.frameUrls) && item.frameUrls.length) })) }; } function summarizeEvidenceQuality(criteria) { const cards = criteria.evidenceCards || []; const verified = cards.filter(item => item.sourceLevel && !/pending|待补|fallback/i.test(String(item.sourceLevel || ''))); const cardsWithSignals = cards.filter(item => { const content = item.contentEvidence || {}; return (content.textSignals || []).length || (content.asrSignals || []).length || (content.visualSignals || []).length || (content.frameEvidence || []).length; }); return { cardCount: cards.length, verifiedCardCount: verified.length, cardsWithSignals: cardsWithSignals.length }; } function buildAcceptanceGates(summary) { return [ gate('真实参考资源已加载', summary.acceptance.realReferenceResourceLoaded, summary.enhanced.resourceEvidence.referenceResources, '> 0'), gate('真实视频 URL 已加载', summary.acceptance.realVideoResourceLoaded, summary.enhanced.resourceEvidence.videoUrlCount, '> 0'), gate('封面/ASR/帧图至少一类已加载', summary.acceptance.coverOrFrameOrAsrLoaded, { coverUrlCount: summary.enhanced.resourceEvidence.coverUrlCount, asrTextCount: summary.enhanced.resourceEvidence.asrTextCount, frameUrlCount: summary.enhanced.resourceEvidence.frameUrlCount }, '任一 > 0'), gate('视频分析 provider 正常', summary.acceptance.videoAnalysisProviderOk, summary.enhanced.provider.evidence.providerStatus, 'ok'), gate('证据卡已生成', summary.acceptance.evidenceCardsLoaded, summary.enhanced.counts.evidenceCards, '> 0'), gate('证据卡不是待补占位', summary.acceptance.evidenceCardsNotPending, summary.enhanced.evidenceQuality.verifiedCardCount, '> 0'), gate('证据卡包含可解释信号', summary.acceptance.evidenceCardsHaveSignals, summary.enhanced.evidenceQuality.cardsWithSignals, '> 0'), gate('Top10 证据命中提升且分数不下降', summary.acceptance.top10EvidenceImproved, summary.delta, 'evidence + score/reference >= 0'), gate('强推荐数量不下降', summary.acceptance.strongNotDegraded, summary.delta.strong, '>= 0') ]; } function computeVideoAcceptance(summary) { const acceptance = { realReferenceResourceLoaded: summary.enhanced.resourceEvidence.referenceResources > 0, realVideoResourceLoaded: summary.enhanced.resourceEvidence.videoUrlCount > 0, coverOrFrameOrAsrLoaded: summary.enhanced.resourceEvidence.coverUrlCount > 0 || summary.enhanced.resourceEvidence.asrTextCount > 0 || summary.enhanced.resourceEvidence.frameUrlCount > 0, videoAnalysisProviderOk: summary.enhanced.provider.evidence.providerStatus === 'ok', evidenceCardsLoaded: summary.enhanced.counts.evidenceCards > 0, evidenceCardsNotPending: summary.enhanced.evidenceQuality.verifiedCardCount > 0, evidenceCardsHaveSignals: summary.enhanced.evidenceQuality.cardsWithSignals > 0, top10EvidenceImproved: summary.delta.top10EvidenceHitCandidates > 0 && summary.delta.top10AvgScore >= 0 && summary.delta.top10AvgReferenceStyleFitScore >= 0, strongNotDegraded: summary.delta.strong >= 0 }; acceptance.passed = Object.values(acceptance).every(Boolean); return acceptance; } function gate(name, pass, value, target) { return { name, status: pass ? 'pass' : 'fail', value, target }; } function renderVideoAbReport(summary) { const lines = [ '# 提号参考视频 A/B 验收报告', '', `- 运行目录:${summary.outputRoot}`, `- Brief:${summary.brief}`, `- 参考视频:${summary.referenceUrl}`, `- result-first:${summary.resultFirstMode}`, `- 总体验收:${summary.acceptance.passed ? '通过' : '未通过'}`, '', '## 核心差异', '', '| 指标 | A 组 baseline | B 组视频增强 | 差值 |', '| --- | ---: | ---: | ---: |', `| 候选数 | ${summary.baseline.counts.candidates} | ${summary.enhanced.counts.candidates} | ${summary.delta.candidates} |`, `| 强推荐数 | ${summary.baseline.counts.strong} | ${summary.enhanced.counts.strong} | ${summary.delta.strong} |`, `| 证据卡数 | ${summary.baseline.counts.evidenceCards} | ${summary.enhanced.counts.evidenceCards} | ${summary.delta.evidenceCards} |`, `| 参考资源数 | ${summary.baseline.counts.referenceResources} | ${summary.enhanced.counts.referenceResources} | ${summary.delta.referenceResources} |`, `| Top10 平均分 | ${summary.baseline.top10.avgScore} | ${summary.enhanced.top10.avgScore} | ${summary.delta.top10AvgScore} |`, `| Top10 平均参考风格分 | ${summary.baseline.top10.avgReferenceStyleFitScore} | ${summary.enhanced.top10.avgReferenceStyleFitScore} | ${summary.delta.top10AvgReferenceStyleFitScore} |`, `| Top10 证据命中候选 | ${summary.baseline.top10.evidenceHitCandidates} | ${summary.enhanced.top10.evidenceHitCandidates} | ${summary.delta.top10EvidenceHitCandidates} |`, '', '## 视频资源证据', '', `- 参考资源数:${summary.enhanced.resourceEvidence.referenceResources}`, `- 视频 URL 数:${summary.enhanced.resourceEvidence.videoUrlCount}`, `- 封面 URL 数:${summary.enhanced.resourceEvidence.coverUrlCount}`, `- ASR/字幕数:${summary.enhanced.resourceEvidence.asrTextCount}`, `- 帧图资源数:${summary.enhanced.resourceEvidence.frameUrlCount}`, '', '## 验收门禁', '', '| 状态 | 门禁 | 当前值 | 目标 |', '| --- | --- | --- | --- |', ...summary.gates.map(item => `| ${item.status} | ${item.name} | ${escapeCell(typeof item.value === 'object' ? JSON.stringify(item.value) : item.value)} | ${escapeCell(item.target)} |`), '', '## 边界说明', '', '- 只有本报告总体验收通过,才可以说本次样本中视频分析对提号结果有正向证据。', '- 即使通过,也仍需用更多真实 Brief 和客户选中数据验证长期命中率。' ]; return lines.join('\n'); } function countStatus(candidates, status) { return candidates.filter(item => item.recommendStatus === status).length; } function avg(values) { const nums = values.map(Number).filter(value => Number.isFinite(value)); return nums.length ? round(nums.reduce((sum, value) => sum + value, 0) / nums.length) : 0; } function round(value) { return Math.round(Number(value || 0) * 10) / 10; } function ratio(numerator, denominator) { return denominator ? Math.round((Number(numerator || 0) / Number(denominator)) * 100) / 100 : 0; } function scanForLeaks(root, secrets) { const realSecrets = [...new Set(secrets.filter(Boolean))]; for (const file of listFiles(root)) { const text = fs.readFileSync(file, 'utf8'); for (const secret of realSecrets) { assert(!text.includes(secret), `Secret leaked into ${file}`); } assert(!/Authorization\s*:/i.test(text), `Authorization header leaked into ${file}`); } } function withBom(text) { return `\uFEFF${text}`; } function escapeCell(value) { return String(value || '').replace(/\|/g, '/').replace(/\n/g, ' '); } function safeErrorMessage(error) { return redactSecrets(error && error.message ? error.message : String(error)).slice(0, 240); } function listFiles(root) { return fs.readdirSync(root, { withFileTypes: true }).flatMap(entry => { const full = path.join(root, entry.name); return entry.isDirectory() ? listFiles(full) : [full]; }); } function assert(condition, message) { if (!condition) throw new Error(message); } if (require.main === module) { main().catch(error => { console.error(redactSecrets(JSON.stringify({ error: 'acceptance:video-ab failed', message: safeErrorMessage(error), nextActions: [ '先运行 npm run live:preflight -- --strict 查看运行环境是否就绪。', '不要把失败输出或 preflight 产物当作 video-ab-live-proof。' ] }, null, 2))); process.exit(1); }); } module.exports = { buildAcceptanceGates, buildRuntimePreflight, collectRuntimeConfig, computeVideoAcceptance, renderRuntimePreflightReport, summarizeEvidenceQuality, summarizeResourceEvidence, summarizeRun };