tihao-ai/claude-code-tihao-sourcing/scripts/real-gap-material-audit.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { buildLocalSeedMaterialIndex, sanitizePublicUrl, sanitizeCell } = require('./local-seed-material-index');
const { buildLocalSeedToIntakeWorklist } = require('./local-seed-to-intake-worklist');
const { buildExperienceTranscriptIndex } = require('./experience-transcript-index');

const ROOT = path.resolve(__dirname, '..');
const DEFAULT_OUTPUT = path.join(ROOT, 'outputs', 'real-gap-material-audit-latest');

function main() {
  const args = parseArgs(process.argv.slice(2));
  const workspaceRoot = path.resolve(args.workspaceRoot || path.join(ROOT, '..'));
  const outputsDir = path.resolve(args.outputs || path.join(ROOT, 'outputs'));
  const transcriptsDir = path.resolve(args.transcriptsDir || path.join(workspaceRoot, 'docs', '提号经验'));
  const outputDir = path.resolve(args.output || DEFAULT_OUTPUT);
  const summary = buildRealGapMaterialAudit({ root: ROOT, workspaceRoot, outputsDir, transcriptsDir });

  fs.mkdirSync(outputDir, { recursive: true });
  const summaryPath = path.join(outputDir, 'real-gap-material-audit-summary.json');
  const reportPath = path.join(outputDir, 'real-gap-material-audit.md');
  const csvPath = path.join(outputDir, 'real-gap-material-audit.csv');
  fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf8');
  fs.writeFileSync(reportPath, withBom(renderReport(summary)), 'utf8');
  fs.writeFileSync(csvPath, withBom(renderCsv(summary)), 'utf8');

  console.log(JSON.stringify({
    outputDir,
    summary: summaryPath,
    report: reportPath,
    csv: csvPath,
    gapCount: summary.gapCount,
    materialFoundCount: summary.materialFoundCount,
    openGapCount: summary.openGapCount,
    canCloseProofGap: summary.canCloseProofGap
  }, null, 2));

  if (args.strict && summary.canCloseProofGap !== false) process.exitCode = 1;
}

function buildRealGapMaterialAudit({ root = ROOT, workspaceRoot = path.join(root, '..'), outputsDir = path.join(root, 'outputs'), transcriptsDir = path.join(workspaceRoot, 'docs', '提号经验') } = {}) {
  const seedIndex = buildLocalSeedMaterialIndex({ root, workspaceRoot, outputsDir });
  const worklist = buildLocalSeedToIntakeWorklist({ root, seedIndex });
  const transcriptIndex = buildExperienceTranscriptIndex({ root, transcriptsDir });
  const proofGapClosure = readJson(path.join(outputsDir, 'proof-gap-closure-latest', 'proof-gap-closure-summary.json')) ||
    readJson(latestFile(outputsDir, 'proof-gap-closure-summary.json'));
  const intakeReadiness = readJson(path.join(outputsDir, 'intake-readiness-latest', 'intake-readiness-summary.json')) ||
    readJson(latestFile(outputsDir, 'intake-readiness-summary.json'));
  const videoReadiness = readJson(path.join(outputsDir, 'video-resource-readiness-latest', 'video-resource-readiness-summary.json')) ||
    readJson(latestFile(outputsDir, 'video-resource-readiness-summary.json'));
  const videoAbPreflight = readJson(path.join(outputsDir, 'video-hit-rate-preflight-latest', 'video-hit-rate-preflight-summary.json')) ||
    readJson(latestFile(outputsDir, 'video-hit-rate-preflight-summary.json'));
  const candidateVideoNotes = extractCandidateVideoNotes(path.join(workspaceRoot, 'output', 'dha-tihao-poc', 'candidate_details_top18.json'));
  const transcriptEvidence = {
    historical: findTranscriptHits(transcriptsDir, [
      /DHA.*(提报|反馈|选新的号)/,
      /没有给我反馈/,
      /没有选新的号/,
      /五六十个/,
      /六七个主页/
    ]),
    video: findTranscriptHits(transcriptsDir, [
      /视频.*(内容|点进去|每一帧|解析)/,
      /封面/,
      /首图/,
      /前十.*笔记/,
      /20篇/
    ]),
    manualCustomer: findTranscriptHits(transcriptsDir, [
      /选中率/,
      /10%.*30%/,
      /50%以上/,
      /客户.*满意/,
      /六七个主页/,
      /人工.*看/
    ])
  };

  const rowsById = Object.fromEntries((proofGapClosure?.rows || []).map(row => [row.id, row]));
  const gaps = [
    buildHistoricalGap({
      root,
      workspaceRoot,
      worklist,
      seedIndex,
      transcriptIndex,
      transcriptEvidence: transcriptEvidence.historical,
      intakeReadiness,
      proofRow: rowsById['historical-dataset']
    }),
    buildVideoAbGap({
      root,
      worklist,
      videoReadiness,
      videoAbPreflight,
      candidateVideoNotes,
      transcriptEvidence: transcriptEvidence.video,
      proofRow: rowsById['video-ab-live-proof']
    }),
    buildManualCustomerGap({
      root,
      worklist,
      transcriptIndex,
      transcriptEvidence: transcriptEvidence.manualCustomer,
      proofRow: rowsById['manual-review-and-customer-effect']
    })
  ];
  const materialFoundCount = gaps.filter(item => item.projectMaterialFound).length;
  const openGapCount = gaps.filter(item => item.currentProofStatus !== 'closed').length;
  return {
    generatedAt: new Date().toISOString(),
    root,
    workspaceRoot,
    outputsDir,
    proofLevel: 'not_business_proof',
    materialType: 'three_real_gap_material_audit',
    directCustomerProof: false,
    canCloseProofGap: false,
    complete: false,
    gapCount: gaps.length,
    materialFoundCount,
    openGapCount,
    sourceState: {
      proofGapClosureComplete: Boolean(proofGapClosure?.complete),
      proofGapOpenCount: Number(proofGapClosure?.openCount ?? gaps.length),
      intakeOverallReady: Boolean(intakeReadiness?.acceptance?.overallReady),
      videoResourceReady: Boolean(videoReadiness?.acceptance?.readyForVideoAbPreflight),
      videoAbPreflightReady: Boolean(videoAbPreflight?.readyForVideoAb),
      experienceRuleCoverage: `${transcriptIndex.coveredRuleCount}/${transcriptIndex.ruleCount}`,
      localSeedHistoryDraftRows: Number(worklist.counts?.historyDraftRows || 0),
      localSeedReviewDraftRows: Number(worklist.counts?.reviewDraftRows || 0),
      candidateVideoNoteCount: candidateVideoNotes.length
    },
    gaps,
    guardrails: [
      '本审计证明“项目资料已找到并接入到补证工作流”，不证明业务效果已经达标。',
      'DHA PoC、候选视频资源和逐字稿经验可以关闭“找不到资料”的问题，但不能自动关闭客户效果、历史数据或 live A/B 证明缺口。',
      '视频资源 readiness=true 只说明 A/B 前置资源够用；video-ab-live-proof 仍必须由 acceptance:video-ab 的 live proofContext 关闭。',
      '逐字稿里的“客户没反馈/没选新号”是重要业务事实，但不是逐条客户选中率、拒绝原因或人工补号量审计。',
      '任一 open gap 存在时，不得宣称提号率提升、客户效果达标或人工补号量下降。'
    ],
    nextActions: [
      '把 real-gap-material-audit-latest 作为三缺口事实底稿，避免后续再说“项目里没有资料”。',
      '把 DHA worklist 中可转写字段晋升到正式 intake 前，必须补齐客户选择、拒绝原因和人工补号量，不得用占位值代替。',
      'provider token/company/video analysis token 就绪后，先跑 acceptance:video-ab，再重跑 proof-gap:closure 和 optimization:completion。',
      '客户效果字段补齐后，重跑 review:metrics、history:audit、customer-effect:audit，再刷新 round:refresh。'
    ]
  };
}

function buildHistoricalGap({ root, workspaceRoot, worklist, seedIndex, transcriptIndex, transcriptEvidence, intakeReadiness, proofRow }) {
  const dhaMaterials = (seedIndex.materials || []).filter(item => item.exists && /dha/i.test(item.id));
  const historyDraftRows = Number(worklist.counts?.historyDraftRows || 0);
  const customerNoFeedback = transcriptEvidence.some(item => /没有给我反馈|没有选新的号/.test(item.snippet));
  const missing = uniqueStrings([
    ...(proofRow?.missingProofRequirements || []),
    ...(intakeReadiness?.history?.issues || []).map(item => item.message)
  ]);
  return {
    id: 'historical-dataset',
    title: '真实历史 Brief 数据集',
    currentProofStatus: proofRow?.status || 'open',
    projectMaterialFound: dhaMaterials.length > 0 || historyDraftRows > 0 || transcriptEvidence.length > 0,
    canCloseNow: false,
    materialSignals: {
      dhaMaterialCount: dhaMaterials.length,
      historyDraftRows,
      transcriptEvidenceCount: transcriptEvidence.length,
      customerNoFeedbackObserved: customerNoFeedback,
      currentBriefCount: extractNumber(proofRow?.evidence, /briefCount=(\d+)/),
      minBriefsRequired: 5
    },
    connectedArtifacts: [
      'outputs/local-seed-to-intake-worklist-latest/history-intake-draft.csv',
      'outputs/local-seed-history-audit-latest/historical-dataset-audit.json',
      'outputs/real-gap-material-audit-latest/real-gap-material-audit-summary.json'
    ],
    projectMaterials: [
      ...dhaMaterials.map(item => ({
        type: 'local_seed_workbook',
        path: item.path,
        contribution: '提供 DHA 小红书真实 Brief、PoC 候选名单、参考账号和数据字段。',
        closureBoundary: '只有一个 DHA/PoC 种子，且缺客户选择、拒绝原因和历史人工补号量。'
      })),
      ...transcriptEvidence.slice(0, 6).map(item => ({
        type: 'experience_transcript',
        path: `${item.path}:${item.lineNumber}`,
        contribution: item.snippet,
        closureBoundary: '只能证明业务过程和经验事实，不能替代结构化历史审计字段。'
      }))
    ],
    blockingRequirements: missing.length ? missing : ['briefCount>=5', 'withCustomerDecision>=5', 'withManualSupplementBaseline>=5'],
    recommendedPromotion: [
      'DHA worklist 可作为 1 条历史 Brief 种子保留。',
      '还需补至少 4 个真实历史 Brief，且每条必须有人工最终名单、客户选中/拒绝、拒绝原因或复核标签、历史人工补号量基线。',
      '逐字稿里“客户没反馈/没选新号”应作为 no-feedback 事实记录，不应伪装成客户选中。'
    ]
  };
}

function buildVideoAbGap({ root, worklist, videoReadiness, videoAbPreflight, candidateVideoNotes, transcriptEvidence, proofRow }) {
  const realCandidateRows = Number(videoReadiness?.counts?.realCandidateRows || 0);
  const realReferenceRows = Number(videoReadiness?.counts?.realReferenceRows || 0);
  const runtimeChecks = (videoAbPreflight?.checks || []).filter(item => item.status === 'fail');
  const missing = uniqueStrings([
    ...(proofRow?.missingProofRequirements || []),
    ...runtimeChecks.map(item => item.missingProofRequirement || item.action)
  ]);
  const topVideoNotes = candidateVideoNotes.slice(0, 6).map(item => ({
    type: 'candidate_video_note',
    path: item.sourcePath,
    contribution: `${item.creatorName} / ${item.title || item.noteId} / ${item.videoUrl}`,
    closureBoundary: '可作为候选视频资源线索；仍需 live A/B 跑出 proofContext。'
  }));
  return {
    id: 'video-ab-live-proof',
    title: '真实视频 A/B live proof',
    currentProofStatus: proofRow?.status || 'open',
    projectMaterialFound: realCandidateRows > 0 || candidateVideoNotes.length > 0 || Number(worklist.counts?.videoWorklistRows || 0) > 0,
    canCloseNow: false,
    materialSignals: {
      videoResourceReady: Boolean(videoReadiness?.acceptance?.readyForVideoAbPreflight),
      realCandidateRows,
      realReferenceRows,
      candidateVideoNoteCount: candidateVideoNotes.length,
      transcriptEvidenceCount: transcriptEvidence.length,
      preflightReadyForVideoAb: Boolean(videoAbPreflight?.readyForVideoAb),
      preflightFailureCount: Number(videoAbPreflight?.failureCount ?? runtimeChecks.length)
    },
    connectedArtifacts: [
      'outputs/video-intake-pack-latest/video-resource-template.csv',
      'outputs/video-resource-readiness-latest/video-resource-readiness-summary.json',
      'outputs/video-hit-rate-preflight-latest/video-hit-rate-preflight-summary.json',
      'outputs/real-gap-material-audit-latest/real-gap-material-audit-summary.json'
    ],
    projectMaterials: [
      {
        type: 'video_resource_readiness',
        path: rel(root, videoReadiness?.inputPath || path.join(root, 'outputs', 'video-intake-pack-latest', 'video-resource-template.csv')),
        contribution: `readyForVideoAbPreflight=${Boolean(videoReadiness?.acceptance?.readyForVideoAbPreflight)}, realCandidateRows=${realCandidateRows}, realReferenceRows=${realReferenceRows}`,
        closureBoundary: '资源 readiness 不等于 acceptance:video-ab live proof。'
      },
      ...topVideoNotes,
      ...transcriptEvidence.slice(0, 4).map(item => ({
        type: 'experience_transcript',
        path: `${item.path}:${item.lineNumber}`,
        contribution: item.snippet,
        closureBoundary: '证明视频/封面深看规则有经验来源，不替代 live provider 验收。'
      }))
    ],
    blockingRequirements: missing.length ? missing : ['proofContext.mode=live', 'proofContext.generatedBy=acceptance:video-ab'],
    recommendedPromotion: [
      '视频资源已到位，下一步不是继续找候选视频，而是补 runtime/company/VOC social/video analysis provider。',
      'provider 就绪后运行 npm run acceptance:video-ab，并要求 video-hit-rate-summary.json 带 live proofContext。',
      '只有 acceptance:video-ab 输出通过，才能关闭 video-ab-live-proof。'
    ]
  };
}

function buildManualCustomerGap({ root, worklist, transcriptIndex, transcriptEvidence, proofRow }) {
  const reviewDraftRows = Number(worklist.counts?.reviewDraftRows || 0);
  const customerEffectRule = (transcriptIndex.ruleCoverage || []).find(item => item.id === 'customer-effect-goal');
  const missing = uniqueStrings(proofRow?.missingProofRequirements || []);
  return {
    id: 'manual-review-and-customer-effect',
    title: '商务复核和客户效果证明',
    currentProofStatus: proofRow?.status || 'open',
    projectMaterialFound: reviewDraftRows > 0 || transcriptEvidence.length > 0 || Boolean(customerEffectRule?.hitCount),
    canCloseNow: false,
    materialSignals: {
      reviewDraftRows,
      transcriptEvidenceCount: transcriptEvidence.length,
      customerEffectRuleHitCount: Number(customerEffectRule?.hitCount || 0),
      currentEvidence: proofRow?.evidence || ''
    },
    connectedArtifacts: [
      'outputs/local-seed-to-intake-worklist-latest/manual-review-draft.csv',
      'outputs/experience-transcript-index-latest/experience-transcript-index-summary.json',
      'outputs/optimization-pipeline-latest/customer-effect-audit/customer-effect-summary.json',
      'outputs/real-gap-material-audit-latest/real-gap-material-audit-summary.json'
    ],
    projectMaterials: [
      {
        type: 'manual_review_draft',
        path: 'outputs/local-seed-to-intake-worklist-latest/manual-review-draft.csv',
        contribution: `DHA PoC 候选已转为 ${reviewDraftRows} 行商务复核草表。`,
        closureBoundary: '草表仍缺人工复核标签、客户选择、反馈原因和本轮人工补号量。'
      },
      ...(customerEffectRule ? [{
        type: 'experience_rule',
        path: 'outputs/experience-transcript-index-latest/experience-transcript-index-summary.json',
        contribution: customerEffectRule.practicalRule,
        closureBoundary: '经验规则说明验收方向，不等于客户效果统计。'
      }] : []),
      ...transcriptEvidence.slice(0, 6).map(item => ({
        type: 'experience_transcript',
        path: `${item.path}:${item.lineNumber}`,
        contribution: item.snippet,
        closureBoundary: '只能作为复核规则和效果目标来源，不能替代客户明细字段。'
      }))
    ],
    blockingRequirements: missing.length ? missing : [
      'customerSelectedRateMeasured=true',
      'referenceCustomerSelectedRateMeasured=true',
      'currentManualSupplement.available=true'
    ],
    recommendedPromotion: [
      '把候选草表交给商务/投放复核，补“可直接发客户/负样本/待补证”等真实标签。',
      '逐条补客户选中/拒绝、归因类型、反馈原因和本轮人工补号量。',
      '绑定通过的 history:audit 后再跑 customer-effect:audit，不能只用经验目标宣称 30%/40%/50% 达标。'
    ]
  };
}

function extractCandidateVideoNotes(file) {
  const data = readJson(file);
  if (!Array.isArray(data)) return [];
  const rows = [];
  for (const item of data) {
    const profile = item.profile || item.search || {};
    for (const [key, rate] of Object.entries({
      note_rate_video: item.note_rate_video,
      note_rate_photo: item.note_rate_photo
    })) {
      const notes = Array.isArray(rate?.notes) ? rate.notes : [];
      for (const note of notes) {
        if (!note?.noteId) continue;
        const isVideo = key === 'note_rate_video' || Number(note.type) === 2;
        if (!isVideo) continue;
        rows.push({
          creatorName: sanitizeCell(item.name || profile.name || ''),
          redId: sanitizeCell(profile.redId || item.search?.redId || ''),
          noteId: sanitizeCell(note.noteId),
          title: sanitizeCell(note.title || ''),
          type: Number(note.type || 0),
          readNum: Number(note.readNum || 0),
          canJump: Boolean(note.canJump),
          coverUrl: sanitizePublicUrl(note.imgUrl || ''),
          videoUrl: sanitizePublicUrl(`https://www.xiaohongshu.com/discovery/item/${note.noteId}`),
          sourcePath: rel(ROOT, file)
        });
      }
    }
  }
  return uniqueBy(rows, item => item.noteId)
    .sort((a, b) => Number(b.readNum || 0) - Number(a.readNum || 0));
}

function findTranscriptHits(transcriptsDir, patterns) {
  if (!fs.existsSync(transcriptsDir)) return [];
  const files = fs.readdirSync(transcriptsDir, { withFileTypes: true })
    .filter(entry => entry.isFile() && /\.md$|\.txt$/.test(entry.name))
    .map(entry => path.join(transcriptsDir, entry.name))
    .sort((a, b) => a.localeCompare(b, 'zh-CN'));
  const rows = [];
  for (const file of files) {
    const lines = fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '').split(/\r?\n/);
    for (let index = 0; index < lines.length; index += 1) {
      const line = lines[index].trim();
      if (!line) continue;
      if (!patterns.some(pattern => pattern.test(line))) continue;
      rows.push({
        path: rel(ROOT, file),
        lineNumber: index + 1,
        snippet: sanitizeSnippet(line)
      });
    }
  }
  return uniqueBy(rows, item => `${item.path}:${item.lineNumber}`).slice(0, 24);
}

function sanitizeSnippet(value) {
  return sanitizeCell(String(value || '')
    .replace(/\*\*/g, '')
    .replace(/^#+\s*/, '')
    .replace(/\s+/g, ' ')
    .slice(0, 180));
}

function renderReport(summary) {
  return [
    '# 三个真实缺口材料审计',
    '',
    `- 生成时间：${summary.generatedAt}`,
    `- proofLevel：${summary.proofLevel}`,
    `- directCustomerProof：${summary.directCustomerProof}`,
    `- canCloseProofGap：${summary.canCloseProofGap}`,
    `- 找到项目材料的缺口：${summary.materialFoundCount}/${summary.gapCount}`,
    `- 仍打开的缺口：${summary.openGapCount}`,
    '',
    '## 当前源状态',
    '',
    ...Object.entries(summary.sourceState).map(([key, value]) => `- ${key}：${value}`),
    '',
    '## 缺口对照',
    '',
    '| Gap | 当前状态 | 项目资料 | 关键材料信号 | 仍缺证明 | 下一步 |',
    '| --- | --- | --- | --- | --- | --- |',
    ...summary.gaps.map(gap => tableRow([
      gap.id,
      gap.currentProofStatus,
      gap.projectMaterialFound,
      formatSignals(gap.materialSignals),
      gap.blockingRequirements.join('；'),
      gap.recommendedPromotion.join('；')
    ])),
    '',
    '## 材料明细',
    '',
    ...summary.gaps.flatMap(gap => [
      `### ${gap.title}`,
      '',
      ...gap.projectMaterials.map(item => `- ${item.type}｜${item.path}｜${item.contribution}｜边界：${item.closureBoundary}`),
      ''
    ]),
    '## 边界',
    '',
    ...summary.guardrails.map(item => `- ${item}`),
    '',
    '## 下一步',
    '',
    ...summary.nextActions.map(item => `- ${item}`),
    ''
  ].join('\n');
}

function renderCsv(summary) {
  const rows = [
    ['gap', 'currentProofStatus', 'projectMaterialFound', 'canCloseNow', 'materialSignals', 'blockingRequirements', 'recommendedPromotion'],
    ...summary.gaps.map(gap => [
      gap.id,
      gap.currentProofStatus,
      gap.projectMaterialFound,
      gap.canCloseNow,
      formatSignals(gap.materialSignals),
      gap.blockingRequirements.join('；'),
      gap.recommendedPromotion.join('；')
    ]),
    [],
    ['gap', 'materialType', 'path', 'contribution', 'closureBoundary'],
    ...summary.gaps.flatMap(gap => gap.projectMaterials.map(item => [
      gap.id,
      item.type,
      item.path,
      item.contribution,
      item.closureBoundary
    ]))
  ];
  return rows.map(row => row.map(csvCell).join(',')).join('\n');
}

function formatSignals(signals) {
  return Object.entries(signals || {})
    .map(([key, value]) => `${key}=${Array.isArray(value) ? value.join('/') : value}`)
    .join('；');
}

function latestFile(outputsDir, fileName) {
  if (!fs.existsSync(outputsDir)) return '';
  const result = [];
  const stack = [outputsDir];
  while (stack.length) {
    const current = stack.pop();
    for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
      const full = path.join(current, entry.name);
      if (entry.isDirectory()) stack.push(full);
      else if (entry.isFile() && entry.name === fileName) result.push(full);
    }
  }
  return result.sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs)[0] || '';
}

function readJson(file) {
  try {
    if (!file || !fs.existsSync(file)) return null;
    return JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
  } catch (_) {
    return null;
  }
}

function uniqueBy(rows, keyFn) {
  const seen = new Set();
  const result = [];
  for (const row of rows || []) {
    const key = keyFn(row);
    if (!key || seen.has(key)) continue;
    seen.add(key);
    result.push(row);
  }
  return result;
}

function uniqueStrings(values) {
  return Array.from(new Set((values || []).map(value => String(value || '').trim()).filter(Boolean)));
}

function extractNumber(value, pattern) {
  const match = pattern.exec(String(value || ''));
  return match ? Number(match[1]) : null;
}

function rel(root, file) {
  if (!file) return '';
  const relative = path.relative(root, file).replace(/\\/g, '/');
  return relative || '.';
}

function parseArgs(argv) {
  const result = {};
  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    if (!arg.startsWith('--')) continue;
    const key = arg.slice(2).replace(/-([a-z])/g, (_, char) => char.toUpperCase());
    const next = argv[index + 1];
    if (!next || next.startsWith('--')) result[key] = true;
    else {
      result[key] = next;
      index += 1;
    }
  }
  return result;
}

function withBom(text) {
  return `\uFEFF${text}`;
}

function tableRow(values) {
  return `| ${values.map(value => escapeCell(value)).join(' | ')} |`;
}

function escapeCell(value) {
  return String(value ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, '<br>');
}

function csvCell(value) {
  return `"${String(value ?? '').replace(/"/g, '""')}"`;
}

if (require.main === module) main();

module.exports = {
  buildRealGapMaterialAudit,
  extractCandidateVideoNotes,
  findTranscriptHits
};