tihao-ai/claude-code-tihao-sourcing/scripts/review-metrics.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');

const POSITIVE_LABELS = new Set(['可直接发客户', '商务复核']);
const NEGATIVE_LABELS = new Set(['跑偏', '硬性规则违规', '调性不符', '主页质感不符', '参考账号不像']);
const SELECTED_LABELS = new Set(['客户选中', '已选中', '选中', '客户通过']);
const ATTRIBUTION_FIELDS = ['归因类型', '失败归因', '归因'];
const REASON_FIELDS = ['反馈原因', '拒绝原因', '复核备注', '备注'];
const REFERENCE_STRATEGIES = new Set(['reference-account', 'homepage-evidence', 'video-enhanced']);
const KEYWORD_BASELINE_STRATEGIES = new Set(['baseline-live', 'brief-only', 'keyword-only']);
const HEADER = [
  'brief编号',
  '策略',
  '排名',
  '平台',
  '博主名称',
  '综合分',
  'brief匹配分',
  '参考风格分',
  '主页证据分',
  '视觉质感分',
  '调性一致分',
  '证据加分',
  '证据风险扣分',
  '推荐理由',
  '风险提示',
  '主页链接',
  '人工复核标签'
];

function main() {
  const args = parseArgs(process.argv.slice(2));
  const input = args.input || args.csv || args.review || process.env.TIHAO_REVIEW_CSV || '';
  if (!input) throw new Error('Usage: node scripts/review-metrics.js --input <reviewed.csv> [--output <dir>] [--strict]');
  const inputPath = path.resolve(input);
  const outputDir = path.resolve(args.output || process.env.TIHAO_REVIEW_OUTPUT || path.join(path.dirname(inputPath), 'review-metrics'));
  const strict = Boolean(args.strict || process.env.TIHAO_REVIEW_STRICT === 'true');
  const rows = readCsv(inputPath);
  const metrics = buildMetrics({ inputPath, rows });
  const report = renderReport(metrics);
  fs.mkdirSync(outputDir, { recursive: true });
  const jsonPath = path.join(outputDir, 'review-metrics-summary.json');
  const mdPath = path.join(outputDir, 'review-metrics-report.md');
  fs.writeFileSync(jsonPath, JSON.stringify(metrics, null, 2), 'utf8');
  fs.writeFileSync(mdPath, withBom(report), 'utf8');
  console.log(JSON.stringify({
    input: inputPath,
    outputDir,
    json: jsonPath,
    report: mdPath,
    total: metrics.total,
    labeled: metrics.labeled,
    businessUsableRate: metrics.businessUsableRate,
    offTargetHardFailRate: metrics.offTargetHardFailRate,
    customerSelectedRate: metrics.customerSelectedRate,
    failureAttributionCoverage: metrics.failureAttributionCoverage,
    pass: metrics.acceptance.overallPass
  }, null, 2));
  if (strict && !metrics.acceptance.overallPass) process.exitCode = 2;
}

function buildMetrics({ inputPath, rows }) {
  const headerOk = HEADER.every((key, index) => rows.header[index] === key);
  const body = rows.body;
  const labeledRows = body.filter(row => normalizeLabel(row['人工复核标签']));
  const positiveRows = labeledRows.filter(row => POSITIVE_LABELS.has(normalizeLabel(row['人工复核标签'])));
  const negativeRows = labeledRows.filter(row => NEGATIVE_LABELS.has(normalizeLabel(row['人工复核标签'])));
  const selectedRows = labeledRows.filter(row => SELECTED_LABELS.has(normalizeLabel(row['客户选择'] || row['客户选中'] || row['最终结果'] || row['人工复核标签'])));
  const attribution = summarizeFailureAttribution(negativeRows);
  const duplicateKeyCount = countDuplicateKeys(body);
  const rankIssues = findRankIssues(body);
  const byBrief = summarizeByBrief(labeledRows);
  const byStrategy = summarizeByStrategy(labeledRows);
  const strategyComparison = compareReferenceAndKeywordStrategies(byStrategy);
  const total = body.length;
  const labeled = labeledRows.length;
  const metrics = {
    inputPath,
    generatedAt: new Date().toISOString(),
    headerOk,
    total,
    labeled,
    unlabeled: total - labeled,
    positiveCount: positiveRows.length,
    negativeCount: negativeRows.length,
    selectedCount: selectedRows.length,
    businessUsableRate: ratio(positiveRows.length, Math.max(labeled, 1)),
    offTargetHardFailRate: ratio(negativeRows.length, Math.max(labeled, 1)),
    customerSelectedRate: ratio(selectedRows.length, Math.max(labeled, 1)),
    failureAttribution: attribution,
    failureAttributionCoverage: ratio(attribution.attributedCount, Math.max(attribution.failureRows, 1)),
    duplicateKeyCount,
    rankContinuous: rankIssues.length === 0,
    rankIssues,
    byBrief,
    byStrategy,
    strategyComparison
  };
  metrics.acceptance = {
    headerOk,
    duplicateKeyCountZero: duplicateKeyCount === 0,
    rankContinuous: metrics.rankContinuous,
    enoughLabeledRows: labeled > 0,
    businessUsableRatePass: metrics.businessUsableRate >= 0.6,
    offTargetHardFailRatePass: metrics.offTargetHardFailRate <= 0.1,
    customerSelectedRateMeasured: selectedRows.length > 0,
    customerSelectedRatePass: selectedRows.length === 0 ? null : metrics.customerSelectedRate >= 0.3,
    referenceCustomerSelectedRatePass: strategyComparison.reference.selected === 0 ? null : strategyComparison.reference.customerSelectedRate >= 0.4,
    referencePassRateHigherThanKeyword: strategyComparison.keyword.total === 0 || strategyComparison.reference.total === 0
      ? null
      : strategyComparison.reference.businessUsableRate > strategyComparison.keyword.businessUsableRate,
    failureAttributionCoveragePass: attribution.failureRows === 0 || attribution.missingAttributionRows.length === 0
  };
  metrics.acceptance.overallPass = metrics.acceptance.headerOk &&
    metrics.acceptance.duplicateKeyCountZero &&
    metrics.acceptance.rankContinuous &&
    metrics.acceptance.enoughLabeledRows &&
    metrics.acceptance.businessUsableRatePass &&
    metrics.acceptance.offTargetHardFailRatePass &&
    metrics.acceptance.failureAttributionCoveragePass &&
    (metrics.acceptance.customerSelectedRatePass !== false) &&
    (metrics.acceptance.referenceCustomerSelectedRatePass !== false) &&
    (metrics.acceptance.referencePassRateHigherThanKeyword !== false);
  return metrics;
}

function summarizeFailureAttribution(rows) {
  const byAttribution = {};
  const missingAttributionRows = [];
  let attributedCount = 0;
  for (const row of rows) {
    const attribution = firstNonEmpty(row, ATTRIBUTION_FIELDS);
    const reason = firstNonEmpty(row, REASON_FIELDS);
    if (attribution) {
      attributedCount += 1;
      byAttribution[attribution] = (byAttribution[attribution] || 0) + 1;
    } else {
      missingAttributionRows.push({
        brief编号: row['brief编号'] || '',
        策略: row['策略'] || '',
        排名: row['排名'] || '',
        平台: row['平台'] || '',
        博主名称: row['博主名称'] || '',
        人工复核标签: row['人工复核标签'] || '',
        主页链接: row['主页链接'] || '',
        反馈原因: reason
      });
    }
  }
  return {
    failureRows: rows.length,
    attributedCount,
    missingAttributionCount: missingAttributionRows.length,
    byAttribution,
    missingAttributionRows
  };
}

function firstNonEmpty(row, fields) {
  for (const field of fields) {
    const value = normalizeLabel(row[field]);
    if (value) return value;
  }
  return '';
}

function summarizeByBrief(rows) {
  const groups = new Map();
  for (const row of rows) {
    const briefId = row['brief编号'] || '未命名brief';
    const group = groups.get(briefId) || { briefId, total: 0, positive: 0, negative: 0, selected: 0, byLabel: {} };
    const label = normalizeLabel(row['人工复核标签']) || '未标注';
    group.total += 1;
    if (POSITIVE_LABELS.has(label)) group.positive += 1;
    if (NEGATIVE_LABELS.has(label)) group.negative += 1;
    if (SELECTED_LABELS.has(normalizeLabel(row['客户选择'] || row['客户选中'] || row['最终结果'] || row['人工复核标签']))) group.selected += 1;
    group.byLabel[label] = (group.byLabel[label] || 0) + 1;
    groups.set(briefId, group);
  }
  return [...groups.values()].map(group => ({
    ...group,
    businessUsableRate: ratio(group.positive, Math.max(group.total, 1)),
    offTargetHardFailRate: ratio(group.negative, Math.max(group.total, 1)),
    customerSelectedRate: ratio(group.selected, Math.max(group.total, 1))
  }));
}

function summarizeByStrategy(rows) {
  const groups = new Map();
  for (const row of rows) {
    const strategy = normalizeStrategy(row['策略']);
    const group = groups.get(strategy) || { strategy, total: 0, positive: 0, negative: 0, selected: 0, byLabel: {} };
    const label = normalizeLabel(row['人工复核标签']) || '未标注';
    group.total += 1;
    if (POSITIVE_LABELS.has(label)) group.positive += 1;
    if (NEGATIVE_LABELS.has(label)) group.negative += 1;
    if (SELECTED_LABELS.has(normalizeLabel(row['客户选择'] || row['客户选中'] || row['最终结果'] || row['人工复核标签']))) group.selected += 1;
    group.byLabel[label] = (group.byLabel[label] || 0) + 1;
    groups.set(strategy, group);
  }
  return [...groups.values()].map(group => ({
    ...group,
    businessUsableRate: ratio(group.positive, Math.max(group.total, 1)),
    offTargetHardFailRate: ratio(group.negative, Math.max(group.total, 1)),
    customerSelectedRate: ratio(group.selected, Math.max(group.total, 1))
  }));
}

function compareReferenceAndKeywordStrategies(byStrategy) {
  const referenceRows = byStrategy.filter(item => REFERENCE_STRATEGIES.has(item.strategy));
  const keywordRows = byStrategy.filter(item => KEYWORD_BASELINE_STRATEGIES.has(item.strategy));
  return {
    reference: aggregateStrategyGroup('reference', referenceRows),
    keyword: aggregateStrategyGroup('keyword-baseline', keywordRows)
  };
}

function aggregateStrategyGroup(name, groups) {
  const total = groups.reduce((sum, item) => sum + item.total, 0);
  const positive = groups.reduce((sum, item) => sum + item.positive, 0);
  const negative = groups.reduce((sum, item) => sum + item.negative, 0);
  const selected = groups.reduce((sum, item) => sum + item.selected, 0);
  return {
    name,
    strategies: groups.map(item => item.strategy),
    total,
    positive,
    negative,
    selected,
    businessUsableRate: ratio(positive, Math.max(total, 1)),
    offTargetHardFailRate: ratio(negative, Math.max(total, 1)),
    customerSelectedRate: ratio(selected, Math.max(total, 1))
  };
}

function countDuplicateKeys(rows) {
  const seen = new Set();
  let duplicates = 0;
  for (const row of rows) {
    const key = row['主页链接']
      ? `${row['brief编号']}|${row['平台']}|${row['主页链接']}`
      : `${row['brief编号']}|${row['平台']}|${row['博主名称']}`;
    if (seen.has(key)) duplicates += 1;
    seen.add(key);
  }
  return duplicates;
}

function findRankIssues(rows) {
  const groups = new Map();
  for (const row of rows) {
    const briefId = row['brief编号'] || '未命名brief';
    if (!groups.has(briefId)) groups.set(briefId, []);
    groups.get(briefId).push(Number(row['排名']));
  }
  const issues = [];
  for (const [briefId, ranks] of groups.entries()) {
    const ok = ranks.every((rank, index) => rank === index + 1);
    if (!ok) issues.push({ briefId, ranks: ranks.join(',') });
  }
  return issues;
}

function renderReport(metrics) {
  const lines = [
    '# 人工复核质量指标报告',
    '',
    `- 输入文件：${metrics.inputPath}`,
    `- 生成时间：${metrics.generatedAt}`,
    `- 总行数：${metrics.total}`,
    `- 已标注：${metrics.labeled}`,
    `- 未标注：${metrics.unlabeled}`,
    `- 总体验收：${metrics.acceptance.overallPass ? '通过' : '未通过'}`,
    '',
    '## 核心指标',
    '',
    '| 指标 | 当前值 | 目标 | 状态 |',
    '| --- | ---: | ---: | --- |',
    `| 商务可用率 | ${pct(metrics.businessUsableRate)} | >= 60% | ${passFail(metrics.acceptance.businessUsableRatePass)} |`,
    `| 负样本率 | ${pct(metrics.offTargetHardFailRate)} | <= 10% | ${passFail(metrics.acceptance.offTargetHardFailRatePass)} |`,
    `| 客户选中率 | ${metrics.selectedCount ? pct(metrics.customerSelectedRate) : '未标注'} | >= 30% | ${metrics.acceptance.customerSelectedRatePass === null ? '待补客户选择' : passFail(metrics.acceptance.customerSelectedRatePass)} |`,
    `| 负样本归因覆盖率 | ${metrics.failureAttribution.failureRows ? pct(metrics.failureAttributionCoverage) : '无负样本'} | 100% | ${passFail(metrics.acceptance.failureAttributionCoveragePass)} |`,
    `| 参考链路客户选中率 | ${metrics.strategyComparison.reference.selected ? pct(metrics.strategyComparison.reference.customerSelectedRate) : '未标注'} | >= 40% | ${metrics.acceptance.referenceCustomerSelectedRatePass === null ? '待补客户选择' : passFail(metrics.acceptance.referenceCustomerSelectedRatePass)} |`,
    `| 参考链路通过率高于关键词基线 | ${formatReferenceVsKeyword(metrics.strategyComparison)} | > 关键词基线 | ${metrics.acceptance.referencePassRateHigherThanKeyword === null ? '待补对照组' : passFail(metrics.acceptance.referencePassRateHigherThanKeyword)} |`,
    `| 重复键 | ${metrics.duplicateKeyCount} | 0 | ${passFail(metrics.acceptance.duplicateKeyCountZero)} |`,
    `| 排名连续 | ${metrics.rankContinuous ? '是' : '否'} | 是 | ${passFail(metrics.acceptance.rankContinuous)} |`,
    '',
    '## 分 Brief 指标',
    '',
    '| Brief | 已标注 | 商务可用率 | 负样本率 | 客户选中率 |',
    '| --- | ---: | ---: | ---: | ---: |',
    ...metrics.byBrief.map(item => `| ${escapeCell(item.briefId)} | ${item.total} | ${pct(item.businessUsableRate)} | ${pct(item.offTargetHardFailRate)} | ${item.selected ? pct(item.customerSelectedRate) : '未标注'} |`),
    '',
    '## 分策略指标',
    '',
    '| 策略 | 已标注 | 商务可用率 | 负样本率 | 客户选中率 |',
    '| --- | ---: | ---: | ---: | ---: |',
    ...metrics.byStrategy.map(item => `| ${escapeCell(item.strategy)} | ${item.total} | ${pct(item.businessUsableRate)} | ${pct(item.offTargetHardFailRate)} | ${item.selected ? pct(item.customerSelectedRate) : '未标注'} |`),
    '',
    '## 参考链路对照',
    '',
    '| 组别 | 策略 | 已标注 | 商务可用率 | 客户选中率 |',
    '| --- | --- | ---: | ---: | ---: |',
    `| 参考/主页/视频增强 | ${escapeCell(metrics.strategyComparison.reference.strategies.join('、') || '无')} | ${metrics.strategyComparison.reference.total} | ${pct(metrics.strategyComparison.reference.businessUsableRate)} | ${metrics.strategyComparison.reference.selected ? pct(metrics.strategyComparison.reference.customerSelectedRate) : '未标注'} |`,
    `| 关键词/基础基线 | ${escapeCell(metrics.strategyComparison.keyword.strategies.join('、') || '无')} | ${metrics.strategyComparison.keyword.total} | ${pct(metrics.strategyComparison.keyword.businessUsableRate)} | ${metrics.strategyComparison.keyword.selected ? pct(metrics.strategyComparison.keyword.customerSelectedRate) : '未标注'} |`,
    '',
    '## 负样本归因',
    '',
    `- 负样本数：${metrics.failureAttribution.failureRows}`,
    `- 已归因：${metrics.failureAttribution.attributedCount}`,
    `- 缺归因：${metrics.failureAttribution.missingAttributionCount}`,
    '',
    '| 归因类型 | 数量 |',
    '| --- | ---: |',
    ...Object.entries(metrics.failureAttribution.byAttribution).map(([key, value]) => `| ${escapeCell(key)} | ${value} |`),
    ...(Object.keys(metrics.failureAttribution.byAttribution).length ? [] : ['| 无 | 0 |']),
    '',
    '## 缺归因负样本',
    '',
    '| Brief | 排名 | 平台 | 博主名称 | 标签 | 反馈原因 |',
    '| --- | ---: | --- | --- | --- | --- |',
    ...metrics.failureAttribution.missingAttributionRows.slice(0, 50).map(row => `| ${escapeCell(row['brief编号'])} | ${escapeCell(row['排名'])} | ${escapeCell(row['平台'])} | ${escapeCell(row['博主名称'])} | ${escapeCell(row['人工复核标签'])} | ${escapeCell(row['反馈原因'])} |`),
    ...(metrics.failureAttribution.missingAttributionRows.length ? [] : ['| 无 |  |  |  |  |  |']),
    '',
    '## 结论',
    '',
    '- “可直接发客户 + 商务复核”用于衡量短期商务可用率。',
    '- 负样本标签用于衡量需求解析、隐性规则、主页质感、参考风格和调性命中问题。',
    '- 负样本必须填写“归因类型/失败归因/归因”，否则不能进入可反哺优化的复盘样本。',
    '- 客户选中率需要客户最终选择字段；没有该字段时不能宣称已达到 30%/50%。',
    '- 参考链路是否优于纯关键词召回，必须同时存在参考策略和关键词/基础基线策略的标注样本才能判断。'
  ];
  return lines.join('\n');
}

function readCsv(file) {
  const text = fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '');
  const lines = text.split(/\r?\n/).filter(Boolean);
  const header = parseCsvLine(lines.shift() || '');
  return {
    header,
    body: lines.map(line => {
      const cells = parseCsvLine(line);
      const row = {};
      header.forEach((key, index) => {
        row[key] = cells[index] || '';
      });
      return row;
    })
  };
}

function parseCsvLine(line) {
  const cells = [];
  let current = '';
  let quoted = false;
  for (let i = 0; i < line.length; i += 1) {
    const char = line[i];
    if (char === '"' && quoted && line[i + 1] === '"') {
      current += '"';
      i += 1;
    } else if (char === '"') quoted = !quoted;
    else if (char === ',' && !quoted) {
      cells.push(current);
      current = '';
    } else current += char;
  }
  cells.push(current);
  return cells;
}

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i += 1) {
    const raw = argv[i];
    if (!raw.startsWith('--')) continue;
    const key = raw.slice(2).replace(/-([a-z])/g, (_, char) => char.toUpperCase());
    const next = argv[i + 1];
    if (!next || next.startsWith('--')) args[key] = true;
    else {
      args[key] = next;
      i += 1;
    }
  }
  return args;
}

function normalizeLabel(value) {
  return String(value || '').trim();
}

function normalizeStrategy(value) {
  return normalizeLabel(value).toLowerCase();
}

function ratio(numerator, denominator) {
  return denominator ? Math.round((Number(numerator || 0) / Number(denominator)) * 10000) / 10000 : 0;
}

function pct(value) {
  return `${Math.round(Number(value || 0) * 100)}%`;
}

function passFail(value) {
  return value ? '通过' : '未通过';
}

function formatReferenceVsKeyword(strategyComparison) {
  if (!strategyComparison.reference.total || !strategyComparison.keyword.total) return '待补对照组';
  return `${pct(strategyComparison.reference.businessUsableRate)} / ${pct(strategyComparison.keyword.businessUsableRate)}`;
}

function escapeCell(value) {
  return String(value || '').replace(/\|/g, '/').replace(/\n/g, ' ');
}

function withBom(text) {
  return `\uFEFF${text}`;
}

if (require.main === module) {
  try {
    main();
  } catch (error) {
    console.error(error && error.stack ? error.stack : String(error));
    process.exit(1);
  }
}

module.exports = {
  buildMetrics,
  readCsv
};