fmode-voc-projects/scripts/tools/ks-lanlingqiuzhi/ks-filter.js

/**
 * KS-VOC 数据筛选脚本
 * 功能：读取 raw/KS/*.json，筛选高质量评论，按假设/主题分类，输出到 markdown
 *
 * 使用方式：
 *   node ks-filter.js                    # 默认输出到 docs/KS-Chanel_analysis/5.数据筛选报告.md
 *   node ks-filter.js --output ./test.md  # 指定输出路径
 *   node ks-filter.js --min-likes 5       # 最小点赞数（默认3）
 *   node ks-filter.js --min-chars 10      # 最小评论字数（默认8）
 */

const fs = require('fs');
const path = require('path');

// ============ 配置 ============
const RAW_DIR = path.join(__dirname, '..', '..', 'raw', 'KS');
const OUTPUT_PATH = process.argv.includes('--output')
  ? process.argv[process.argv.indexOf('--output') + 1]
  : path.join(__dirname, '..', '5.数据筛选报告.md');

const MIN_LIKES = parseInt(process.argv[process.argv.indexOf('--min-likes') + 1]) || 3;
const MIN_CHARS = parseInt(process.argv[process.argv.indexOf('--min-chars') + 1]) || 8;

// 分类映射
const CATEGORY_MAP = {
  '快聘': { hypothesis: 'H1', theme: '快聘核心场景', color: 'amber' },
  '直播带岗': { hypothesis: 'H1', theme: '快聘核心场景', color: 'amber' },
  '蓝领找工作': { hypothesis: 'H1', theme: '快聘核心场景', color: 'amber' },
  '工资靠谱': { hypothesis: 'H3', theme: '薪资透明度', color: 'green' },
  '真实薪资': { hypothesis: 'H3', theme: '薪资透明度', color: 'green' },
  '工资日结': { hypothesis: 'H8', theme: '薪资结算方式', color: 'purple' },
  '包吃包住': { hypothesis: 'H4', theme: '食宿条件', color: 'blue' },
  '当天入职': { hypothesis: 'H5', theme: '入职便捷性', color: 'blue' },
  '求职被骗': { hypothesis: 'H6', theme: '求职被骗焦虑', color: 'rose' },
  '入职被坑': { hypothesis: 'H6', theme: '求职被骗焦虑', color: 'rose' },
  '黑中介': { hypothesis: 'H7', theme: '黑中介痛点', color: 'rose' },
  '押金不退': { hypothesis: 'H7', theme: '黑中介痛点', color: 'rose' },
};

// 噪音词过滤
const SPAM_PATTERNS = [
  /^[\[\]\🌹👍]+$/,                    // 纯emoji/符号
  /^(催|来|在|哪|要|报|这|那|啥|吗|么|呢|吧|啊|哈|噢)$/,  // 单字重复
  /^嗯{2,}/,                           // 重复语气词
  /^(支持|感谢|点赞|收藏|分享)+$/,      // 灌水词
];

// ============ 工具函数 ============
function isValidComment(comment) {
  const content = comment.content?.trim() || '';

  // 长度过滤
  if (content.length < MIN_CHARS) return false;

  // 过滤纯数字/字母
  if (/^[\d\w\s\.,]+$/.test(content)) return false;

  // 噪音词过滤
  for (const pattern of SPAM_PATTERNS) {
    if (pattern.test(content)) return false;
  }

  // 过滤纯@提及
  if (/^@{[\w]+}$/.test(content.trim())) return false;

  // 过滤太短的疑问（可能是灌水）
  if (content.length < 10 && content.includes('？') && content.length < 8) return false;

  return true;
}

function extractKeyword(filename) {
  return path.basename(filename, '.json');
}

function formatDate(timestamp) {
  return new Date(timestamp).toLocaleDateString('zh-CN', {
    year: 'numeric',
    month: '2-digit',
    day: '2-digit',
  });
}

function escapeMarkdown(text) {
  return text
    .replace(/\\/g, '\\\\')
    .replace(/`/g, '\\`')
    .replace(/\*/g, '\\*')
    .replace(/_/g, '\\_')
    .replace(/\[/g, '\\[')
    .replace(/\]/g, '\\]');
}

// ============ 读取数据 ============
function loadAllData() {
  const files = fs.readdirSync(RAW_DIR).filter(f => f.endsWith('.json') && f !== '-.json');

  const allData = [];
  for (const file of files) {
    const filepath = path.join(RAW_DIR, file);
    try {
      const data = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
      data._file = file;
      allData.push(data);
    } catch (e) {
      console.warn(`⚠️ 跳过损坏文件: ${file}`);
    }
  }

  return allData;
}

// ============ 筛选逻辑 ============
function filterComments(data) {
  const validComments = [];

  for (const [videoId, comments] of Object.entries(data.comments || {})) {
    for (const comment of comments) {
      if (isValidComment(comment) && (comment.like_count || 0) >= MIN_LIKES) {
        validComments.push({
          ...comment,
          _video_id: videoId,
          _video_caption: data.top_videos?.find(v => v.id === videoId)?.caption || '',
        });
      }
    }
  }

  return validComments;
}

// ============ 生成报告 ============
function generateMarkdown(allFilteredData, stats) {
  const now = new Date().toLocaleString('zh-CN');

  let md = '';
  md += `# 快手快聘 · 数据筛选报告\n\n`;
  md += `> **生成时间**：${now}  \n`;
  md += `> **筛选条件**：点赞 ≥ ${MIN_LIKES} | 字数 ≥ ${MIN_CHARS} | 噪音过滤已启用  \n`;
  md += `> **数据来源**：\`${RAW_DIR.replace(/\\/g, '\\\\')}\`\n\n`;
  md += `---\n\n`;

  // 统计概览
  md += `## 一、筛选统计\n\n`;
  md += `| 指标 | 数值 |\n`;
  md += `| --- | --- |\n`;
  md += `| 关键词文件数 | ${stats.totalFiles} |\n`;
  md += `| 视频总数 | ${stats.totalVideos} |\n`;
  md += `| 原始评论总数 | ${stats.totalComments} |\n`;
  md += `| 筛选后评论数 | ${stats.filteredComments} |\n`;
  md += `| 过滤掉 | ${stats.totalComments - stats.filteredComments} (${((1 - stats.filteredComments / stats.totalComments) * 100).toFixed(1)}%) |\n\n`;

  // 按假设分组
  const byHypothesis = {};
  for (const item of allFilteredData) {
    const cat = CATEGORY_MAP[item._keyword] || { hypothesis: 'H?', theme: '未分类', color: 'gray' };
    const h = cat.hypothesis;
    if (!byHypothesis[h]) {
      byHypothesis[h] = { comments: [], theme: cat.theme, color: cat.color };
    }
    byHypothesis[h].comments.push(item);
  }

  md += `## 二、按假设分组（H1-H8）\n\n`;

  const sortedH = Object.keys(byHypothesis).sort();
  for (const h of sortedH) {
    const group = byHypothesis[h];
    const icon = h === 'H1' ? '💼' : h === 'H3' ? '💰' : h === 'H6' ? '⚠️' : '📌';

    md += `### ${icon} ${h} · ${group.theme}\n\n`;
    md += `**评论数：${group.comments.length}条**\n\n`;

    // 按点赞数排序，取前10
    const topComments = group.comments
      .sort((a, b) => (b.like_count || 0) - (a.like_count || 0))
      .slice(0, 10);

    for (const comment of topComments) {
      const likes = comment.like_count || 0;
      const date = formatDate(comment.create_time);
      const content = escapeMarkdown(comment.content);
      const ip = comment.ip_location ? ` [${comment.ip_location}]` : '';

      md += `- **快手 ♥${likes}**${ip} "${content}"\n`;
    }

    md += `\n`;
  }

  // 高赞评论TOP20
  md += `## 三、高赞评论 TOP20\n\n`;
  md += `| 排名 | 平台 | 点赞 | 评论内容摘要 | 关键词 |\n`;
  md += `| --- | --- | --- | --- | --- |\n`;

  const allTop = allFilteredData
    .sort((a, b) => (b.like_count || 0) - (a.like_count || 0))
    .slice(0, 20);

  allTop.forEach((c, i) => {
    const content = c.content.length > 40 ? c.content.slice(0, 40) + '...' : c.content;
    md += `| ${i + 1} | 快手 | ♥${c.like_count || 0} | ${escapeMarkdown(content)} | ${c._keyword} |\n`;
  });

  md += `\n`;

  // 原始数据样例
  md += `## 四、各关键词数据样例\n\n`;

  for (const data of stats.keywordDetails) {
    if (data.filteredCount === 0) continue;

    const cat = CATEGORY_MAP[data.keyword] || { theme: '未分类', color: 'gray' };
    md += `### ${data.keyword} · ${cat.theme}\n\n`;
    md += `> 视频数: ${data.videoCount} | 评论数: ${data.commentCount} | 有效评论: ${data.filteredCount}\n\n`;

    const top3 = data.comments
      .sort((a, b) => (b.like_count || 0) - (a.like_count || 0))
      .slice(0, 3);

    for (const c of top3) {
      const likes = c.like_count || 0;
      const content = escapeMarkdown(c.content);
      md += `- **♥${likes}** "${content}"\n`;
    }

    md += `\n`;
  }

  md += `---\n\n`;
  md += `*报告由 ks-filter.js 自动生成 · ${now}*\n`;

  return md;
}

// ============ 主流程 ============
function main() {
  console.log(`\n📊 KS-VOC 数据筛选脚本`);
  console.log(`   筛选条件: 点赞≥${MIN_LIKES}, 字数≥${MIN_CHARS}`);
  console.log(`   数据源: ${RAW_DIR}`);
  console.log(`   输出: ${OUTPUT_PATH}\n`);

  // 1. 读取所有JSON
  console.log(`📂 读取数据文件...`);
  const allData = loadAllData();
  console.log(`   读取了 ${allData.length} 个关键词文件`);

  // 2. 统计
  const stats = {
    totalFiles: allData.length,
    totalVideos: 0,
    totalComments: 0,
    filteredComments: 0,
    keywordDetails: [],
  };

  // 3. 筛选
  const allFilteredData = [];

  for (const data of allData) {
    const keyword = extractKeyword(data._file);
    const cat = CATEGORY_MAP[keyword] || { hypothesis: 'H?', theme: '未分类', color: 'gray' };

    const videoCount = data.top_videos?.length || 0;
    const comments = data.comments || {};
    const commentList = Object.values(comments).flat();
    const commentCount = commentList.length;

    stats.totalVideos += videoCount;
    stats.totalComments += commentCount;

    // 筛选
    const filteredComments = filterComments(data);
    stats.filteredComments += filteredComments.length;

    // 附加关键词信息
    for (const c of filteredComments) {
      c._keyword = keyword;
    }
    allFilteredData.push(...filteredComments);

    stats.keywordDetails.push({
      keyword,
      theme: cat.theme,
      videoCount,
      commentCount,
      filteredCount: filteredComments.length,
      comments: filteredComments,
    });

    console.log(`   ${keyword}: ${commentCount} → ${filteredComments.length} 条有效`);
  }

  console.log(`\n📈 统计结果:`);
  console.log(`   总视频: ${stats.totalVideos}`);
  console.log(`   总评论: ${stats.totalComments}`);
  console.log(`   有效评论: ${stats.filteredComments}`);
  console.log(`   过滤率: ${((1 - stats.filteredComments / stats.totalComments) * 100).toFixed(1)}%`);

  // 4. 生成Markdown
  console.log(`\n📝 生成Markdown报告...`);
  const markdown = generateMarkdown(allFilteredData, stats);

  // 5. 写入文件
  const outputDir = path.dirname(OUTPUT_PATH);
  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir, { recursive: true });
  }
  fs.writeFileSync(OUTPUT_PATH, '\uFEFF' + markdown, 'utf-8'); // BOM for Chinese

  console.log(`\n✅ 报告已生成: ${OUTPUT_PATH}`);
  console.log(`\n---`);
  console.log(`📌 下一步:`);
  console.log(`   1. 审查 docs/KS-Chanel_analysis/5.数据筛选报告.md`);
  console.log(`   2. 填充 VOC 洞察报告的证据锚点`);
  console.log(`   3. 运行 gen-report.js 生成可视化报告\n`);
}

main();