fmode-voc-projects/scripts/tools/voc-validation/collect.js

#!/usr/bin/env node
/**
 * 江中肝纯片 · VOC 验证数据扩采脚本
 *
 * 目标：围绕《1.VOC方向定位.md》5 模块 × 4 渠道 × 6 竞品，
 *       采集高质量小红书评论（评论颗粒度），补充现有 jiangzhong-liver-voc.json。
 *
 * 输出：data/jiangzhong-liver-voc-extended.json
 *   - by_module.{package|pricing|positioning|scene|competitor}.{keyword}.{notes|comments|users}
 *
 * 策略：
 *   - 分批次（argv: 第 N 批，每批 3 关键词）避免限流
 *   - 每关键词取搜索结果按 comments_count 降序前 5 篇笔记，抓前 3 篇的评论
 *   - 支持续跑：已有关键词跳过（--force 强制重抓）
 */
const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

// ============================================================
// 配置
// ============================================================
const XHS_SKILL_CONFIG = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
const xhsConf = JSON.parse(fs.readFileSync(XHS_SKILL_CONFIG, 'utf-8'));
const TIKHUB_TOKEN = xhsConf.endpoint.headers.Authorization.replace(/^Bearer\s+/, '');

const DATA_DIR = path.resolve(__dirname, '..', '..', 'data');
const OUT_PATH = path.join(DATA_DIR, 'jiangzhong-liver-voc-extended.json');
const LOG_PATH = path.join(DATA_DIR, 'voc-validation-collect.log');

// 5 模块 × 关键词（按优先级分组，每批 3-4 关键词）
const KEYWORD_GROUPS = {
    // ===== 定位与话术模块（最高优先级 · 差异化验证）=====
    positioning: [
        { kw: '藤茶 护肝', note: '核心差异化·藤茶概念认知' },
        { kw: '二氢杨梅素', note: '核心成分·DMY 认知度' },
        { kw: '解酒药', note: '核心场景·餐饮社交' },
    ],
    // ===== 场景适配模块 =====
    scene: [
        { kw: '熬夜 护肝', note: '日常养护场景·女性+打工人' },
        { kw: '送礼 保健品', note: '礼品渠道·体面感' },
        { kw: '应酬 护肝', note: '餐饮渠道·男性酒局' },
    ],
    // ===== 竞品认知模块 =====
    competitor: [
        { kw: '护肝片 副作用', note: '药品竞品·护肝片吐槽' },
        { kw: '易善复', note: '西药竞品·胃肠反应' },
        { kw: 'Swisse 水飞蓟', note: '保健品竞品·海外对标' },
        { kw: '海王金樽', note: '失败案例·解酒品复盘' },
        { kw: '汤臣倍健 护肝', note: '国产保健品竞品' },
    ],
    // ===== 竞品扩展（Batch 2 补采：片仔癀 / 熊胆粉 / 葵花 / 解酒神器）=====
    competitor_ext: [
        { kw: '片仔癀 护肝', note: '名贵中成药·天价对标' },
        { kw: '熊胆粉', note: '名贵药材·寒凉/伦理敏感' },
        { kw: '葵花护肝片', note: '国产老牌药·品牌老化' },
        { kw: '解酒神器', note: '餐饮大池·酒局场景 VOC' },
    ],
    // ===== 包装 & 定价模块 =====
    package_pricing: [
        { kw: '护肝片 包装', note: '包装吐槽·"土"的印证' },
        { kw: '保健品 送礼装', note: '礼品包装偏好' },
    ],
};

// ============================================================
// HTTP helper
// ============================================================
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function httpGet(host, urlPath, auth, attempt = 1, maxAttempts = 3) {
    return new Promise((resolve, reject) => {
        const headers = { Accept: 'application/json' };
        if (auth) headers.Authorization = `Bearer ${auth}`;
        const req = https.request({
            hostname: host, path: urlPath, method: 'GET', headers, timeout: 45000,
        }, (res) => {
            const chunks = [];
            res.on('data', (c) => chunks.push(c));
            res.on('end', async () => {
                const body = Buffer.concat(chunks).toString('utf-8');
                if (res.statusCode === 200) {
                    try { resolve(JSON.parse(body)); }
                    catch (e) { resolve({ _error: { parse: e.message, body: body.slice(0, 300) } }); }
                } else if ([400, 429, 500, 502, 503, 504].includes(res.statusCode) && attempt < maxAttempts) {
                    const wait = 1500 * Math.pow(1.8, attempt - 1);
                    console.log(`    ↻ retry#${attempt} in ${wait}ms (status=${res.statusCode})`);
                    await sleep(wait);
                    resolve(httpGet(host, urlPath, auth, attempt + 1, maxAttempts));
                } else {
                    resolve({ _error: { status: res.statusCode, body: body.slice(0, 300) } });
                }
            });
        });
        req.on('error', async (err) => {
            if (attempt < maxAttempts) {
                await sleep(2000 * attempt);
                resolve(httpGet(host, urlPath, auth, attempt + 1, maxAttempts));
            } else reject(err);
        });
        req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
        req.end();
    });
}

// ============================================================
// 小红书采集（单关键词）
// ============================================================
async function collectXhsForKeyword(kw, opts = {}) {
    const { topNotes = 5, commentsPerNote = 50 } = opts;

    console.log(`\n  [xhs] 🔍 "${kw}"`);
    const searchRes = await httpGet('server.fmode.cn',
        `/thapi/v1/xiaohongshu/app/search_notes?keyword=${encodeURIComponent(kw)}&page=1`,
        TIKHUB_TOKEN);

    const items = searchRes?.data?.data?.items || [];
    const notes = items.map((w) => w.note).filter((n) => n?.id);
    console.log(`    ✓ ${notes.length} 条笔记`);

    // 按评论数降序取前 N 篇
    const topNotesArr = notes
        .filter((n) => (n.comments_count || 0) > 3)
        .sort((a, b) => (b.comments_count || 0) - (a.comments_count || 0))
        .slice(0, topNotes);

    // 抓评论（可翻页到目标数量）
    const comments = {};
    for (const note of topNotesArr) {
        await sleep(600);
        const preview = String(note.title || note.desc || note.id).slice(0, 30);
        console.log(`    💬 "${preview}" (${note.comments_count} 评论)`);
        const noteComments = [];
        let cursor = '';
        let pages = 0;
        const maxPages = Math.ceil(commentsPerNote / 10);
        while (pages < maxPages) {
            try {
                const cRes = await httpGet('server.fmode.cn',
                    `/thapi/v1/xiaohongshu/app/get_note_comments?note_id=${note.id}&cursor=${encodeURIComponent(cursor)}`,
                    TIKHUB_TOKEN);
                const batch = cRes?.data?.data?.comments || [];
                if (!batch.length) break;
                noteComments.push(...batch);
                cursor = cRes?.data?.data?.cursor || '';
                if (!cursor || !cRes?.data?.data?.has_more) break;
                pages++;
                await sleep(400);
            } catch (e) {
                console.log(`      ↻ ${note.id} err=${e.message}`);
                break;
            }
        }
        comments[note.id] = noteComments;
        console.log(`      → ${noteComments.length} 条评论`);
    }

    // 收集作者 ID（不深挖用户详情，节省 API）
    const userIds = new Set();
    topNotesArr.forEach((n) => n?.user?.userid && userIds.add(n.user.userid));

    return { notes, top_notes: topNotesArr.map((n) => n.id), comments, user_ids: Array.from(userIds) };
}

// ============================================================
// Main
// ============================================================
async function main() {
    const args = process.argv.slice(2);
    const force = args.includes('--force');
    const groupArg = args.find((a) => !a.startsWith('--'));

    // 读已有数据（续跑）
    let extended = {};
    if (fs.existsSync(OUT_PATH)) {
        try { extended = JSON.parse(fs.readFileSync(OUT_PATH, 'utf-8')); }
        catch { extended = {}; }
    }
    if (!extended.by_module) extended.by_module = {};
    if (!extended.meta) extended.meta = { created_at: new Date().toISOString() };

    const logLines = [];
    function log(msg) {
        console.log(msg);
        logLines.push(`[${new Date().toISOString()}] ${msg}`);
    }

    log('╔══════════════════════════════════════════════════════════╗');
    log('║   江中肝纯片 · VOC 验证扩采（多平台评论颗粒度）            ║');
    log('╚══════════════════════════════════════════════════════════╝');
    log(`  输出: ${OUT_PATH}`);
    log(`  TikHub token: ${TIKHUB_TOKEN.slice(0, 8)}...`);

    const groups = groupArg ? [groupArg] : Object.keys(KEYWORD_GROUPS);
    log(`  目标模块: ${groups.join(', ')}`);
    log(`  强制重抓: ${force}`);
    log('');

    const t0 = Date.now();
    let totalNotes = 0, totalComments = 0;

    for (const groupName of groups) {
        if (!KEYWORD_GROUPS[groupName]) {
            log(`  ⚠ 未知模块: ${groupName}`);
            continue;
        }
        log(`\n━━━━━ 模块: ${groupName} ━━━━━`);
        if (!extended.by_module[groupName]) extended.by_module[groupName] = {};

        for (const { kw, note } of KEYWORD_GROUPS[groupName]) {
            if (!force && extended.by_module[groupName][kw]?.notes?.length > 0) {
                log(`  ⏭  跳过（已采集）: "${kw}"  —— ${extended.by_module[groupName][kw].notes.length} 笔记 / ${Object.values(extended.by_module[groupName][kw].comments || {}).flat().length} 评论`);
                continue;
            }
            try {
                const result = await collectXhsForKeyword(kw, { topNotes: 5, commentsPerNote: 50 });
                result._meta = { keyword: kw, purpose: note, collected_at: new Date().toISOString() };
                extended.by_module[groupName][kw] = result;
                const nCnt = result.notes.length;
                const cCnt = Object.values(result.comments).flat().length;
                totalNotes += nCnt;
                totalComments += cCnt;
                log(`  ✅ "${kw}"  —— ${nCnt} 笔记 / ${cCnt} 评论`);

                // 每个关键词采完就保存，避免中断丢失
                fs.writeFileSync(OUT_PATH, JSON.stringify(extended, null, 2), 'utf-8');
                await sleep(1200);
            } catch (e) {
                log(`  ❌ "${kw}"  —— ${e.message}`);
                extended.by_module[groupName][kw] = { _error: e.message, _meta: { keyword: kw } };
            }
        }
    }

    extended.meta.last_updated = new Date().toISOString();
    extended.meta.total_notes = totalNotes;
    extended.meta.total_comments = totalComments;

    fs.writeFileSync(OUT_PATH, JSON.stringify(extended, null, 2), 'utf-8');
    fs.writeFileSync(LOG_PATH, logLines.join('\n'), 'utf-8');

    const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
    log('');
    log('╔══════════════════════════════════════════════════════════╗');
    log(`║  ✅ 扩采完成 (${elapsed}s)  ${totalNotes} 笔记 / ${totalComments} 评论`);
    log('╚══════════════════════════════════════════════════════════╝');
    log(`  输出: ${OUT_PATH}  (${(fs.statSync(OUT_PATH).size / 1024).toFixed(1)} KB)`);
    log(`  日志: ${LOG_PATH}`);

    // 保存最终 log
    fs.writeFileSync(LOG_PATH, logLines.join('\n'), 'utf-8');
}

main().catch((err) => {
    console.error('❌ 致命错误:', err);
    process.exit(1);
});