fmode-voc-projects/scripts/tools/hongchengdaojia/hongcheng-collect-xhs.js

#!/usr/bin/env node
/**
 * 洪城到家 · 小红书（XHS）数据采集脚本
 *
 * 使用 TikHub API (via server.fmode.cn)
 * 参考 fmode-voc-projects/scripts/tools/lactic-collect.js
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', '洪城到家', 'raw');
const XHS_DIR = path.join(RAW_DIR, 'xhs');
const AUDIT_LOG = path.join(RAW_DIR, 'audit.log');

const API_CONFIG = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(p)) return null;
    try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; }
})();

const TIKHUB_TOKEN = API_CONFIG?.currentToken
    || API_CONFIG?.endpoint?.headers?.Authorization?.replace(/^Bearer\s+/, '')
    || 'gqsZHfMWgAiMwV+ITbmZy0qALADWBZVS7QnV7kKJe9CwzgWgJG+7bwK+GQ==';

const API_HOST = 'server.fmode.cn';

[RAW_DIR, XHS_DIR].forEach((d) => {
    if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
});

const BATCHES = {
    1: {
        name: 'P0 · 品牌关键词',
        xhs: [
            { kw: '洪城到家', notes: 20, commentPages: 2, hypotheses: ['H1', 'H4', 'H6'] },
            { kw: '月嫂', notes: 20, commentPages: 2, hypotheses: ['H1', 'H2', 'H4'] },
            { kw: '多喜娃', notes: 20, commentPages: 2, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '天鹅到家', notes: 20, commentPages: 2, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '好孕妈妈', notes: 20, commentPages: 2, hypotheses: ['H1', 'H5', 'H7'] },
        ],
    },
};

const HYPOTHESIS_KEYWORDS = {
    H1: ['医院', '产检', '待产', '生孩子', '妇幼', '生产', '月嫂怎么找'],
    H2: ['价格', '多少钱', '收费', '报价', '性价比', '便宜', '贵'],
    H3: ['短剧', '抖音', '视频', '小红书', '看到'],
    H4: ['专业', '资质', '证书', '星级', '靠谱', '放心', '正规'],
    H5: ['社区', '小店', '私人', '对比', '选择'],
    H6: ['朋友', '推荐', '介绍', '转介绍', '口碑', '好评'],
    H7: ['美团', '大众点评', '搜索', '排名', '评价'],
    H8: ['换', '退', '不满意', '保障', '售后', '风险'],
};

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function sanitizeFilename(kw) {
    return kw.replace(/[\/\\:*?"<>|\s]+/g, '-');
}

function auditLog(line) {
    const ts = new Date().toISOString();
    try { fs.appendFileSync(AUDIT_LOG, `[${ts}] ${line}\n`); } catch {}
}

function fileExistsNonEmpty(p) {
    if (!fs.existsSync(p)) return false;
    return fs.statSync(p).size > 100;
}

function tikhubGet(apiPath, params = {}) {
    const qs = Object.entries(params)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const url = qs ? `${apiPath}?${qs}` : apiPath;
    return new Promise((resolve) => {
        const opts = {
            hostname: API_HOST,
            path: url,
            method: 'GET',
            headers: {
                Authorization: `Bearer ${TIKHUB_TOKEN}`,
                Accept: 'application/json',
            },
        };
        const req = https.request(opts, (res) => {
            const chunks = [];
            res.on('data', (c) => chunks.push(c));
            res.on('end', () => {
                const body = Buffer.concat(chunks).toString('utf-8');
                try {
                    const j = JSON.parse(body);
                    resolve({ data: j, status: res.statusCode });
                } catch (e) {
                    resolve({ data: { _parse_error: e.message, _raw: body.slice(0, 500) }, status: res.statusCode });
                }
            });
        });
        req.on('error', (e) => resolve({ data: { _error: e.message }, status: 0 }));
        req.setTimeout(45000, () => { req.destroy(); resolve({ data: { _error: 'timeout' }, status: 0 }); });
        req.end();
    });
}

async function collectXhs(spec, { force = false } = {}) {
    const outPath = path.join(XHS_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [xhs] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`xhs SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [xhs] 🔍 搜索 "${spec.kw}" · notes=${spec.notes} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    const searchRes1 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'general' });
    await sleep(600);
    const searchRes2 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'popularity_descending' });

    const items1 = searchRes1?.data?.data?.items || [];
    const items2 = searchRes2?.data?.data?.items || [];
    const noteMap = new Map();
    for (const it of [...items1, ...items2]) {
        const n = it?.note;
        if (n?.id && !noteMap.has(n.id)) noteMap.set(n.id, n);
    }
    const allNotes = Array.from(noteMap.values());
    console.log(`    [xhs] 去重后 ${allNotes.length} 条笔记 (综合 ${items1.length} + 最热 ${items2.length})`);

    if (allNotes.length === 0) {
        const errOut = {
            platform: 'xiaohongshu',
            keyword: spec.kw,
            _error: searchRes1?.data?._error || searchRes2?.data?._error || searchRes1?.data?.detail || searchRes2?.data?.detail || 'empty',
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`xhs FAIL ${spec.kw} empty/err=${JSON.stringify(errOut._error).slice(0, 80)}`);
        return { skipped: false, notes: 0, comments: 0, path: outPath };
    }

    const topNotes = [...allNotes]
        .sort((a, b) => ((b.comments_count || 0) * 2 + (b.liked_count || 0)) - ((a.comments_count || 0) * 2 + (a.liked_count || 0)))
        .slice(0, spec.notes);

    const commentsByNoteId = {};
    let totalComments = 0;
    for (const note of topNotes) {
        if (!note?.id) continue;
        commentsByNoteId[note.id] = [];
        let cursor = '';
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await tikhubGet('/thapi/v1/xiaohongshu/app/get_note_comments', { note_id: note.id, cursor });
            const cmts = cRes?.data?.data?.comments || [];
            for (const c of cmts) {
                commentsByNoteId[note.id].push({
                    id: c.id,
                    content: c.content,
                    create_time: c.create_time,
                    like_count: c.like_count,
                    sub_comment_count: c.sub_comment_count,
                    ip_location: c.ip_location,
                    user: c.user_info ? { user_id: c.user_info.user_id, nickname: c.user_info.nickname } : null,
                    sub_comments: (c.sub_comments || []).slice(0, 3).map((s) => ({
                        content: s.content,
                        like_count: s.like_count,
                        nickname: s.user_info?.nickname,
                    })),
                });
            }
            totalComments += cmts.length;
            const hasMore = cRes?.data?.data?.has_more;
            cursor = cRes?.data?.data?.cursor || '';
            if (!hasMore || !cursor) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [xhs] ✓ ${spec.kw}: ${topNotes.length} 笔记 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`xhs OK ${spec.kw} notes=${topNotes.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'xiaohongshu',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_notes_found: allNotes.length,
        top_notes: topNotes.map((n) => ({
            id: n.id,
            type: n.type,
            title: n.title,
            desc: n.desc,
            timestamp: n.timestamp,
            liked_count: n.liked_count,
            comments_count: n.comments_count,
            collected_count: n.collected_count,
            shared_count: n.shared_count,
            cover: n.images_list?.[0]?.url,
            user: n.user ? {
                userid: n.user.userid,
                nickname: n.user.nickname,
                red_id: n.user.red_id,
                verified: n.user.red_official_verified,
            } : null,
        })),
        comments: commentsByNoteId,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, notes: topNotes.length, comments: totalComments, path: outPath };
}

function inferHypotheses(text, keywordHypotheses) {
    const results = new Set(keywordHypotheses || []);
    const t = String(text || '').toLowerCase();
    for (const [h, kws] of Object.entries(HYPOTHESIS_KEYWORDS)) {
        for (const kw of kws) {
            if (t.includes(kw.toLowerCase())) { results.add(h); break; }
        }
    }
    return Array.from(results);
}

const TAG_RULES = [
    { tag: '价格敏感', re: /价格|多少钱|贵|便宜|性价比|收费/ },
    { tag: '专业度关注', re: /专业|资质|证书|星级|培训/ },
    { tag: '安全保障', re: /放心|靠谱|安全|保障|正规/ },
    { tag: '医院渠道', re: /医院|产检|妇幼|待产|生孩子/ },
    { tag: '熟人推荐', re: /朋友推荐|介绍|口碑|好评|亲戚/ },
    { tag: '线上搜索', re: /美团|小红书|抖音|搜索/ },
    { tag: '服务担忧', re: /换|退|不满意|售后|保障/ },
    { tag: '婆媳关系', re: /婆婆|奶奶|家里|老人/ },
    { tag: '职场妈妈', re: /上班|工作|复工|职场/ },
    { tag: '新手爸妈', re: /新手|第一次|第一次当妈妈/ },
];

function inferTags(text) {
    const tags = [];
    for (const r of TAG_RULES) {
        if (r.re.test(text || '')) tags.push(r.tag);
    }
    return tags;
}

const SENTIMENT_POS = /好|推荐|满意|专业|靠谱|放心|值得|不错|棒|优秀|喜欢|感谢/;
const SENTIMENT_NEG = /差|坑|骗|贵|不专业|不满意|后悔|吐槽|垃圾|失望|糟糕|骗人/;
const SENTIMENT_CONFLICT = /但是|可是|纠结|担心|犹豫|想又怕/;

function inferSentiment(text) {
    const t = String(text || '');
    const pos = SENTIMENT_POS.test(t);
    const neg = SENTIMENT_NEG.test(t);
    const conf = SENTIMENT_CONFLICT.test(t);
    if (conf && (pos || neg)) return 'conflicted';
    if (pos && !neg) return 'positive';
    if (neg && !pos) return 'negative';
    return 'neutral';
}

function buildItem({ id, platform, product, keyword, type, nickname, ip, content, likes, rating, kwHypos }) {
    if (!content || content.length < 3) return null;
    const hypotheses = inferHypotheses(content, kwHypos);
    const tags = inferTags(content);
    const sentiment = inferSentiment(content);
    return { id, platform, product, keyword, type, nickname, ip, content, likes, rating, hypothesis: hypotheses, tags, sentiment, source: 'real-collected' };
}

function mergeAll() {
    const out = {
        meta: {
            collectedAt: new Date().toISOString(),
            platforms: {},
            products: {},
            hypotheses: {},
            keywords: {},
            stage: 'batch-real',
            sourceTier: 'real-collected',
        },
        items: [],
    };
    const flat = [];

    if (fs.existsSync(XHS_DIR)) {
        for (const f of fs.readdirSync(XHS_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(XHS_DIR, f), 'utf8'));
            if (raw._error || !raw.comments) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const [noteId, cmts] of Object.entries(raw.comments)) {
                for (const c of cmts) {
                    if (!c.content) continue;
                    const item = buildItem({
                        id: `xhs_${noteId}_${c.id}`,
                        platform: 'xhs',
                        product: kw,
                        keyword: kw,
                        type: 'comment',
                        nickname: c.user?.nickname || '匿名',
                        ip: c.ip_location || '',
                        content: c.content.slice(0, 500),
                        likes: c.like_count || 0,
                        rating: null,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); }
                }
            }
            out.meta.keywords[kw] = (out.meta.keywords[kw] || 0) + (raw.total_notes_found || 0);
        }
    }

    out.meta.platforms['xhs'] = out.items.length;
    out.meta.productsCount = Object.keys(out.meta.keywords).length;
    out.meta.keywordsCount = Object.keys(out.meta.keywords).length;
    out.meta.comments = out.items.length;

    for (const it of out.items) {
        for (const h of (it.hypothesis || [])) {
            out.meta.hypotheses[h] = (out.meta.hypotheses[h] || 0) + 1;
        }
    }

    fs.writeFileSync(path.join(RAW_DIR, '_merged.json'), JSON.stringify(out, null, 2), 'utf8');
    fs.writeFileSync(path.join(RAW_DIR, 'comments-flat.jsonl'), flat.map((it) => JSON.stringify(it)).join('\n'), 'utf8');
    console.log(`  ✅ merged: ${out.items.length} items | ${out.meta.productsCount} keywords`);
    return out;
}

async function runBatch(batchNum, opts) {
    const batch = BATCHES[batchNum];
    if (!batch) throw new Error(`unknown batch: ${batchNum}`);
    console.log(`\n▶ Batch ${batchNum}: ${batch.name}`);

    let totalNotes = 0;
    let totalComments = 0;
    for (const task of batch.xhs || []) {
        try {
            const r = await collectXhs(task, opts);
            totalNotes += r.notes || 0;
            totalComments += r.comments || 0;
        } catch (e) {
            console.log(`    ✗ xhs:${task.kw}: ${e.message}`);
            auditLog(`xhs EXC ${task.kw}: ${e.message}`);
        }
        await sleep(500);
    }
    console.log(`  📊 Batch ${batchNum} 合计: ${totalNotes} 笔记 / ${totalComments} 评论`);
}

async function main() {
    const argv = process.argv.slice(2);
    const opts = { force: argv.includes('--force') };
    const batchArg = argv.find((a) => a.startsWith('--batch='));
    const isMerge = argv.includes('--merge');

    console.log('\n╔═══════════════════════════════════════════════════════════╗');
    console.log('║  洪城到家 · 小红书（XHS）数据采集                       ║');
    console.log('╚═══════════════════════════════════════════════════════════╝');
    console.log(`  TikHub Token: ${TIKHUB_TOKEN ? '✓' : '✗'}`);

    if (batchArg) {
        const bn = batchArg.split('=')[1];
        if (bn === 'all') {
            for (const k of Object.keys(BATCHES)) await runBatch(k, opts);
        } else {
            await runBatch(bn, opts);
        }
    }

    if (isMerge || batchArg) {
        console.log('\n▶ Merging...');
        mergeAll();
    }

    if (!batchArg && !isMerge) {
        console.log('\nUsage:');
        console.log('  --batch=1   执行小红书采集');
        console.log('  --merge     合并数据');
        console.log('  --force     强制重抓');
        console.log('\n关键词: 洪城到家、月嫂、多喜娃、天鹅到家、好孕妈妈');
    }
}

if (require.main === module) {
    main().catch((e) => { console.error('fatal:', e); process.exit(1); });
}

module.exports = { BATCHES, mergeAll, collectXhs };