fmode-voc-projects/scripts/tools/lactic-collect.js

#!/usr/bin/env node
/**
 * 江中乳酸菌素片儿童版 · VOC 多平台采集（真实采集）
 *
 * 对齐 docs/乳酸菌/3.采集矩阵.md 定义的关键词矩阵，
 * 采用 collect-jiangzhong-liver-deep / houguyin-collect 已验证的 fmode.cn 接入模式：
 *   - 小红书：TikHub · https://server.fmode.cn/thapi/v1/xiaohongshu/app/...
 *   - 抖音：VOC · https://server.fmode.cn/api/voc-social/douyin/...
 *
 * 用法：
 *   node scripts/tools/lactic-collect.js --batch=1
 *   node scripts/tools/lactic-collect.js --batch=all
 *   node scripts/tools/lactic-collect.js --merge
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

// ============================================================
// 路径
// ============================================================
const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', '乳酸菌', 'raw');
const XHS_DIR = path.join(RAW_DIR, 'xhs');
const DY_DIR = path.join(RAW_DIR, 'douyin');
const AUDIT_LOG = path.join(RAW_DIR, 'audit.log');

[RAW_DIR, XHS_DIR, DY_DIR].forEach((d) => {
    if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
});

// ============================================================
// 凭据
// ============================================================
const TIKHUB_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(p)) return null;
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return (c.endpoint?.headers?.Authorization || '').replace(/^Bearer\s+/, '');
    } catch { return null; }
})();

const VOC_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'voc-credentials.json');
    if (!fs.existsSync(p)) return '';
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return c.vocToken || c.sessionToken || '';
    } catch { return ''; }
})();

// ============================================================
// 关键词矩阵（对标 3.采集矩阵.md · 聚焦 xhs + douyin 两大主力平台）
// ============================================================
const BATCHES = {
    1: {
        name: 'P0 · 本品 + 药品益生菌 + 食品益生菌头部',
        xhs: [
            // 本品 · H1/H3
            { kw: '乳酸菌素片',       notes: 6, commentPages: 3, hypotheses: ['H1', 'H3'] },
            { kw: '乳酸菌素片儿童',   notes: 5, commentPages: 3, hypotheses: ['H1', 'H3', 'H5'] },
            { kw: '江中乳酸菌素片',   notes: 5, commentPages: 2, hypotheses: ['H1'] },
            // 药品益生菌竞品 · H1
            { kw: '妈咪爱',           notes: 6, commentPages: 3, hypotheses: ['H1'] },
            { kw: '妈咪爱 宝宝',      notes: 5, commentPages: 2, hypotheses: ['H1', 'H3'] },
            { kw: '亿活 儿童',        notes: 5, commentPages: 2, hypotheses: ['H1'] },
            { kw: '金双岐',           notes: 4, commentPages: 2, hypotheses: ['H1'] },
            // 食品益生菌头部 · H2
            { kw: '合生元益生菌',     notes: 6, commentPages: 3, hypotheses: ['H2'] },
            { kw: '万益蓝益生菌',     notes: 5, commentPages: 3, hypotheses: ['H2'] },
            { kw: '拜奥益生菌',       notes: 5, commentPages: 2, hypotheses: ['H2'] },
            { kw: '儿童益生菌推荐',   notes: 6, commentPages: 3, hypotheses: ['H2', 'H3'] },
        ],
        douyin: [
            { kw: '妈咪爱',           videos: 4, commentPages: 2, hypotheses: ['H1'] },
            { kw: '乳酸菌素片',       videos: 4, commentPages: 2, hypotheses: ['H1', 'H3'] },
            { kw: '儿童益生菌',       videos: 4, commentPages: 2, hypotheses: ['H2', 'H3'] },
            { kw: '合生元益生菌',     videos: 3, commentPages: 2, hypotheses: ['H2'] },
            { kw: '万益蓝益生菌',     videos: 3, commentPages: 2, hypotheses: ['H2'] },
        ],
    },
    2: {
        name: 'P1 · 食品长尾 + 儿童肠道刚需',
        xhs: [
            // 食品益生菌长尾 · H2
            { kw: 'inne 噗噗宝',      notes: 4, commentPages: 2, hypotheses: ['H2', 'H3'] },
            { kw: '小胖瓶益生菌',     notes: 4, commentPages: 2, hypotheses: ['H2'] },
            { kw: '康萃乐益生菌',     notes: 4, commentPages: 2, hypotheses: ['H2'] },
            { kw: 'lifespace 儿童',   notes: 4, commentPages: 2, hypotheses: ['H2'] },
            { kw: '宝乐安',           notes: 4, commentPages: 2, hypotheses: ['H1'] },
            // 儿童肠道刚需 · H3
            { kw: '宝宝便秘',         notes: 6, commentPages: 3, hypotheses: ['H3'] },
            { kw: '宝宝积食',         notes: 6, commentPages: 3, hypotheses: ['H3'] },
            { kw: '宝宝腹泻',         notes: 5, commentPages: 2, hypotheses: ['H3'] },
            { kw: '儿童挑食',         notes: 5, commentPages: 2, hypotheses: ['H3'] },
            { kw: '抗生素 益生菌',    notes: 5, commentPages: 2, hypotheses: ['H4'] },
        ],
        douyin: [
            { kw: '宝宝便秘',         videos: 3, commentPages: 2, hypotheses: ['H3'] },
            { kw: '宝宝积食',         videos: 3, commentPages: 2, hypotheses: ['H3'] },
            { kw: '抗生素 益生菌',    videos: 3, commentPages: 2, hypotheses: ['H4'] },
        ],
    },
    3: {
        name: 'P2 · 场景 + 决策 + 营养吸收',
        xhs: [
            // 场景 · H4
            { kw: '宝宝入园生病',     notes: 5, commentPages: 2, hypotheses: ['H4'] },
            { kw: '宝宝换季腹泻',     notes: 4, commentPages: 2, hypotheses: ['H4'] },
            // 喂药依从性 · H5/H6
            { kw: '喂药难',           notes: 5, commentPages: 2, hypotheses: ['H5'] },
            { kw: '儿童咀嚼片',       notes: 4, commentPages: 2, hypotheses: ['H5', 'H8'] },
            // 营养吸收 · H7
            { kw: '儿童营养软糖',     notes: 6, commentPages: 3, hypotheses: ['H7', 'H8'] },
            { kw: '儿童DHA',          notes: 5, commentPages: 2, hypotheses: ['H7'] },
            { kw: '瓶瓶罐罐',         notes: 4, commentPages: 2, hypotheses: ['H7'] },
            { kw: '儿童补钙',         notes: 5, commentPages: 2, hypotheses: ['H7'] },
            // 糖分焦虑 · H8
            { kw: '儿童无糖',         notes: 4, commentPages: 2, hypotheses: ['H8'] },
            { kw: '宝宝专用',         notes: 4, commentPages: 2, hypotheses: ['H6'] },
        ],
        douyin: [
            { kw: '儿童营养软糖',     videos: 3, commentPages: 2, hypotheses: ['H7', 'H8'] },
            { kw: '瓶瓶罐罐',         videos: 3, commentPages: 2, hypotheses: ['H7'] },
        ],
    },
};

// ============================================================
// HTTP 工具（对齐 houguyin-collect 的已验证实现）
// ============================================================
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function sanitizeFilename(kw) {
    return kw.replace(/[\/\\:*?"<>|\s]+/g, '-');
}

function auditLog(line) {
    const ts = new Date().toISOString();
    try {
        fs.appendFileSync(AUDIT_LOG, `[${ts}] ${line}\n`);
    } catch (e) {
        if (e.code === 'EBUSY' || e.code === 'EACCES' || e.code === 'EPERM') {
            try {
                fs.appendFileSync(AUDIT_LOG + '.alt', `[${ts}] ${line}\n`);
            } catch { /* ignore */ }
        }
    }
}

function fileExistsNonEmpty(p) {
    if (!fs.existsSync(p)) return false;
    return fs.statSync(p).size > 50;
}

function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        const chunks = [];
                        response.on('data', (c) => chunks.push(c));
                        response.on('end', () => res({
                            status: response.statusCode,
                            body: Buffer.concat(chunks).toString('utf-8'),
                        }));
                    });
                    req.on('error', rej);
                    req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    await sleep(1500 * Math.pow(1.8, attempt - 1));
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await sleep(2000 * attempt);
            }
        }
    });
}

async function tikhubGet(apiPath, paramsObj = {}) {
    if (!TIKHUB_TOKEN) return { _error: 'no TikHub token' };
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${TIKHUB_TOKEN}`,
        },
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyPostV2(apiPath, bodyObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const bodyStr = JSON.stringify(bodyObj);
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: apiPath,
        method: 'POST',
        headers: {
            Accept: 'application/json',
            'Content-Type': 'application/json',
            'Content-Length': Buffer.byteLength(bodyStr),
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyGetV3(apiPath, paramsObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

// ============================================================
// 小红书采集
// ============================================================
async function collectXhs(spec, { force = false } = {}) {
    const outPath = path.join(XHS_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [xhs] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`xhs SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [xhs] 🔍 搜索 "${spec.kw}" · notes=${spec.notes} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    const searchRes1 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'general' });
    await sleep(600);
    const searchRes2 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'popularity_descending' });

    const items1 = searchRes1?.data?.data?.items || [];
    const items2 = searchRes2?.data?.data?.items || [];
    const noteMap = new Map();
    for (const it of [...items1, ...items2]) {
        const n = it?.note;
        if (n?.id && !noteMap.has(n.id)) noteMap.set(n.id, n);
    }
    const allNotes = Array.from(noteMap.values());
    console.log(`    [xhs] 去重后 ${allNotes.length} 条笔记 (综合 ${items1.length} + 最热 ${items2.length})`);

    if (allNotes.length === 0) {
        const errOut = {
            platform: 'xiaohongshu', keyword: spec.kw,
            _error: searchRes1?._error || searchRes2?._error || 'empty',
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`xhs FAIL ${spec.kw} empty/err=${JSON.stringify(errOut._error).slice(0, 80)}`);
        return { skipped: false, notes: 0, comments: 0, path: outPath };
    }

    const topNotes = [...allNotes]
        .sort((a, b) => ((b.comments_count || 0) * 2 + (b.liked_count || 0)) - ((a.comments_count || 0) * 2 + (a.liked_count || 0)))
        .slice(0, spec.notes);

    const commentsByNoteId = {};
    let totalComments = 0;
    for (const note of topNotes) {
        if (!note?.id) continue;
        commentsByNoteId[note.id] = [];
        let cursor = '';
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await tikhubGet('/thapi/v1/xiaohongshu/app/get_note_comments', { note_id: note.id, cursor });
            const cmts = cRes?.data?.data?.comments || [];
            for (const c of cmts) {
                commentsByNoteId[note.id].push({
                    id: c.id,
                    content: c.content,
                    create_time: c.create_time,
                    like_count: c.like_count,
                    sub_comment_count: c.sub_comment_count,
                    ip_location: c.ip_location,
                    user: c.user_info ? { user_id: c.user_info.user_id, nickname: c.user_info.nickname } : null,
                    sub_comments: (c.sub_comments || []).slice(0, 3).map((s) => ({
                        content: s.content,
                        like_count: s.like_count,
                        nickname: s.user_info?.nickname,
                    })),
                });
            }
            totalComments += cmts.length;
            const hasMore = cRes?.data?.data?.has_more;
            cursor = cRes?.data?.data?.cursor || '';
            if (!hasMore || !cursor) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [xhs] ✓ ${spec.kw}: ${topNotes.length} 笔记 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`xhs OK ${spec.kw} notes=${topNotes.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'xiaohongshu',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_notes_found: allNotes.length,
        top_notes: topNotes.map((n) => ({
            id: n.id,
            type: n.type,
            title: n.title,
            desc: n.desc,
            timestamp: n.timestamp,
            liked_count: n.liked_count,
            comments_count: n.comments_count,
            collected_count: n.collected_count,
            shared_count: n.shared_count,
            cover: n.images_list?.[0]?.url,
            user: n.user ? {
                userid: n.user.userid,
                nickname: n.user.nickname,
                red_id: n.user.red_id,
                verified: n.user.red_official_verified,
            } : null,
        })),
        comments_by_note_id: commentsByNoteId,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, notes: topNotes.length, comments: totalComments, path: outPath };
}

// ============================================================
// 抖音采集
// ============================================================
async function collectDouyin(spec, { force = false } = {}) {
    const outPath = path.join(DY_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [dy] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`dy SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [dy] 🔍 搜索 "${spec.kw}" · videos=${spec.videos} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    const searchRes = await dyPostV2('/api/voc-social/douyin/search/fetch_general_search_v2', {
        keyword: spec.kw,
        cursor: 0,
        sort_type: '1',
        publish_time: '0',
        content_type: '1',
        filter_duration: '0',
        search_id: '',
        backtrace: '',
    });

    const businessData = searchRes?.data?.business_data || [];
    const validVideos = businessData
        .map((wrap) => wrap?.data?.aweme_info || wrap?.aweme_info)
        .filter((v) => v?.aweme_id);
    console.log(`    [dy] 搜索到 ${validVideos.length} 条视频 (wrappers=${businessData.length})`);

    if (validVideos.length === 0) {
        const errOut = {
            platform: 'douyin', keyword: spec.kw,
            _search_error: searchRes?._error || searchRes?.mess || 'empty',
            _search_code: searchRes?.code,
            _search_sample: JSON.stringify(searchRes).slice(0, 400),
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`dy FAIL ${spec.kw} search_empty code=${searchRes?.code}`);
        return { skipped: false, videos: 0, comments: 0, path: outPath };
    }

    const topVideos = [...validVideos]
        .sort((a, b) => (b.statistics?.comment_count || 0) - (a.statistics?.comment_count || 0))
        .slice(0, spec.videos);

    const commentsByAweme = {};
    let totalComments = 0;
    for (const v of topVideos) {
        commentsByAweme[v.aweme_id] = [];
        let cursor = 0;
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await dyGetV3('/api/voc-social/douyin/app/v3/fetch_video_comments', {
                aweme_id: v.aweme_id,
                cursor,
                count: 20,
            });
            const cmts = cRes?.data?.comments || cRes?.comments || [];
            for (const c of cmts) {
                commentsByAweme[v.aweme_id].push({
                    cid: c.cid,
                    text: c.text,
                    digg_count: c.digg_count,
                    create_time: c.create_time,
                    ip_label: c.ip_label,
                    reply_comment_total: c.reply_comment_total,
                    user: c.user ? { nickname: c.user.nickname, uid: c.user.uid } : null,
                });
            }
            totalComments += cmts.length;
            const hasMore = (cRes?.data?.has_more ?? cRes?.has_more) === 1;
            cursor = cRes?.data?.cursor ?? cRes?.cursor ?? 0;
            if (!hasMore) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [dy] ✓ ${spec.kw}: ${topVideos.length} 视频 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`dy OK ${spec.kw} videos=${topVideos.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'douyin',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_videos_found: validVideos.length,
        top_videos: topVideos.map((v) => ({
            aweme_id: v.aweme_id,
            desc: v.desc,
            create_time: v.create_time,
            statistics: v.statistics,
            author: v.author ? {
                nickname: v.author.nickname,
                sec_uid: v.author.sec_uid,
                uid: v.author.uid,
                follower_count: v.author.follower_count,
            } : null,
            cover: v.video?.cover?.url_list?.[0],
            text_extra: (v.text_extra || []).map((t) => t.hashtag_name).filter(Boolean),
        })),
        comments_by_aweme_id: commentsByAweme,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, videos: topVideos.length, comments: totalComments, path: outPath };
}

// ============================================================
// 假设 + tag + 情感推理（儿童乳酸菌素片专属规则）
// ============================================================
const HYPOTHESIS_KEYWORDS = {
    H1: ['妈咪爱', '亿活', '金双岐', '宝乐安', '枯草杆菌', '活菌', '冷链', '冷藏', '处方', '医生开'],
    H2: ['合生元', '万益蓝', '拜奥', 'inne', '噗噗宝', '小胖瓶', '康萃乐', 'lifespace', '食品益生菌', '保健食品', '没效果', '智商税'],
    H3: ['便秘', '腹泻', '积食', '挑食', '厌食', '消化不良', '肚子', '肠胃', '拉肚子', '不爱吃饭', '大便'],
    H4: ['入园', '幼儿园', '换季', '抗生素', '感冒', '发烧', '开学', '集体生活', '交叉感染', '开园'],
    H5: ['喂药', '不吃', '抗拒', '咀嚼', '独立包装', '像打仗', '哄着', '加在', '偷偷放', '吐出来', '难喂'],
    H6: ['包装', '卡通', '颜值', '分龄', 'ip', '可爱', '娃娃头', '专用', '辨识度', '分年龄'],
    H7: ['DHA', '叶黄素', '补钙', '钙片', '维生素', '营养软糖', '营养包', '瓶瓶罐罐', '吸收', '吃了不吸收', '补了白补'],
    H8: ['无糖', '木糖醇', '含糖', '糖分', '龋齿', '蛀牙', '防腐剂', '香精', '色素', '甜度'],
};

function inferHypotheses(text, keywordHypotheses) {
    const results = new Set(keywordHypotheses || []);
    const t = String(text || '').toLowerCase();
    for (const [h, kws] of Object.entries(HYPOTHESIS_KEYWORDS)) {
        for (const kw of kws) {
            if (t.includes(kw.toLowerCase())) { results.add(h); break; }
        }
    }
    return Array.from(results);
}

const TAG_RULES = [
    // 药品益生菌痛点
    { tag: '冷链焦虑',       re: /冷藏|冷链|2-8度|常温|失活|温度/ },
    { tag: '处方门槛',       re: /处方|医生开|医嘱|儿科开|挂号/ },
    { tag: '活菌依赖',       re: /停了|停药|反弹|长期吃|依赖/ },
    { tag: '妈咪爱好评',     re: /(妈咪爱|亿活).*(好|有效|对症|管用|推荐)/ },
    { tag: '妈咪爱吐槽',     re: /(妈咪爱|亿活).*(贵|苦|难喂|不好|没用)/ },
    // 食品益生菌
    { tag: '食品级怀疑',     re: /(合生元|万益蓝|拜奥|inne).*(没效果|智商税|安慰剂|噱头|浪费)/ },
    { tag: '食品级推荐',     re: /(合生元|万益蓝|拜奥|inne).*(好吃|爱吃|小朋友喜欢|果味)/ },
    // 儿童肠道刚需
    { tag: '便秘痛点',       re: /便秘|便干|排便困难|几天不拉|拉不出/ },
    { tag: '腹泻痛点',       re: /腹泻|拉稀|拉肚子|水样便|反复拉/ },
    { tag: '积食痛点',       re: /积食|不消化|舌苔厚|口气|肚子胀/ },
    { tag: '挑食厌食',       re: /挑食|厌食|不爱吃饭|不好好吃|吃得少/ },
    // 场景
    { tag: '入园生病',       re: /入园|幼儿园.*(生病|感冒|腹泻)/ },
    { tag: '换季肠胃',       re: /换季|春秋|季节.*(腹泻|拉肚)/ },
    { tag: '抗生素后',       re: /抗生素|消炎药|头孢|阿莫西林|输液/ },
    // 决策 / 依从
    { tag: '喂药困难',       re: /喂药|不吃|抗拒|吐出|哄着|偷偷/ },
    { tag: '咀嚼片认可',     re: /咀嚼|嚼|小朋友爱吃|主动要吃|像糖/ },
    { tag: '颜值决策',       re: /包装|卡通|ip|颜值|可爱|好看/ },
    // 营养吸收
    { tag: '瓶瓶罐罐焦虑',   re: /瓶瓶罐罐|一堆|各种营养|十几种|追着喂/ },
    { tag: '吸收焦虑',       re: /吸收|补了.*(白补|没效果|缺营养)|不吸收/ },
    // 糖分焦虑
    { tag: '糖分抵触',       re: /无糖|糖分|龋齿|蛀牙|含糖量/ },
    { tag: '添加剂焦虑',     re: /防腐剂|香精|色素|添加剂|配料表/ },
];

function inferTags(text) {
    const tags = [];
    for (const r of TAG_RULES) {
        if (r.re.test(text || '')) tags.push(r.tag);
    }
    return tags;
}

const SENTIMENT_POS = /好喝|推荐|有效|舒服|喜欢|好用|回购|复购|靠谱|安心|爱吃|管用|对症|好转|有改善/;
const SENTIMENT_NEG = /难喝|不推荐|智商税|贵|骗|假|副作用|反弹|没用|没效果|失望|难喂|抗拒|吐出/;
const SENTIMENT_CONFLICT = /想停|但是|可是|纠结|矛盾|害怕|担心|又.*又|不知道.*好/;

function inferSentiment(text) {
    const t = String(text || '');
    const pos = SENTIMENT_POS.test(t);
    const neg = SENTIMENT_NEG.test(t);
    const conf = SENTIMENT_CONFLICT.test(t);
    if (conf && (pos || neg)) return 'conflicted';
    if (pos && !neg) return 'positive';
    if (neg && !pos) return 'negative';
    return 'neutral';
}

// ============================================================
// 合并 → _merged.json + comments-flat.jsonl
// ============================================================
function mergeAll() {
    const out = {
        meta: {
            collectedAt: new Date().toISOString(),
            platforms: {},
            products: {},
            hypotheses: {},
            keywords: {},
            stage: 'batch-real',
            sourceTier: 'real-collected',
        },
        items: [],
    };
    const flat = [];

    // 小红书
    if (fs.existsSync(XHS_DIR)) {
        for (const f of fs.readdirSync(XHS_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(XHS_DIR, f), 'utf8'));
            if (raw._error) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const note of (raw.top_notes || [])) {
                const noteId = note.id;
                const cmts = (raw.comments_by_note_id || {})[noteId] || [];
                if (note.desc || note.title) {
                    const noteContent = [note.title, note.desc].filter(Boolean).join(' · ').slice(0, 400);
                    const item = buildItem({
                        id: `xhs_${noteId}`,
                        platform: 'xhs',
                        product: kw,
                        keyword: kw,
                        type: 'note',
                        nickname: note.user?.nickname || '匿名',
                        ip: '',
                        content: noteContent,
                        likes: note.liked_count || 0,
                        rating: null,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
                for (const c of cmts) {
                    if (!c.content) continue;
                    const item = buildItem({
                        id: `xhs_${noteId}_${c.id}`,
                        platform: 'xhs',
                        product: kw,
                        keyword: kw,
                        type: 'comment',
                        nickname: c.user?.nickname || '匿名',
                        ip: c.ip_location || '',
                        content: c.content.slice(0, 500),
                        likes: c.like_count || 0,
                        rating: null,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
            }
        }
    }

    // 抖音
    if (fs.existsSync(DY_DIR)) {
        for (const f of fs.readdirSync(DY_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(DY_DIR, f), 'utf8'));
            if (raw._error || raw._search_error) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const v of (raw.top_videos || [])) {
                const awemeId = v.aweme_id;
                const cmts = (raw.comments_by_aweme_id || {})[awemeId] || [];
                if (v.desc) {
                    const item = buildItem({
                        id: `dy_${awemeId}`,
                        platform: 'douyin',
                        product: kw,
                        keyword: kw,
                        type: 'video',
                        nickname: v.author?.nickname || '匿名',
                        ip: '',
                        content: v.desc.slice(0, 400),
                        likes: v.statistics?.digg_count || 0,
                        rating: null,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
                for (const c of cmts) {
                    if (!c.text) continue;
                    const item = buildItem({
                        id: `dy_${awemeId}_${c.cid}`,
                        platform: 'douyin',
                        product: kw,
                        keyword: kw,
                        type: 'comment',
                        nickname: c.user?.nickname || '匿名',
                        ip: c.ip_label || '',
                        content: c.text.slice(0, 500),
                        likes: c.digg_count || 0,
                        rating: null,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
            }
        }
    }

    // cid/id 级去重
    const seen = new Set();
    const dedupeItems = [];
    const dedupeFlat = [];
    for (const it of out.items) {
        if (seen.has(it.id)) continue;
        seen.add(it.id);
        dedupeItems.push(it);
    }
    for (const it of flat) {
        if (dedupeFlat.find((x) => x.id === it.id)) continue;
        dedupeFlat.push(it);
    }
    out.items = dedupeItems;

    out.meta.productsCount = Object.keys(out.meta.products).length;
    out.meta.keywordsCount = Object.keys(out.meta.keywords).length;
    out.meta.comments = out.items.length;

    fs.writeFileSync(path.join(RAW_DIR, '_merged.json'), JSON.stringify(out, null, 2), 'utf8');
    fs.writeFileSync(path.join(RAW_DIR, 'comments-flat.jsonl'), dedupeFlat.map((it) => JSON.stringify(it)).join('\n'), 'utf8');
    console.log(`  ✅ merged: ${out.items.length} items | ${out.meta.productsCount} products | ${Object.keys(out.meta.platforms).length} platforms`);
    console.log(`     假设覆盖: ${Object.entries(out.meta.hypotheses).map(([k, v]) => `${k}:${v}`).join(' / ')}`);
    return out;
}

function buildItem({ id, platform, product, keyword, type, nickname, ip, content, likes, rating, kwHypos }) {
    if (!content || content.length < 3) return null;
    const hypotheses = inferHypotheses(content, kwHypos);
    const tags = inferTags(content);
    const sentiment = inferSentiment(content);
    return {
        id, platform, product, keyword, type,
        nickname, ip, content, likes, rating,
        hypothesis: hypotheses,
        tags,
        sentiment,
        source: 'real-collected',
    };
}

function countMeta(out, it) {
    out.meta.platforms[it.platform] = (out.meta.platforms[it.platform] || 0) + 1;
    out.meta.products[it.product] = (out.meta.products[it.product] || 0) + 1;
    out.meta.keywords[it.keyword] = (out.meta.keywords[it.keyword] || 0) + 1;
    for (const h of (it.hypothesis || [])) {
        out.meta.hypotheses[h] = (out.meta.hypotheses[h] || 0) + 1;
    }
}

// ============================================================
// 批次执行
// ============================================================
async function runBatch(batchNum, opts) {
    const batch = BATCHES[batchNum];
    if (!batch) throw new Error(`unknown batch: ${batchNum}`);
    console.log(`\n▶ Batch ${batchNum}: ${batch.name}`);
    auditLog(`BATCH-START ${batchNum} ${batch.name}`);

    for (const task of batch.xhs || []) {
        try { await collectXhs(task, opts); }
        catch (e) { console.log(`    [xhs] ✗ ${task.kw}: ${e.message}`); auditLog(`xhs EXC ${task.kw} ${e.message.slice(0, 80)}`); }
        await sleep(800);
    }
    for (const task of batch.douyin || []) {
        try { await collectDouyin(task, opts); }
        catch (e) { console.log(`    [dy] ✗ ${task.kw}: ${e.message}`); auditLog(`dy EXC ${task.kw} ${e.message.slice(0, 80)}`); }
        await sleep(600);
    }
    auditLog(`BATCH-END ${batchNum}`);
}

// ============================================================
// CLI
// ============================================================
async function main() {
    const argv = process.argv.slice(2);
    const opts = { force: argv.includes('--force') };
    const batchArg = argv.find((a) => a.startsWith('--batch='));
    const isMerge = argv.includes('--merge');
    const isTest = argv.includes('--test');

    console.log('╔═══════════════════════════════════════════════════════════╗');
    console.log('║  江中乳酸菌素片儿童版 · VOC 多平台真实采集                 ║');
    console.log('╚═══════════════════════════════════════════════════════════╝');
    console.log(`  TikHub token: ${TIKHUB_TOKEN ? '✓ ' + TIKHUB_TOKEN.slice(0, 8) + '...' : '✗'}`);
    console.log(`  VOC token:    ${VOC_TOKEN ? '✓ ' + VOC_TOKEN.slice(0, 8) + '...' : '✗'}`);

    if (isTest) {
        console.log('\n▶ Test mode · 跑通单关键词 xhs["乳酸菌素片"]');
        await collectXhs({ kw: '乳酸菌素片', notes: 3, commentPages: 1, hypotheses: ['H1', 'H3'] }, { force: true });
        return;
    }

    if (batchArg) {
        const bn = batchArg.split('=')[1];
        if (bn === 'all') {
            for (const k of Object.keys(BATCHES)) await runBatch(k, opts);
        } else {
            await runBatch(bn, opts);
        }
    }

    if (isMerge || batchArg) {
        console.log('\n▶ Merging all collected raw data...');
        mergeAll();
    }

    if (!batchArg && !isMerge && !isTest) {
        console.log('\nUsage:');
        console.log('  --test              跑通单关键词测试 API 链路');
        console.log('  --batch=1|2|3|all   执行采集批次');
        console.log('  --merge             合并 raw/*/*.json → _merged.json');
        console.log('  --force             强制重抓（忽略已存在文件）');
    }
}

if (require.main === module) {
    main().catch((e) => { console.error('fatal:', e); process.exit(1); });
}

module.exports = { BATCHES, mergeAll, collectXhs, collectDouyin };