fmode-voc-projects/scripts/tools/houguyin-collect.js

#!/usr/bin/env node
/**
 * 江中猴菇饮 OTC · VOC 多平台采集（真实采集执行版）
 *
 * 对齐 docs/jiangzhong-houguyin/2.采集矩阵.md 定义的关键词矩阵，
 * 采用 collect-jiangzhong-liver-deep.js 已验证的 fmode.cn 接入模式：
 *   - 小红书：TikHub · https://server.fmode.cn/thapi/v1/xiaohongshu/app/...
 *   - 抖音：VOC · https://server.fmode.cn/api/voc-social/douyin/...
 *
 * 特性：
 *   - 分批次执行 (--batch=1|2|3|all)
 *   - 每关键词独立 JSON 落盘 docs/jiangzhong-houguyin/raw/{platform}/{kw}.json
 *   - 断点续跑：已存在文件跳过，--force 强制重跑
 *   - 3 次重试 + 指数退避
 *   - 审计日志 docs/jiangzhong-houguyin/raw/audit.log
 *   - --merge 合并 → _merged.json + comments-flat.jsonl（供分析器消费）
 *
 * 前置凭据：
 *   - ~/.openclaw/skills/xiaohongshu-search-notes/api-config.json （TikHub token）
 *   - ~/.openclaw/voc-credentials.json （VOC token for 抖音）
 *
 * 用法：
 *   node scripts/tools/houguyin-collect.js --batch=1
 *   node scripts/tools/houguyin-collect.js --batch=all
 *   node scripts/tools/houguyin-collect.js --merge
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

// ============================================================
// 路径
// ============================================================
const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', 'jiangzhong-houguyin', 'raw');
const XHS_DIR = path.join(RAW_DIR, 'xhs');
const DY_DIR = path.join(RAW_DIR, 'douyin');
const AUDIT_LOG = path.join(RAW_DIR, 'audit.log');

[RAW_DIR, XHS_DIR, DY_DIR].forEach((d) => {
    if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
});

// ============================================================
// 凭据
// ============================================================
const TIKHUB_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(p)) return null;
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return (c.endpoint?.headers?.Authorization || '').replace(/^Bearer\s+/, '');
    } catch { return null; }
})();

const VOC_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'voc-credentials.json');
    if (!fs.existsSync(p)) return '';
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return c.vocToken || c.sessionToken || '';
    } catch { return ''; }
})();

// ============================================================
// 关键词矩阵（对标 2.采集矩阵.md · 聚焦 xhs + douyin 两大主力平台）
// ============================================================
const BATCHES = {
    1: {
        name: 'P0 · 内部参照 + 药品主流',
        xhs: [
            { kw: '江中猴姑米稀',     notes: 6, commentPages: 3, hypotheses: ['H1', 'H5'] },
            { kw: '猴头菇米稀',       notes: 5, commentPages: 2, hypotheses: ['H1'] },
            { kw: '养胃米糊',         notes: 5, commentPages: 2, hypotheses: ['H1', 'H5'] },
            { kw: '江中猴姑饼干',     notes: 5, commentPages: 2, hypotheses: ['H2'] },
            { kw: '三九胃泰',         notes: 5, commentPages: 2, hypotheses: ['H3'] },
            { kw: '奥美拉唑 副作用',  notes: 5, commentPages: 2, hypotheses: ['H3', 'H4'] },
            { kw: '雷贝拉唑',         notes: 4, commentPages: 2, hypotheses: ['H3', 'H4'] },
        ],
        douyin: [
            { kw: '猴姑米稀',         videos: 3, commentPages: 2, hypotheses: ['H1'] },
            { kw: '三九胃泰',         videos: 3, commentPages: 2, hypotheses: ['H3'] },
            { kw: '拉唑 长期吃',      videos: 3, commentPages: 2, hypotheses: ['H3', 'H4'] },
        ],
    },
    2: {
        name: 'P1 · 同名对标 + 场景 + 长尾药品',
        xhs: [
            { kw: '猴菇饮',           notes: 5, commentPages: 2, hypotheses: ['H5'] },
            { kw: '猴头菇饮料',       notes: 4, commentPages: 2, hypotheses: ['H5'] },
            { kw: '康复新液',         notes: 4, commentPages: 2, hypotheses: ['H3'] },
            { kw: '年轻人 胃病',      notes: 5, commentPages: 2, hypotheses: ['H5', 'H6'] },
            { kw: '熬夜 胃不舒服',    notes: 5, commentPages: 2, hypotheses: ['H5'] },
            { kw: '应酬 胃难受',      notes: 4, commentPages: 2, hypotheses: ['H5'] },
            { kw: '办公室 养胃',      notes: 4, commentPages: 2, hypotheses: ['H5'] },
        ],
        douyin: [
            { kw: '胃病 年轻人',      videos: 2, commentPages: 2, hypotheses: ['H5'] },
            { kw: '养胃',             videos: 3, commentPages: 2, hypotheses: ['H5', 'H6'] },
        ],
    },
    3: {
        name: 'P2 · 细分人群 + 礼品类比',
        xhs: [
            { kw: '药店 胃药推荐',    notes: 4, commentPages: 2, hypotheses: ['H6'] },
            { kw: '糖尿病 养胃',      notes: 4, commentPages: 2, hypotheses: ['H7'] },
            { kw: '糖友 胃不好',      notes: 4, commentPages: 2, hypotheses: ['H7'] },
            { kw: '送父母 胃药',      notes: 4, commentPages: 2, hypotheses: ['H8'] },
            { kw: '东阿阿胶 送礼',    notes: 3, commentPages: 2, hypotheses: ['H8'] },
            { kw: '长期吃 拉唑',      notes: 5, commentPages: 2, hypotheses: ['H3', 'H4'] },
            { kw: '拉唑 停药反弹',    notes: 4, commentPages: 2, hypotheses: ['H4'] },
        ],
        douyin: [
            { kw: '送父母礼品',       videos: 2, commentPages: 2, hypotheses: ['H8'] },
        ],
    },
};

// ============================================================
// HTTP 工具（对齐 collect-jiangzhong-liver-deep.js 的已验证实现）
// ============================================================
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function sanitizeFilename(kw) {
    return kw.replace(/[\/\\:*?"<>|\s]+/g, '-');
}

function auditLog(line) {
    const ts = new Date().toISOString();
    try {
        fs.appendFileSync(AUDIT_LOG, `[${ts}] ${line}\n`);
    } catch (e) {
        // audit.log 被锁时不要中断采集（VSCode 打开会锁）
        if (e.code === 'EBUSY' || e.code === 'EACCES' || e.code === 'EPERM') {
            try {
                fs.appendFileSync(AUDIT_LOG + '.alt', `[${ts}] ${line}\n`);
            } catch { /* 忽略 */ }
        }
    }
}

function fileExistsNonEmpty(p) {
    if (!fs.existsSync(p)) return false;
    return fs.statSync(p).size > 50;
}

function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        const chunks = [];
                        response.on('data', (c) => chunks.push(c));
                        response.on('end', () => res({
                            status: response.statusCode,
                            body: Buffer.concat(chunks).toString('utf-8'),
                        }));
                    });
                    req.on('error', rej);
                    req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    await sleep(1500 * Math.pow(1.8, attempt - 1));
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await sleep(2000 * attempt);
            }
        }
    });
}

async function tikhubGet(apiPath, paramsObj = {}) {
    if (!TIKHUB_TOKEN) return { _error: 'no TikHub token' };
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${TIKHUB_TOKEN}`,
        },
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyPostV2(apiPath, bodyObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const bodyStr = JSON.stringify(bodyObj);
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: apiPath,
        method: 'POST',
        headers: {
            Accept: 'application/json',
            'Content-Type': 'application/json',
            'Content-Length': Buffer.byteLength(bodyStr),
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyGetV3(apiPath, paramsObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

// ============================================================
// 小红书采集（单关键词深度）
// ============================================================
async function collectXhs(spec, { force = false } = {}) {
    const outPath = path.join(XHS_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [xhs] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`xhs SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [xhs] 🔍 搜索 "${spec.kw}" · notes=${spec.notes} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    // 综合 + 最热两种排序，取并集
    const searchRes1 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'general' });
    await sleep(600);
    const searchRes2 = await tikhubGet('/thapi/v1/xiaohongshu/app/search_notes', { keyword: spec.kw, page: 1, sort: 'popularity_descending' });

    const items1 = searchRes1?.data?.data?.items || [];
    const items2 = searchRes2?.data?.data?.items || [];
    const noteMap = new Map();
    for (const it of [...items1, ...items2]) {
        const n = it?.note;
        if (n?.id && !noteMap.has(n.id)) noteMap.set(n.id, n);
    }
    const allNotes = Array.from(noteMap.values());
    console.log(`    [xhs] 去重后 ${allNotes.length} 条笔记 (综合 ${items1.length} + 最热 ${items2.length})`);

    if (allNotes.length === 0) {
        const errOut = {
            platform: 'xiaohongshu', keyword: spec.kw,
            _error: searchRes1?._error || searchRes2?._error || 'empty',
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`xhs FAIL ${spec.kw} empty/err=${JSON.stringify(errOut._error).slice(0, 80)}`);
        return { skipped: false, notes: 0, comments: 0, path: outPath };
    }

    const topNotes = [...allNotes]
        .sort((a, b) => ((b.comments_count || 0) * 2 + (b.liked_count || 0)) - ((a.comments_count || 0) * 2 + (a.liked_count || 0)))
        .slice(0, spec.notes);

    const commentsByNoteId = {};
    let totalComments = 0;
    for (const note of topNotes) {
        if (!note?.id) continue;
        commentsByNoteId[note.id] = [];
        let cursor = '';
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await tikhubGet('/thapi/v1/xiaohongshu/app/get_note_comments', { note_id: note.id, cursor });
            const cmts = cRes?.data?.data?.comments || [];
            for (const c of cmts) {
                commentsByNoteId[note.id].push({
                    id: c.id,
                    content: c.content,
                    create_time: c.create_time,
                    like_count: c.like_count,
                    sub_comment_count: c.sub_comment_count,
                    ip_location: c.ip_location,
                    user: c.user_info ? { user_id: c.user_info.user_id, nickname: c.user_info.nickname } : null,
                    sub_comments: (c.sub_comments || []).slice(0, 3).map((s) => ({
                        content: s.content,
                        like_count: s.like_count,
                        nickname: s.user_info?.nickname,
                    })),
                });
            }
            totalComments += cmts.length;
            const hasMore = cRes?.data?.data?.has_more;
            cursor = cRes?.data?.data?.cursor || '';
            if (!hasMore || !cursor) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [xhs] ✓ ${spec.kw}: ${topNotes.length} 笔记 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`xhs OK ${spec.kw} notes=${topNotes.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'xiaohongshu',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_notes_found: allNotes.length,
        top_notes: topNotes.map((n) => ({
            id: n.id,
            type: n.type,
            title: n.title,
            desc: n.desc,
            timestamp: n.timestamp,
            liked_count: n.liked_count,
            comments_count: n.comments_count,
            collected_count: n.collected_count,
            shared_count: n.shared_count,
            cover: n.images_list?.[0]?.url,
            user: n.user ? {
                userid: n.user.userid,
                nickname: n.user.nickname,
                red_id: n.user.red_id,
                verified: n.user.red_official_verified,
            } : null,
        })),
        comments_by_note_id: commentsByNoteId,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, notes: topNotes.length, comments: totalComments, path: outPath };
}

// ============================================================
// 抖音采集（单关键词 · Top 视频 + 评论）
// ============================================================
async function collectDouyin(spec, { force = false } = {}) {
    const outPath = path.join(DY_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [dy] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`dy SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [dy] 🔍 搜索 "${spec.kw}" · videos=${spec.videos} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    const searchRes = await dyPostV2('/api/voc-social/douyin/search/fetch_general_search_v2', {
        keyword: spec.kw,
        cursor: 0,
        sort_type: '1',
        publish_time: '0',
        content_type: '1',
        filter_duration: '0',
        search_id: '',
        backtrace: '',
    });

    const businessData = searchRes?.data?.business_data || [];
    const validVideos = businessData
        .map((wrap) => wrap?.data?.aweme_info || wrap?.aweme_info)
        .filter((v) => v?.aweme_id);
    console.log(`    [dy] 搜索到 ${validVideos.length} 条视频 (wrappers=${businessData.length})`);

    if (validVideos.length === 0) {
        const errOut = {
            platform: 'douyin', keyword: spec.kw,
            _search_error: searchRes?._error || searchRes?.mess || 'empty',
            _search_code: searchRes?.code,
            _search_sample: JSON.stringify(searchRes).slice(0, 400),
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`dy FAIL ${spec.kw} search_empty code=${searchRes?.code}`);
        return { skipped: false, videos: 0, comments: 0, path: outPath };
    }

    const topVideos = [...validVideos]
        .sort((a, b) => (b.statistics?.comment_count || 0) - (a.statistics?.comment_count || 0))
        .slice(0, spec.videos);

    const commentsByAweme = {};
    let totalComments = 0;
    for (const v of topVideos) {
        commentsByAweme[v.aweme_id] = [];
        let cursor = 0;
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await dyGetV3('/api/voc-social/douyin/app/v3/fetch_video_comments', {
                aweme_id: v.aweme_id,
                cursor,
                count: 20,
            });
            const cmts = cRes?.data?.comments || cRes?.comments || [];
            for (const c of cmts) {
                commentsByAweme[v.aweme_id].push({
                    cid: c.cid,
                    text: c.text,
                    digg_count: c.digg_count,
                    create_time: c.create_time,
                    ip_label: c.ip_label,
                    reply_comment_total: c.reply_comment_total,
                    user: c.user ? { nickname: c.user.nickname, uid: c.user.uid } : null,
                });
            }
            totalComments += cmts.length;
            const hasMore = (cRes?.data?.has_more ?? cRes?.has_more) === 1;
            cursor = cRes?.data?.cursor ?? cRes?.cursor ?? 0;
            if (!hasMore) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [dy] ✓ ${spec.kw}: ${topVideos.length} 视频 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`dy OK ${spec.kw} videos=${topVideos.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'douyin',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_videos_found: validVideos.length,
        top_videos: topVideos.map((v) => ({
            aweme_id: v.aweme_id,
            desc: v.desc,
            create_time: v.create_time,
            statistics: v.statistics,
            author: v.author ? {
                nickname: v.author.nickname,
                sec_uid: v.author.sec_uid,
                uid: v.author.uid,
                follower_count: v.author.follower_count,
            } : null,
            cover: v.video?.cover?.url_list?.[0],
            text_extra: (v.text_extra || []).map((t) => t.hashtag_name).filter(Boolean),
        })),
        comments_by_aweme_id: commentsByAweme,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, videos: topVideos.length, comments: totalComments, path: outPath };
}

// ============================================================
// 假设 + tag 推理（纯规则，基于关键词 / 文本内容）
// ============================================================
const HYPOTHESIS_KEYWORDS = {
    H1: ['米稀', '猴头菇', '养胃', '糯', '暖胃', '早餐', '粥', '糊'],
    H2: ['饼干', '硬', '油', '反胃', '甜', '卫健委', '噱头', '加工'],
    H3: ['胃泰', '拉唑', '奥美', '雷贝', '艾司', '康复新', '中成药', '颗粒', '苦', '起效', '疗程'],
    H4: ['停药', '反弹', '依赖', '戒毒', '减量', 'PPI', '骨密度', '副作用', '长期吃'],
    H5: ['应酬', '熬夜', '办公', '工位', '差旅', '出差', '饭局', '酒后', '咖啡', '便携', '即饮', '应急'],
    H6: ['药店', '店员', '柜台', '推荐', '挂牌', '医保', 'OTC'],
    H7: ['糖尿', '糖友', '血糖', '无糖', '木糖醇'],
    H8: ['送礼', '送父母', '送长辈', '阿胶', '燕窝', '黄芪', '礼盒', '节日'],
};

function inferHypotheses(text, keywordHypotheses) {
    const results = new Set(keywordHypotheses || []);
    const t = String(text || '').toLowerCase();
    for (const [h, kws] of Object.entries(HYPOTHESIS_KEYWORDS)) {
        for (const kw of kws) {
            if (t.includes(kw.toLowerCase())) { results.add(h); break; }
        }
    }
    return Array.from(results);
}

const TAG_RULES = [
    { tag: '米稀好评', re: /好喝|糯|舒服|推荐|正品|复购/i },
    { tag: '米稀局限性', re: /太甜|腻|结块|冲泡|贵/i },
    { tag: '饼干失败', re: /饼干.*(硬|油|反|胃酸|噱头)/i },
    { tag: '胃泰口感差', re: /(胃泰|颗粒).*苦|苦.*(胃泰|颗粒)|难喝/i },
    { tag: 'PPI 依赖焦虑', re: /(拉唑|ppi).*(依赖|停|戒|反弹|骨)/i },
    { tag: '应酬场景', re: /应酬|饭局|酒后|白酒|喝多/i },
    { tag: '熬夜场景', re: /熬夜|加班|通宵|凌晨/i },
    { tag: '办公室场景', re: /办公室|工位|午饭|午餐|加班餐/i },
    { tag: '差旅场景', re: /出差|差旅|高铁|飞机|酒店/i },
    { tag: '药店推荐', re: /药店|店员|柜台|挂牌/i },
    { tag: '糖尿病', re: /糖友|糖尿|血糖|无糖|木糖醇/i },
    { tag: '送礼', re: /送父母|送长辈|送礼|阿胶|燕窝/i },
    { tag: '温和', re: /温和|不刺激|不苦|顺滑/i },
    { tag: '便携', re: /便携|包里|小支|随身|铝膜/i },
];

function inferTags(text) {
    const tags = [];
    for (const r of TAG_RULES) {
        if (r.re.test(text || '')) tags.push(r.tag);
    }
    return tags;
}

const SENTIMENT_POS = /好喝|推荐|值得|有效|舒服|喜欢|好用|回购|复购|相信|靠谱|安心/;
const SENTIMENT_NEG = /难喝|不推荐|智商税|贵|骗|假|副作用|反弹|依赖|呕|反胃|腹泻|差评|失望/;
const SENTIMENT_CONFLICT = /想停|但是|可是|纠结|矛盾|害怕|担心|又.*又/;

function inferSentiment(text) {
    const t = String(text || '');
    const pos = SENTIMENT_POS.test(t);
    const neg = SENTIMENT_NEG.test(t);
    const conf = SENTIMENT_CONFLICT.test(t);
    if (conf && (pos || neg)) return 'conflicted';
    if (pos && !neg) return 'positive';
    if (neg && !pos) return 'negative';
    return 'neutral';
}

// ============================================================
// 合并 → _merged.json + comments-flat.jsonl
// ============================================================
function mergeAll() {
    const out = {
        meta: {
            collectedAt: new Date().toISOString(),
            platforms: {},
            products: {},
            hypotheses: {},
            keywords: {},
            stage: 'batch-2-real',
            sourceTier: 'real-collected',
        },
        items: [],
    };
    const flat = [];

    // 小红书
    if (fs.existsSync(XHS_DIR)) {
        for (const f of fs.readdirSync(XHS_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(XHS_DIR, f), 'utf8'));
            if (raw._error) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const note of (raw.top_notes || [])) {
                const noteId = note.id;
                const cmts = (raw.comments_by_note_id || {})[noteId] || [];
                // 笔记本身也作为一条 item
                if (note.desc || note.title) {
                    const noteContent = [note.title, note.desc].filter(Boolean).join(' · ').slice(0, 400);
                    const item = buildItem({
                        id: `xhs_${noteId}`,
                        platform: 'xhs',
                        product: kw,
                        keyword: kw,
                        type: 'note',
                        nickname: note.user?.nickname || '匿名',
                        ip: '',
                        content: noteContent,
                        likes: note.liked_count || 0,
                        rating: null,
                        noteId,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
                for (const c of cmts) {
                    if (!c.content) continue;
                    const item = buildItem({
                        id: `xhs_${noteId}_${c.id}`,
                        platform: 'xhs',
                        product: kw,
                        keyword: kw,
                        type: 'comment',
                        nickname: c.user?.nickname || '匿名',
                        ip: c.ip_location || '',
                        content: c.content.slice(0, 500),
                        likes: c.like_count || 0,
                        rating: null,
                        noteId,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
            }
        }
    }

    // 抖音
    if (fs.existsSync(DY_DIR)) {
        for (const f of fs.readdirSync(DY_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(DY_DIR, f), 'utf8'));
            if (raw._error || raw._search_error) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const v of (raw.top_videos || [])) {
                const awemeId = v.aweme_id;
                const cmts = (raw.comments_by_aweme_id || {})[awemeId] || [];
                if (v.desc) {
                    const item = buildItem({
                        id: `dy_${awemeId}`,
                        platform: 'douyin',
                        product: kw,
                        keyword: kw,
                        type: 'video',
                        nickname: v.author?.nickname || '匿名',
                        ip: '',
                        content: v.desc.slice(0, 400),
                        likes: v.statistics?.digg_count || 0,
                        rating: null,
                        awemeId,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
                for (const c of cmts) {
                    if (!c.text) continue;
                    const item = buildItem({
                        id: `dy_${awemeId}_${c.cid}`,
                        platform: 'douyin',
                        product: kw,
                        keyword: kw,
                        type: 'comment',
                        nickname: c.user?.nickname || '匿名',
                        ip: c.ip_label || '',
                        content: c.text.slice(0, 500),
                        likes: c.digg_count || 0,
                        rating: null,
                        awemeId,
                        kwHypos,
                    });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
            }
        }
    }

    // 与 seed 合并（如果 seed 存在，以真实为优先）
    const seedPath = path.join(RAW_DIR, '_seed.json');
    if (fs.existsSync(seedPath)) {
        const seed = JSON.parse(fs.readFileSync(seedPath, 'utf8'));
        for (const it of (seed.items || [])) {
            if (!out.items.find((x) => x.id === it.id)) {
                out.items.push(it);
                flat.push(it);
                countMeta(out, it);
            }
        }
        out.meta.includesSeedFallback = true;
    }

    out.meta.productsCount = Object.keys(out.meta.products).length;
    out.meta.keywordsCount = Object.keys(out.meta.keywords).length;
    out.meta.comments = out.items.length;

    fs.writeFileSync(path.join(RAW_DIR, '_merged.json'), JSON.stringify(out, null, 2), 'utf8');
    fs.writeFileSync(path.join(RAW_DIR, 'comments-flat.jsonl'), flat.map((it) => JSON.stringify(it)).join('\n'), 'utf8');
    console.log(`  ✅ merged: ${out.items.length} items | ${out.meta.productsCount} products | ${Object.keys(out.meta.platforms).length} platforms`);
    return out;
}

function buildItem({ id, platform, product, keyword, type, nickname, ip, content, likes, rating, kwHypos }) {
    if (!content || content.length < 3) return null;
    const hypotheses = inferHypotheses(content, kwHypos);
    const tags = inferTags(content);
    const sentiment = inferSentiment(content);
    return {
        id, platform, product, keyword, type,
        nickname, ip, content, likes, rating,
        hypothesis: hypotheses,
        tags,
        sentiment,
        source: 'real-collected',
    };
}

function countMeta(out, it) {
    out.meta.platforms[it.platform] = (out.meta.platforms[it.platform] || 0) + 1;
    out.meta.products[it.product] = (out.meta.products[it.product] || 0) + 1;
    out.meta.keywords[it.keyword] = (out.meta.keywords[it.keyword] || 0) + 1;
    for (const h of (it.hypothesis || [])) {
        out.meta.hypotheses[h] = (out.meta.hypotheses[h] || 0) + 1;
    }
}

// ============================================================
// 批次执行
// ============================================================
async function runBatch(batchNum, opts) {
    const batch = BATCHES[batchNum];
    if (!batch) throw new Error(`unknown batch: ${batchNum}`);
    console.log(`\n▶ Batch ${batchNum}: ${batch.name}`);
    auditLog(`BATCH-START ${batchNum} ${batch.name}`);

    for (const task of batch.xhs || []) {
        try { await collectXhs(task, opts); }
        catch (e) { console.log(`    [xhs] ✗ ${task.kw}: ${e.message}`); auditLog(`xhs EXC ${task.kw} ${e.message.slice(0, 80)}`); }
        await sleep(800);
    }
    for (const task of batch.douyin || []) {
        try { await collectDouyin(task, opts); }
        catch (e) { console.log(`    [dy] ✗ ${task.kw}: ${e.message}`); auditLog(`dy EXC ${task.kw} ${e.message.slice(0, 80)}`); }
        await sleep(600);
    }
    auditLog(`BATCH-END ${batchNum}`);
}

// ============================================================
// CLI
// ============================================================
async function main() {
    const argv = process.argv.slice(2);
    const opts = { force: argv.includes('--force') };
    const batchArg = argv.find((a) => a.startsWith('--batch='));
    const isMerge = argv.includes('--merge');
    const isTest = argv.includes('--test');

    console.log('╔═══════════════════════════════════════════════════════════╗');
    console.log('║  江中猴菇饮 OTC · VOC 多平台真实采集                          ║');
    console.log('╚═══════════════════════════════════════════════════════════╝');
    console.log(`  TikHub token: ${TIKHUB_TOKEN ? '✓ ' + TIKHUB_TOKEN.slice(0, 8) + '...' : '✗'}`);
    console.log(`  VOC token:    ${VOC_TOKEN ? '✓ ' + VOC_TOKEN.slice(0, 8) + '...' : '✗'}`);

    if (isTest) {
        console.log('\n▶ Test mode · 跑通单关键词 xhs["江中猴姑米稀"]');
        await collectXhs({ kw: '江中猴姑米稀', notes: 3, commentPages: 1, hypotheses: ['H1', 'H5'] }, { force: true });
        return;
    }

    if (batchArg) {
        const bn = batchArg.split('=')[1];
        if (bn === 'all') {
            for (const k of Object.keys(BATCHES)) await runBatch(k, opts);
        } else {
            await runBatch(bn, opts);
        }
    }

    if (isMerge || batchArg) {
        console.log('\n▶ Merging all collected raw data...');
        mergeAll();
    }

    if (!batchArg && !isMerge && !isTest) {
        console.log('\nUsage:');
        console.log('  --test              跑通单关键词测试 API 链路');
        console.log('  --batch=1|2|3|all   执行采集批次');
        console.log('  --merge             合并 raw/*/*.json → _merged.json');
        console.log('  --force             强制重抓（忽略已存在文件）');
    }
}

if (require.main === module) {
    main().catch((e) => { console.error('fatal:', e); process.exit(1); });
}

module.exports = { BATCHES, mergeAll, collectXhs, collectDouyin };