fmode-voc-projects/scripts/tools/hongchengdaojia/hongcheng-collect-douy.js

#!/usr/bin/env node
/**
 * 洪城到家 · VOC 多平台采集（抖音为主）
 *
 * 覆盖竞品：天鹅到家、好孕妈妈、宜尔宝、妈咪无忧、爱侬家政、
 *          叮当找阿姨、优护佳、大爱天使、多喜娃、爱的果实
 *
 * 用法：
 *   node scripts/tools/hongcheng-collect.js --batch=1
 *   node scripts/tools/hongcheng-collect.js --batch=all
 *   node scripts/tools/hongcheng-collect.js --merge
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', '洪城到家', 'raw');
const XHS_DIR = path.join(RAW_DIR, 'xhs');
const DY_DIR = path.join(RAW_DIR, 'douyin');
const MEITUAN_DIR = path.join(RAW_DIR, 'meituan');
const AUDIT_LOG = path.join(RAW_DIR, 'audit.log');

[RAW_DIR, XHS_DIR, DY_DIR, MEITUAN_DIR].forEach((d) => {
    if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
});

const VOC_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'voc-credentials.json');
    if (!fs.existsSync(p)) return '';
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return c.vocToken || c.sessionToken || '';
    } catch { return ''; }
})();

const TIKHUB_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(p)) return '';
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf8'));
        return c.currentToken || '';
    } catch { return ''; }
})();

console.log(`  VOC Token: ${VOC_TOKEN ? VOC_TOKEN.slice(0,8)+'...' : '❌ missing'}`);
console.log(`  TikHub Token: ${TIKHUB_TOKEN ? TIKHUB_TOKEN.slice(0,8)+'...' : '❌ missing'}`);

const BATCHES = {
    1: {
        name: 'P0 · 本品 + 全国头部竞品',
        douyin: [
            { kw: '洪诚到家月嫂', videos: 8, commentPages: 30, hypotheses: ['H1', 'H4', 'H6'] },
            { kw: '天鹅到家月嫂', videos: 8, commentPages: 30, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '好孕妈妈月嫂', videos: 8, commentPages: 30, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '多喜娃月嫂', videos: 8, commentPages: 30, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '妈咪无忧月嫂', videos: 8, commentPages: 30, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '江西月嫂', videos: 6, commentPages: 25, hypotheses: ['H1'] },
            { kw: '南昌月嫂哪家好', videos: 6, commentPages: 25, hypotheses: ['H5', 'H7'] },
        ],
    },
    2: {
        name: 'P1 · 场景痛点 + 更多竞品',
        douyin: [
            { kw: '新手爸妈找月嫂', videos: 6, commentPages: 25, hypotheses: ['H1', 'H4'] },
            { kw: '坐月子注意事项', videos: 6, commentPages: 25, hypotheses: ['H2', 'H4'] },
            { kw: '月嫂多少钱一个月', videos: 6, commentPages: 25, hypotheses: ['H2'] },
            { kw: '职场妈妈月子', videos: 5, commentPages: 20, hypotheses: ['H3'] },
            { kw: '新手爸妈', videos: 5, commentPages: 20, hypotheses: ['H1', 'H4'] },
            { kw: '宜尔宝月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
            { kw: '爱侬家政月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
            { kw: '优护佳月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
        ],
    },
    3: {
        name: 'P2 · 长尾 + 竞品对比 + 短剧',
        douyin: [
            { kw: '社区月嫂', videos: 5, commentPages: 20, hypotheses: ['H5'] },
            { kw: '南昌月嫂对比', videos: 5, commentPages: 20, hypotheses: ['H5', 'H7'] },
            { kw: '月嫂怎么找靠谱', videos: 5, commentPages: 20, hypotheses: ['H4', 'H8'] },
            { kw: '月嫂不满意可以换吗', videos: 4, commentPages: 15, hypotheses: ['H8'] },
            { kw: '朋友推荐月嫂', videos: 4, commentPages: 15, hypotheses: ['H6'] },
            { kw: '南昌短剧', videos: 5, commentPages: 20, hypotheses: ['H3'] },
            { kw: '婆媳关系坐月子', videos: 5, commentPages: 20, hypotheses: ['H3'] },
            { kw: '叮当找阿姨月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
            { kw: '大爱天使月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
            { kw: '爱的果实月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H5'] },
        ],
    },
    4: {
        name: 'P3 · 价格/决策 + 更多长尾',
        douyin: [
            { kw: '南昌月嫂价格', videos: 5, commentPages: 20, hypotheses: ['H2'] },
            { kw: '月嫂服务', videos: 4, commentPages: 15, hypotheses: ['H4'] },
            { kw: '婆婆不让请月嫂', videos: 4, commentPages: 15, hypotheses: ['H1'] },
            { kw: '月嫂和保姆的区别', videos: 4, commentPages: 15, hypotheses: ['H4'] },
            { kw: '坐月子请月嫂', videos: 5, commentPages: 20, hypotheses: ['H1', 'H2'] },
            { kw: '月嫂真实评价', videos: 4, commentPages: 15, hypotheses: ['H4', 'H6'] },
            { kw: '月嫂一天工作流程', videos: 4, commentPages: 15, hypotheses: ['H4'] },
            { kw: '新生儿护理', videos: 5, commentPages: 20, hypotheses: ['H4'] },
            { kw: '产妇护理', videos: 4, commentPages: 15, hypotheses: ['H4'] },
            { kw: '月子餐做法', videos: 4, commentPages: 15, hypotheses: ['H4'] },
        ],
    },
};

const XHS_BATCHES = {
    1: {
        name: 'P0 · 本品 + 头部竞品',
        xhs: [
            { kw: '洪诚到家月嫂', maxNotes: 15, maxComments: 10, hypotheses: ['H1', 'H4', 'H6'] },
            { kw: '天鹅到家月嫂', maxNotes: 15, maxComments: 10, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '好孕妈妈月嫂', maxNotes: 15, maxComments: 10, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '多喜娃月嫂', maxNotes: 12, maxComments: 8, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '妈咪无忧月嫂', maxNotes: 12, maxComments: 8, hypotheses: ['H1', 'H5', 'H7'] },
            { kw: '南昌月嫂哪家好', maxNotes: 12, maxComments: 8, hypotheses: ['H5', 'H7'] },
        ],
    },
    2: {
        name: 'P1 · 场景痛点',
        xhs: [
            { kw: '新手爸妈找月嫂', maxNotes: 12, maxComments: 8, hypotheses: ['H1', 'H4'] },
            { kw: '坐月子注意事项', maxNotes: 12, maxComments: 8, hypotheses: ['H2', 'H4'] },
            { kw: '月嫂多少钱一个月', maxNotes: 12, maxComments: 8, hypotheses: ['H2'] },
            { kw: '月嫂怎么找靠谱', maxNotes: 10, maxComments: 6, hypotheses: ['H4', 'H8'] },
            { kw: '婆婆不让请月嫂', maxNotes: 10, maxComments: 6, hypotheses: ['H1'] },
        ],
    },
    3: {
        name: 'P2 · 长尾 + 补全',
        xhs: [
            { kw: '社区月嫂', maxNotes: 10, maxComments: 6, hypotheses: ['H5'] },
            { kw: '南昌月嫂对比', maxNotes: 10, maxComments: 6, hypotheses: ['H5', 'H7'] },
            { kw: '月嫂不满意可以换吗', maxNotes: 8, maxComments: 5, hypotheses: ['H8'] },
            { kw: '朋友推荐月嫂', maxNotes: 8, maxComments: 5, hypotheses: ['H6'] },
            { kw: '江西月嫂', maxNotes: 10, maxComments: 6, hypotheses: ['H1'] },
        ],
    },
};

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function sanitizeFilename(kw) {
    return kw.replace(/[\/\\:*?"<>|\s]+/g, '-');
}

function auditLog(line) {
    const ts = new Date().toISOString();
    try {
        fs.appendFileSync(AUDIT_LOG, `[${ts}] ${line}\n`);
    } catch {}
}

function fileExistsNonEmpty(p) {
    if (!fs.existsSync(p)) return false;
    return fs.statSync(p).size > 50;
}

function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        const chunks = [];
                        response.on('data', (c) => chunks.push(c));
                        response.on('end', () => res({
                            status: response.statusCode,
                            body: Buffer.concat(chunks).toString('utf-8'),
                        }));
                    });
                    req.on('error', rej);
                    req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
                    req.setTimeout(60000);
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    await sleep(1500 * Math.pow(1.8, attempt - 1));
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await sleep(2000 * attempt);
            }
        }
    });
}

async function dyPostV2(apiPath, bodyObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const bodyStr = JSON.stringify(bodyObj);
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: apiPath,
        method: 'POST',
        headers: {
            Accept: 'application/json',
            'Content-Type': 'application/json',
            'Content-Length': Buffer.byteLength(bodyStr),
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyGetV3(apiPath, paramsObj = {}) {
    if (!VOC_TOKEN) return { _error: 'no VOC token' };
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${VOC_TOKEN}`,
        },
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function xhsSearchNotes(keyword, maxNotes = 15) {
    if (!TIKHUB_TOKEN) return { _error: 'no TikHub token' };
    const bodyStr = JSON.stringify({
        keyword,
        sort_type: 'general',
        note_type: 'normal',
        total_number: maxNotes,
    });
    const { status, body } = await httpRequest({
        hostname: 'api.tikhub.io',
        path: '/api/v1/xiaohongshu/web_v1/search_notes',
        method: 'POST',
        headers: {
            Accept: 'application/json',
            'Content-Type': 'application/json',
            'Content-Length': Buffer.byteLength(bodyStr),
            Authorization: `Bearer ${TIKHUB_TOKEN}`,
        },
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function xhsNoteComments(noteId, maxComments = 10) {
    if (!TIKHUB_TOKEN) return { _error: 'no TikHub token' };
    let cursor = '';
    const allComments = [];
    let hasMore = true;
    let attempts = 0;
    while (hasMore && allComments.length < maxComments && attempts < 5) {
        const bodyStr = JSON.stringify({ note_id: noteId, cursor });
        const { status, body } = await httpRequest({
            hostname: 'api.tikhub.io',
            path: '/api/v1/xiaohongshu/web_v1/note_comments',
            method: 'POST',
            headers: {
                Accept: 'application/json',
                'Content-Type': 'application/json',
                'Content-Length': Buffer.byteLength(bodyStr),
                Authorization: `Bearer ${TIKHUB_TOKEN}`,
            },
        }, bodyStr);
        if (status !== 200 || !body) break;
        try {
            const res = JSON.parse(body);
            const comments = res?.data?.comments || res?.comments || [];
            allComments.push(...comments.slice(0, maxComments - allComments.length));
            cursor = res?.data?.cursor || res?.cursor || '';
            hasMore = !!(res?.data?.has_more || res?.has_more);
        } catch { break; }
        attempts++;
        await sleep(300);
    }
    return allComments;
}

async function collectDouyin(spec, { force = false } = {}) {
    const outPath = path.join(DY_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`dy SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  🔍 "${spec.kw}" · videos=${spec.videos} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    const searchRes = await dyPostV2('/api/voc-social/douyin/search/fetch_general_search_v2', {
        keyword: spec.kw,
        cursor: 0,
        sort_type: '1',
        publish_time: '0',
        content_type: '1',
        filter_duration: '0',
        search_id: '',
        backtrace: '',
    });

    const businessData = searchRes?.data?.business_data || [];
    const validVideos = businessData
        .map((wrap) => wrap?.data?.aweme_info || wrap?.aweme_info)
        .filter((v) => v?.aweme_id);
    console.log(`    搜索到 ${validVideos.length} 条视频`);

    if (validVideos.length === 0) {
        const errOut = {
            platform: 'douyin', keyword: spec.kw,
            _search_error: searchRes?._error || searchRes?.msg || 'empty',
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`dy FAIL ${spec.kw} search_empty`);
        return { skipped: false, videos: 0, comments: 0, path: outPath };
    }

    const topVideos = [...validVideos]
        .sort((a, b) => (b.statistics?.comment_count || 0) - (a.statistics?.comment_count || 0))
        .slice(0, spec.videos);

    const commentsByAweme = {};
    let totalComments = 0;
    for (const v of topVideos) {
        commentsByAweme[v.aweme_id] = [];
        let cursor = 0;
        for (let p = 0; p < (spec.commentPages || 2); p++) {
            await sleep(700);
            const cRes = await dyGetV3('/api/voc-social/douyin/app/v3/fetch_video_comments', {
                aweme_id: v.aweme_id,
                cursor,
                count: 20,
            });
            const cmts = cRes?.data?.comments || cRes?.comments || [];
            for (const c of cmts) {
                commentsByAweme[v.aweme_id].push({
                    cid: c.cid,
                    text: c.text,
                    digg_count: c.digg_count,
                    create_time: c.create_time,
                    ip_label: c.ip_label,
                    reply_comment_total: c.reply_comment_total,
                    user: c.user ? { nickname: c.user.nickname, uid: c.user.uid } : null,
                });
            }
            totalComments += cmts.length;
            const hasMore = (cRes?.data?.has_more ?? cRes?.has_more) === 1;
            cursor = cRes?.data?.cursor ?? cRes?.cursor ?? 0;
            if (!hasMore) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    ✓ ${spec.kw}: ${topVideos.length} 视频 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`dy OK ${spec.kw} videos=${topVideos.length} comments=${totalComments}`);

    const out = {
        platform: 'douyin',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_videos_found: validVideos.length,
        top_videos: topVideos.map((v) => ({
            aweme_id: v.aweme_id,
            desc: v.desc,
            create_time: v.create_time,
            statistics: v.statistics,
            author: v.author ? { nickname: v.author.nickname, sec_uid: v.author.sec_uid, uid: v.author.uid } : null,
            cover: v.video?.cover?.url_list?.[0],
        })),
        comments_by_aweme_id: commentsByAweme,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, videos: topVideos.length, comments: totalComments, path: outPath };
}

async function collectXhs(spec, { force = false } = {}) {
    const outPath = path.join(XHS_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  ⏭️  xhs:${spec.kw} (已存在, skip)`);
        auditLog(`xhs SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  🔍 xhs:"${spec.kw}" · notes=${spec.maxNotes} × comments=${spec.maxComments}`);
    const startTs = Date.now();

    const searchRes = await xhsSearchNotes(spec.kw, spec.maxNotes);
    if (searchRes._error) {
        console.log(`    ✗ 搜索失败: ${searchRes._error.status || searchRes._error.message}`);
        auditLog(`xhs FAIL ${spec.kw} search_error`);
        fs.writeFileSync(outPath, JSON.stringify({ platform: 'xhs', keyword: spec.kw, _error: searchRes._error, collected_at: new Date().toISOString() }, null, 2));
        return { skipped: false, notes: 0, comments: 0 };
    }

    const notes = searchRes?.data?.notes || searchRes?.notes || [];
    console.log(`    搜索到 ${notes.length} 篇笔记`);

    if (notes.length === 0) {
        fs.writeFileSync(outPath, JSON.stringify({ platform: 'xhs', keyword: spec.kw, _empty: true, collected_at: new Date().toISOString() }, null, 2));
        auditLog(`xhs EMPTY ${spec.kw}`);
        return { skipped: false, notes: 0, comments: 0 };
    }

    const items = [];
    for (const note of notes.slice(0, spec.maxNotes)) {
        await sleep(250);
        const comments = await xhsNoteComments(note.id, spec.maxComments);
        for (const c of comments) {
            items.push({
                platform: 'xhs',
                keyword: spec.kw,
                hypotheses: spec.hypotheses,
                note_id: note.id,
                note_title: note.title || note.desc || '',
                note_likes: note.liked_count || 0,
                comment_id: c.id || c.comment_id || '',
                content: c.content || c.text || '',
                user_nickname: c.user_info?.nickname || c.user_nickname || '匿名',
                ip_location: c.ip_location || c.ip_label || '',
                comment_likes: c.like_count || c.liked_count || 0,
                collected_at: new Date().toISOString(),
            });
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    ✓ xhs:${spec.kw}: ${notes.length} 笔记 / ${items.length} 评论 (${elapsed}s)`);
    auditLog(`xhs OK ${spec.kw} notes=${notes.length} comments=${items.length}`);

    const out = { platform: 'xhs', keyword: spec.kw, hypotheses: spec.hypotheses, collected_at: new Date().toISOString(), elapsed_seconds: Number(elapsed), notes_found: notes.length, notes_processed: Math.min(notes.length, spec.maxNotes), comments: items };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, notes: notes.length, comments: items.length, path: outPath };
}

const HYPOTHESIS_KEYWORDS = {
    H1: ['医院', '产检', '待产', '生孩子', '妇幼', '生产', '月嫂怎么找'],
    H2: ['价格', '多少钱', '收费', '报价', '性价比', '便宜', '贵'],
    H3: ['短剧', '抖音', '视频', '小红书', '看到'],
    H4: ['专业', '资质', '证书', '星级', '靠谱', '放心', '正规'],
    H5: ['社区', '小店', '私人', '对比', '选择'],
    H6: ['朋友', '推荐', '介绍', '转介绍', '口碑', '好评'],
    H7: ['美团', '大众点评', '搜索', '排名', '评价'],
    H8: ['换', '退', '不满意', '保障', '售后', '风险'],
};

function inferHypotheses(text, keywordHypotheses) {
    const results = new Set(keywordHypotheses || []);
    const t = String(text || '').toLowerCase();
    for (const [h, kws] of Object.entries(HYPOTHESIS_KEYWORDS)) {
        for (const kw of kws) {
            if (t.includes(kw.toLowerCase())) { results.add(h); break; }
        }
    }
    return Array.from(results);
}

const TAG_RULES = [
    { tag: '价格敏感', re: /价格|多少钱|贵|便宜|性价比|收费/ },
    { tag: '专业度关注', re: /专业|资质|证书|星级|培训/ },
    { tag: '安全保障', re: /放心|靠谱|安全|保障|正规/ },
    { tag: '医院渠道', re: /医院|产检|妇幼|待产|生孩子/ },
    { tag: '熟人推荐', re: /朋友推荐|介绍|口碑|好评|亲戚/ },
    { tag: '线上搜索', re: /美团|小红书|抖音|搜索/ },
    { tag: '服务担忧', re: /换|退|不满意|售后|保障/ },
    { tag: '婆媳关系', re: /婆婆|奶奶|家里|老人/ },
    { tag: '职场妈妈', re: /上班|工作|复工|职场/ },
    { tag: '新手爸妈', re: /新手|第一次|第一次当妈妈/ },
];

function inferTags(text) {
    const tags = [];
    for (const r of TAG_RULES) {
        if (r.re.test(text || '')) tags.push(r.tag);
    }
    return tags;
}

const SENTIMENT_POS = /好|推荐|满意|专业|靠谱|放心|值得|不错|棒|优秀|喜欢|感谢/;
const SENTIMENT_NEG = /差|坑|骗|贵|不专业|不满意|后悔|吐槽|垃圾|失望|糟糕|骗人/;
const SENTIMENT_CONFLICT = /但是|可是|纠结|担心|犹豫|想又怕/;

function inferSentiment(text) {
    const t = String(text || '');
    const pos = SENTIMENT_POS.test(t);
    const neg = SENTIMENT_NEG.test(t);
    const conf = SENTIMENT_CONFLICT.test(t);
    if (conf && (pos || neg)) return 'conflicted';
    if (pos && !neg) return 'positive';
    if (neg && !pos) return 'negative';
    return 'neutral';
}

function mergeAll() {
    const out = {
        meta: {
            collectedAt: new Date().toISOString(),
            platforms: {},
            products: {},
            hypotheses: {},
            keywords: {},
            stage: 'batch-real',
            sourceTier: 'real-collected',
        },
        items: [],
    };
    const flat = [];

    if (fs.existsSync(DY_DIR)) {
        for (const f of fs.readdirSync(DY_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(DY_DIR, f), 'utf8'));
            if (raw._error || raw._search_error) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            for (const v of (raw.top_videos || [])) {
                const awemeId = v.aweme_id;
                const cmts = (raw.comments_by_aweme_id || {})[awemeId] || [];
                if (v.desc) {
                    const item = buildItem({ id: `dy_${awemeId}`, platform: 'douyin', product: kw, keyword: kw, type: 'video', nickname: v.author?.nickname || '匿名', ip: '', content: v.desc.slice(0, 400), likes: v.statistics?.digg_count || 0, rating: null, kwHypos });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
                for (const c of cmts) {
                    if (!c.text) continue;
                    const item = buildItem({ id: `dy_${awemeId}_${c.cid}`, platform: 'douyin', product: kw, keyword: kw, type: 'comment', nickname: c.user?.nickname || '匿名', ip: c.ip_label || '', content: c.text.slice(0, 500), likes: c.digg_count || 0, rating: null, kwHypos });
                    if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
                }
            }
        }
    }

    if (fs.existsSync(XHS_DIR)) {
        for (const f of fs.readdirSync(XHS_DIR)) {
            if (!f.endsWith('.json')) continue;
            const raw = JSON.parse(fs.readFileSync(path.join(XHS_DIR, f), 'utf8'));
            if (raw._error || raw._empty) continue;
            const kw = raw.keyword;
            const kwHypos = raw.hypotheses || [];
            const comments = raw.comments || [];
            for (const c of comments) {
                if (!c.content) continue;
                const item = buildItem({
                    id: `xhs_${c.note_id}_${c.comment_id}`,
                    platform: 'xhs',
                    product: kw,
                    keyword: kw,
                    type: 'comment',
                    nickname: c.user_nickname || '匿名',
                    ip: c.ip_location || '',
                    content: c.content.slice(0, 500),
                    likes: c.comment_likes || 0,
                    rating: null,
                    kwHypos,
                    note_title: c.note_title || '',
                    note_likes: c.note_likes || 0,
                });
                if (item) { out.items.push(item); flat.push(item); countMeta(out, item); }
            }
        }
    }

    const seen = new Set();
    out.items = out.items.filter((it) => { if (seen.has(it.id)) return false; seen.add(it.id); return true; });

    out.meta.productsCount = Object.keys(out.meta.products).length;
    out.meta.keywordsCount = Object.keys(out.meta.keywords).length;
    out.meta.comments = out.items.length;

    fs.writeFileSync(path.join(RAW_DIR, '_merged.json'), JSON.stringify(out, null, 2), 'utf8');
    fs.writeFileSync(path.join(RAW_DIR, 'comments-flat.jsonl'), flat.map((it) => JSON.stringify(it)).join('\n'), 'utf8');
    console.log(`  ✅ merged: ${out.items.length} items | ${out.meta.productsCount} products`);
    return out;
}

function buildItem({ id, platform, product, keyword, type, nickname, ip, content, likes, rating, kwHypos }) {
    if (!content || content.length < 3) return null;
    const hypotheses = inferHypotheses(content, kwHypos);
    const tags = inferTags(content);
    const sentiment = inferSentiment(content);
    return { id, platform, product, keyword, type, nickname, ip, content, likes, rating, hypothesis: hypotheses, tags, sentiment, source: 'real-collected' };
}

function countMeta(out, it) {
    out.meta.platforms[it.platform] = (out.meta.platforms[it.platform] || 0) + 1;
    out.meta.products[it.product] = (out.meta.products[it.product] || 0) + 1;
    out.meta.keywords[it.keyword] = (out.meta.keywords[it.keyword] || 0) + 1;
    for (const h of (it.hypothesis || [])) {
        out.meta.hypotheses[h] = (out.meta.hypotheses[h] || 0) + 1;
    }
}

async function runBatch(batchNum, opts) {
    const batch = BATCHES[batchNum];
    if (!batch) throw new Error(`unknown batch: ${batchNum}`);
    console.log(`\n▶ Batch ${batchNum}: ${batch.name}`);

    for (const task of batch.douyin || []) {
        try { await collectDouyin(task, opts); } catch (e) { console.log(`    ✗ ${task.kw}: ${e.message}`); auditLog(`dy EXC ${task.kw}`); }
        await sleep(800);
    }
}

async function runXhsBatch(batchNum, opts) {
    const batch = XHS_BATCHES[batchNum];
    if (!batch) throw new Error(`unknown xhs batch: ${batchNum}`);
    console.log(`\n▶ XHS Batch ${batchNum}: ${batch.name}`);

    for (const task of batch.xhs || []) {
        try { await collectXhs(task, opts); } catch (e) { console.log(`    ✗ xhs:${task.kw}: ${e.message}`); auditLog(`xhs EXC ${task.kw}`); }
        await sleep(350);
    }
}

async function main() {
    const argv = process.argv.slice(2);
    const opts = { force: argv.includes('--force') };
    const batchArg = argv.find((a) => a.startsWith('--batch='));
    const xhsBatchArg = argv.find((a) => a.startsWith('--xhs='));
    const isMerge = argv.includes('--merge');

    console.log('╔═══════════════════════════════════════════════════════════╗');
    console.log('║  洪城到家 · VOC 多平台采集（抖音+小红书）              ║');
    console.log('╚═══════════════════════════════════════════════════════════╝');
    console.log(`  VOC Token: ${VOC_TOKEN ? '✓' : '✗'}`);
    console.log(`  TikHub Token: ${TIKHUB_TOKEN ? '✓' : '✗'}`);

    if (batchArg) {
        const bn = batchArg.split('=')[1];
        if (bn === 'all') {
            for (const k of Object.keys(BATCHES)) await runBatch(k, opts);
        } else {
            await runBatch(bn, opts);
        }
    }

    if (xhsBatchArg) {
        const bn = xhsBatchArg.split('=')[1];
        if (bn === 'all') {
            for (const k of Object.keys(XHS_BATCHES)) await runXhsBatch(k, opts);
        } else {
            await runXhsBatch(bn, opts);
        }
    }

    if (isMerge || batchArg || xhsBatchArg) {
        console.log('\n▶ Merging...');
        mergeAll();
    }

    if (!batchArg && !xhsBatchArg && !isMerge) {
        console.log('\nUsage:');
        console.log('  --batch=1|2|3|4|all   执行抖音采集批次');
        console.log('  --xhs=1|2|3|all       执行小红书采集批次');
        console.log('  --merge                 合并数据');
        console.log('  --force                强制重抓');
        console.log('\n抖音关键词覆盖:');
        console.log('  - 全国龙头: 天鹅到家、好孕妈妈');
        console.log('  - 区域强者: 多喜娃、妈咪无忧、宜尔宝、爱侬家政、优护佳');
        console.log('  - 长尾品牌: 叮当找阿姨、大爱天使、爱的果实');
        console.log('  - 场景词: 新手爸妈、坐月子、月子餐、产妇护理');
        console.log('\n小红书关键词覆盖:');
        console.log('  - 本品: 洪诚到家月嫂');
        console.log('  - 竞品: 天鹅到家、好孕妈妈、多喜娃、妈咪无忧');
        console.log('  - 场景: 新手爸妈找月嫂、月嫂多少钱一个月、月嫂怎么找靠谱');
    }
}

if (require.main === module) {
    main().catch((e) => { console.error('fatal:', e); process.exit(1); });
}

module.exports = { BATCHES, XHS_BATCHES, mergeAll, collectDouyin, collectXhs };