fmode-voc-projects/scripts/tools/collect-jiangzhong-liver-deep.js

#!/usr/bin/env node
/**
 * 江中肝纯片 VOC 深度评论采集（多平台×多假设×多关键词）
 *
 * 对标 docs/jiangzhong/2.数据采集矩阵.md 定义的 42 个关键词，
 * 在小红书 / 抖音 / Amazon 三平台跑到评论颗粒度。
 *
 * 特性:
 *   - 分批次执行 (--batch=1/2/3/all), 每批内按平台并行
 *   - 每关键词独立 JSON 落盘到 docs/jiangzhong/raw/{platform}/{kw}.json
 *   - 断点续跑：已存在文件跳过，--force 强制重跑
 *   - 3 次重试 + 指数退避，适应 TikHub 瞬时 400/超时
 *   - 审计日志 docs/jiangzhong/raw/audit.log 每关键词一行
 *   - --merge 合并所有关键词 → _merged.json + comments-flat.jsonl
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

// ============================================================
// 配置
// ============================================================

const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', 'jiangzhong', 'raw');
const XHS_DIR = path.join(RAW_DIR, 'xhs');
const DY_DIR = path.join(RAW_DIR, 'douyin');
const AMZ_DIR = path.join(RAW_DIR, 'amazon');
const AUDIT_LOG = path.join(RAW_DIR, 'audit.log');

// 确保目录存在
[RAW_DIR, XHS_DIR, DY_DIR, AMZ_DIR].forEach((d) => {
    if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
});

// TikHub token（与 xiaohongshu-search-notes skill 共用）
const TIKHUB_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(p)) {
        console.error('❌ TikHub 凭据不存在: ' + p);
        console.error('   请先部署 xiaohongshu-search-notes skill 到 ~/.openclaw/skills/');
        process.exit(2);
    }
    const c = JSON.parse(fs.readFileSync(p, 'utf8'));
    return c.endpoint.headers.Authorization.replace(/^Bearer\s+/, '');
})();

// 抖音 cookie（可选，无则自动降级到仅 share_url 解析）
const DY_COOKIE = (() => {
    const p = path.join(ROOT, 'data', 'douyin-cookie.txt');
    if (!fs.existsSync(p)) return '';
    return fs.readFileSync(p, 'utf-8').trim()
        .split('\n').filter((l) => l && !l.startsWith('#')).join('; ').trim();
})();

// VOC Token（抖音搜索/评论 skill 共用，走 server.fmode.cn/api/voc-social/）
const VOC_TOKEN = (() => {
    const p = path.join(os.homedir(), '.openclaw', 'voc-credentials.json');
    if (!fs.existsSync(p)) return '';
    try {
        const c = JSON.parse(fs.readFileSync(p, 'utf-8'));
        return c.vocToken || c.sessionToken || '';
    } catch (e) { return ''; }
})();

// ============================================================
// 关键词矩阵（对标 2.数据采集矩阵.md）
// ============================================================

const BATCHES = {
    1: {
        name: '本品 & 核心竞品',
        xhs: [
            { kw: '肝纯片',         notes: 5, commentPages: 2, hypotheses: ['H1', 'H2'] },
            { kw: '江中肝纯片',     notes: 5, commentPages: 2, hypotheses: ['H1'] },
            { kw: '护肝片',         notes: 5, commentPages: 2, hypotheses: ['H3', 'H4'] },
            { kw: 'swisse护肝片',   notes: 5, commentPages: 2, hypotheses: ['H5', 'H7'] },
            { kw: '易善复',         notes: 5, commentPages: 2, hypotheses: ['H3', 'H5'] },
            { kw: '葵花护肝片',     notes: 5, commentPages: 2, hypotheses: ['H3'] },
            { kw: '海王金樽',       notes: 5, commentPages: 2, hypotheses: ['H2'] },
            { kw: '解酒神器',       notes: 5, commentPages: 2, hypotheses: ['H2'] },
        ],
        douyin: [
            { kw: '肝纯片',       videos: 2, commentPages: 2, hypotheses: ['H1', 'H2'] },
            { kw: '江中肝纯片',   videos: 2, commentPages: 2, hypotheses: ['H1'] },
            { kw: '护肝片',       videos: 2, commentPages: 2, hypotheses: ['H3', 'H4'] },
            { kw: '解酒神器',     videos: 2, commentPages: 2, hypotheses: ['H2'] },
            { kw: '海王金樽',     videos: 2, commentPages: 2, hypotheses: ['H2'] },
        ],
        amazon: [
            { kw: 'milk thistle',              pages: 3, topDetail: 6, topReviews: 5, hypotheses: ['H5', 'H8'] },
            { kw: 'silymarin',                 pages: 3, topDetail: 6, topReviews: 5, hypotheses: ['H5', 'H8'] },
            { kw: 'liver support supplement',  pages: 3, topDetail: 6, topReviews: 5, hypotheses: ['H8'] },
            { kw: 'dihydromyricetin',          pages: 3, topDetail: 6, topReviews: 5, hypotheses: ['H1'] },
        ],
    },
    2: {
        name: '场景 & 人群',
        xhs: [
            { kw: '脂肪肝',             notes: 4, commentPages: 2, hypotheses: ['H4'] },
            { kw: '熬夜护肝',           notes: 4, commentPages: 2, hypotheses: ['H4'] },
            { kw: '应酬解酒',           notes: 4, commentPages: 2, hypotheses: ['H2'] },
            { kw: '送长辈保健品',       notes: 4, commentPages: 2, hypotheses: ['H6', 'H7'] },
            { kw: '水飞蓟',             notes: 4, commentPages: 2, hypotheses: ['H5', 'H3'] },
            { kw: '奶蓟草',             notes: 4, commentPages: 2, hypotheses: ['H5'] },
            { kw: '片仔癀护肝',         notes: 4, commentPages: 2, hypotheses: ['H5'] },
            { kw: '汤臣倍健护肝',       notes: 4, commentPages: 2, hypotheses: ['H5'] },
            { kw: '酒局必备',           notes: 4, commentPages: 2, hypotheses: ['H2'] },
            { kw: '藤茶',               notes: 4, commentPages: 2, hypotheses: ['H1'] },
        ],
        douyin: [
            { kw: '脂肪肝',       videos: 2, commentPages: 2, hypotheses: ['H4'] },
            { kw: '熬夜伤肝',     videos: 2, commentPages: 2, hypotheses: ['H4'] },
            { kw: '应酬解酒',     videos: 2, commentPages: 2, hypotheses: ['H2'] },
        ],
        amazon: [
            { kw: 'liver detox',    pages: 3, topDetail: 5, topReviews: 5, hypotheses: ['H8'] },
            { kw: 'hangover pills', pages: 3, topDetail: 5, topReviews: 5, hypotheses: ['H2', 'H8'] },
        ],
    },
    3: {
        name: '长尾',
        xhs: [
            { kw: '二氢杨梅素',     notes: 3, commentPages: 1, hypotheses: ['H1'] },
            { kw: '熊胆粉',         notes: 3, commentPages: 1, hypotheses: ['H5'] },
            { kw: '保健品礼盒',     notes: 3, commentPages: 1, hypotheses: ['H7'] },
            { kw: '父母保健品',     notes: 3, commentPages: 1, hypotheses: ['H6'] },
            { kw: '养肝',           notes: 3, commentPages: 1, hypotheses: ['H4'] },
            { kw: '肝不好',         notes: 3, commentPages: 1, hypotheses: ['H4'] },
            { kw: '酒后护肝',       notes: 3, commentPages: 1, hypotheses: ['H2'] },
            { kw: '进口保健品礼盒', notes: 3, commentPages: 1, hypotheses: ['H7'] },
            { kw: '保肝片',         notes: 3, commentPages: 1, hypotheses: ['H3'] },
            { kw: '肝损伤',         notes: 3, commentPages: 1, hypotheses: ['H4'] },
        ],
        douyin: [],
        amazon: [],
    },
    4: {
        name: '竞品深度补采',
        xhs: [
            // Swisse 护肝片 —— 需要更多正面/负面体验帖
            { kw: 'swisse护肝片 测评',       notes: 8, commentPages: 3, hypotheses: ['H5'] },
            { kw: 'swisse护肝片 有用吗',     notes: 8, commentPages: 3, hypotheses: ['H5'] },
            { kw: 'swisse奶蓟草 效果',       notes: 6, commentPages: 3, hypotheses: ['H5'] },
            // 葵花护肝片 —— 国民品牌评价
            { kw: '葵花护肝片 效果',          notes: 8, commentPages: 3, hypotheses: ['H3'] },
            { kw: '葵花护肝片 副作用',        notes: 6, commentPages: 3, hypotheses: ['H3'] },
            // 海王金樽 —— 解酒品类一哥
            { kw: '海王金樽 有用吗',          notes: 8, commentPages: 3, hypotheses: ['H2'] },
            { kw: '海王金樽 测评',            notes: 6, commentPages: 3, hypotheses: ['H2'] },
            { kw: '解酒药 有用吗',            notes: 6, commentPages: 3, hypotheses: ['H2'] },
            // 易善复 —— 处方药用户体验
            { kw: '易善复 效果',              notes: 8, commentPages: 3, hypotheses: ['H3'] },
            { kw: '易善复 副作用',            notes: 6, commentPages: 3, hypotheses: ['H3'] },
            { kw: '多烯磷脂酰胆碱 测评',     notes: 5, commentPages: 2, hypotheses: ['H3'] },
            // 片仔癀 —— 高端护肝品牌
            { kw: '片仔癀 护肝 效果',         notes: 8, commentPages: 3, hypotheses: ['H5'] },
            { kw: '片仔癀 值得买吗',          notes: 6, commentPages: 3, hypotheses: ['H5'] },
            // 汤臣倍健 —— 保健品大牌
            { kw: '汤臣倍健护肝片 测评',      notes: 6, commentPages: 2, hypotheses: ['H5'] },
        ],
        douyin: [
            { kw: 'swisse护肝片',     videos: 3, commentPages: 3, hypotheses: ['H5'] },
            { kw: '葵花护肝片',       videos: 3, commentPages: 3, hypotheses: ['H3'] },
            { kw: '海王金樽',         videos: 3, commentPages: 3, hypotheses: ['H2'] },
            { kw: '易善复',           videos: 3, commentPages: 3, hypotheses: ['H3'] },
            { kw: '片仔癀护肝',       videos: 3, commentPages: 3, hypotheses: ['H5'] },
        ],
        amazon: [],
    },
};

// ============================================================
// 通用工具
// ============================================================

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function sanitizeFilename(kw) {
    return kw.replace(/[\/\\:*?"<>|\s]+/g, '-');
}

function auditLog(line) {
    const ts = new Date().toISOString();
    fs.appendFileSync(AUDIT_LOG, `[${ts}] ${line}\n`);
}

function fileExistsNonEmpty(p) {
    if (!fs.existsSync(p)) return false;
    const st = fs.statSync(p);
    return st.size > 50;
}

function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        const chunks = [];
                        response.on('data', (c) => chunks.push(c));
                        response.on('end', () => res({
                            status: response.statusCode,
                            body: Buffer.concat(chunks).toString('utf-8'),
                        }));
                    });
                    req.on('error', rej);
                    req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    const wait = 1500 * Math.pow(1.8, attempt - 1);
                    await sleep(wait);
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await sleep(2000 * attempt);
            }
        }
    });
}

async function tikhubGet(apiPath, paramsObj = {}) {
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers: {
            Accept: 'application/json',
            Authorization: `Bearer ${TIKHUB_TOKEN}`,
        },
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyPostV2(apiPath, bodyObj = {}) {
    const bodyStr = JSON.stringify(bodyObj);
    const headers = {
        Accept: 'application/json',
        'Content-Type': 'application/json',
        'Content-Length': Buffer.byteLength(bodyStr),
    };
    if (VOC_TOKEN) headers.Authorization = `Bearer ${VOC_TOKEN}`;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: apiPath,
        method: 'POST',
        headers,
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function dyGetV3(apiPath, paramsObj = {}) {
    const qs = Object.entries(paramsObj)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const headers = { Accept: 'application/json' };
    if (VOC_TOKEN) headers.Authorization = `Bearer ${VOC_TOKEN}`;
    const { status, body } = await httpRequest({
        hostname: 'server.fmode.cn',
        path: fullPath,
        method: 'GET',
        headers,
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 200) } };
    try { return JSON.parse(body); }
    catch (e) { return { _error: { parse: e.message, body: body.slice(0, 200) } }; }
}

async function sorftimeCall(apiPath, body = {}, domain = 1) {
    const bodyStr = JSON.stringify({ path: apiPath, method: 'POST', body, query: { domain } });
    const { status, body: respBody } = await httpRequest({
        hostname: 'server-msq.fmode.cn',
        path: '/api/sorftime/forward',
        method: 'POST',
        headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(bodyStr) },
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: respBody.slice(0, 200) } };
    try {
        const j = JSON.parse(respBody);
        return j?.Data ?? j?.data ?? j;
    } catch (e) { return { _error: { parse: e.message } }; }
}

// ============================================================
// 小红书采集（单关键词深度）
// ============================================================

async function collectXhsForKeyword(spec, { force = false } = {}) {
    const outPath = path.join(XHS_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [xhs] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`xhs SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [xhs] 🔍 搜索 "${spec.kw}" · notes=${spec.notes} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    // 1) 综合 + 最热 两种排序各抓一页，取并集提高笔记质量
    const searchRes1 = await tikhubGet(
        '/thapi/v1/xiaohongshu/app/search_notes',
        { keyword: spec.kw, page: 1, sort: 'general' },
    );
    await sleep(600);
    const searchRes2 = await tikhubGet(
        '/thapi/v1/xiaohongshu/app/search_notes',
        { keyword: spec.kw, page: 1, sort: 'popularity_descending' },
    );

    const items1 = searchRes1?.data?.data?.items || [];
    const items2 = searchRes2?.data?.data?.items || [];
    const noteMap = new Map();
    for (const it of [...items1, ...items2]) {
        const n = it?.note;
        if (n?.id && !noteMap.has(n.id)) noteMap.set(n.id, n);
    }
    const allNotes = Array.from(noteMap.values());
    console.log(`    [xhs] 去重后 ${allNotes.length} 条笔记 (综合 ${items1.length} + 最热 ${items2.length})`);

    if (allNotes.length === 0) {
        const errOut = { keyword: spec.kw, _error: searchRes1?._error || searchRes2?._error || 'empty', collected_at: new Date().toISOString() };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`xhs FAIL ${spec.kw} empty/err=${JSON.stringify(errOut._error).slice(0, 80)}`);
        return { skipped: false, notes: 0, comments: 0, path: outPath };
    }

    // 2) 按互动数/评论数排序，取 Top K
    const topNotes = [...allNotes]
        .sort((a, b) => {
            const sa = (a.comments_count || 0) * 2 + (a.liked_count || 0);
            const sb = (b.comments_count || 0) * 2 + (b.liked_count || 0);
            return sb - sa;
        })
        .slice(0, spec.notes);

    // 3) 抓每条 Top 笔记的评论（多页）
    const commentsByNoteId = {};
    let totalComments = 0;
    for (const note of topNotes) {
        if (!note?.id) continue;
        const noteId = note.id;
        commentsByNoteId[noteId] = [];
        let cursor = '';
        for (let p = 0; p < spec.commentPages; p++) {
            await sleep(700);
            const cRes = await tikhubGet(
                '/thapi/v1/xiaohongshu/app/get_note_comments',
                { note_id: noteId, cursor },
            );
            const cmts = cRes?.data?.data?.comments || [];
            // 保存精简字段，供后续合并
            for (const c of cmts) {
                commentsByNoteId[noteId].push({
                    id: c.id,
                    content: c.content,
                    create_time: c.create_time,
                    like_count: c.like_count,
                    sub_comment_count: c.sub_comment_count,
                    ip_location: c.ip_location,
                    user: c.user_info ? {
                        user_id: c.user_info.user_id,
                        nickname: c.user_info.nickname,
                    } : null,
                    sub_comments: (c.sub_comments || []).slice(0, 3).map((s) => ({
                        content: s.content,
                        like_count: s.like_count,
                        nickname: s.user_info?.nickname,
                    })),
                });
            }
            totalComments += cmts.length;
            const hasMore = cRes?.data?.data?.has_more;
            cursor = cRes?.data?.data?.cursor || '';
            if (!hasMore || !cursor) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [xhs] ✓ ${spec.kw}: ${topNotes.length} 笔记 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`xhs OK ${spec.kw} notes=${topNotes.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'xiaohongshu',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_notes_found: allNotes.length,
        top_notes: topNotes.map((n) => ({
            id: n.id,
            type: n.type,
            title: n.title,
            desc: n.desc,
            timestamp: n.timestamp,
            liked_count: n.liked_count,
            comments_count: n.comments_count,
            collected_count: n.collected_count,
            shared_count: n.shared_count,
            cover: n.images_list?.[0]?.url,
            user: n.user ? {
                userid: n.user.userid,
                nickname: n.user.nickname,
                red_id: n.user.red_id,
                verified: n.user.red_official_verified,
            } : null,
        })),
        comments_by_note_id: commentsByNoteId,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, notes: topNotes.length, comments: totalComments, path: outPath };
}

// ============================================================
// 抖音采集（单关键词搜索 + Top 视频评论）
// ============================================================

async function collectDouyinForKeyword(spec, { force = false } = {}) {
    const outPath = path.join(DY_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [dy] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`dy SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [dy] 🔍 搜索 "${spec.kw}" · videos=${spec.videos} × commentPages=${spec.commentPages}`);
    const startTs = Date.now();

    // 抖音搜索 v2：POST /api/voc-social/douyin/search/fetch_general_search_v2
    // 响应结构: { code, data: { business_data: [{data: {aweme_info: {...}}}], has_more, cursor } }
    const searchRes = await dyPostV2('/api/voc-social/douyin/search/fetch_general_search_v2', {
        keyword: spec.kw,
        cursor: 0,
        sort_type: '1',         // 最多点赞
        publish_time: '0',      // 不限
        content_type: '1',      // 视频
        filter_duration: '0',
        search_id: '',
        backtrace: '',
    });

    // 从 business_data 中抽出 aweme_info
    const businessData = searchRes?.data?.business_data || [];
    const validVideos = businessData
        .map((wrap) => wrap?.data?.aweme_info || wrap?.aweme_info)
        .filter((v) => v?.aweme_id);
    console.log(`    [dy] 搜索到 ${validVideos.length} 条视频 (business_data wrappers=${businessData.length})`);

    if (validVideos.length === 0) {
        const errOut = {
            platform: 'douyin',
            keyword: spec.kw,
            _search_error: searchRes?._error || searchRes?.mess || 'empty',
            _search_code: searchRes?.code,
            _search_sample: JSON.stringify(searchRes).slice(0, 500),
            collected_at: new Date().toISOString(),
        };
        fs.writeFileSync(outPath, JSON.stringify(errOut, null, 2), 'utf-8');
        auditLog(`dy FAIL ${spec.kw} search_empty code=${searchRes?.code}`);
        return { skipped: false, videos: 0, comments: 0, path: outPath };
    }

    // 按评论数排序取 Top K
    const topVideos = [...validVideos]
        .sort((a, b) => (b.statistics?.comment_count || 0) - (a.statistics?.comment_count || 0))
        .slice(0, spec.videos);

    // 对每条 Top 视频抓评论
    // 响应: { code, data: { comments: [...], cursor, has_more } }
    const commentsByAweme = {};
    let totalComments = 0;
    for (const v of topVideos) {
        const awemeId = v.aweme_id;
        commentsByAweme[awemeId] = [];
        let cursor = 0;
        for (let p = 0; p < spec.commentPages; p++) {
            await sleep(700);
            const cRes = await dyGetV3('/api/voc-social/douyin/app/v3/fetch_video_comments', {
                aweme_id: awemeId,
                cursor,
                count: 20,
            });
            const cmts = cRes?.data?.comments || cRes?.comments || [];
            for (const c of cmts) {
                commentsByAweme[awemeId].push({
                    cid: c.cid,
                    text: c.text,
                    digg_count: c.digg_count,
                    create_time: c.create_time,
                    ip_label: c.ip_label,
                    reply_comment_total: c.reply_comment_total,
                    user: c.user ? {
                        nickname: c.user.nickname,
                        uid: c.user.uid,
                    } : null,
                });
            }
            totalComments += cmts.length;
            const hasMore = (cRes?.data?.has_more ?? cRes?.has_more) === 1;
            cursor = cRes?.data?.cursor ?? cRes?.cursor ?? 0;
            if (!hasMore) break;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [dy] ✓ ${spec.kw}: ${topVideos.length} 视频 / ${totalComments} 评论 (${elapsed}s)`);
    auditLog(`dy OK ${spec.kw} videos=${topVideos.length} comments=${totalComments} (${elapsed}s)`);

    const out = {
        platform: 'douyin',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_videos_found: validVideos.length,
        top_videos: topVideos.map((v) => ({
            aweme_id: v.aweme_id,
            desc: v.desc,
            create_time: v.create_time,
            statistics: v.statistics,
            author: v.author ? {
                nickname: v.author.nickname,
                sec_uid: v.author.sec_uid,
                uid: v.author.uid,
                follower_count: v.author.follower_count,
            } : null,
            cover: v.video?.cover?.url_list?.[0],
            text_extra: (v.text_extra || []).map((t) => t.hashtag_name).filter(Boolean),
        })),
        comments_by_aweme_id: commentsByAweme,
        total_comments: totalComments,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, videos: topVideos.length, comments: totalComments, path: outPath };
}

// ============================================================
// Amazon 采集（Sorftime：多页产品 + Top 评论）
// ============================================================

async function collectAmazonForKeyword(spec, { force = false } = {}) {
    const outPath = path.join(AMZ_DIR, `${sanitizeFilename(spec.kw)}.json`);
    if (!force && fileExistsNonEmpty(outPath)) {
        console.log(`  [amz] ⏭️  ${spec.kw} (已存在, skip)`);
        auditLog(`amz SKIP ${spec.kw} (exists)`);
        return { skipped: true, path: outPath };
    }

    console.log(`  [amz] 🔎 "${spec.kw}" · pages=${spec.pages} × topDetail=${spec.topDetail} × topReviews=${spec.topReviews}`);
    const startTs = Date.now();

    const allProducts = [];
    for (let page = 1; page <= spec.pages; page++) {
        const r = await sorftimeCall('/api/ProductQuery', {
            Page: page, Query: '1', QueryType: '7', Pattern: spec.kw,
        });
        if (r?._error) {
            console.log(`    [amz] p${page} 失败: ${JSON.stringify(r._error).slice(0, 120)}`);
            break;
        }
        const prods = r?.Products || [];
        allProducts.push(...prods);
        console.log(`    [amz] p${page} +${prods.length} · total=${r?.PageCount || '?'}`);
        if (!prods.length) break;
        await sleep(500);
    }

    // 去重（按 ASIN 取 SalesVolume 最大的那条）
    const asinMap = new Map();
    for (const p of allProducts) {
        if (!p.Asin) continue;
        const cur = asinMap.get(p.Asin);
        if (!cur || (p.ListingSalesVolumeOfMonth || 0) > (cur.ListingSalesVolumeOfMonth || 0)) {
            asinMap.set(p.Asin, p);
        }
    }
    const uniqueProducts = Array.from(asinMap.values())
        .sort((a, b) => (b.ListingSalesVolumeOfMonth || 0) - (a.ListingSalesVolumeOfMonth || 0));

    // Top N 抓详情
    const details = {};
    for (const p of uniqueProducts.slice(0, spec.topDetail)) {
        await sleep(400);
        const d = await sorftimeCall('/api/ProductRequest', {
            ASIN: p.Asin, Trend: 1, QueryTrendStartDt: '', QueryTrendEndDt: '',
        });
        if (!d?._error) details[p.Asin] = d;
    }

    // Top M 抓评论 （Sorftime 返回扁数组：[{ConsumerName, Star, Title, Content, Helpful, ReviewsDate, IsVP, Asin, ReviewsLink, ...}]）
    const reviewsByAsin = {};
    let totalReviews = 0;
    for (const p of uniqueProducts.slice(0, spec.topReviews)) {
        await sleep(500);
        const r = await sorftimeCall('/api/ProductReviewsQuery', { ASIN: p.Asin });
        if (!r?._error) {
            // 响应有三种形态: 数组 / {Reviews:[...]} / {reviews:[...]}
            let arr = Array.isArray(r) ? r : (r?.Reviews || r?.reviews || []);
            // Sorftime 用数字 key 的对象（Object.values 可以还原）
            if (!Array.isArray(arr) && typeof r === 'object') {
                const vals = Object.values(r);
                if (vals.length && typeof vals[0] === 'object' && (vals[0].Title || vals[0].Content || vals[0].Star)) {
                    arr = vals;
                }
            }
            reviewsByAsin[p.Asin] = arr;
            totalReviews += arr.length;
        }
    }

    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`    [amz] ✓ ${spec.kw}: ${uniqueProducts.length} 产品 / ${Object.keys(details).length} 详情 / ${totalReviews} 评论 (${elapsed}s)`);
    auditLog(`amz OK ${spec.kw} products=${uniqueProducts.length} reviews=${totalReviews} (${elapsed}s)`);

    const out = {
        platform: 'amazon',
        keyword: spec.kw,
        hypotheses: spec.hypotheses,
        collected_at: new Date().toISOString(),
        elapsed_seconds: Number(elapsed),
        total_products: uniqueProducts.length,
        top_products: uniqueProducts.slice(0, Math.max(spec.topDetail, spec.topReviews)),
        product_details: details,
        reviews_by_asin: reviewsByAsin,
        total_reviews: totalReviews,
    };
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    return { skipped: false, products: uniqueProducts.length, reviews: totalReviews, path: outPath };
}

// ============================================================
// Batch 执行器
// ============================================================

async function runBatch(batchId, opts = {}) {
    const batch = BATCHES[batchId];
    if (!batch) throw new Error(`Unknown batch: ${batchId}`);

    const only = opts.only; // 'xhs' | 'dy' | 'amz' | null
    const runXhs = !only || only === 'xhs';
    const runDy = !only || only === 'dy' || only === 'douyin';
    const runAmz = !only || only === 'amz' || only === 'amazon';

    console.log('\n' + '━'.repeat(60));
    console.log(`🦐 Batch ${batchId}: ${batch.name}` + (only ? ` · only=${only}` : ''));
    console.log(`   XHS=${runXhs ? batch.xhs.length : 'skip'}kw · DY=${runDy ? batch.douyin.length : 'skip'}kw · AMZ=${runAmz ? batch.amazon.length : 'skip'}kw`);
    console.log('━'.repeat(60));
    auditLog(`=== BATCH ${batchId} START ${batch.name}${only ? ' only=' + only : ''} ===`);

    const batchStart = Date.now();
    const stats = { xhs: [], douyin: [], amazon: [] };

    // 平台间并行 (XHS / DY / AMZ)；平台内串行
    await Promise.all([
        (async () => {
            if (!runXhs) return;
            for (const spec of batch.xhs) {
                try {
                    const r = await collectXhsForKeyword(spec, opts);
                    stats.xhs.push({ kw: spec.kw, ...r });
                } catch (e) {
                    console.log(`  [xhs] ❌ ${spec.kw}: ${e.message}`);
                    auditLog(`xhs ERR ${spec.kw} ${e.message}`);
                    stats.xhs.push({ kw: spec.kw, error: e.message });
                }
                await sleep(500);
            }
        })(),
        (async () => {
            if (!runDy) return;
            for (const spec of batch.douyin) {
                try {
                    const r = await collectDouyinForKeyword(spec, opts);
                    stats.douyin.push({ kw: spec.kw, ...r });
                } catch (e) {
                    console.log(`  [dy] ❌ ${spec.kw}: ${e.message}`);
                    auditLog(`dy ERR ${spec.kw} ${e.message}`);
                    stats.douyin.push({ kw: spec.kw, error: e.message });
                }
                await sleep(500);
            }
        })(),
        (async () => {
            if (!runAmz) return;
            for (const spec of batch.amazon) {
                try {
                    const r = await collectAmazonForKeyword(spec, opts);
                    stats.amazon.push({ kw: spec.kw, ...r });
                } catch (e) {
                    console.log(`  [amz] ❌ ${spec.kw}: ${e.message}`);
                    auditLog(`amz ERR ${spec.kw} ${e.message}`);
                    stats.amazon.push({ kw: spec.kw, error: e.message });
                }
                await sleep(500);
            }
        })(),
    ]);

    const elapsed = ((Date.now() - batchStart) / 1000).toFixed(1);
    console.log(`\n🦐 Batch ${batchId} 完成 (${elapsed}s)`);
    const xhsCmts = stats.xhs.reduce((s, x) => s + (x.comments || 0), 0);
    const dyCmts = stats.douyin.reduce((s, x) => s + (x.comments || 0), 0);
    const amzRev = stats.amazon.reduce((s, x) => s + (x.reviews || 0), 0);
    console.log(`   xhs: ${stats.xhs.filter(x => !x.error).length}/${batch.xhs.length} kw · ${xhsCmts} 评论`);
    console.log(`   dy : ${stats.douyin.filter(x => !x.error).length}/${batch.douyin.length} kw · ${dyCmts} 评论`);
    console.log(`   amz: ${stats.amazon.filter(x => !x.error).length}/${batch.amazon.length} kw · ${amzRev} 评论`);
    auditLog(`=== BATCH ${batchId} END xhsCmts=${xhsCmts} dyCmts=${dyCmts} amzRev=${amzRev} (${elapsed}s) ===`);
    return stats;
}

// ============================================================
// 合并
// ============================================================

function mergeAll() {
    console.log('\n📦 合并所有关键词 → _merged.json + comments-flat.jsonl');

    const merged = {
        product: '江中肝纯片',
        collected_at: new Date().toISOString(),
        xhs: {},
        douyin: {},
        amazon: {},
        stats: {},
    };

    const flatPath = path.join(RAW_DIR, 'comments-flat.jsonl');
    const flatStream = fs.createWriteStream(flatPath);

    let xhsCmts = 0, dyCmts = 0, amzCmts = 0;
    let xhsNotes = 0, dyVideos = 0, amzProducts = 0;

    // XHS
    for (const f of fs.readdirSync(XHS_DIR).filter((f) => f.endsWith('.json'))) {
        const d = JSON.parse(fs.readFileSync(path.join(XHS_DIR, f), 'utf-8'));
        merged.xhs[d.keyword] = d;
        xhsNotes += (d.top_notes?.length || 0);
        const cmtMap = d.comments_by_note_id || {};
        for (const [noteId, cmts] of Object.entries(cmtMap)) {
            for (const c of cmts) {
                xhsCmts++;
                flatStream.write(JSON.stringify({
                    platform: 'xhs',
                    keyword: d.keyword,
                    hypotheses: d.hypotheses,
                    note_id: noteId,
                    cid: c.id,
                    content: c.content,
                    like: c.like_count,
                    sub_count: c.sub_comment_count,
                    ip: c.ip_location,
                    user: c.user?.nickname,
                }) + '\n');
            }
        }
    }

    // Douyin
    for (const f of fs.readdirSync(DY_DIR).filter((f) => f.endsWith('.json'))) {
        const d = JSON.parse(fs.readFileSync(path.join(DY_DIR, f), 'utf-8'));
        merged.douyin[d.keyword] = d;
        dyVideos += (d.top_videos?.length || 0);
        const cmtMap = d.comments_by_aweme_id || {};
        for (const [awemeId, cmts] of Object.entries(cmtMap)) {
            for (const c of cmts) {
                dyCmts++;
                flatStream.write(JSON.stringify({
                    platform: 'douyin',
                    keyword: d.keyword,
                    hypotheses: d.hypotheses,
                    aweme_id: awemeId,
                    cid: c.cid,
                    content: c.text,
                    like: c.digg_count,
                    ip: c.ip_label,
                    user: c.user?.nickname,
                }) + '\n');
            }
        }
    }

    // Amazon （Sorftime 返回的扁数组：{ConsumerName, Star, Title, Content, Helpful, ReviewsDate, IsVP, Asin, ReviewsLink, AsinProperty}）
    for (const f of fs.readdirSync(AMZ_DIR).filter((f) => f.endsWith('.json'))) {
        const d = JSON.parse(fs.readFileSync(path.join(AMZ_DIR, f), 'utf-8'));
        merged.amazon[d.keyword] = d;
        amzProducts += (d.top_products?.length || 0);
        for (const [asin, rObj] of Object.entries(d.reviews_by_asin || {})) {
            // 支持 3 种格式：直接数组 / {Reviews:[]} / {'0':{...},'1':{...}}
            let reviews = [];
            if (Array.isArray(rObj)) {
                reviews = rObj;
            } else if (rObj?.Reviews || rObj?.reviews) {
                reviews = rObj.Reviews || rObj.reviews;
            } else if (typeof rObj === 'object') {
                const vals = Object.values(rObj);
                if (vals.length && typeof vals[0] === 'object' && (vals[0].Title || vals[0].Content || vals[0].Star)) {
                    reviews = vals;
                }
            }
            for (const r of reviews) {
                amzCmts++;
                flatStream.write(JSON.stringify({
                    platform: 'amazon',
                    keyword: d.keyword,
                    hypotheses: d.hypotheses,
                    asin: r.Asin || asin,
                    parent_asin: asin,
                    title: r.Title,
                    content: r.Content,
                    rating: r.Star,
                    verified: r.IsVP,
                    helpful: r.Helpful,
                    date: r.ReviewsDate,
                    reviewer: r.ConsumerName,
                    review_link: r.ReviewsLink,
                    variant: r.AsinProperty,
                }) + '\n');
            }
        }
    }

    flatStream.end();

    merged.stats = {
        xhs: { keywords: Object.keys(merged.xhs).length, notes: xhsNotes, comments: xhsCmts },
        douyin: { keywords: Object.keys(merged.douyin).length, videos: dyVideos, comments: dyCmts },
        amazon: { keywords: Object.keys(merged.amazon).length, products: amzProducts, reviews: amzCmts },
        total_comments: xhsCmts + dyCmts + amzCmts,
    };

    const mergedPath = path.join(RAW_DIR, '_merged.json');
    fs.writeFileSync(mergedPath, JSON.stringify(merged, null, 2), 'utf-8');

    console.log(`\n✓ 合并完成:`);
    console.log(`   XHS    : ${merged.stats.xhs.keywords} kw · ${xhsNotes} 笔记 · ${xhsCmts} 评论`);
    console.log(`   Douyin : ${merged.stats.douyin.keywords} kw · ${dyVideos} 视频 · ${dyCmts} 评论`);
    console.log(`   Amazon : ${merged.stats.amazon.keywords} kw · ${amzProducts} 产品 · ${amzCmts} 评论`);
    console.log(`   TOTAL  : ${merged.stats.total_comments} 条评论\n`);
    console.log(`   → ${mergedPath}`);
    console.log(`   → ${flatPath}`);

    auditLog(`=== MERGE xhs=${xhsCmts} dy=${dyCmts} amz=${amzCmts} total=${merged.stats.total_comments} ===`);
    return merged;
}

// ============================================================
// CLI
// ============================================================

function parseArgs() {
    const args = { batch: 'all', force: false, merge: false, only: null };
    for (const a of process.argv.slice(2)) {
        if (a.startsWith('--batch=')) args.batch = a.split('=')[1];
        else if (a === '--force') args.force = true;
        else if (a === '--merge') args.merge = true;
        else if (a.startsWith('--only=')) args.only = a.split('=')[1];
    }
    return args;
}

async function main() {
    const args = parseArgs();
    console.log('╔══════════════════════════════════════════════════════════╗');
    console.log('║   江中肝纯片 VOC 深度评论采集 (多平台×42 关键词)           ║');
    console.log('╚══════════════════════════════════════════════════════════╝');
    console.log(`  📅 ${new Date().toLocaleString()}`);
    console.log(`  📁 输出: ${RAW_DIR}`);
    console.log(`  🔑 TikHub: ${TIKHUB_TOKEN.slice(0, 8)}...`);
    console.log(`  🍪 抖音 cookie: ${DY_COOKIE ? `已加载 (${DY_COOKIE.length} chars)` : '未配置'}`);
    console.log(`  ⚙️  batch=${args.batch} force=${args.force} merge=${args.merge}`);

    if (args.merge) {
        mergeAll();
        return;
    }

    const targets = args.batch === 'all' ? ['1', '2', '3', '4'] : [args.batch];
    for (const b of targets) {
        await runBatch(b, { force: args.force, only: args.only });
    }

    // 默认在 batch=all 或指定时自动合并一次
    if (args.batch === 'all') {
        mergeAll();
    } else {
        console.log(`\n💡 提示: 跑完所有批次后执行 \`node scripts/tools/collect-jiangzhong-liver-deep.js --merge\` 合并`);
    }
}

if (require.main === module) {
    main().catch((e) => {
        console.error('❌ 致命错误:', e);
        auditLog(`FATAL ${e.message} ${e.stack?.slice(0, 300)}`);
        process.exit(1);
    });
}

module.exports = { runBatch, mergeAll, BATCHES };