fmode-voc-projects/scripts/tools/jiangzhong-voc-collect.js

#!/usr/bin/env node
/**
 * 江中三产品 VOC 数据采集脚本 · v2（增强版）
 *
 * 目标：为「江中肝纯片 / 儿童乳酸菌素片 / 江中猴菇饮」并行采集：
 *   - 小红书：3 关键词搜索 + Top 笔记评论 + 作者画像
 *   - 抖音：官方分享链 → aweme + 评论 + 作者主页（如存在 cookie 再加关键词搜索）
 *   - Amazon：4 英文关键词 × 3 页 Sorftime + 产品详情 + 评论
 *
 * 输出：./data/jiangzhong-{产品}-voc.json
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const { collectAmazonForProduct } = require('./collect-amazon');
const { collectDouyinForProduct, loadCookie, COOKIE_PATH } = require('./collect-douyin');

// ============================================================
// 配置
// ============================================================

const XHS_SKILL_CONFIG = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
const xhsConf = JSON.parse(fs.readFileSync(XHS_SKILL_CONFIG, 'utf-8'));
const TIKHUB_TOKEN = xhsConf.endpoint.headers.Authorization.replace(/^Bearer\s+/, '');

const DATA_DIR = path.resolve(__dirname, '..', '..', 'data');
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });

// ============================================================
// 三个产品配置
// ============================================================
// ⚠ douyin_share_urls 按「官方号 + 品牌竞品号 + 垂类博主号」三卡顺序填写
//   收到用户 6 条分享链后，在对应位置取消注释并填入
const PRODUCTS = [
    {
        id: 'liver',
        name: '江中肝纯片',
        category: 'OTC 保肝片剂',
        keywords_cn: ['肝纯片', '江中肝纯片', '护肝片'],
        keywords_en: ['milk thistle', 'liver support supplement', 'liver cleanse detox', 'silymarin'],
        douyin_share_url: 'https://v.douyin.com/BMklXPTyiUM/', // 向下兼容
        douyin_share_urls: [
            { share_url: 'https://v.douyin.com/BMklXPTyiUM/', role: '官方号', label: '江中（现绑）' },
            // { share_url: '___待填___', role: '品牌竞品', label: '葵花护肝片 / 葵花药业' },
            // { share_url: '___待填___', role: '垂类博主', label: '肝病科医生 / 护肝博主' },
        ],
        sorftime_pages: 3, sorftime_top_detail: 6, sorftime_top_reviews: 3,
        outfile: 'jiangzhong-liver-voc.json',
    },
    {
        id: 'probiotic',
        name: '儿童乳酸菌素片',
        category: '儿童功能食品 / 肠道调理',
        keywords_cn: ['乳酸菌素片', '儿童乳酸菌素片', '儿童肠胃调理'],
        keywords_en: ['kids probiotics', 'children probiotics gummy', 'probiotics gummies for kids', 'childrens digestive supplement'],
        douyin_share_url: 'https://v.douyin.com/1eV76OP0FNM/',
        douyin_share_urls: [
            { share_url: 'https://v.douyin.com/1eV76OP0FNM/', role: '官方号', label: '江中食疗（现绑）' },
            // { share_url: '___待填___', role: '品牌竞品', label: '妈咪爱 / 亿活 / 合生元' },
            // { share_url: '___待填___', role: '垂类博主', label: '儿科医生 / 育儿达人' },
        ],
        sorftime_pages: 3, sorftime_top_detail: 6, sorftime_top_reviews: 3,
        outfile: 'jiangzhong-probiotic-voc.json',
    },
    {
        id: 'monkey',
        name: '江中猴菇饮',
        category: '胃肠道养胃饮品',
        keywords_cn: ['猴菇饮', '江中猴菇', '养胃饮'],
        keywords_en: ['lions mane mushroom supplement', 'lion mane mushroom', 'stomach gut health supplement', 'lions mane gummies'],
        douyin_share_url: 'https://v.douyin.com/C3mMbhRYKHw/',
        douyin_share_urls: [
            { share_url: 'https://v.douyin.com/C3mMbhRYKHw/', role: '官方号', label: '江中制药（现绑）' },
            // { share_url: '___待填___', role: '品牌竞品', label: '康恩贝 / 太阳神 / 三九' },
            // { share_url: '___待填___', role: '垂类博主', label: '胃肠科医生 / 养胃博主' },
        ],
        sorftime_pages: 3, sorftime_top_detail: 6, sorftime_top_reviews: 3,
        outfile: 'jiangzhong-monkey-voc.json',
    },
];

// ============================================================
// HTTP 请求助手（带重试）
// ============================================================
function httpRequest(options, bodyData = null, maxAttempts = 3) {
    return new Promise(async (resolve, reject) => {
        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
            try {
                const result = await new Promise((res, rej) => {
                    const req = https.request(options, (response) => {
                        let data = [];
                        response.on('data', (chunk) => data.push(chunk));
                        response.on('end', () => {
                            const body = Buffer.concat(data).toString('utf-8');
                            res({ status: response.statusCode, body });
                        });
                    });
                    req.on('error', rej);
                    req.on('timeout', () => {
                        req.destroy();
                        rej(new Error('timeout'));
                    });
                    if (bodyData) req.write(bodyData);
                    req.end();
                });
                if (result.status === 200) return resolve(result);
                if ([400, 429, 500, 502, 503, 504].includes(result.status) && attempt < maxAttempts) {
                    const wait = 1500 * Math.pow(1.8, attempt - 1);
                    console.log(`    ↻ ${options.path.slice(0, 60)} → ${result.status} retry#${attempt} in ${wait}ms`);
                    await new Promise((r) => setTimeout(r, wait));
                    continue;
                }
                return resolve(result);
            } catch (e) {
                if (attempt === maxAttempts) return reject(e);
                await new Promise((r) => setTimeout(r, 2000 * attempt));
            }
        }
    });
}

async function getJson(host, path, auth) {
    const headers = { Accept: 'application/json' };
    if (auth) headers.Authorization = `Bearer ${auth}`;
    const { status, body } = await httpRequest({
        hostname: host,
        path,
        method: 'GET',
        headers,
        timeout: 60000,
    });
    if (status !== 200) return { _error: { status, body: body.slice(0, 300) } };
    try {
        return JSON.parse(body);
    } catch (e) {
        return { _error: { parse: e.message, body: body.slice(0, 300) } };
    }
}

async function postJson(host, path, bodyObj, auth, extraHeaders = {}) {
    const bodyStr = JSON.stringify(bodyObj);
    const headers = {
        Accept: 'application/json',
        'Content-Type': 'application/json',
        'Content-Length': Buffer.byteLength(bodyStr),
        ...extraHeaders,
    };
    if (auth) headers.Authorization = `Bearer ${auth}`;
    const { status, body } = await httpRequest({
        hostname: host,
        path,
        method: 'POST',
        headers,
        timeout: 60000,
    }, bodyStr);
    if (status !== 200) return { _error: { status, body: body.slice(0, 300) } };
    try {
        return JSON.parse(body);
    } catch (e) {
        return { _error: { parse: e.message, body: body.slice(0, 300) } };
    }
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ============================================================
// 小红书采集器
// ============================================================
async function collectXhsForKeyword(kw) {
    console.log(`  [xhs] 🔍 搜索 "${kw}"`);
    const searchRes = await getJson('server.fmode.cn',
        `/thapi/v1/xiaohongshu/app/search_notes?keyword=${encodeURIComponent(kw)}&page=1`,
        TIKHUB_TOKEN);
    const items = searchRes?.data?.data?.items || [];
    const notes = items.map((w) => w.note).filter((n) => n?.id);
    console.log(`  [xhs] ✓ ${notes.length} 条笔记`);

    // 取前 5 条最高评论数的笔记去抓评论
    const topNotes = notes
        .filter((n) => n.comments_count > 5)
        .sort((a, b) => (b.comments_count || 0) - (a.comments_count || 0))
        .slice(0, 5);

    const comments = {};
    const userIds = new Set();
    for (const note of topNotes) {
        if (!note?.id) continue;
        await sleep(800);
        const preview = String(note.title || note.desc || note.id).slice(0, 30);
        console.log(`    [xhs] 💬 评论 "${preview}" (${note.comments_count})`);
        try {
            const cRes = await getJson('server.fmode.cn',
                `/thapi/v1/xiaohongshu/app/get_note_comments?note_id=${note.id}&cursor=`,
                TIKHUB_TOKEN);
            comments[note.id] = cRes?.data?.data?.comments || [];
        } catch (e) {
            console.log(`      ↻ ${note.id} ${e.message}`);
            comments[note.id] = [];
        }
        if (note.user?.userid) userIds.add(note.user.userid);
    }
    // 收集其余笔记的作者 ID
    notes.forEach((n) => n?.user?.userid && userIds.add(n.user.userid));

    // 抓 top 3 笔记作者信息
    const users = {};
    const topUserIds = Array.from(userIds).slice(0, 5);
    for (const uid of topUserIds) {
        await sleep(800);
        console.log(`    [xhs] 👤 用户 ${uid}`);
        const uRes = await getJson('server.fmode.cn',
            `/thapi/v1/xiaohongshu/app/get_user_info?user_id=${uid}`,
            TIKHUB_TOKEN);
        users[uid] = uRes?.data?.data || { _error: uRes?._error };
    }

    return { notes, comments, users };
}

// ============================================================
// 单产品采集编排（使用模块化 collect-amazon + collect-douyin）
// ============================================================
async function collectOneProduct(product, cookie) {
    console.log('');
    console.log(`🦐 ===== ${product.name} (${product.category}) =====`);
    const startTs = Date.now();

    const out = {
        product: product.name,
        product_id: product.id,
        category: product.category,
        keywords: { cn: product.keywords_cn, en: product.keywords_en },
        douyin_share_url: product.douyin_share_url,
        xiaohongshu: { by_keyword: {} },
        douyin: null,
        amazon: null,
        collected_at: new Date().toISOString(),
    };

    // —— 小红书：3 个 CN 关键词 ——
    for (const kw of product.keywords_cn.slice(0, 3)) {
        try {
            out.xiaohongshu.by_keyword[kw] = await collectXhsForKeyword(kw);
        } catch (e) {
            console.log(`    [xhs:${kw}] ❌ ${e.message}`);
            out.xiaohongshu.by_keyword[kw] = { _error: e.message };
        }
        await sleep(1000);
    }

    // —— Amazon：调模块 ——
    try {
        out.amazon = await collectAmazonForProduct(product);
    } catch (e) {
        console.log(`  [amz] ❌ ${e.message}`);
        out.amazon = { _error: e.message };
    }

    // —— 抖音：调模块（按 share URL + 可选 cookie） ——
    try {
        out.douyin = await collectDouyinForProduct(product, cookie);
    } catch (e) {
        console.log(`  [dy] ❌ ${e.message}`);
        out.douyin = { _error: e.message };
    }

    // 保存
    const outPath = path.join(DATA_DIR, product.outfile);
    fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf-8');
    const elapsed = ((Date.now() - startTs) / 1000).toFixed(1);
    console.log(`🦐 ===== ${product.name} ✅ 完成 (${elapsed}s) → ${outPath} =====`);

    return out;
}

// ============================================================
// Main
// ============================================================
async function main() {
    console.log('');
    console.log('╔══════════════════════════════════════════════════════════╗');
    console.log('║   江中三产品 VOC 数据采集 v2 (XHS + Amazon + Douyin)       ║');
    console.log('╚══════════════════════════════════════════════════════════╝');
    console.log(`  📅 ${new Date().toLocaleString()}`);
    console.log(`  📁 输出目录: ${DATA_DIR}`);
    console.log(`  🔑 TikHub token: ${TIKHUB_TOKEN.slice(0, 8)}...`);

    // 加载 Douyin cookie (可选)
    const cookie = loadCookie();
    if (cookie) {
        console.log(`  🍪 Douyin cookie: 已加载 (${cookie.length} chars)`);
    } else {
        console.log(`  🍪 Douyin cookie: 未提供 (预计分享链解析会失败，可后续在 ${COOKIE_PATH} 提供)`);
    }

    const totalStart = Date.now();

    // CLI: 允许传单个产品 id 来只跑某个产品
    const arg = process.argv[2];
    const targets = arg ? PRODUCTS.filter((p) => p.id === arg) : PRODUCTS;
    if (arg && !targets.length) {
        console.error(`❌ unknown product: ${arg}, available: ${PRODUCTS.map((p) => p.id).join(',')}`);
        process.exit(1);
    }
    console.log(`  🎯 目标产品: ${targets.map((p) => p.id).join(', ')}`);

    // 产品间并发启动（避免总时长过长），产品内仍是 sequential
    const results = await Promise.all(targets.map((p) => collectOneProduct(p, cookie)));

    const totalElapsed = ((Date.now() - totalStart) / 1000).toFixed(1);
    console.log('');
    console.log('╔══════════════════════════════════════════════════════════╗');
    console.log(`║   ✅ 全部完成 (${totalElapsed}s)`);
    console.log('╚══════════════════════════════════════════════════════════╝');

    for (const r of results) {
        const xhsKws = Object.keys(r.xiaohongshu.by_keyword);
        const xhsNotes = xhsKws.reduce((s, k) => s + (r.xiaohongshu.by_keyword[k]?.notes?.length || 0), 0);
        const xhsCmts = xhsKws.reduce((s, k) => s + Object.values(r.xiaohongshu.by_keyword[k]?.comments || {}).flat().length, 0);
        const amzKws = Object.keys(r.amazon?.by_keyword || {});
        const amzProducts = r.amazon?.merged_products?.length || 0;
        const amzDetails = Object.keys(r.amazon?.product_details || {}).length;
        const amzReviews = Object.values(r.amazon?.reviews_by_asin || {})
            .reduce((s, v) => s + ((v?.Reviews?.length || v?.reviews?.length) || 0), 0);
        const dyComments = r.douyin?.comments?.length || 0;
        const dyUserPosts = r.douyin?.user_posts?.length || 0;
        const dySearches = Object.values(r.douyin?.searches || {}).reduce((s, v) => s + (Array.isArray(v) ? v.length : 0), 0);
        console.log(`  📊 ${r.product}:`);
        console.log(`    xhs    : ${xhsNotes} 笔记 / ${xhsCmts} 评论 / ${xhsKws.length} 关键词`);
        console.log(`    amazon : ${amzProducts} 产品 (${amzKws.length} kw × 3页) · ${amzDetails} 详情 · ${amzReviews} 评论`);
        console.log(`    douyin : ${r.douyin?.video ? '✓视频' : '✗视频'} / ${dyComments} 评论 / ${dyUserPosts} 作者作品 / ${dySearches} 搜索视频`);
    }
}

main().catch((err) => {
    console.error('❌ 致命错误:', err);
    process.exit(1);
});