fmode-voc-projects/docs/_methodology/voc-deep-insight-workflow/templates/collect.template.js

// ==============================================================================
// <品类名> · 多平台 VOC 真实采集脚本（模板）
// ==============================================================================
// 用法：
//   node scripts/tools/<品类>-collect.js --test        # 测试单 kw
//   node scripts/tools/<品类>-collect.js --batch=1     # P0 批次
//   node scripts/tools/<品类>-collect.js --batch=2     # P1 批次
//   node scripts/tools/<品类>-collect.js --batch=3     # P2 批次
//   node scripts/tools/<品类>-collect.js --batch=all   # 全部
//   node scripts/tools/<品类>-collect.js --merge       # 合并 → _merged.json
//   node scripts/tools/<品类>-collect.js --force       # 强制重抓（忽略缓存）
// ==============================================================================

const fs = require('fs');
const path = require('path');
const https = require('https');
const os = require('os');
const crypto = require('crypto');

// ==========================================================
// 1. 常量 + 路径
// ==========================================================
const CATEGORY = '<品类>';                              // TODO: 改为实际品类目录名
const PROJECT_TAG = '<project-tag>';                    // TODO: 简短英文标签，用于日志

const ROOT = path.resolve(__dirname, '..', '..');
const RAW = path.join(ROOT, 'docs', CATEGORY, 'raw');
const XHS_DIR = path.join(RAW, 'xhs');
const DY_DIR = path.join(RAW, 'douyin');
const MERGED = path.join(RAW, '_merged.json');
const AUDIT = path.join(RAW, 'audit.log');

// 确保目录存在
for (const dir of [RAW, XHS_DIR, DY_DIR]) {
    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}

// ==========================================================
// 2. API 凭据加载
// ==========================================================
function loadTikHubKey() {
    const fp = path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json');
    if (!fs.existsSync(fp)) throw new Error(`TikHub 凭据缺失: ${fp}`);
    return JSON.parse(fs.readFileSync(fp, 'utf8')).currentToken;
}

function loadVocToken() {
    const fp = path.join(os.homedir(), '.openclaw', 'voc-credentials.json');
    if (!fs.existsSync(fp)) throw new Error(`VOC 凭据缺失: ${fp}`);
    return JSON.parse(fs.readFileSync(fp, 'utf8')).sessionToken;
}

// ==========================================================
// 3. 批次定义（BATCHES）
// TODO: 按 `3.采集矩阵.md` 里的关键词矩阵填充
// ==========================================================
const BATCHES = {
    1: {
        // P0 · 本品 + 头部竞品
        xhs: [
            { keyword: '<本品名>', max_notes: 15, max_comments_per: 10, product: '<本品>' },
            { keyword: '<竞品A>', max_notes: 12, max_comments_per: 8, product: '<竞品A>' },
            // ... 10-15 个 kw
        ],
        douyin: [
            { keyword: '<本品名>', max_items: 25, product: '<本品>' },
            // ... 3-6 个 kw
        ],
    },
    2: {
        // P1 · 场景 + 长尾
        xhs: [/* ... */],
        douyin: [/* ... */],
    },
    3: {
        // P2 · 决策 + 相邻赛道
        xhs: [/* ... */],
        douyin: [/* ... */],
    },
};

// ==========================================================
// 4. HTTP 工具
// ==========================================================
function httpRequest({ method = 'GET', host, path: urlPath, headers = {}, body = null }) {
    return new Promise((resolve, reject) => {
        const opts = { method, hostname: host, path: urlPath, headers };
        const req = https.request(opts, (res) => {
            let data = '';
            res.on('data', (c) => (data += c));
            res.on('end', () => {
                if (res.statusCode >= 200 && res.statusCode < 300) resolve({ status: res.statusCode, body: data });
                else reject(new Error(`HTTP ${res.statusCode}: ${data.slice(0, 200)}`));
            });
        });
        req.on('error', reject);
        if (body) req.write(typeof body === 'string' ? body : JSON.stringify(body));
        req.end();
    });
}

async function retry(fn, times = 2, baseDelayMs = 1000) {
    for (let i = 0; i <= times; i++) {
        try { return await fn(); }
        catch (e) {
            if (i === times) throw e;
            await sleep(baseDelayMs * Math.pow(3, i));
        }
    }
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ==========================================================
// 5. TikHub · 小红书接口
// ==========================================================
async function tikhubSearchNotes(keyword, total = 10) {
    const token = loadTikHubKey();
    const res = await httpRequest({
        method: 'POST',
        host: 'api.tikhub.io',
        path: '/api/v1/xiaohongshu/web_v1/search_notes',
        headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
        body: { keyword, sort_type: 'general', note_type: 'normal', total_number: total },
    });
    const parsed = JSON.parse(res.body);
    return (parsed.data?.items || parsed.data?.notes || []).slice(0, total);
}

async function tikhubNoteComments(noteId, max = 10) {
    const token = loadTikHubKey();
    const res = await httpRequest({
        method: 'POST',
        host: 'api.tikhub.io',
        path: '/api/v1/xiaohongshu/web_v1/note_comments',
        headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
        body: { note_id: noteId, cursor: '' },
    });
    const parsed = JSON.parse(res.body);
    return (parsed.data?.comments || []).slice(0, max);
}

// ==========================================================
// 6. 小红书采集器
// ==========================================================
async function collectXhs(task) {
    const fp = path.join(XHS_DIR, `${safeName(task.keyword)}.json`);
    if (fs.existsSync(fp) && !FORCE) {
        logAudit(`SKIP xhs/${task.keyword} → cached`);
        return;
    }
    const start = Date.now();
    try {
        const notes = await retry(() => tikhubSearchNotes(task.keyword, task.max_notes));
        await sleep(250);
        const items = [];
        for (const note of notes) {
            try {
                const comments = await retry(() => tikhubNoteComments(note.id || note.note_id, task.max_comments_per));
                for (const c of comments) {
                    items.push(buildItem({
                        platform: 'xhs',
                        product: task.product,
                        keyword: task.keyword,
                        type: 'comment',
                        nickname: c.user_info?.nickname || c.user_nickname || '匿名',
                        ip: c.ip_location,
                        content: c.content,
                        likes: c.like_count,
                        note_id: note.id || note.note_id,
                        note_title: note.display_title || note.title,
                        note_author: note.user?.nickname,
                    }));
                }
                await sleep(200);
            } catch (e) {
                logAudit(`ERR xhs/${task.keyword}/note ${note.id}: ${e.message.slice(0, 80)}`);
            }
        }
        fs.writeFileSync(fp, JSON.stringify(items, null, 2), 'utf8');
        logAudit(`OK xhs/${task.keyword} → ${items.length} (${Date.now() - start}ms)`);
    } catch (e) {
        logAudit(`ERR xhs/${task.keyword}: ${e.message.slice(0, 80)}`);
    }
}

// ==========================================================
// 7. 抖音采集器（通过 VOC Skill）
// TODO: 根据实际 VOC Skill API 调整
// ==========================================================
async function collectDouyin(task) {
    const fp = path.join(DY_DIR, `${safeName(task.keyword)}.json`);
    if (fs.existsSync(fp) && !FORCE) {
        logAudit(`SKIP douyin/${task.keyword} → cached`);
        return;
    }
    const start = Date.now();
    try {
        // 具体 API 因项目而定，此处伪代码
        const videos = await retry(() => vocApiHotVideos(task.keyword, task.max_items));
        await sleep(250);
        const items = [];
        for (const video of videos) {
            try {
                const comments = await retry(() => vocApiVideoComments(video.id, 10));
                for (const c of comments) {
                    items.push(buildItem({
                        platform: 'douyin',
                        product: task.product,
                        keyword: task.keyword,
                        type: 'comment',
                        nickname: c.nickname || '匿名',
                        ip: c.ip_label,
                        content: c.text,
                        likes: c.digg_count,
                        video_id: video.id,
                        video_title: video.desc,
                        video_author: video.author?.nickname,
                    }));
                }
                await sleep(200);
            } catch (e) {
                logAudit(`ERR douyin/${task.keyword}/video ${video.id}: ${e.message.slice(0, 80)}`);
            }
        }
        fs.writeFileSync(fp, JSON.stringify(items, null, 2), 'utf8');
        logAudit(`OK douyin/${task.keyword} → ${items.length} (${Date.now() - start}ms)`);
    } catch (e) {
        logAudit(`ERR douyin/${task.keyword}: ${e.message.slice(0, 80)}`);
    }
}

// VOC Skill API stubs - 根据实际 skill 替换
async function vocApiHotVideos(keyword, max) { /* TODO */ return []; }
async function vocApiVideoComments(videoId, max) { /* TODO */ return []; }

// ==========================================================
// 8. 统一 Item 构造
// ==========================================================
function buildItem({ platform, product, keyword, type, nickname, ip, content, likes, ...rest }) {
    const idSource = `${platform}:${nickname}:${(content || '').slice(0, 60)}`;
    const id = crypto.createHash('md5').update(idSource).digest('hex').slice(0, 16);
    return {
        id,
        platform,
        product,
        keyword,
        type,
        nickname: nickname || '匿名',
        ip: ip || '',
        content: String(content || '').trim(),
        likes: Number(likes) || 0,
        tags: inferTags(content),
        hypotheses: inferHypotheses(content, keyword),
        sentiment: inferSentiment(content),
        collected_at: Date.now(),
        ...rest,
    };
}

// ==========================================================
// 9. 标签推断（按品类定制）
// TODO: 修改 regex 以匹配品类特征词
// ==========================================================
function inferTags(content) {
    const c = String(content || '');
    const tags = [];
    if (/有效|管用|好了|改善|通了/.test(c)) tags.push('有效');
    if (/没用|假的|没效果|坑|避雷/.test(c)) tags.push('无效');
    if (/推荐|安利|必买/.test(c)) tags.push('推荐');
    if (/贵|便宜|性价比|价格/.test(c)) tags.push('价格');
    if (/孩子|宝宝|儿童|娃/.test(c)) tags.push('儿童');
    // TODO: 按品类加其他标签
    return tags;
}

function inferHypotheses(content, keyword) {
    const c = String(content || '');
    const tags = [];
    // TODO: 按 H1-H8 各自的判断逻辑
    if (/<H1 regex>/.test(c)) tags.push('H1');
    if (/<H2 regex>/.test(c)) tags.push('H2');
    if (/<H3 regex>/.test(c)) tags.push('H3');
    if (/<H4 regex>/.test(c)) tags.push('H4');
    if (/<H5 regex>/.test(c)) tags.push('H5');
    if (/<H6 regex>/.test(c)) tags.push('H6');
    if (/<H7 regex>/.test(c)) tags.push('H7');
    if (/<H8 regex>/.test(c)) tags.push('H8');
    return tags;
}

function inferSentiment(content) {
    const c = String(content || '');
    if (/没用|假的|避雷|坑|差评|不行|退货/.test(c)) return 'negative';
    if (/有效|好用|推荐|爱|治好|管用|回购/.test(c)) return 'positive';
    return 'neutral';
}

// ==========================================================
// 10. 合并函数
// ==========================================================
function mergeAll() {
    console.log('\n▶ Merging all collected raw data...');
    const allItems = [];
    scanDir(XHS_DIR, 'xhs', allItems);
    scanDir(DY_DIR, 'douyin', allItems);

    // 去重（同 id 合并，保留 likes 较高的版本）
    const dedup = new Map();
    for (const it of allItems) {
        const ex = dedup.get(it.id);
        if (!ex || (it.likes || 0) > (ex.likes || 0)) dedup.set(it.id, it);
    }
    const items = [...dedup.values()];

    // 统计
    const platforms = {};
    const products = new Set();
    const hypTally = { H1: 0, H2: 0, H3: 0, H4: 0, H5: 0, H6: 0, H7: 0, H8: 0 };
    for (const it of items) {
        platforms[it.platform] = (platforms[it.platform] || 0) + 1;
        products.add(it.product);
        for (const h of it.hypotheses || []) if (hypTally[h] !== undefined) hypTally[h]++;
    }

    const merged = {
        meta: {
            count: items.length,
            platforms,
            products: [...products],
            hypotheses: hypTally,
            collected_at: Date.now(),
            category: CATEGORY,
        },
        items,
    };
    fs.writeFileSync(MERGED, JSON.stringify(merged, null, 2), 'utf8');
    fs.writeFileSync(
        path.join(RAW, 'comments-flat.jsonl'),
        items.map((x) => JSON.stringify(x)).join('\n'),
        'utf8'
    );
    console.log(`  ✅ merged: ${items.length} items | ${products.size} products | ${Object.keys(platforms).length} platforms`);
    console.log(`     假设覆盖: ${Object.entries(hypTally).map(([k, v]) => `${k}:${v}`).join(' / ')}`);
}

function scanDir(dir, platform, arr) {
    if (!fs.existsSync(dir)) return;
    for (const f of fs.readdirSync(dir)) {
        if (!f.endsWith('.json')) continue;
        try {
            const data = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
            if (Array.isArray(data)) arr.push(...data);
        } catch (e) {
            console.warn(`  ⚠️ 无法解析 ${platform}/${f}: ${e.message}`);
        }
    }
}

// ==========================================================
// 11. 工具函数
// ==========================================================
function safeName(s) {
    return String(s).replace(/[\\/:*?"<>|\s]/g, '_').slice(0, 100);
}

function logAudit(line) {
    const ts = new Date().toISOString();
    const full = `[${ts}] ${line}\n`;
    fs.appendFileSync(AUDIT, full, 'utf8');
    console.log('  ' + line);
}

// ==========================================================
// 12. 主入口 + CLI
// ==========================================================
let FORCE = false;

async function runBatch(n) {
    const b = BATCHES[n];
    if (!b) { console.error(`❌ Batch ${n} 未定义`); return; }
    console.log(`\n▶ Running Batch ${n}...`);
    for (const task of (b.xhs || [])) {
        await collectXhs(task);
        await sleep(300);
    }
    for (const task of (b.douyin || [])) {
        await collectDouyin(task);
        await sleep(300);
    }
}

async function testSingle() {
    const first = BATCHES[1].xhs[0];
    if (first) { console.log('Test mode → 采 1 个 XHS kw'); await collectXhs(first); }
    mergeAll();
}

async function main() {
    const args = process.argv.slice(2);
    FORCE = args.includes('--force');

    console.log('╔══════════════════════════════════════════════╗');
    console.log(`║  ${CATEGORY} · VOC 多平台真实采集 [${PROJECT_TAG}]`.padEnd(48) + '║');
    console.log('╚══════════════════════════════════════════════╝');

    try { console.log('  TikHub token: ✓', loadTikHubKey().slice(0, 8) + '...'); }
    catch { console.log('  TikHub token: ✗（XHS 采集不可用）'); }
    try { console.log('  VOC token:    ✓', loadVocToken().slice(0, 8) + '...'); }
    catch { console.log('  VOC token:    ✗（抖音采集不可用）'); }

    if (args.includes('--test')) return testSingle();
    if (args.includes('--merge')) return mergeAll();

    const batchArg = args.find((a) => a.startsWith('--batch='))?.split('=')[1];
    if (batchArg === 'all') {
        for (const n of Object.keys(BATCHES)) await runBatch(Number(n));
    } else if (batchArg) {
        await runBatch(Number(batchArg));
    } else {
        console.log('\n用法: --test | --batch=1|2|3|all | --merge | --force');
        return;
    }

    mergeAll();
}

main().catch((e) => { console.error('❌ FATAL:', e); process.exit(1); });