fmode-voc-projects/scripts/tools/collect-tiktok.js

#!/usr/bin/env node
/**
 * TikTok 采集模块（TikHub）
 *
 * TikTok 是 Amazon 的内容端对标 —— 用英文关键词驱动：
 *   - milk thistle / liver support        → 肝纯片
 *   - kids probiotics                     → 儿童乳酸菌
 *   - lions mane mushroom                 → 猴菇饮
 *
 * 流程（每产品）：
 *   1) 对每个 keyword_en 走 fetch_general_search_result，累计视频
 *   2) 去重后取 Top 8 高赞视频，每个抓前 20 条评论
 *   3) 合并所有视频的 author，按 follower_count 取 Top 5 作者，抓 profile + Top 10 作品
 *   4) 合并用户搜索 (/web/fetch_search_user) 拿到品类头部账号
 *
 * 不需要 cookie（TikTok 海外版）
 */
const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const TIKHUB_TOKEN = JSON.parse(
    fs.readFileSync(
        path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json'),
        'utf-8'
    )
).endpoint.headers.Authorization.replace(/^Bearer\s+/, '');

function get(apiPath, params = {}) {
    const qs = Object.entries(params)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    return new Promise((resolve) => {
        const opts = {
            hostname: 'api.tikhub.io',
            path: fullPath,
            method: 'GET',
            headers: { Authorization: `Bearer ${TIKHUB_TOKEN}`, Accept: 'application/json' },
        };
        const req = https.request(opts, (res) => {
            const chunks = [];
            res.on('data', (c) => chunks.push(c));
            res.on('end', () => {
                const body = Buffer.concat(chunks).toString('utf-8');
                try { resolve({ status: res.statusCode, json: JSON.parse(body) }); }
                catch (e) { resolve({ status: res.statusCode, json: { _raw: body.slice(0, 200), _parseError: e.message } }); }
            });
        });
        req.on('error', (e) => resolve({ status: 0, json: { _error: e.message } }));
        req.setTimeout(60000, () => { req.destroy(); resolve({ status: 0, json: { _error: 'timeout' } }); });
        req.end();
    });
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ============================================================
// 基础工具：解析搜索结果为扁平的视频数组
// ============================================================
function parseSearchVideos(json) {
    const d = json?.data;
    if (!d) return [];
    // TikTok search 返回 { data: { data: [...], cursor, ... } }
    let list = Array.isArray(d) ? d : (Array.isArray(d.data) ? d.data : (Array.isArray(d.aweme_list) ? d.aweme_list : []));
    return list
        .map((item) => item?.aweme_info || item)
        .filter((v) => v && v.aweme_id);
}

function slimVideo(v) {
    return {
        aweme_id: v.aweme_id,
        desc: v.desc,
        create_time: v.create_time,
        duration: v.duration,
        statistics: v.statistics ? {
            digg_count: v.statistics.digg_count || 0,
            comment_count: v.statistics.comment_count || 0,
            play_count: v.statistics.play_count || 0,
            share_count: v.statistics.share_count || 0,
            collect_count: v.statistics.collect_count || 0,
            download_count: v.statistics.download_count || 0,
        } : null,
        author: v.author ? {
            sec_uid: v.author.sec_uid,
            uid: v.author.uid,
            nickname: v.author.nickname,
            unique_id: v.author.unique_id,
            signature: v.author.signature,
            follower_count: v.author.follower_count,
            total_favorited: v.author.total_favorited,
            aweme_count: v.author.aweme_count,
            avatar: v.author.avatar_thumb?.url_list?.[0] || v.author.avatar_larger?.url_list?.[0],
            verification_type: v.author.verification_type,
            custom_verify: v.author.custom_verify,
        } : null,
        cover: v.video?.cover?.url_list?.[0] || v.video?.origin_cover?.url_list?.[0],
        hashtags: (v.text_extra || []).map((t) => t.hashtag_name).filter(Boolean),
        region: v.region,
    };
}

// ============================================================
// 主入口：单产品采集
// ============================================================
async function collectTikTokForProduct(product) {
    console.log('');
    console.log(`  [tt] 🎵 TikTok for ${product.name}`);
    const keywords = (product.keywords_en || []).slice(0, 4);
    if (!keywords.length) {
        return { _error: 'no keywords_en provided' };
    }

    const result = {
        product: product.name,
        keywords_used: keywords,
        by_keyword: {},
        merged_videos: [],
        top_authors: [],
        author_profiles: {},
        author_posts: {},
        top_comments_by_video: {},
        category_users: {},
        collected_at: new Date().toISOString(),
    };

    // === 1) 对每个关键词做 2 轮 search（拿 ~20 条） ===
    const allVideos = [];
    const seenAwemeIds = new Set();
    for (const kw of keywords) {
        result.by_keyword[kw] = { videos: [], pages: 0 };
        let cursor = 0;
        for (let page = 0; page < 2; page++) {
            console.log(`    [tt] 🔍 search "${kw}" page=${page + 1} cursor=${cursor}`);
            const r = await get('/api/v1/tiktok/app/v3/fetch_general_search_result', {
                keyword: kw, cursor, count: 10,
            });
            await sleep(1000);
            const videos = parseSearchVideos(r.json);
            console.log(`        → ${videos.length} videos`);
            videos.forEach((v) => {
                if (!seenAwemeIds.has(v.aweme_id)) {
                    seenAwemeIds.add(v.aweme_id);
                    const slim = slimVideo(v);
                    slim._from_keyword = kw;
                    allVideos.push(slim);
                    result.by_keyword[kw].videos.push(slim);
                }
            });
            result.by_keyword[kw].pages = page + 1;
            cursor = r.json?.data?.cursor || r.json?.data?.next_cursor || 0;
            if (!r.json?.data?.has_more || videos.length === 0) break;
        }
    }
    result.merged_videos = allVideos;
    console.log(`    [tt] ✓ merged ${allVideos.length} unique videos (keywords=${keywords.length})`);

    if (!allVideos.length) {
        console.log(`    [tt] ⚠ 无视频数据，返回空结果`);
        return result;
    }

    // === 2) 按点赞取 Top 8 视频，每个抓前 20 评论 ===
    const topVideos = [...allVideos]
        .sort((a, b) => (b.statistics?.digg_count || 0) - (a.statistics?.digg_count || 0))
        .slice(0, 8);
    for (const v of topVideos) {
        await sleep(900);
        console.log(`    [tt] 💬 comments ${v.aweme_id} · ${(v.desc || '').slice(0, 40)}`);
        const rC = await get('/api/v1/tiktok/app/v3/fetch_video_comments', {
            aweme_id: v.aweme_id, cursor: 0, count: 20,
        });
        const raw = rC.json?.data?.comments || [];
        const comments = raw.map((c) => ({
            cid: c.cid,
            text: c.text,
            digg_count: c.digg_count || 0,
            create_time: c.create_time,
            reply_comment_total: c.reply_comment_total || 0,
            user: c.user ? {
                nickname: c.user.nickname,
                sec_uid: c.user.sec_uid,
                unique_id: c.user.unique_id,
                follower_count: c.user.follower_count,
                region: c.user.region,
            } : null,
        }));
        result.top_comments_by_video[v.aweme_id] = comments;
        console.log(`        → ${comments.length} comments`);
    }

    // === 3) 合并作者 → Top 5 按粉丝，各抓 profile + Top 10 posts ===
    const authorMap = new Map();
    allVideos.forEach((v) => {
        if (v.author?.sec_uid && !authorMap.has(v.author.sec_uid)) {
            authorMap.set(v.author.sec_uid, v.author);
        }
    });
    const topAuthors = [...authorMap.values()]
        .sort((a, b) => (b.follower_count || 0) - (a.follower_count || 0))
        .slice(0, 5);
    result.top_authors = topAuthors.map((a) => ({
        sec_uid: a.sec_uid,
        nickname: a.nickname,
        unique_id: a.unique_id,
        follower_count: a.follower_count,
        total_favorited: a.total_favorited,
        aweme_count: a.aweme_count,
        custom_verify: a.custom_verify,
        avatar: a.avatar,
    }));

    for (const a of topAuthors) {
        await sleep(900);
        console.log(`    [tt] 👤 profile @${a.unique_id || a.nickname} (${(a.follower_count || 0).toLocaleString()}粉)`);
        const rP = await get('/api/v1/tiktok/app/v3/handler_user_profile', {
            sec_user_id: a.sec_uid,
        });
        const u = rP.json?.data?.user;
        if (u) {
            result.author_profiles[a.sec_uid] = {
                nickname: u.nickname,
                unique_id: u.unique_id,
                sec_uid: u.sec_uid,
                signature: u.signature,
                follower_count: u.follower_count,
                following_count: u.following_count,
                total_favorited: u.total_favorited,
                aweme_count: u.aweme_count,
                custom_verify: u.custom_verify,
                verification_type: u.verification_type,
                region: u.region,
                avatar: u.avatar_larger?.url_list?.[0] || u.avatar_thumb?.url_list?.[0],
            };
        }
        await sleep(700);
        console.log(`    [tt] 📹 posts @${a.unique_id || a.nickname}`);
        const rPost = await get('/api/v1/tiktok/app/v3/fetch_user_post_videos', {
            sec_user_id: a.sec_uid, max_cursor: 0, count: 10,
        });
        const awemeList = rPost.json?.data?.aweme_list || [];
        result.author_posts[a.sec_uid] = awemeList.map(slimVideo);
        console.log(`        → ${result.author_posts[a.sec_uid].length} posts`);
    }

    // === 4) 品类用户搜索（可选，拿头部品牌/垂类号）===
    await sleep(1000);
    const primaryKw = keywords[0];
    console.log(`    [tt] 🔎 user search "${primaryKw}"`);
    const rUser = await get('/api/v1/tiktok/web/fetch_search_user', {
        keyword: primaryKw, cursor: 0, count: 10,
    });
    const userList = rUser.json?.data;
    const userArr = Array.isArray(userList) ? userList : (Array.isArray(userList?.data) ? userList.data : (Array.isArray(userList?.user_list) ? userList.user_list : []));
    if (userArr.length) {
        result.category_users[primaryKw] = userArr.slice(0, 10).map((u) => {
            const ui = u.user_info || u;
            return {
                sec_uid: ui.sec_uid,
                unique_id: ui.unique_id,
                nickname: ui.nickname,
                follower_count: ui.follower_count,
                total_favorited: ui.total_favorited,
                aweme_count: ui.aweme_count,
                signature: ui.signature,
                custom_verify: ui.custom_verify,
                avatar: ui.avatar_thumb?.url_list?.[0] || ui.avatar_larger?.url_list?.[0],
            };
        });
        console.log(`        → ${result.category_users[primaryKw].length} category users`);
    } else {
        console.log(`        → 0 users (empty or error)`);
    }

    return result;
}

module.exports = { collectTikTokForProduct };