fmode-voc-projects/scripts/tools/collect-douyin.js

#!/usr/bin/env node
/**
 * 抖音采集模块（TikHub via /thapi/v1/ 代理）
 *
 * 策略：
 *   1) 如果存在 data/douyin-cookie.txt，加载用户 cookie → 注入到每个 TikHub 请求的 ?cookie= 参数
 *   2) 对每个产品 share_url：
 *        a. fetch_one_video_by_share_url → 拿 aweme_id / sec_uid / 视频元数据
 *        b. fetch_video_comments → 评论
 *        c. fetch_user_post_videos → 作者其他作品
 *        d. handler_user_profile → 作者画像
 *   3) 如果 cookie 可用，再对每个中文关键词做一次 fetch_general_search_result
 *
 * 无 cookie 时：记录 _pending_cookie: true 优雅降级
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const TIKHUB_TOKEN = JSON.parse(fs.readFileSync(
    path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json'),
    'utf-8'
)).endpoint.headers.Authorization.replace(/^Bearer\s+/, '');

// Cookie file：用户登录 www.douyin.com 后从 DevTools 复制整条 Cookie header
const COOKIE_PATH = path.resolve(__dirname, '..', '..', 'data', 'douyin-cookie.txt');

function loadCookie() {
    if (!fs.existsSync(COOKIE_PATH)) return '';
    const raw = fs.readFileSync(COOKIE_PATH, 'utf-8').trim();
    // 过滤掉注释行（#开头）
    return raw.split('\n').filter((l) => !l.startsWith('#') && l.trim()).join('; ').trim();
}

function buildUrl(base, params) {
    const qs = Object.entries(params)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    return qs ? `${base}?${qs}` : base;
}

async function tikhubGet(apiPath, params = {}, cookie = '') {
    if (cookie) params.cookie = cookie;
    const url = buildUrl(apiPath, params);
    return new Promise((resolve) => {
        const opts = {
            hostname: 'api.tikhub.io', path: url, method: 'GET',
            headers: { Authorization: `Bearer ${TIKHUB_TOKEN}`, Accept: 'application/json' },
        };
        const req = https.request(opts, (res) => {
            const chunks = [];
            res.on('data', (c) => chunks.push(c));
            res.on('end', () => {
                const body = Buffer.concat(chunks).toString('utf-8');
                try {
                    const j = JSON.parse(body);
                    resolve({ status: res.statusCode, json: j });
                } catch (e) {
                    resolve({ status: res.statusCode, json: { _parse_error: e.message, _raw: body.slice(0, 300) } });
                }
            });
        });
        req.on('error', (e) => resolve({ status: 0, json: { _error: e.message } }));
        req.setTimeout(45000, () => { req.destroy(); resolve({ status: 0, json: { _error: 'timeout' } }); });
        req.end();
    });
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function extractVideoInfo(j) {
    if (!j || j.code !== 200 || !j.data) return null;
    const info = j.data.aweme_detail || j.data.aweme_info || j.data;
    if (!info?.aweme_id) return null;
    return info;
}

// 手动解析 v.douyin.com 短链 → 支持两种类型：
//   type=video → 拿 aweme_id（视频分享链）
//   type=user  → 拿 sec_uid（作者主页分享链）
function resolveShareUrlHead(shareUrl) {
    return new Promise((resolve) => {
        try {
            const u = new URL(shareUrl);
            const opts = {
                hostname: u.hostname,
                path: u.pathname + u.search,
                method: 'GET',
                headers: {
                    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
                    Accept: 'text/html,*/*',
                },
            };
            const req = https.request(opts, (res) => {
                const loc = res.headers.location || '';
                // 视频分享链：/share/video/{aweme_id}/
                const mVideo = loc.match(/\/share\/video\/(\d+)/) || loc.match(/\/video\/(\d+)/);
                // 作者分享链：/share/user/{sec_uid}/  or  ?sec_uid={sec_uid}
                const mUser = loc.match(/\/share\/user\/([A-Za-z0-9_\-]+)/);
                const mSecUid = loc.match(/[?&]sec_uid=([A-Za-z0-9_\-]+)/);
                if (mVideo) {
                    res.resume();
                    resolve({ type: 'video', aweme_id: mVideo[1], location: loc, status: res.statusCode });
                    return;
                }
                if (mSecUid || mUser) {
                    res.resume();
                    resolve({ type: 'user', sec_uid: (mSecUid && mSecUid[1]) || mUser[1], location: loc, status: res.statusCode });
                    return;
                }
                // 兜底：读 body 找线索
                const chunks = [];
                res.on('data', (c) => chunks.push(c));
                res.on('end', () => {
                    const body = Buffer.concat(chunks).toString('utf-8');
                    const b1 = body.match(/"aweme_id"\s*:\s*"(\d+)"/) || body.match(/\/video\/(\d{15,})/);
                    const b2 = body.match(/"sec_uid"\s*:\s*"([A-Za-z0-9_\-]+)"/);
                    if (b1) resolve({ type: 'video', aweme_id: b1[1], location: loc, status: res.statusCode });
                    else if (b2) resolve({ type: 'user', sec_uid: b2[1], location: loc, status: res.statusCode });
                    else resolve({ type: 'unknown', location: loc, status: res.statusCode, body_len: body.length });
                });
            });
            req.on('error', (e) => resolve({ _error: e.message }));
            req.setTimeout(20000, () => { req.destroy(); resolve({ _error: 'timeout' }); });
            req.end();
        } catch (e) {
            resolve({ _error: e.message });
        }
    });
}

// ============================================================
// 规范化账号 specs：支持 douyin_share_urls (array) / douyin_share_url (string)
// ============================================================
function normalizeAccountSpecs(product) {
    // 优先用 douyin_share_urls 数组
    if (Array.isArray(product.douyin_share_urls) && product.douyin_share_urls.length) {
        return product.douyin_share_urls.map((item, idx) => {
            if (typeof item === 'string') {
                return { share_url: item, role: idx === 0 ? '官方号' : '补充号', label: idx === 0 ? '官方号' : `账号 ${idx + 1}` };
            }
            return {
                share_url: item.share_url || item.url,
                role: item.role || (idx === 0 ? '官方号' : '补充号'),
                label: item.label || item.name || (idx === 0 ? '官方号' : `账号 ${idx + 1}`),
            };
        }).filter((s) => s.share_url);
    }
    // 向下兼容单个 share_url
    if (product.douyin_share_url) {
        return [{ share_url: product.douyin_share_url, role: '官方号', label: '官方号' }];
    }
    return [];
}

// ============================================================
// 单个账号采集（原 collectDouyinForProduct 主体逻辑抽出来）
// ============================================================
async function collectSingleAccount(spec, product, cookie, { runSearches = false } = {}) {
    const result = {
        share_url: spec.share_url,
        role: spec.role,
        label: spec.label,
        share_type: null,
        video: null,
        comments: [],
        user_posts: [],
        user_profile: null,
        searches: {},
    };

    // 1) 手动解析短链（TikHub 的 resolver 不稳定，直接跟重定向）
    console.log(`    [dy·${spec.label}] 🎬 解析 ${spec.share_url}`);
    const parsed = await resolveShareUrlHead(spec.share_url);
    if (parsed._error || parsed.type === 'unknown') {
        result._resolve_error = parsed._error || `unknown type, location=${parsed.location}`;
        console.log(`      ❌ 解析失败: ${String(result._resolve_error).slice(0, 160)}`);
        return result;
    }
    console.log(`      ✓ type=${parsed.type} | ${parsed.aweme_id || parsed.sec_uid}`);
    result.share_type = parsed.type;

    let aweme_id = parsed.aweme_id || '';
    let sec_uid = parsed.sec_uid || '';
    await sleep(500);

    // 2) 若是视频分享链 → 先抓视频详情拿到 aweme_detail + author.sec_uid
    if (parsed.type === 'video' && aweme_id) {
        const rV = await tikhubGet(
            '/api/v1/douyin/app/v3/fetch_one_video',
            { aweme_id }, cookie
        );
        const video = extractVideoInfo(rV.json);
        if (video) {
            result.video = {
                aweme_id: video.aweme_id,
                desc: video.desc,
                create_time: video.create_time,
                duration: video.duration,
                statistics: video.statistics,
                author: video.author ? {
                    nickname: video.author.nickname,
                    sec_uid: video.author.sec_uid,
                    uid: video.author.uid,
                    avatar: video.author.avatar_thumb?.url_list?.[0] || video.author.avatar_larger?.url_list?.[0],
                    signature: video.author.signature,
                    follower_count: video.author.follower_count,
                    aweme_count: video.author.aweme_count,
                } : null,
                cover: video.video?.cover?.url_list?.[0] || video.video?.origin_cover?.url_list?.[0],
                text_extra: video.text_extra?.map((t) => t.hashtag_name).filter(Boolean) || [],
            };
            sec_uid = sec_uid || video.author?.sec_uid || '';
            console.log(`      [video] ${video.author?.nickname} · digg=${video.statistics?.digg_count} comment=${video.statistics?.comment_count}`);
        } else {
            console.log(`      [video] ❌ detail failed: ${JSON.stringify(rV.json).slice(0, 120)}`);
        }
        await sleep(700);

        // 3) 视频评论（3 页累计最多 60）
        let cursor = 0;
        for (let page = 0; page < 3; page++) {
            const r2 = await tikhubGet(
                '/api/v1/douyin/app/v3/fetch_video_comments',
                { aweme_id, cursor, count: 20 }, cookie
            );
            if (r2.json?.code === 200 && r2.json?.data) {
                const cmts = r2.json.data.comments || [];
                cmts.forEach((c) => {
                    result.comments.push({
                        cid: c.cid,
                        text: c.text,
                        digg_count: c.digg_count,
                        create_time: c.create_time,
                        ip_label: c.ip_label,
                        user: c.user ? {
                            nickname: c.user.nickname,
                            sec_uid: c.user.sec_uid,
                            follower_count: c.user.follower_count,
                        } : null,
                    });
                });
                console.log(`      [comments p${page + 1}] +${cmts.length} total=${result.comments.length}`);
                if (!r2.json.data.has_more) break;
                cursor = r2.json.data.cursor || 0;
            } else {
                console.log(`      [comments p${page + 1}] ERR ${JSON.stringify(r2.json).slice(0, 150)}`);
                break;
            }
            await sleep(700);
        }
    }

    if (!sec_uid) {
        console.log(`      ⚠ 无 sec_uid，跳过作者板块`);
        return result;
    }
    await sleep(500);

    // 4) 作者主页作品
    const r3 = await tikhubGet(
        '/api/v1/douyin/app/v3/fetch_user_post_videos',
        { sec_user_id: sec_uid, max_cursor: 0, count: 20 }, cookie
    );
    if (r3.json?.code === 200 && r3.json?.data) {
        const vids = r3.json.data.aweme_list || [];
        result.user_posts = vids.map((v) => ({
            aweme_id: v.aweme_id,
            desc: v.desc,
            create_time: v.create_time,
            statistics: v.statistics,
            cover: v.video?.cover?.url_list?.[0] || v.video?.origin_cover?.url_list?.[0],
            duration: v.duration,
        }));
        console.log(`      [user_posts] ${result.user_posts.length} 条`);
    } else {
        console.log(`      [user_posts] ERR ${JSON.stringify(r3.json).slice(0, 150)}`);
    }
    await sleep(500);

    // 5) 作者画像
    const r4 = await tikhubGet(
        '/api/v1/douyin/app/v3/handler_user_profile',
        { sec_user_id: sec_uid }, cookie
    );
    if (r4.json?.code === 200 && r4.json?.data) {
        const u = r4.json.data.user || r4.json.data;
        result.user_profile = {
            nickname: u.nickname,
            signature: u.signature,
            follower_count: u.follower_count,
            following_count: u.following_count,
            aweme_count: u.aweme_count,
            total_favorited: u.total_favorited,
            avatar: u.avatar_larger?.url_list?.[0] || u.avatar_thumb?.url_list?.[0],
            ip_location: u.ip_location,
            verification_type: u.verification_type,
            custom_verify: u.custom_verify,
        };
        console.log(`      [profile] ${u.nickname} · 粉丝 ${(u.follower_count || 0).toLocaleString()} · 获赞 ${(u.total_favorited || 0).toLocaleString()}`);
    } else {
        console.log(`      [profile] ERR ${JSON.stringify(r4.json).slice(0, 150)}`);
    }
    await sleep(500);

    // 6) 若是作者分享链 → 对作者的 Top 3 热门作品抓评论
    if (parsed.type === 'user' && result.user_posts.length) {
        const topPosts = [...result.user_posts]
            .sort((a, b) => (b.statistics?.digg_count || 0) - (a.statistics?.digg_count || 0))
            .slice(0, 3);
        for (const p of topPosts) {
            await sleep(600);
            const rC = await tikhubGet(
                '/api/v1/douyin/app/v3/fetch_video_comments',
                { aweme_id: p.aweme_id, cursor: 0, count: 20 }, cookie
            );
            if (rC.json?.code === 200 && rC.json?.data) {
                const cmts = rC.json.data.comments || [];
                cmts.forEach((c) => {
                    result.comments.push({
                        cid: c.cid,
                        aweme_id: p.aweme_id,
                        text: c.text,
                        digg_count: c.digg_count,
                        create_time: c.create_time,
                        ip_label: c.ip_label,
                        user: c.user ? {
                            nickname: c.user.nickname,
                            sec_uid: c.user.sec_uid,
                            follower_count: c.user.follower_count,
                        } : null,
                    });
                });
                console.log(`      [comments on ${p.aweme_id}] +${cmts.length}`);
            }
        }
    }

    // 5) 如果启用（通常只在主号跑），再做关键词搜索覆盖品类全景
    //    注意：抖音 web search 需要带设备指纹的 cookie，多数情况下返回"用户未登录"。
    //    保留代码作为 best-effort，失败不影响其他数据。
    if (runSearches && cookie && product.keywords_cn) {
        for (const kw of product.keywords_cn.slice(0, 2)) {
            await sleep(800);
            console.log(`      [search "${kw}"]`);
            const r5 = await tikhubGet(
                '/api/v1/douyin/app/v3/fetch_general_search_result',
                { keyword: kw, offset: 0, count: 20, sort_type: 1, publish_time: 180 }, cookie
            );
            if (r5.json?.code === 200 && r5.json?.data) {
                const data = r5.json.data.data || r5.json.data;
                const videos = Array.isArray(data) ? data.filter((x) => x.aweme_info).map((x) => x.aweme_info) : [];
                result.searches[kw] = videos.map((v) => ({
                    aweme_id: v.aweme_id,
                    desc: v.desc,
                    create_time: v.create_time,
                    statistics: v.statistics,
                    author: v.author ? {
                        nickname: v.author.nickname,
                        sec_uid: v.author.sec_uid,
                        follower_count: v.author.follower_count,
                    } : null,
                    cover: v.video?.cover?.url_list?.[0],
                }));
                console.log(`        → ${result.searches[kw].length} 条视频`);
            } else {
                const st = r5.json?.data?.[0]?.status_msg || r5.json?.message_zh || 'unknown';
                console.log(`        ❌ ${String(st).slice(0, 80)}`);
                result.searches[kw] = { _error: st };
            }
        }
    }

    return result;
}

// ============================================================
// 多账号采集编排（对外主入口）
// ============================================================
async function collectDouyinForProduct(product, cookie) {
    const specs = normalizeAccountSpecs(product);
    const result = {
        product: product.name,
        cookie_loaded: !!cookie,
        primary_share_url: specs[0]?.share_url || null,
        accounts_total: specs.length,
        accounts: [],
    };

    if (specs.length === 0) {
        result._error = 'no share_url/share_urls provided';
        return result;
    }

    console.log(`  [dy] 📦 ${specs.length} 个账号待采集: ${specs.map((s) => s.label).join(' · ')}`);

    for (let i = 0; i < specs.length; i++) {
        const spec = specs[i];
        try {
            const acc = await collectSingleAccount(spec, product, cookie, {
                runSearches: i === 0, // 只在首个账号跑搜索
            });
            result.accounts.push(acc);
        } catch (e) {
            console.log(`    [dy·${spec.label}] ❌ ${e.message}`);
            result.accounts.push({ ...spec, _error: e.message });
        }
        if (i < specs.length - 1) await sleep(1500);
    }

    // 向下兼容：把首个账号的关键字段打平到根，老 render/analyze 还能用
    const primary = result.accounts[0];
    if (primary) {
        result.share_url = primary.share_url;
        result.share_type = primary.share_type;
        result.video = primary.video;
        result.comments = primary.comments;
        result.user_posts = primary.user_posts;
        result.user_profile = primary.user_profile;
        result.searches = primary.searches || {};
    }

    return result;
}

module.exports = { collectDouyinForProduct, collectSingleAccount, normalizeAccountSpecs, loadCookie, COOKIE_PATH };