fmode-voc-projects/scripts/tools/collect-market-intel.js

#!/usr/bin/env node
/**
 * Market Intel 采集模块（TikHub）
 *
 * 两条数据线：
 *  A. 公共 Douyin 站级热榜快照（3 产品共享、每次采集 1 份）
 *     - fetch_hot_total_hot_word_list    → 全站热词榜
 *     - web_fetch_hot_search_result       → 抖音 Web 首页热搜榜
 *     - fetch_hot_total_high_topic_list   → 全站热挑战榜
 *     - fetch_hot_total_high_like_list    → 全站热赞视频榜
 *     存到 data/douyin-trends-snapshot.json
 *
 *  B. TikTok Ads 官方情报（按产品英文关键词）
 *     - get_related_keywords  → Creative Center 相关词网络（Top 50 w/ score）
 *     - get_keyword_insights  → 单关键词市场情报（impression/cost/ctr/cvr/post_change）
 *     合并到产品 VOC JSON 的 market_intel 字段
 */
const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');

const TIKHUB_TOKEN = JSON.parse(
    fs.readFileSync(
        path.join(os.homedir(), '.openclaw', 'skills', 'xiaohongshu-search-notes', 'api-config.json'),
        'utf-8'
    )
).endpoint.headers.Authorization.replace(/^Bearer\s+/, '');

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

function request(method, apiPath, { params = {}, body = null } = {}) {
    const qs = Object.entries(params)
        .filter(([, v]) => v !== undefined && v !== null && v !== '')
        .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
        .join('&');
    const fullPath = qs ? `${apiPath}?${qs}` : apiPath;
    const bodyStr = body ? JSON.stringify(body) : null;
    return new Promise((resolve) => {
        const headers = { Authorization: `Bearer ${TIKHUB_TOKEN}`, Accept: 'application/json' };
        if (bodyStr) {
            headers['Content-Type'] = 'application/json';
            headers['Content-Length'] = Buffer.byteLength(bodyStr);
        }
        const opts = { hostname: 'api.tikhub.io', path: fullPath, method, headers };
        const req = https.request(opts, (res) => {
            const chunks = [];
            res.on('data', (c) => chunks.push(c));
            res.on('end', () => {
                const raw = Buffer.concat(chunks).toString('utf-8');
                try { resolve({ status: res.statusCode, json: JSON.parse(raw) }); }
                catch (e) { resolve({ status: res.statusCode, json: { _raw: raw.slice(0, 300) } }); }
            });
        });
        req.on('error', (e) => resolve({ status: 0, json: { _error: e.message } }));
        req.setTimeout(30000, () => { req.destroy(); resolve({ status: 0, json: { _error: 'timeout' } }); });
        if (bodyStr) req.write(bodyStr);
        req.end();
    });
}

// TikHub 嵌套规律：resp.data.data.xxx
function pickData(j) { return j?.data?.data; }

// ============================================================
// A. Douyin 公共站级热榜（3 产品共享）
// ============================================================
async function collectDouyinTrendsOnce() {
    console.log('\n=== [A] Douyin 站级热榜快照（3 产品共享）===');
    const out = { collected_at: new Date().toISOString() };

    // 1) 热词榜（title/score/trends）
    console.log('  📊 fetch_hot_total_hot_word_list ...');
    const rHotWord = await request('POST', '/api/v1/douyin/billboard/fetch_hot_total_hot_word_list',
        { body: { date_window: 24, page: 1, page_size: 30 } });
    out.hot_word_list = pickData(rHotWord.json)?.word_list || [];
    console.log(`    → ${out.hot_word_list.length} 热词`);
    await sleep(600);

    // 2) Web 首页热搜榜（word/view_count/discuss_video_count）
    console.log('  📊 web/fetch_hot_search_result ...');
    const rWebHot = await request('GET', '/api/v1/douyin/web/fetch_hot_search_result');
    out.web_hot_search = pickData(rWebHot.json)?.word_list || [];
    console.log(`    → ${out.web_hot_search.length} 热搜榜`);
    await sleep(600);

    // 3) 热挑战榜（challenge_name/play_cnt/publish_cnt）
    console.log('  📊 fetch_hot_total_high_topic_list ...');
    const rHotTopic = await request('POST', '/api/v1/douyin/billboard/fetch_hot_total_high_topic_list',
        { body: { date_window: 24, page: 1, page_size: 30 } });
    out.hot_topic_list = pickData(rHotTopic.json)?.objs || [];
    console.log(`    → ${out.hot_topic_list.length} 热挑战`);
    await sleep(600);

    // 4) 热赞视频榜（item_title/nick_name/play_cnt/like_cnt）
    console.log('  📊 fetch_hot_total_high_like_list ...');
    const rHotLike = await request('POST', '/api/v1/douyin/billboard/fetch_hot_total_high_like_list',
        { body: { date_window: 24, page: 1, page_size: 30 } });
    out.hot_like_videos = pickData(rHotLike.json)?.objs || [];
    console.log(`    → ${out.hot_like_videos.length} 热赞视频`);
    await sleep(600);

    // 5) 热播视频榜（补强）
    console.log('  📊 fetch_hot_total_high_play_list ...');
    const rHotPlay = await request('POST', '/api/v1/douyin/billboard/fetch_hot_total_high_play_list',
        { body: { date_window: 24, page: 1, page_size: 30 } });
    out.hot_play_videos = pickData(rHotPlay.json)?.objs || [];
    console.log(`    → ${out.hot_play_videos.length} 热播视频`);

    return out;
}

// ============================================================
// 本地品类关键词过滤器（把全站热榜筛到本品类相关）
// ============================================================
function filterByKeywords(items, keywords, getText) {
    if (!items || !items.length || !keywords || !keywords.length) return [];
    const kws = keywords.map((k) => String(k).toLowerCase()).filter(Boolean);
    return items.filter((item) => {
        const text = String(getText(item) || '').toLowerCase();
        return kws.some((kw) => text.includes(kw));
    });
}

function buildDouyinCategoryFiltered(trends, productKeywords) {
    const kws = productKeywords || [];
    return {
        hot_words: filterByKeywords(trends.hot_word_list || [], kws, (w) => w.title),
        web_hot_search: filterByKeywords(trends.web_hot_search || [], kws, (w) => w.word),
        hot_topics: filterByKeywords(trends.hot_topic_list || [], kws, (t) => t.challenge_name),
        hot_like_videos: filterByKeywords(trends.hot_like_videos || [], kws, (v) => v.item_title),
        hot_play_videos: filterByKeywords(trends.hot_play_videos || [], kws, (v) => v.item_title),
    };
}

// ============================================================
// B. TikTok Ads 情报（按英文关键词）
// ============================================================
async function collectTikTokAdsForKeyword(keywordEn, opts = {}) {
    const { period = 30, countryCode = 'US' } = opts;
    const out = { keyword: keywordEn, period, country_code: countryCode };

    console.log(`    🔗 related_keywords "${keywordEn}"`);
    const rRelated = await request('GET', '/api/v1/tiktok/ads/get_related_keywords',
        { params: { keyword: keywordEn, period, country_code: countryCode } });
    const relatedInner = rRelated.json?.data?.data;
    // related: data.list [{ name, score }] 50 条
    out.related_keywords = Array.isArray(relatedInner?.list) ? relatedInner.list.slice(0, 50) : [];
    console.log(`       → ${out.related_keywords.length} 相关词`);
    await sleep(600);

    console.log(`    💡 keyword_insights "${keywordEn}"`);
    const rInsights = await request('GET', '/api/v1/tiktok/ads/get_keyword_insights',
        { params: { keyword: keywordEn, period, country_code: countryCode } });
    const insightsInner = rInsights.json?.data?.data;
    // insights: data.keyword_list[0] { impression, cost, ctr, cvr, post, post_change, comment, share, like, video_list }
    out.insights = Array.isArray(insightsInner?.keyword_list) && insightsInner.keyword_list.length
        ? insightsInner.keyword_list[0]
        : null;
    if (out.insights) {
        console.log(`       → impression ${out.insights.impression?.toLocaleString?.() || out.insights.impression} · CTR ${out.insights.ctr}% · post +${out.insights.post_change}%`);
    } else {
        console.log(`       → 无 insights 数据`);
    }
    await sleep(600);

    return out;
}

// ============================================================
// 对外入口：每产品跑 market intel
// ============================================================
async function collectMarketIntelForProduct(productName, cfg) {
    // cfg: { keywords_en: ['milk thistle', ...], douyin_filter_keywords: ['肝', '解酒', ...] }
    console.log(`\n── [market] ${productName} ──`);
    const enKws = (cfg.keywords_en || []).slice(0, 2); // 最多 2 个英文关键词，每个 2 次 API
    const adsByKeyword = {};
    for (const kw of enKws) {
        adsByKeyword[kw] = await collectTikTokAdsForKeyword(kw, { period: 30, countryCode: 'US' });
    }
    return {
        ads_by_keyword: adsByKeyword,
        douyin_filter_keywords: cfg.douyin_filter_keywords || [],
    };
}

module.exports = {
    collectDouyinTrendsOnce,
    buildDouyinCategoryFiltered,
    collectTikTokAdsForKeyword,
    collectMarketIntelForProduct,
};