openclaw-voc-skill/scripts/tools/probe-dy-web-search.js

#!/usr/bin/env node
/**
 * 直接用 douyin.com web 搜索（绕开 TikHub），从返回 HTML 里抽 sec_uid
 */
const fs = require('fs');
const path = require('path');
const https = require('https');
const zlib = require('zlib');

const COOKIE = fs.readFileSync(path.resolve('data/douyin-cookie.txt'), 'utf-8').trim();

function fetchHtml(url, cookie) {
  const u = new URL(url);
  return new Promise((resolve) => {
    const opts = {
      hostname: u.hostname,
      path: u.pathname + u.search,
      method: 'GET',
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Referer': 'https://www.douyin.com/',
        Cookie: cookie,
      },
    };
    const req = https.request(opts, (res) => {
      const chunks = [];
      res.on('data', (c) => chunks.push(c));
      res.on('end', () => {
        let buf = Buffer.concat(chunks);
        const enc = res.headers['content-encoding'];
        try {
          if (enc === 'gzip') buf = zlib.gunzipSync(buf);
          else if (enc === 'deflate') buf = zlib.inflateSync(buf);
          else if (enc === 'br') buf = zlib.brotliDecompressSync(buf);
        } catch (e) {
          // ignore
        }
        resolve({ status: res.statusCode, body: buf.toString('utf-8'), headers: res.headers });
      });
    });
    req.on('error', (e) => resolve({ status: 0, error: e.message }));
    req.setTimeout(20000, () => { req.destroy(); resolve({ status: 0, error: 'timeout' }); });
    req.end();
  });
}

function extractSecUids(html) {
  const results = [];
  const seen = new Set();
  // 抖音 HTML 里 sec_uid 通常出现在 "sec_uid":"MS4wLjABAAAA..." 或 url: /user/MS4wLjABAAAA...
  const regexes = [
    /"sec_uid"\s*:\s*"(MS4wLjABAAAA[A-Za-z0-9_\-]+)"/g,
    /\/user\/(MS4wLjABAAAA[A-Za-z0-9_\-]+)/g,
  ];
  for (const re of regexes) {
    let m;
    while ((m = re.exec(html)) !== null) {
      if (!seen.has(m[1])) {
        seen.add(m[1]);
        results.push(m[1]);
      }
    }
  }
  return results;
}

function extractNicknameForSecUid(html, secUid) {
  // 寻找 sec_uid 附近的 nickname
  const re = new RegExp('"sec_uid"\\s*:\\s*"' + secUid + '"[\\s\\S]{0,800}?"nickname"\\s*:\\s*"([^"]{1,80})"');
  const m = html.match(re);
  if (m) return m[1];
  // 反向
  const re2 = new RegExp('"nickname"\\s*:\\s*"([^"]{1,80})"[\\s\\S]{0,800}?"sec_uid"\\s*:\\s*"' + secUid + '"');
  const m2 = html.match(re2);
  return m2 ? m2[1] : '';
}

function extractFansForSecUid(html, secUid) {
  const re = new RegExp('"sec_uid"\\s*:\\s*"' + secUid + '"[\\s\\S]{0,1500}?"follower_count"\\s*:\\s*(\\d+)');
  const m = html.match(re);
  return m ? +m[1] : 0;
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

const candidates = [
  { product: 'liver', role: '品牌竞品', keyword: '葵花药业' },
  { product: 'liver', role: '品牌竞品', keyword: '葵花护肝片' },
  { product: 'liver', role: '垂类博主', keyword: '肝病科' },
  { product: 'probiotic', role: '品牌竞品', keyword: '妈咪爱' },
  { product: 'probiotic', role: '品牌竞品', keyword: '亿活益生菌' },
  { product: 'probiotic', role: '品牌竞品', keyword: '合生元' },
  { product: 'probiotic', role: '垂类博主', keyword: '儿科医生' },
  { product: 'monkey', role: '品牌竞品', keyword: '康恩贝' },
  { product: 'monkey', role: '品牌竞品', keyword: '太阳神' },
  { product: 'monkey', role: '品牌竞品', keyword: '三九胃泰' },
  { product: 'monkey', role: '垂类博主', keyword: '养胃' },
];

(async () => {
  const out = [];
  for (const c of candidates) {
    await sleep(1500);
    const url = `https://www.douyin.com/search/${encodeURIComponent(c.keyword)}?type=user`;
    console.log(`\n[${c.product} · ${c.role}] ${c.keyword}`);
    const r = await fetchHtml(url, COOKIE);
    if (r.error) {
      console.log('  ❌ fetch error:', r.error);
      out.push({ ...c, _error: r.error });
      continue;
    }
    console.log('  status:', r.status, '| body size:', r.body.length);
    // 保存首个样本用于调试
    if (out.length === 0) {
      fs.writeFileSync('data/dy-web-search-sample.html', r.body.slice(0, 200000), 'utf-8');
    }
    const secUids = extractSecUids(r.body);
    console.log('  sec_uids found:', secUids.length);
    if (secUids.length === 0) {
      const dataMatch = r.body.match(/<script\s+id="RENDER_DATA"[^>]*>([\s\S]{0,300})/);
      console.log('  snippet:', dataMatch ? dataMatch[1].slice(0, 200) : r.body.slice(0, 300));
      out.push({ ...c, hits: [] });
      continue;
    }
    const top = secUids.slice(0, 5).map((sec_uid) => ({
      sec_uid,
      nickname: extractNicknameForSecUid(r.body, sec_uid),
      follower_count: extractFansForSecUid(r.body, sec_uid),
      share_url: 'https://www.douyin.com/user/' + sec_uid,
    }));
    top.forEach((t, i) => console.log(`    #${i + 1} ${t.nickname || '(no name)'} · 粉丝 ${t.follower_count.toLocaleString()} · ${t.sec_uid.slice(0, 20)}...`));
    out.push({ ...c, hits: top });
  }
  fs.writeFileSync('data/dy-web-search-out.json', JSON.stringify(out, null, 2), 'utf-8');
  console.log('\n✓ Wrote data/dy-web-search-out.json');
})();