openclaw-voc-skill/openclaw-skills/industry-trend-intelligence/scripts/xiaohongshu-trend-collector.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const os = require('os');
const https = require('https');
const { URL } = require('url');
const {
  buildSampleDataset,
  keywordMatrixFromProfile
} = require('./industry-trend-report');

const DEFAULT_RESULT_PREFIX = 'XIAOHONGSHU_TREND_COLLECTOR_RESULT';
const PROJECT_ROOT = path.resolve(__dirname, '..', '..');
const XHS_PROJECT_DIR = path.join(PROJECT_ROOT, 'xiaohongshu');
const OPENCLAW_SKILLS_DIR = path.join(os.homedir(), '.openclaw', 'skills');

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i++) {
    const token = argv[i];
    if (!token.startsWith('--')) continue;
    const eq = token.indexOf('=');
    if (eq >= 0) {
      args[token.slice(2, eq)] = token.slice(eq + 1);
    } else {
      const key = token.slice(2);
      const next = argv[i + 1];
      if (next && !next.startsWith('--')) {
        args[key] = next;
        i++;
      } else {
        args[key] = true;
      }
    }
  }
  return args;
}

function readJson(filePath) {
  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
}

function writeJson(filePath, data) {
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
  fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
}

function toBool(value, fallback = false) {
  if (typeof value === 'boolean') return value;
  if (value === undefined || value === null || value === '') return fallback;
  return ['1', 'true', 'yes', 'y', 'on'].includes(String(value).toLowerCase());
}

function toInt(value, fallback) {
  const parsed = Number.parseInt(String(value ?? ''), 10);
  return Number.isFinite(parsed) ? parsed : fallback;
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

function uniqueBy(items, keyFn) {
  const seen = new Set();
  const out = [];
  items.forEach(item => {
    const key = keyFn(item);
    if (!key || seen.has(key)) return;
    seen.add(key);
    out.push(item);
  });
  return out;
}

function normalizeCount(value) {
  if (value === undefined || value === null || value === '') return 0;
  if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
  const text = String(value).trim().toLowerCase();
  if (!text) return 0;
  const match = text.match(/([\d.]+)/);
  if (!match) return 0;
  const base = Number.parseFloat(match[1]);
  if (!Number.isFinite(base)) return 0;
  if (text.includes('w') || text.includes('万')) return Math.round(base * 10000);
  if (text.includes('k') || text.includes('千')) return Math.round(base * 1000);
  return Math.round(base);
}

function noteScore(note) {
  return normalizeCount(note.liked_count || note.likeCount) +
    normalizeCount(note.collected_count || note.collectCount) * 1.5 +
    normalizeCount(note.comments_count || note.commentCount) * 3 +
    normalizeCount(note.shared_count || note.shareCount) * 2;
}

function loadSkillConfig(name) {
  const installedPath = path.join(OPENCLAW_SKILLS_DIR, name, 'api-config.json');
  const projectPath = path.join(XHS_PROJECT_DIR, name, 'api-config.json');
  const configPath = fs.existsSync(installedPath) ? installedPath : projectPath;
  if (!fs.existsSync(configPath)) {
    throw new Error(`Missing Xiaohongshu skill config: ${name}`);
  }
  return { config: readJson(configPath), source: configPath };
}

function buildUrl(skillConfig, params) {
  const url = new URL(skillConfig.endpoint.url);
  Object.entries(params).forEach(([key, value]) => {
    if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
  });
  return url.toString();
}

async function requestJson(url, headers, options = {}) {
  const maxAttempts = options.maxAttempts || 3;
  const retryOn = new Set([400, 408, 429, 500, 502, 503, 504]);
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
    try {
      const result = await new Promise((resolve, reject) => {
        const u = new URL(url);
        const req = https.request(
          {
            hostname: u.hostname,
            path: u.pathname + u.search,
            method: 'GET',
            headers,
            timeout: options.timeoutMs || 60000
          },
          res => {
            let body = '';
            res.on('data', chunk => {
              body += chunk;
            });
            res.on('end', () => resolve({ status: res.statusCode, body }));
          }
        );
        req.on('error', reject);
        req.on('timeout', () => {
          req.destroy();
          reject(new Error('request timeout'));
        });
        req.end();
      });

      let json;
      try {
        json = JSON.parse(result.body);
      } catch {
        json = undefined;
      }

      if (result.status === 200) {
        return { status: result.status, body: result.body, json };
      }

      if (retryOn.has(result.status) && attempt < maxAttempts) {
        await sleep((options.retryDelayMs || 1200) * attempt);
        continue;
      }

      return { status: result.status, body: result.body, json };
    } catch (error) {
      if (attempt >= maxAttempts) throw error;
      await sleep((options.retryDelayMs || 1200) * attempt);
    }
  }
  throw new Error('max retries exceeded');
}

function unwrapData(json) {
  if (!json) return {};
  return json.data?.data || json.data || json;
}

function pickTags(note) {
  const rawTags = note.tag_info || note.tagList || note.tags || [];
  if (!Array.isArray(rawTags)) return [];
  return rawTags.map(tag => tag.name || tag.tag_name || tag.title || tag).filter(Boolean).map(String);
}

function pickImages(note) {
  const images = note.images_list || note.image_list || note.images || [];
  if (!Array.isArray(images)) return [];
  return images
    .map(image => image.url || image.url_default || image.info_list?.[0]?.url || image)
    .filter(Boolean)
    .map(String);
}

function normalizeNote(raw, keyword, detail) {
  const note = detail && detail.id ? detail : raw;
  const id = note.id || raw.id || raw.note_id;
  const author = note.user || raw.user || {};
  return {
    id,
    platform: 'xiaohongshu',
    keyword,
    title: note.title || raw.title || '',
    content: note.desc || raw.desc || note.content || '',
    author: author.nickname || author.name || '',
    authorId: author.userid || author.user_id || '',
    type: note.type || raw.type || '',
    likeCount: normalizeCount(note.liked_count ?? raw.liked_count),
    collectCount: normalizeCount(note.collected_count ?? raw.collected_count),
    commentCount: normalizeCount(note.comments_count ?? raw.comments_count),
    shareCount: normalizeCount(note.shared_count ?? raw.shared_count),
    timestamp: note.timestamp || raw.timestamp || '',
    ipLocation: note.ip_location || raw.ip_location || '',
    url: id ? `https://www.xiaohongshu.com/explore/${id}` : '',
    tags: pickTags(note).length ? pickTags(note) : pickTags(raw),
    images: pickImages(note).length ? pickImages(note) : pickImages(raw),
    raw
  };
}

function normalizeComment(raw, noteId, keyword) {
  const user = raw.user_info || raw.user || {};
  return {
    id: raw.id || raw.comment_id || `${noteId}-${Math.random().toString(36).slice(2, 8)}`,
    noteId,
    platform: 'xiaohongshu',
    keyword,
    text: raw.content || raw.text || '',
    likeCount: normalizeCount(raw.like_count || raw.liked_count || raw.likeCount),
    author: user.nickname || user.name || '',
    authorId: user.user_id || user.userid || '',
    ipLocation: raw.ip_location || '',
    createTime: raw.create_time || '',
    subCommentCount: normalizeCount(raw.sub_comment_count || raw.subCommentCount),
    raw
  };
}

function keywordRows(keywordMatrix, keywordLimit) {
  return (keywordMatrix || [])
    .map(row => (typeof row === 'string' ? { keyword: row } : row))
    .filter(row => row && row.keyword)
    .slice(0, keywordLimit);
}

async function collectXiaohongshuDataset(options = {}) {
  const profile = options.profile || {};
  const keywordMatrix = options.keywordMatrix || keywordMatrixFromProfile(profile);
  const mode = options.mode || 'sample';
  const output = options.output ? path.resolve(options.output) : undefined;
  const fallbackSample = options.fallbackSample !== false;

  if (mode !== 'live') {
    const dataset = buildSampleDataset(profile, keywordMatrix);
    if (output) writeJson(output, dataset);
    return {
      status: 'ok',
      mode: 'sample',
      dataset,
      output,
      noteCount: dataset.notes.length,
      commentCount: dataset.comments.length,
      warnings: ['当前使用 P0 样例数据；需要真实采集时设置 collectionMode=live。']
    };
  }

  const errors = [];
  const warnings = [];
  const notes = [];
  const comments = [];
  const keywordLimit = Math.max(1, toInt(options.keywordLimit, 3));
  const searchPages = Math.max(1, toInt(options.searchPages, 1));
  const notesPerKeyword = Math.max(1, toInt(options.notesPerKeyword, 3));
  const maxCommentPages = Math.max(0, toInt(options.maxCommentPages, 1));
  const delayMs = Math.max(0, toInt(options.delayMs, 900));
  const sort = options.sort || 'popularity_descending';
  const noteType = options.noteType || '_0';

  let searchSkill;
  let detailSkill;
  let commentsSkill;
  try {
    searchSkill = loadSkillConfig('xiaohongshu-search-notes');
    detailSkill = loadSkillConfig('xiaohongshu-note-detail');
    commentsSkill = loadSkillConfig('xiaohongshu-note-comments');
  } catch (error) {
    if (!fallbackSample) throw error;
    const dataset = buildSampleDataset(profile, keywordMatrix);
    if (output) writeJson(output, dataset);
    return {
      status: 'fallback_sample',
      mode: 'sample',
      dataset,
      output,
      noteCount: dataset.notes.length,
      commentCount: dataset.comments.length,
      warnings: [`真实小红书采集配置不可用，已退回样例数据：${error.message}`],
      errors: [{ stage: 'load_config', message: error.message }]
    };
  }

  const rows = keywordRows(keywordMatrix, keywordLimit);
  for (const row of rows) {
    const keyword = row.keyword;
    const searched = [];
    for (let page = 1; page <= searchPages; page++) {
      const url = buildUrl(searchSkill.config, { keyword, page, sort, note_type: noteType });
      try {
        const response = await requestJson(url, searchSkill.config.endpoint.headers, searchSkill.config.retry || {});
        if (response.status !== 200) {
          errors.push({
            stage: 'search_notes',
            keyword,
            page,
            status: response.status,
            message: response.body.slice(0, 240)
          });
          continue;
        }
        const inner = unwrapData(response.json);
        const items = Array.isArray(inner.items) ? inner.items : [];
        searched.push(...items.map(item => item.note || item).filter(Boolean));
      } catch (error) {
        errors.push({ stage: 'search_notes', keyword, page, message: error.message });
      }
      if (delayMs) await sleep(delayMs);
    }

    const selected = uniqueBy(searched, item => item.id || item.note_id)
      .sort((a, b) => noteScore(b) - noteScore(a))
      .slice(0, notesPerKeyword);

    for (const rawNote of selected) {
      const noteId = rawNote.id || rawNote.note_id;
      if (!noteId) continue;
      let detailNote;
      try {
        const detailUrl = buildUrl(detailSkill.config, { note_id: noteId });
        const detailResponse = await requestJson(detailUrl, detailSkill.config.endpoint.headers, detailSkill.config.retry || {});
        if (detailResponse.status === 200) {
          const inner = unwrapData(detailResponse.json);
          detailNote = inner.note_list?.[0] || inner.note || undefined;
          const embeddedComments = Array.isArray(inner.comment_list) ? inner.comment_list : [];
          comments.push(...embeddedComments.map(comment => normalizeComment(comment, noteId, keyword)));
        } else {
          warnings.push(`笔记详情失败：${noteId} / HTTP ${detailResponse.status}`);
        }
      } catch (error) {
        warnings.push(`笔记详情失败：${noteId} / ${error.message}`);
      }
      notes.push(normalizeNote(rawNote, keyword, detailNote));
      if (delayMs) await sleep(delayMs);

      let cursor = '';
      for (let page = 1; page <= maxCommentPages; page++) {
        try {
          const commentsUrl = buildUrl(commentsSkill.config, { note_id: noteId, cursor });
          const commentsResponse = await requestJson(commentsUrl, commentsSkill.config.endpoint.headers, commentsSkill.config.retry || {});
          if (commentsResponse.status !== 200) {
            warnings.push(`评论采集失败：${noteId} / HTTP ${commentsResponse.status}`);
            break;
          }
          const inner = unwrapData(commentsResponse.json);
          const pageComments = Array.isArray(inner.comments) ? inner.comments : [];
          comments.push(...pageComments.map(comment => normalizeComment(comment, noteId, keyword)));
          if (!inner.has_more || !inner.cursor) break;
          cursor = inner.cursor;
        } catch (error) {
          warnings.push(`评论采集失败：${noteId} / ${error.message}`);
          break;
        }
        if (delayMs) await sleep(delayMs);
      }
    }
  }

  const dataset = {
    metadata: {
      platform: 'xiaohongshu',
      dataNature: 'live xiaohongshu trend collection',
      generatedAt: new Date().toISOString(),
      keywordLimit,
      searchPages,
      notesPerKeyword,
      maxCommentPages
    },
    notes: uniqueBy(notes, note => note.id),
    comments: uniqueBy(comments, comment => comment.id)
  };

  if (!dataset.notes.length && fallbackSample) {
    const sample = buildSampleDataset(profile, keywordMatrix);
    if (output) writeJson(output, sample);
    return {
      status: 'fallback_sample',
      mode: 'sample',
      dataset: sample,
      output,
      noteCount: sample.notes.length,
      commentCount: sample.comments.length,
      warnings: ['真实小红书采集未拿到有效笔记，已退回样例数据。', ...warnings],
      errors
    };
  }

  if (output) writeJson(output, dataset);
  return {
    status: errors.length ? 'partial' : 'ok',
    mode: 'live',
    dataset,
    output,
    noteCount: dataset.notes.length,
    commentCount: dataset.comments.length,
    warnings,
    errors
  };
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const output = path.resolve(args.output || 'outputs/industry-trend-intelligence/xiaohongshu-sample/raw-input.json');
  const profile = args.profile ? readJson(path.resolve(args.profile)) : {};
  const keywordMatrix = args['keyword-matrix'] ? readJson(path.resolve(args['keyword-matrix'])) : keywordMatrixFromProfile(profile);
  const mode = args.mode || args['collection-mode'] || args.collectionMode || (toBool(args.live) ? 'live' : 'sample');
  const result = await collectXiaohongshuDataset({
    profile,
    keywordMatrix,
    mode,
    output,
    fallbackSample: !toBool(args['no-fallback-sample']),
    keywordLimit: args['keyword-limit'] || args.keywordLimit,
    searchPages: args['search-pages'] || args.searchPages,
    notesPerKeyword: args['notes-per-keyword'] || args.notesPerKeyword,
    maxCommentPages: args['max-comment-pages'] || args.maxCommentPages,
    delayMs: args['delay-ms'] || args.delayMs,
    sort: args.sort,
    noteType: args['note-type'] || args.noteType
  });

  const prefix = args['result-prefix'] || args.resultPrefix || DEFAULT_RESULT_PREFIX;
  console.log(`${prefix}=${JSON.stringify({
    status: result.status,
    mode: result.mode,
    output: result.output,
    noteCount: result.noteCount,
    commentCount: result.commentCount,
    warningCount: (result.warnings || []).length,
    errorCount: (result.errors || []).length,
    warnings: result.warnings || [],
    errors: result.errors || []
  })}`);
}

if (require.main === module) {
  main().catch(error => {
    console.error(error && error.stack ? error.stack : String(error));
    process.exit(1);
  });
}

module.exports = {
  collectXiaohongshuDataset
};