openclaw-voc-skill/claude-code/claude-code-voc-intelligence/mcp/src/features/xiaohongshu-trend/live-collector.js

const { XiaohongshuApi } = require('../../providers/xiaohongshu-api');

function normalizeCount(value) {
  if (value === undefined || value === null || value === '') return 0;
  if (typeof value === 'number') return Number.isFinite(value) ? value : 0;
  const text = String(value).trim().toLowerCase();
  const match = text.match(/([\d.]+)/);
  if (!match) return 0;
  const base = Number.parseFloat(match[1]);
  if (!Number.isFinite(base)) return 0;
  if (text.includes('w') || text.includes('万')) return Math.round(base * 10000);
  if (text.includes('k') || text.includes('千')) return Math.round(base * 1000);
  return Math.round(base);
}

function noteScore(note) {
  return normalizeCount(note.likeCount) +
    normalizeCount(note.collectCount) * 1.5 +
    normalizeCount(note.commentCount) * 3 +
    normalizeCount(note.shareCount) * 2;
}

function pickTags(note) {
  const tags = note.tag_info || note.tagList || note.tags || [];
  if (!Array.isArray(tags)) return [];
  return tags
    .map(tag => tag.name || tag.tag_name || tag.title || tag)
    .filter(Boolean)
    .map(String);
}

function pickFirstString(...values) {
  return values
    .flat()
    .filter(value => typeof value === 'string')
    .map(value => value.trim())
    .find(Boolean) || '';
}

function pickFirstVerifiedUrl(...values) {
  return values
    .flat()
    .filter(value => typeof value === 'string')
    .map(value => value.trim())
    .find(value => value && hasXsecToken(value)) || '';
}

function nestedValue(object, path) {
  return path.split('.').reduce((current, key) => (
    current && typeof current === 'object' ? current[key] : undefined
  ), object);
}

function hasXsecToken(url) {
  return /[?&]xsec_token=/i.test(String(url || ''));
}

function pickWebUrl(note = {}, raw = {}) {
  const candidates = [
    'mini_program_info.webpage_url',
    'note_card.mini_program_info.webpage_url',
    'share_info.link',
    'note_card.share_info.link',
    'shareLink',
    'share_url',
    'shareUrl',
    'link',
    'webpage_url',
    'webUrl',
    'url'
  ].flatMap(field => [nestedValue(note, field), nestedValue(raw, field)]);
  const withXsecToken = candidates.find(hasXsecToken);
  return withXsecToken || pickFirstString(candidates);
}

function looksLikeImageUrl(value) {
  return /^https?:\/\//i.test(String(value || ''));
}

function collectImageUrlsFromValue(value, result = []) {
  if (!value) return result;
  if (typeof value === 'string') {
    if (looksLikeImageUrl(value) && !result.includes(value)) result.push(value);
    return result;
  }
  if (Array.isArray(value)) {
    value.forEach(item => collectImageUrlsFromValue(item, result));
    return result;
  }
  if (typeof value !== 'object') return result;

  [
    'url',
    'url_default',
    'url_pre',
    'original_url',
    'original',
    'large',
    'large_url',
    'middle',
    'middle_url',
    'thumbnail',
    'image_url',
    'imageUrl',
    'cover',
    'cover_url',
    'coverUrl',
    'poster',
    'poster_url',
    'posterUrl',
    'master_url'
  ].forEach(key => collectImageUrlsFromValue(value[key], result));

  [
    'url_list',
    'urlList',
    'urlListHigh',
    'info_list',
    'infoList',
    'images',
    'image_list',
    'imageList',
    'image_info_list',
    'imageInfoList',
    'image_infos',
    'imageInfos',
    'display_image_info',
    'displayImageInfo',
    'image',
    'image_info',
    'imageInfo',
    'multiMedia',
    'multimedia'
  ].forEach(key => collectImageUrlsFromValue(value[key], result));

  return result;
}

function extractImageUrls(note = {}, raw = {}) {
  const result = [];
  [
    note.cover,
    raw.cover,
    note.note_card && note.note_card.cover,
    raw.note_card && raw.note_card.cover,
    note.note_card && note.note_card.display_image_info,
    raw.note_card && raw.note_card.display_image_info,
    note.note_card && note.note_card.image_info_list,
    raw.note_card && raw.note_card.image_info_list,
    note.image_list,
    raw.image_list,
    note.image_info_list,
    raw.image_info_list,
    note.display_image_info,
    raw.display_image_info,
    note.cover_info,
    raw.cover_info,
    note.images_list,
    raw.images_list,
    note.imageList,
    raw.imageList,
    note.note_card && note.note_card.image_list,
    raw.note_card && raw.note_card.image_list
  ].forEach(value => collectImageUrlsFromValue(value, result));
  return result;
}

function contentExcerpt(text, limit = 120) {
  const value = String(text || '').replace(/\s+/g, ' ').trim();
  if (value.length <= limit) return value;
  return `${value.slice(0, limit - 1)}...`;
}

function normalizeNote(raw, keyword, detail) {
  const note = detail && (detail.id || detail.note_id) ? detail : raw;
  const id = note.id || note.note_id || raw.id || raw.note_id;
  const user = note.user || raw.user || note.user_info || raw.user_info || {};
  const accessibleUrl = pickWebUrl(note, raw);
  const verifiedUrl = pickFirstVerifiedUrl(accessibleUrl);
  const fallbackUrl = id ? `https://www.xiaohongshu.com/discovery/item/${id}` : '';
  const imageUrls = extractImageUrls(note, raw);
  const content = note.desc || raw.desc || note.content || raw.content || '';
  return {
    id,
    platform: 'xiaohongshu',
    keyword,
    title: note.title || raw.title || '',
    content,
    contentExcerpt: contentExcerpt(content),
    rawExcerpt: contentExcerpt(content),
    author: user.nickname || user.name || '',
    authorId: user.userid || user.user_id || '',
    type: note.type || raw.type || '',
    likeCount: normalizeCount(note.liked_count ?? raw.liked_count ?? note.likeCount ?? raw.likeCount),
    collectCount: normalizeCount(note.collected_count ?? raw.collected_count ?? note.collectCount ?? raw.collectCount),
    commentCount: normalizeCount(note.comments_count ?? raw.comments_count ?? note.commentCount ?? raw.commentCount),
    shareCount: normalizeCount(note.shared_count ?? raw.shared_count ?? note.shareCount ?? raw.shareCount),
    timestamp: note.timestamp || raw.timestamp || '',
    url: verifiedUrl,
    webUrl: verifiedUrl,
    shareUrl: pickFirstString(
      nestedValue(note, 'share_info.link'),
      nestedValue(raw, 'share_info.link'),
      nestedValue(note, 'note_card.share_info.link'),
      nestedValue(raw, 'note_card.share_info.link')
    ),
    sourceUrl: verifiedUrl,
    sourceUrlVerified: Boolean(verifiedUrl),
    candidateUrl: accessibleUrl || fallbackUrl,
    fallbackUrl,
    coverUrl: imageUrls[0] || '',
    imageUrls,
    tags: pickTags(note).length ? pickTags(note) : pickTags(raw)
  };
}

function normalizeComment(raw, noteId, keyword) {
  const user = raw.user_info || raw.user || {};
  return {
    id: raw.id || raw.comment_id || `${noteId}-${Math.random().toString(36).slice(2, 8)}`,
    noteId,
    platform: 'xiaohongshu',
    keyword,
    text: raw.content || raw.text || '',
    likeCount: normalizeCount(raw.like_count || raw.liked_count || raw.likeCount),
    author: user.nickname || user.name || '',
    authorId: user.user_id || user.userid || '',
    createTime: raw.create_time || '',
    ipLocation: raw.ip_location || '',
    subCommentCount: normalizeCount(raw.sub_comment_count || raw.subCommentCount)
  };
}

function extractSearchNotes(data) {
  const items = data.items || data.note_list || data.notes || [];
  if (!Array.isArray(items)) return [];
  return items
    .map(item => item.note || item)
    .filter(note => note && (note.id || note.note_id));
}

function extractDetailNote(data) {
  const list = data.note_list || data.notes || [];
  return Array.isArray(list) && list[0] ? list[0] : data.note || data;
}

function extractComments(data) {
  const comments = data.comments || data.comment_list || [];
  return Array.isArray(comments) ? comments : [];
}

async function collectLiveDataset({ profile, token, options = {} }) {
  const api = new XiaohongshuApi({ token, baseUrl: options.baseUrl });
  const keywordLimit = Math.max(1, Number(options.keywordLimit || 1));
  const notesPerKeyword = Math.max(1, Number(options.notesPerKeyword || 1));
  const maxCommentPages = Math.max(0, Number(options.maxCommentPages ?? 1));
  const keywords = (profile.keywords || []).slice(0, keywordLimit);
  const notes = [];
  const comments = [];
  const warnings = [];
  const errors = [];

  for (const keyword of keywords) {
    try {
      const search = await api.searchNotes({
        keyword,
        page: 1,
        sort: options.sort || 'popularity_descending',
        noteType: options.noteType || '_0'
      });
      const rawNotes = extractSearchNotes(search.data)
        .map(raw => normalizeNote(raw, keyword))
        .sort((a, b) => noteScore(b) - noteScore(a))
        .slice(0, notesPerKeyword);

      for (const rawNote of rawNotes) {
        let note = rawNote;
        try {
          const detail = await api.getNoteDetail({ noteId: rawNote.id });
          note = normalizeNote(rawNote, keyword, extractDetailNote(detail.data));
        } catch (error) {
          warnings.push({
            stage: 'note_detail',
            keyword,
            noteId: rawNote.id,
            message: error.message,
            httpStatus: error.httpStatus
          });
        }
        notes.push(note);

        let cursor = '';
        for (let page = 0; page < maxCommentPages; page++) {
          try {
            const commentResult = await api.getNoteComments({ noteId: note.id, cursor });
            extractComments(commentResult.data).forEach(rawComment => {
              comments.push(normalizeComment(rawComment, note.id, keyword));
            });
            cursor = commentResult.data.cursor || '';
            if (!commentResult.data.has_more || !cursor) break;
          } catch (error) {
            warnings.push({
              stage: 'note_comments',
              keyword,
              noteId: note.id,
              message: error.message,
              httpStatus: error.httpStatus
            });
            break;
          }
        }
      }
    } catch (error) {
      errors.push({
        stage: 'search_notes',
        keyword,
        message: error.message,
        httpStatus: error.httpStatus
      });
    }
  }

  return {
    notes,
    comments,
    warnings,
    errors
  };
}

module.exports = {
  collectLiveDataset,
  normalizeNote,
  pickWebUrl,
  extractImageUrls,
  hasXsecToken
};