openclaw-voc-skill/tools/collect_centrum_douyin_live.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { collectLiveDataset } = require('../claude-code/claude-code-voc-intelligence/mcp/src/features/douyin-trend/live-collector');

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token.startsWith('--')) continue;
    const eq = token.indexOf('=');
    if (eq > 0) {
      args[token.slice(2, eq)] = token.slice(eq + 1);
      continue;
    }
    const key = token.slice(2);
    const next = argv[i + 1];
    if (next && !next.startsWith('--')) {
      args[key] = next;
      i += 1;
    } else {
      args[key] = true;
    }
  }
  return args;
}

function splitList(value) {
  if (!value) return [];
  if (Array.isArray(value)) return value;
  return String(value)
    .split(/[,，、\n\r]+/)
    .map(item => item.trim())
    .filter(Boolean);
}

function numberArg(value, fallback) {
  const number = Number(value);
  return Number.isFinite(number) && number >= 0 ? Math.round(number) : fallback;
}

function videoKey(video) {
  return String(video.id || video.videoId || video.url || '').trim();
}

function commentKey(comment) {
  return String(comment.id || comment.cid || `${comment.videoId}:${comment.text}`).trim();
}

function dedupeBy(items, keyFn) {
  const map = new Map();
  for (const item of items) {
    const key = keyFn(item);
    if (!key) continue;
    if (!map.has(key)) map.set(key, item);
  }
  return [...map.values()];
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const token = process.env.VOC_TOKEN || process.env.VOC_DOUYIN_TOKEN || process.env.DOUYIN_TOKEN || '';
  if (!token) {
    throw new Error('VOC_TOKEN is required in process env.');
  }

  const output = path.resolve(args.output || 'outputs/centrum_douyin_audit_2026-06-08/live-full/live-raw-dataset.json');
  const keywords = splitList(args.keywords);
  if (!keywords.length) throw new Error('--keywords is required.');

  const profile = {
    project: args.project || 'Centrum Douyin VOC audit',
    industry: args.industry || 'nutrition supplements',
    targetAudience: splitList(args.audience || 'adults,30+,silver age,parents'),
    keywords,
    mustTrackSignals: ['price', 'authenticity', 'version', 'effect', 'energy', 'brain', 'pregnancy', 'dosage', 'ingredients']
  };

  const startedAt = new Date().toISOString();
  const result = await collectLiveDataset({
    profile,
    token,
    options: {
      keywordLimit: numberArg(args.keywordLimit || args['keyword-limit'], keywords.length),
      videosPerKeyword: numberArg(args.videosPerKeyword || args['videos-per-keyword'], 10),
      maxCommentPages: numberArg(args.maxCommentPages || args['max-comment-pages'], 3),
      sortType: args.sortType || args.sort || args['sort-type'] || '1',
      publishTime: args.publishTime || args['publish-time'] || '180',
      filterDuration: args.filterDuration || args['filter-duration'] || '0',
      contentType: args.contentType || args['content-type'] || '1',
      baseUrl: args.baseUrl || args['base-url'] || process.env.DOUYIN_API_BASE_URL || process.env.VOC_DOUYIN_API_BASE_URL
    }
  });

  const videos = dedupeBy(result.videos || [], videoKey);
  const comments = dedupeBy(result.comments || [], commentKey);
  const doc = {
    generatedAt: new Date().toISOString(),
    startedAt,
    profile,
    requested: {
      keywordCount: keywords.length,
      keywordLimit: numberArg(args.keywordLimit || args['keyword-limit'], keywords.length),
      videosPerKeyword: numberArg(args.videosPerKeyword || args['videos-per-keyword'], 10),
      maxCommentPages: numberArg(args.maxCommentPages || args['max-comment-pages'], 3)
    },
    effectiveKeywords: result.keywords || [],
    summary: {
      videoCount: videos.length,
      commentCount: comments.length,
      warningCount: (result.warnings || []).length,
      errorCount: (result.errors || []).length
    },
    videos,
    comments,
    warnings: result.warnings || [],
    errors: result.errors || []
  };

  fs.mkdirSync(path.dirname(output), { recursive: true });
  fs.writeFileSync(output, JSON.stringify(doc, null, 2), 'utf8');
  console.log(JSON.stringify({
    status: 'ok',
    output,
    summary: doc.summary,
    effectiveKeywordCount: doc.effectiveKeywords.length
  }, null, 2));
}

main().catch(error => {
  console.error(JSON.stringify({ status: 'error', message: error.message }, null, 2));
  process.exit(1);
});