openclaw-voc-skill/scripts/tools/voc-data-normalizer.js

const fs = require('fs');
const path = require('path');

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i++) {
    const token = argv[i];
    if (!token.startsWith('--')) continue;
    const eq = token.indexOf('=');
    if (eq >= 0) {
      args[token.slice(2, eq)] = token.slice(eq + 1);
    } else {
      const key = token.slice(2);
      const next = argv[i + 1];
      if (next && !next.startsWith('--')) {
        args[key] = next;
        i++;
      } else {
        args[key] = true;
      }
    }
  }
  return args;
}

function usage() {
  return [
    'Usage:',
    '  node scripts/tools/voc-data-normalizer.js --input <raw-dir> [--output <out-dir>] [--project <name>] [--category <name>]',
    '',
    'Outputs:',
    '  _merged.json',
    '  comments-flat.jsonl',
    '  normalizer-audit.json'
  ].join('\n');
}

function readJson(filePath) {
  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
}

function readJsonl(filePath) {
  return fs.readFileSync(filePath, 'utf8')
    .split(/\r?\n/)
    .map(line => line.trim())
    .filter(Boolean)
    .map(line => JSON.parse(line));
}

function ensureDir(dirPath) {
  fs.mkdirSync(dirPath, { recursive: true });
}

function listDataFiles(inputDir) {
  return fs.readdirSync(inputDir)
    .filter(name => name.endsWith('.json') || name.endsWith('.jsonl'))
    .filter(name => !['_merged.json', 'normalizer-audit.json'].includes(name))
    .filter(name => !['comments-flat.jsonl', 'comments-flat.normalized.jsonl'].includes(name))
    .map(name => path.join(inputDir, name));
}

function asArray(value) {
  if (!value) return [];
  return Array.isArray(value) ? value : [value];
}

function firstDefined(...values) {
  for (const value of values) {
    if (value !== undefined && value !== null && value !== '') return value;
  }
  return undefined;
}

function toNumber(value) {
  if (value === undefined || value === null || value === '') return undefined;
  const number = Number(value);
  return Number.isFinite(number) ? number : undefined;
}

function normalizeTime(value) {
  if (value === undefined || value === null || value === '') return undefined;
  if (typeof value === 'string') return value;
  if (typeof value !== 'number') return String(value);
  const ms = value > 100000000000 ? value : value * 1000;
  return new Date(ms).toISOString();
}

function cleanText(value) {
  if (value === undefined || value === null) return '';
  return String(value).replace(/\s+/g, ' ').trim();
}

function uniq(values) {
  return Array.from(new Set(values.filter(value => value !== undefined && value !== null && value !== '')));
}

function makeId(prefix, value, fallbackIndex) {
  const base = firstDefined(value, `${prefix}_${fallbackIndex}`);
  return `${prefix}_${String(base).replace(/[^a-zA-Z0-9_-]+/g, '_')}`;
}

function sourceUrl(record) {
  return firstDefined(record.url, record.sourceUrl, record.link, record.share_url, record.ReviewsLink, record.detailUrl);
}

function authorName(record) {
  if (record && typeof record === 'object' && !Array.isArray(record)) {
    const nested = firstDefined(
      record.nickname,
      record.name,
      record.user?.nickname,
      record.user_info?.nickname,
      record.author?.nickname,
      record.author?.name,
      record.ConsumerName,
      record.shop_name
    );
    if (nested !== undefined) return nested;
  }
  return firstDefined(
    record.author,
    record.nickname,
    record.user?.nickname,
    record.user_info?.nickname,
    record.user?.unique_id,
    record.ConsumerName,
    record.shop_name
  );
}

function xhsNoteToItem(note, source, index) {
  const metrics = note.metrics || {};
  return {
    id: makeId('xhs_note', firstDefined(note.noteId, note.id, note.note_id), index),
    platform: 'xiaohongshu',
    sourceType: firstDefined(note.type, 'note'),
    keyword: firstDefined(note.keyword, source.keyword, source.metadata?.keyword),
    batch: firstDefined(note.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(note.hypothesisTags, source.hypothesisTags)),
    author: authorName(note),
    title: note.title,
    content: cleanText(firstDefined(note.desc, note.content, note.text, note.title)),
    likeCount: toNumber(firstDefined(metrics.likeCount, note.likeCount, note.liked_count)),
    commentCount: toNumber(firstDefined(metrics.commentCount, note.commentCount, note.comments_count)),
    publishTime: normalizeTime(firstDefined(note.publishedAt, note.publishTime, note.timestamp, note.create_time)),
    collectedAt: firstDefined(note.collectedAt, source.metadata?.generatedAt, new Date().toISOString()),
    url: sourceUrl(note),
    productId: firstDefined(note.noteId, note.id, note.note_id),
    brand: note.brand,
    competitor: note.competitor,
    sentiment: note.sentiment,
    raw: note
  };
}

function xhsCommentToVoc(comment, source, index) {
  const user = comment.user || comment.user_info || {};
  return {
    id: makeId('xhs_comment', firstDefined(comment.commentId, comment.id, comment.comment_id), index),
    platform: 'xiaohongshu',
    sourceType: 'comment',
    keyword: firstDefined(comment.keyword, source.keyword, source.metadata?.keyword),
    batch: firstDefined(comment.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(comment.hypothesisTags, source.hypothesisTags)),
    author: firstDefined(comment.author, user.nickname, user.name),
    text: cleanText(firstDefined(comment.content, comment.text, comment.desc)),
    likeCount: toNumber(firstDefined(comment.likeCount, comment.like_count, comment.digg_count)),
    replyCount: toNumber(firstDefined(comment.subCommentCount, comment.sub_comment_count, comment.reply_comment_total)),
    rating: toNumber(comment.rating),
    publishTime: normalizeTime(firstDefined(comment.createdAt, comment.create_time, comment.publishTime)),
    collectedAt: firstDefined(comment.collectedAt, source.metadata?.generatedAt, new Date().toISOString()),
    url: sourceUrl(comment),
    parentId: firstDefined(comment.noteId, comment.note_id, comment.parentId),
    commentId: firstDefined(comment.commentId, comment.id, comment.comment_id),
    ipLocation: firstDefined(comment.ipLocation, comment.ip_location, comment.ip_label),
    sentiment: comment.sentiment,
    theme: comment.theme,
    raw: comment
  };
}

function douyinVideoToItem(video, source, index) {
  const stats = video.statistics || video.stats || {};
  return {
    id: makeId('douyin_video', firstDefined(video.aweme_id, video.videoId, video.id), index),
    platform: 'douyin',
    sourceType: 'video',
    keyword: firstDefined(video.keyword, source.keyword, source.metadata?.keyword),
    batch: firstDefined(video.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(video.hypothesisTags, source.hypothesisTags)),
    author: authorName(video.author || video),
    title: firstDefined(video.title, video.desc),
    content: cleanText(firstDefined(video.desc, video.title, video.text)),
    likeCount: toNumber(firstDefined(stats.digg_count, video.digg_count, video.likeCount)),
    commentCount: toNumber(firstDefined(stats.comment_count, video.comment_count, video.commentCount)),
    publishTime: normalizeTime(firstDefined(video.create_time, video.publishedAt, video.publishTime)),
    collectedAt: firstDefined(video.collectedAt, source.metadata?.generatedAt, new Date().toISOString()),
    url: sourceUrl(video),
    productId: firstDefined(video.aweme_id, video.videoId, video.id),
    brand: video.brand,
    competitor: video.competitor,
    sentiment: video.sentiment,
    raw: video
  };
}

function douyinCommentToVoc(comment, source, index) {
  return {
    id: makeId('douyin_comment', firstDefined(comment.cid, comment.commentId, comment.id), index),
    platform: 'douyin',
    sourceType: 'comment',
    keyword: firstDefined(comment.keyword, source.keyword, source.metadata?.keyword),
    batch: firstDefined(comment.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(comment.hypothesisTags, source.hypothesisTags)),
    author: authorName(comment),
    text: cleanText(firstDefined(comment.text, comment.content, comment.desc)),
    likeCount: toNumber(firstDefined(comment.digg_count, comment.likeCount, comment.like_count)),
    replyCount: toNumber(firstDefined(comment.reply_comment_total, comment.replyCount, comment.sub_comment_count)),
    rating: undefined,
    publishTime: normalizeTime(firstDefined(comment.create_time, comment.createdAt, comment.publishTime)),
    collectedAt: firstDefined(comment.collectedAt, source.metadata?.generatedAt, new Date().toISOString()),
    url: sourceUrl(comment),
    parentId: firstDefined(comment.aweme_id, comment.videoId, comment.parentId),
    commentId: firstDefined(comment.cid, comment.commentId, comment.id),
    ipLocation: firstDefined(comment.ip_label, comment.ipLocation, comment.ip_location),
    sentiment: comment.sentiment,
    theme: comment.theme,
    raw: comment
  };
}

function amazonReviewToVoc(review, source, index) {
  return {
    id: makeId('amazon_review', firstDefined(review.ReviewId, review.reviewId, review.ReviewsLink, review.Id), index),
    platform: 'amazon',
    sourceType: 'review',
    keyword: firstDefined(review.keyword, source.keyword, source.ASIN, review.Asin, review.ASIN),
    batch: firstDefined(review.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(review.hypothesisTags, source.hypothesisTags)),
    author: firstDefined(review.ConsumerName, review.author, review.nickname),
    text: cleanText([review.Title, review.Content, review.content, review.text].filter(Boolean).join(' ')),
    likeCount: toNumber(firstDefined(review.Helpful, review.likeCount)),
    replyCount: undefined,
    rating: toNumber(firstDefined(review.Star, review.rating)),
    publishTime: normalizeTime(firstDefined(review.ReviewsDate, review.publishTime, review.date)),
    collectedAt: firstDefined(review.collectedAt, source.metadata?.generatedAt, review.UpdateTime, new Date().toISOString()),
    url: sourceUrl(review),
    parentId: firstDefined(review.Asin, review.ASIN, source.ASIN),
    commentId: firstDefined(review.ReviewId, review.reviewId, review.ReviewsLink),
    ipLocation: firstDefined(review.ReviewedCountry, review.ipLocation),
    sentiment: review.sentiment,
    theme: review.theme,
    raw: review
  };
}

function amazonProductToItem(product, source, index) {
  return {
    id: makeId('amazon_product', firstDefined(product.Asin, product.ASIN, product.asin, product.productId), index),
    platform: 'amazon',
    sourceType: 'product',
    keyword: firstDefined(product.keyword, source.keyword, source.ASIN, product.Asin, product.ASIN),
    batch: firstDefined(product.batch, source.batch, 'P0'),
    hypothesisTags: asArray(firstDefined(product.hypothesisTags, source.hypothesisTags)),
    author: firstDefined(product.Brand, product.brand, product.shop_name),
    title: firstDefined(product.Title, product.title, product.productTitle),
    content: cleanText(firstDefined(product.Description, product.description, product.Title, product.title)),
    likeCount: undefined,
    commentCount: toNumber(firstDefined(product.ReviewsCount, product.reviewCount, product.commentCount)),
    rating: toNumber(firstDefined(product.Rating, product.rating)),
    publishTime: normalizeTime(firstDefined(product.UpdateTime, product.publishTime)),
    collectedAt: firstDefined(product.collectedAt, source.metadata?.generatedAt, new Date().toISOString()),
    url: sourceUrl(product),
    productId: firstDefined(product.Asin, product.ASIN, product.asin, product.productId),
    brand: firstDefined(product.Brand, product.brand),
    competitor: product.competitor,
    sentiment: product.sentiment,
    raw: product
  };
}

function collectXiaohongshu(data, filePath) {
  const items = [];
  const comments = [];
  if (Array.isArray(data.notes)) {
    data.notes.forEach((note, index) => items.push(xhsNoteToItem(note, data, index + 1)));
  }
  if (Array.isArray(data.commentsByNote)) {
    data.commentsByNote.forEach(group => {
      asArray(group.comments).forEach((comment, index) => {
        comments.push(xhsCommentToVoc({ ...comment, noteId: firstDefined(comment.noteId, group.noteId), keyword: firstDefined(comment.keyword, group.keyword) }, data, comments.length + index + 1));
      });
    });
  }
  if (Array.isArray(data.comments)) {
    data.comments.forEach((comment, index) => comments.push(xhsCommentToVoc(comment, data, index + 1)));
  }
  return { items, comments, source: sourceRecord(filePath, 'xiaohongshu', items.length, comments.length) };
}

function collectDouyin(data, filePath) {
  const items = [];
  const comments = [];
  const videos = asArray(firstDefined(data.videos, data.aweme_list, data.data?.aweme_list, data.data?.videos));
  videos.forEach((video, index) => items.push(douyinVideoToItem(video, data, index + 1)));
  const rawComments = asArray(firstDefined(data.comments, data.data?.comments));
  rawComments.forEach((comment, index) => comments.push(douyinCommentToVoc(comment, data, index + 1)));
  return { items, comments, source: sourceRecord(filePath, 'douyin', items.length, comments.length) };
}

function collectAmazon(data, filePath) {
  const items = [];
  const comments = [];
  const products = asArray(firstDefined(data.Products, data.products, data.Items, data.items));
  products.forEach((product, index) => items.push(amazonProductToItem(product, data, index + 1)));
  const reviews = asArray(firstDefined(data.Reviews, data.reviews, data.data?.Reviews, data.data?.reviews));
  reviews.forEach((review, index) => comments.push(amazonReviewToVoc(review, data, index + 1)));
  return { items, comments, source: sourceRecord(filePath, 'amazon', items.length, comments.length) };
}

function collectJsonl(records, filePath) {
  const items = [];
  const comments = [];
  records.forEach((record, index) => {
    const platform = String(firstDefined(record.platform, '')).toLowerCase();
    if (platform.includes('xiaohongshu') || platform === 'xhs') comments.push(xhsCommentToVoc(record, {}, index + 1));
    else if (platform.includes('douyin')) comments.push(douyinCommentToVoc(record, {}, index + 1));
    else if (platform.includes('amazon')) comments.push(amazonReviewToVoc(record, {}, index + 1));
    else comments.push(genericVoc(record, index + 1));
  });
  return { items, comments, source: sourceRecord(filePath, 'jsonl', items.length, comments.length) };
}

function genericVoc(record, index) {
  return {
    id: makeId('voc', firstDefined(record.id, record.commentId, record.reviewId), index),
    platform: firstDefined(record.platform, 'unknown'),
    sourceType: firstDefined(record.sourceType, 'comment'),
    keyword: record.keyword,
    batch: firstDefined(record.batch, 'P0'),
    hypothesisTags: asArray(record.hypothesisTags),
    author: authorName(record),
    text: cleanText(firstDefined(record.text, record.content, record.comment)),
    likeCount: toNumber(firstDefined(record.likeCount, record.like_count, record.digg_count, record.Helpful)),
    replyCount: toNumber(firstDefined(record.replyCount, record.subCommentCount, record.reply_comment_total)),
    rating: toNumber(firstDefined(record.rating, record.Star)),
    publishTime: normalizeTime(firstDefined(record.publishTime, record.createdAt, record.create_time, record.ReviewsDate)),
    collectedAt: firstDefined(record.collectedAt, new Date().toISOString()),
    url: sourceUrl(record),
    parentId: firstDefined(record.parentId, record.noteId, record.videoId, record.Asin, record.ASIN),
    commentId: firstDefined(record.commentId, record.id, record.reviewId, record.ReviewId),
    ipLocation: firstDefined(record.ipLocation, record.ip_location, record.ip_label, record.ReviewedCountry),
    sentiment: record.sentiment,
    theme: record.theme,
    raw: record
  };
}

function sourceRecord(filePath, platform, itemCount, commentCount) {
  return {
    file: path.basename(filePath),
    platform,
    itemCount,
    commentCount
  };
}

function inferCollector(data, filePath) {
  const name = path.basename(filePath).toLowerCase();
  const platform = String(firstDefined(data.metadata?.platform, data.platform, '')).toLowerCase();
  if (name.endsWith('.jsonl')) return collectJsonl(data, filePath);
  if (platform.includes('xiaohongshu') || platform === 'xhs' || name.includes('xhs') || name.includes('xiaohongshu') || Array.isArray(data.notes) || Array.isArray(data.commentsByNote)) return collectXiaohongshu(data, filePath);
  if (platform.includes('douyin') || name.includes('douyin') || data.data?.comments || data.data?.aweme_list) return collectDouyin(data, filePath);
  if (platform.includes('amazon') || name.includes('amazon') || Array.isArray(data.Reviews) || Array.isArray(data.reviews) || data.ASIN || data.TotalCount !== undefined) return collectAmazon(data, filePath);
  return { items: [], comments: [], source: sourceRecord(filePath, 'unknown', 0, 0) };
}

function dedupe(records, keyFn) {
  const seen = new Map();
  const duplicates = [];
  const output = [];
  records.forEach(record => {
    const key = keyFn(record);
    if (seen.has(key)) {
      duplicates.push({ key, keptId: seen.get(key).id, duplicateId: record.id });
      return;
    }
    seen.set(key, record);
    output.push(record);
  });
  return { output, duplicates };
}

function missingRequired(record, fields) {
  return fields.filter(field => {
    const value = record[field];
    if (Array.isArray(value)) return value.length === 0;
    return value === undefined || value === null || value === '';
  });
}

function main() {
  const args = parseArgs(process.argv.slice(2));
  if (args.help || !args.input) {
    console.log(usage());
    process.exit(args.help ? 0 : 1);
  }

  const inputDir = path.resolve(args.input);
  const outputDir = path.resolve(args.output || inputDir);
  if (!fs.existsSync(inputDir) || !fs.statSync(inputDir).isDirectory()) {
    throw new Error(`input directory not found: ${inputDir}`);
  }
  ensureDir(outputDir);

  const files = listDataFiles(inputDir);
  const sources = [];
  const items = [];
  const comments = [];
  const errors = [];

  files.forEach(filePath => {
    try {
      const data = filePath.endsWith('.jsonl') ? readJsonl(filePath) : readJson(filePath);
      const collected = inferCollector(data, filePath);
      sources.push(collected.source);
      items.push(...collected.items);
      comments.push(...collected.comments);
    } catch (error) {
      errors.push({ file: path.basename(filePath), message: error.message });
    }
  });

  const itemDedupe = dedupe(items.filter(item => cleanText(item.content)), item => `${item.platform}|${item.id}|${cleanText(item.content).slice(0, 120)}`);
  const commentDedupe = dedupe(comments.filter(comment => cleanText(comment.text)), comment => `${comment.platform}|${firstDefined(comment.commentId, comment.id)}|${cleanText(comment.text).slice(0, 120)}`);

  const normalizedItems = itemDedupe.output;
  const normalizedComments = commentDedupe.output;
  const missing = {
    items: normalizedItems.map(item => ({ id: item.id, missing: missingRequired(item, ['id', 'platform', 'sourceType', 'keyword', 'batch', 'hypothesisTags', 'content', 'collectedAt']) })).filter(row => row.missing.length),
    comments: normalizedComments.map(comment => ({ id: comment.id, missing: missingRequired(comment, ['id', 'platform', 'sourceType', 'keyword', 'batch', 'hypothesisTags', 'text', 'collectedAt']) })).filter(row => row.missing.length)
  };

  const platforms = uniq([...normalizedItems.map(item => item.platform), ...normalizedComments.map(comment => comment.platform)]);
  const keywords = uniq([...normalizedItems.map(item => item.keyword), ...normalizedComments.map(comment => comment.keyword)]);
  const batches = uniq([...normalizedItems.map(item => item.batch), ...normalizedComments.map(comment => comment.batch)]);
  const hypothesisTags = uniq([...normalizedItems.flatMap(item => item.hypothesisTags), ...normalizedComments.flatMap(comment => comment.hypothesisTags)]);

  const audit = {
    generatedAt: new Date().toISOString(),
    inputDir,
    outputDir,
    files: files.map(file => path.basename(file)),
    sources,
    rawItemCount: items.length,
    rawVocCount: comments.length,
    validItemCount: normalizedItems.length,
    validVocCount: normalizedComments.length,
    duplicateItemCount: itemDedupe.duplicates.length,
    duplicateVocCount: commentDedupe.duplicates.length,
    emptyItemDroppedCount: items.length - items.filter(item => cleanText(item.content)).length,
    emptyVocDroppedCount: comments.length - comments.filter(comment => cleanText(comment.text)).length,
    missingRequired: missing,
    errors,
    dedupeRule: 'platform + id/commentId + first 120 normalized text characters'
  };

  const merged = {
    metadata: {
      project: firstDefined(args.project, path.basename(inputDir)),
      category: firstDefined(args.category, ''),
      generatedAt: audit.generatedAt,
      platforms,
      keywords,
      batches,
      hypothesisTags,
      rawSampleCount: items.length + comments.length,
      rawItemCount: items.length,
      rawVocCount: comments.length,
      itemCount: normalizedItems.length,
      validVocCount: normalizedComments.length,
      dedupeRule: audit.dedupeRule
    },
    sources,
    items: normalizedItems,
    comments: normalizedComments,
    audit: {
      duplicateItemCount: audit.duplicateItemCount,
      duplicateVocCount: audit.duplicateVocCount,
      emptyItemDroppedCount: audit.emptyItemDroppedCount,
      emptyVocDroppedCount: audit.emptyVocDroppedCount,
      missingRequiredCount: missing.items.length + missing.comments.length,
      errorCount: errors.length
    }
  };

  fs.writeFileSync(path.join(outputDir, '_merged.json'), JSON.stringify(merged, null, 2) + '\n', 'utf8');
  fs.writeFileSync(path.join(outputDir, 'comments-flat.jsonl'), normalizedComments.map(comment => JSON.stringify(comment)).join('\n') + (normalizedComments.length ? '\n' : ''), 'utf8');
  fs.writeFileSync(path.join(outputDir, 'normalizer-audit.json'), JSON.stringify(audit, null, 2) + '\n', 'utf8');

  console.log(JSON.stringify({
    outputDir,
    files: ['_merged.json', 'comments-flat.jsonl', 'normalizer-audit.json'],
    itemCount: normalizedItems.length,
    validVocCount: normalizedComments.length,
    duplicateItemCount: audit.duplicateItemCount,
    duplicateVocCount: audit.duplicateVocCount,
    errors: errors.length
  }, null, 2));
}

if (require.main === module) {
  try {
    main();
  } catch (error) {
    console.error(error.message);
    process.exit(1);
  }
}

module.exports = {
  parseArgs,
  inferCollector,
  collectXiaohongshu,
  collectDouyin,
  collectAmazon,
  dedupe
};