tihao-ai/claude-code-tihao-sourcing/scripts/history-dataset-from-csv.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');

function main() {
  const args = parseArgs(process.argv.slice(2));
  const input = args.input || args.csv || process.env.TIHAO_HISTORY_CSV || '';
  if (!input) throw new Error('Usage: node scripts/history-dataset-from-csv.js --input <history.csv> --output <history-dataset-dir>');
  const inputPath = path.resolve(input);
  const outputDir = path.resolve(args.output || process.env.TIHAO_HISTORY_DATASET_OUTPUT || path.join(path.dirname(inputPath), 'history-dataset'));
  const rows = readCsv(inputPath);
  const records = buildRecords(rows);
  fs.mkdirSync(outputDir, { recursive: true });

  const files = [];
  for (const record of records) {
    const file = path.join(outputDir, `${safeFileName(record.id)}.json`);
    fs.writeFileSync(file, JSON.stringify(record, null, 2), 'utf8');
    files.push(file);
  }

  const summary = {
    input: inputPath,
    outputDir,
    generatedAt: new Date().toISOString(),
    briefCount: records.length,
    creatorRows: rows.body.length,
    files
  };
  const summaryPath = path.join(outputDir, 'history-dataset-from-csv-summary.json');
  fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf8');
  console.log(JSON.stringify({ ...summary, summaryPath }, null, 2));
}

function buildRecords(rows) {
  const groups = new Map();
  for (const row of rows.body) {
    const id = firstNonEmpty(row, ['brief编号', 'briefId', 'id', '项目编号']);
    if (!id) continue;
    const group = groups.get(id) || {
      id,
      name: firstNonEmpty(row, ['项目名称', 'brief名称', 'name']) || id,
      category: firstNonEmpty(row, ['类目', 'category', '品类']),
      briefText: firstNonEmpty(row, ['客户原始Brief', 'briefText', 'brief', 'originalBrief']),
      manualSupplementBaseline: numberOrNull(firstNonEmpty(row, ['历史人工补号量基线', 'manualSupplementBaseline', 'manualSupplementCount', 'manualAddedCount'])),
      referenceLinks: [],
      manualFinalList: [],
      customerFeedback: []
    };
    mergeReferenceLinks(group, firstNonEmpty(row, ['参考账号或视频', '参考链接', 'referenceLinks', 'referenceVideos', 'referenceAccounts']));
    const creatorName = firstNonEmpty(row, ['博主名称', 'creatorName', '达人名称', '账号名称']);
    const profileUrl = firstNonEmpty(row, ['主页链接', 'profileUrl', '账号链接']);
    const customerDecision = firstNonEmpty(row, ['客户选择', 'customerDecision', 'finalDecision', '客户最终结果']);
    const rejectReason = firstNonEmpty(row, ['拒绝原因', 'rejectReason', 'feedbackReason', '反馈原因']);
    const manualReviewLabel = firstNonEmpty(row, ['人工复核标签', 'manualReviewLabel', '复核标签']);
    if (creatorName || profileUrl) {
      const item = {
        platform: firstNonEmpty(row, ['平台', 'platform']),
        creatorName,
        profileUrl,
        manualReviewLabel,
        customerDecision
      };
      if (rejectReason) item.rejectReason = rejectReason;
      group.manualFinalList.push(item);
      if (customerDecision || rejectReason) {
        group.customerFeedback.push({
          creatorName,
          finalDecision: customerDecision,
          ...(rejectReason ? { rejectReason } : {})
        });
      }
    }
    groups.set(id, group);
  }
  return [...groups.values()].map(record => {
    if (record.manualSupplementBaseline === null) delete record.manualSupplementBaseline;
    record.referenceLinks = dedupeBy(record.referenceLinks, item => `${item.platform}|${item.url}`);
    record.manualFinalList = dedupeBy(record.manualFinalList, item => `${item.platform}|${item.profileUrl || item.creatorName}`);
    record.customerFeedback = dedupeBy(record.customerFeedback, item => `${item.creatorName}|${item.finalDecision}|${item.rejectReason || ''}`);
    return record;
  });
}

function mergeReferenceLinks(group, value) {
  const links = String(value || '')
    .split(/[;；\n]/)
    .map(item => item.trim())
    .filter(Boolean);
  for (const url of links) {
    group.referenceLinks.push({
      url,
      platform: inferPlatform(url),
      contentType: /video|note|aweme|douyin|tiktok/i.test(url) ? 'video' : 'account'
    });
  }
}

function inferPlatform(value) {
  const text = String(value || '').toLowerCase();
  if (text.includes('xiaohongshu') || text.includes('xhs')) return 'xiaohongshu';
  if (text.includes('douyin')) return 'douyin';
  if (text.includes('tiktok')) return 'tiktok';
  return '';
}

function readCsv(file) {
  const parsed = parseCsv(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
  const header = (parsed[0] || []).map(item => String(item || '').trim());
  const body = parsed.slice(1)
    .filter(row => row.some(cell => String(cell || '').trim()))
    .map(row => Object.fromEntries(header.map((key, index) => [key, row[index] || ''])));
  return { header, body };
}

function parseCsv(text) {
  const rows = [];
  let row = [];
  let cell = '';
  let quoted = false;
  for (let index = 0; index < text.length; index += 1) {
    const char = text[index];
    if (char === '\r') continue;
    if (char === '"' && quoted && text[index + 1] === '"') {
      cell += '"';
      index += 1;
    } else if (char === '"') quoted = !quoted;
    else if (char === ',' && !quoted) {
      row.push(cell);
      cell = '';
    } else if (char === '\n' && !quoted) {
      row.push(cell);
      rows.push(row);
      row = [];
      cell = '';
    } else cell += char;
  }
  if (cell || row.length) {
    row.push(cell);
    rows.push(row);
  }
  return rows;
}

function firstNonEmpty(row, fields) {
  for (const field of fields) {
    const value = String(row[field] || '').trim();
    if (value) return value;
  }
  return '';
}

function numberOrNull(value) {
  if (value === '') return null;
  const numeric = Number(value);
  return Number.isFinite(numeric) && numeric >= 0 ? numeric : null;
}

function dedupeBy(items, keyFn) {
  const seen = new Set();
  return items.filter(item => {
    const key = keyFn(item);
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
}

function safeFileName(value) {
  return String(value || 'history-brief')
    .trim()
    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '-')
    .replace(/\s+/g, '-')
    .slice(0, 80);
}

function parseArgs(argv) {
  const args = {};
  for (let index = 0; index < argv.length; index += 1) {
    const raw = argv[index];
    if (!raw.startsWith('--')) continue;
    const key = raw.slice(2).replace(/-([a-z])/g, (_, char) => char.toUpperCase());
    const next = argv[index + 1];
    if (!next || next.startsWith('--')) args[key] = true;
    else {
      args[key] = next;
      index += 1;
    }
  }
  return args;
}

if (require.main === module) {
  try {
    main();
  } catch (error) {
    console.error(error && error.stack ? error.stack : String(error));
    process.exit(1);
  }
}

module.exports = {
  buildRecords,
  readCsv
};