openclaw-voc-skill/tools/transcribe_centrum_douyin_videos.js

#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');
const { DouyinApi } = require('../claude-code/claude-code-voc-intelligence/mcp/src/providers/douyin-api');

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token.startsWith('--')) continue;
    const eq = token.indexOf('=');
    if (eq > 0) {
      args[token.slice(2, eq)] = token.slice(eq + 1);
      continue;
    }
    const key = token.slice(2);
    const next = argv[i + 1];
    if (next && !next.startsWith('--')) {
      args[key] = next;
      i += 1;
    } else {
      args[key] = true;
    }
  }
  return args;
}

function ensureDir(dir) {
  fs.mkdirSync(dir, { recursive: true });
}

function clean(value) {
  return String(value || '').replace(/\s+/g, ' ').trim();
}

function numberArg(value, fallback) {
  const number = Number(value);
  return Number.isFinite(number) && number >= 0 ? Math.round(number) : fallback;
}

function splitList(value) {
  if (!value) return [];
  return String(value)
    .split(/[,，、\s]+/)
    .map(item => item.trim())
    .filter(Boolean);
}

function isBrandRelevant(video) {
  const text = [
    video.title,
    video.author,
    video.keyword,
    video.skuGroup,
    video.ruleHits,
  ].map(clean).join(' ');
  if (/善存|Centrum|银善存|小佳维|紫瓶|每日营养包|PRO营养包|复合维生素/i.test(text)) {
    return true;
  }
  return false;
}

function isWeakOrInvalid(video) {
  const text = [video.title, video.author, video.skuGroup].map(clean).join(' ');
  if (/善存|Centrum|银善存|小佳维|紫瓶/i.test(text)) return false;
  return /安利|GNC|膳食套餐|磷虾油|牡蛎片|猴父子|Swisse/i.test(text);
}

function priorityScore(video) {
  const riskBoost = { '高': 5000, '中高': 3000, '中': 1200, '低': 0 }[video.riskLevel] || 0;
  const skuBoost = /营养包|PRO|每日营养包/.test(video.skuGroup || '') ? 2000 : 0;
  const officialBoost = /善存.*(旗舰店|官方|店铺|专卖店)/.test(video.author || '') ? 900 : 0;
  const commentBoost = Math.min(Number(video.commentCount || 0) * 2, 2000);
  return Number(video.score || 0) + riskBoost + skuBoost + officialBoost + commentBoost;
}

function decodeMaybeBase64Url(value) {
  const text = String(value || '').trim();
  if (!text) return '';
  if (/^https?:\/\//i.test(text)) return text;
  if (!/^[A-Za-z0-9+/=_-]+$/.test(text) || text.length < 24) return '';
  try {
    const decoded = Buffer.from(text.replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
    return /^https?:\/\//i.test(decoded) ? decoded : '';
  } catch {
    return '';
  }
}

function findAweme(node, awemeId, depth = 0) {
  if (!node || depth > 8) return null;
  if (Array.isArray(node)) {
    for (const item of node) {
      const found = findAweme(item, awemeId, depth + 1);
      if (found) return found;
    }
    return null;
  }
  if (typeof node !== 'object') return null;
  const id = String(node.aweme_id || node.id || '');
  if (id === String(awemeId)) return node;
  for (const value of Object.values(node)) {
    const found = findAweme(value, awemeId, depth + 1);
    if (found) return found;
  }
  return null;
}

function durationFromAweme(item) {
  const raw = item?.video?.duration || item?.duration || item?.video_duration || 0;
  const numeric = Number(raw || 0);
  if (!Number.isFinite(numeric) || numeric <= 0) return 0;
  return numeric > 10000 ? Math.round(numeric) : Math.round(numeric * 1000);
}

function collectVideoCandidates(item) {
  const out = [];
  const video = item?.video || {};
  const bitRates = Array.isArray(video.bit_rate) ? video.bit_rate : [];
  bitRates.forEach((rate, rateIndex) => {
    const urls = rate?.play_addr?.url_list || [];
    urls.forEach((value, urlIndex) => {
      const url = decodeMaybeBase64Url(value);
      if (!url) return;
      out.push({
        url,
        kind: 'video',
        keyPath: `video.bit_rate.${rateIndex}.play_addr.url_list.${urlIndex}`,
        bitRate: Number(rate.bit_rate || rate.bitrate || rate.real_bitrate || 0),
        dataSize: Number(rate.play_addr?.data_size || rate.data_size || 0),
      });
    });
  });
  const downloadUrls = video.download_addr?.url_list || [];
  downloadUrls.forEach((value, index) => {
    const url = decodeMaybeBase64Url(value);
    if (url) out.push({ url, kind: 'video', keyPath: `video.download_addr.url_list.${index}`, bitRate: 0, dataSize: Number(video.download_addr?.data_size || 0) });
  });

  const unique = [];
  const seen = new Set();
  for (const item of out) {
    if (seen.has(item.url)) continue;
    seen.add(item.url);
    unique.push(item);
  }
  unique.sort((a, b) => {
    const hostRank = item => {
      try {
        const host = new URL(item.url).hostname.toLowerCase();
        if (host.includes('api-play.amemv.com')) return 0;
        if (host.includes('api.amemv.com')) return 1;
        if (host.includes('douyinvod.com')) return 2;
        return 3;
      } catch {
        return 4;
      }
    };
    const aHostRank = hostRank(a);
    const bHostRank = hostRank(b);
    if (aHostRank !== bHostRank) return aHostRank - bHostRank;
    const aSize = a.dataSize || Number.MAX_SAFE_INTEGER;
    const bSize = b.dataSize || Number.MAX_SAFE_INTEGER;
    if (aSize !== bSize) return aSize - bSize;
    return (a.bitRate || Number.MAX_SAFE_INTEGER) - (b.bitRate || Number.MAX_SAFE_INTEGER);
  });
  return unique;
}

async function resolveMedia(api, video) {
  const attempts = [
    video.keyword,
    clean(video.title).replace(/#.*$/g, '').slice(0, 50),
    clean(video.title).split(/[，,。#]/)[0].slice(0, 50),
  ].filter(Boolean);
  const tried = [];
  for (const keyword of attempts) {
    if (tried.includes(keyword)) continue;
    tried.push(keyword);
    for (let retry = 0; retry < 3; retry += 1) {
      try {
        const response = await api.searchVideos({
          keyword,
          cursor: 0,
          sortType: '1',
          publishTime: '180',
          filterDuration: '0',
          contentType: '1',
        });
      const aweme = findAweme(response.data, video.videoId);
      if (!aweme) continue;
      const candidates = collectVideoCandidates(aweme);
      if (!candidates.length) continue;
      return {
        status: 'ok',
        keyword,
        durationMs: durationFromAweme(aweme),
        candidate: candidates[0],
        candidates,
        candidateCount: candidates.length,
      };
      } catch (error) {
        tried.push(`${keyword}: ${error.message}`);
        if (retry < 2) {
          await new Promise(resolve => setTimeout(resolve, 1500 * (retry + 1)));
        }
      }
    }
  }
  return { status: 'not_found', tried };
}

function runTranscriber({ video, media, outDir, ffmpegPath, transcriberPath, nodePath, env }) {
  const videoDir = path.join(outDir, 'transcripts', String(video.videoId));
  ensureDir(videoDir);
  const args = [
    transcriberPath,
    '--provider', 'iflytek-gateway',
    '--media-url', media.candidate.url,
    '--aweme-id', String(video.videoId),
    '--duration-ms', String(media.durationMs || 60000),
    '--output', videoDir,
    '--ffmpeg-path', ffmpegPath,
    '--gateway-max-duration-ms', '600000',
    '--max-download-mb', '160',
    '--max-polls', '60',
    '--poll-interval-ms', '4000',
  ];
  const result = spawnSync(nodePath, args, {
    cwd: process.cwd(),
    env,
    encoding: 'utf8',
    windowsHide: true,
    maxBuffer: 1024 * 1024 * 20,
    timeout: 420000,
  });
  let parsed = {};
  const stdout = String(result.stdout || '').trim();
  if (stdout) {
    try {
      parsed = JSON.parse(stdout.split(/\r?\n/).pop());
    } catch {
      parsed = { rawStdout: stdout.slice(-1000) };
    }
  }
  return {
    exitCode: result.status,
    error: result.error ? result.error.message : '',
    stderr: String(result.stderr || '').trim().slice(-2000),
    stdout: stdout.slice(-2000),
    parsed,
    videoDir,
  };
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const token = process.env.VOC_TOKEN || process.env.VOC_DOUYIN_TOKEN || process.env.DOUYIN_TOKEN;
  if (!token) throw new Error('VOC_TOKEN is required in environment');

  const inputPath = path.resolve(args.input || 'outputs/centrum_douyin_audit_2026-06-08/live-full/live-analysis-data.json');
  const outDir = path.resolve(args.output || 'outputs/centrum_douyin_audit_2026-06-08/transcript-full');
  const limit = numberArg(args.limit, 1);
  const nodePath = process.execPath;
  const transcriberPath = path.resolve(args.transcriber || 'scripts/tools/douyin-video-transcriber.js');
  const ffmpegPath = path.resolve(args.ffmpegPath || args['ffmpeg-path'] || '.codex-run/ffmpeg-installer/node_modules/@ffmpeg-installer/win32-x64/ffmpeg.exe');
  ensureDir(outDir);

  const data = JSON.parse(fs.readFileSync(inputPath, 'utf8'));
  const allVideos = Array.isArray(data.videoRows) ? data.videoRows : [];
  const excluded = allVideos.filter(video => !isBrandRelevant(video) || isWeakOrInvalid(video));
  const requestedIds = new Set(splitList(args.ids || args.videoIds || args['video-ids']));
  const eligible = allVideos
    .filter(video => isBrandRelevant(video) && !isWeakOrInvalid(video))
    .map(video => ({ ...video, transcriptPriority: priorityScore(video) }))
    .sort((a, b) => b.transcriptPriority - a.transcriptPriority);
  const selected = requestedIds.size
    ? eligible.filter(video => requestedIds.has(String(video.videoId)))
    : eligible.slice(0, limit);

  const pool = {
    generatedAt: new Date().toISOString(),
    inputPath,
    limit,
    selectedCount: selected.length,
    excludedCount: excluded.length,
    selected: selected.map(video => ({
      videoId: video.videoId,
      title: video.title,
      author: video.author,
      keyword: video.keyword,
      skuGroup: video.skuGroup,
      riskLevel: video.riskLevel,
      score: video.score,
      transcriptPriority: video.transcriptPriority,
    })),
    excluded: excluded.map(video => ({
      videoId: video.videoId,
      title: video.title,
      author: video.author,
      keyword: video.keyword,
      skuGroup: video.skuGroup,
      reason: isWeakOrInvalid(video) ? 'weak_or_invalid_non_centrum' : 'no_clear_brand_signal',
    })),
  };
  fs.writeFileSync(path.join(outDir, 'transcript-sample-pool.json'), JSON.stringify(pool, null, 2), 'utf8');

  const api = new DouyinApi({ token });
  const runLog = [];
  const safeEnv = { ...process.env, VOC_TOKEN: token };
  for (const video of selected) {
    const existing = path.join(outDir, 'transcripts', String(video.videoId), 'transcript.json');
    if (fs.existsSync(existing) && !args.force) {
      const transcript = JSON.parse(fs.readFileSync(existing, 'utf8'));
      runLog.push({ videoId: video.videoId, status: 'skipped_existing', transcriptStatus: transcript.status, textLength: clean(transcript.text).length });
      continue;
    }
    const media = await resolveMedia(api, video);
    if (media.status !== 'ok') {
      runLog.push({ videoId: video.videoId, status: 'media_not_found', media });
      continue;
    }
    const candidateAttempts = [];
    let result = null;
    const candidates = Array.isArray(media.candidates) && media.candidates.length
      ? media.candidates.slice(0, numberArg(args.candidateLimit || args['candidate-limit'], 8))
      : [media.candidate];
    for (const candidate of candidates) {
      const attemptMedia = { ...media, candidate };
      result = runTranscriber({ video, media: attemptMedia, outDir, ffmpegPath, transcriberPath, nodePath, env: safeEnv });
      const transcriptPath = path.join(result.videoDir, 'transcript.json');
      let attemptTranscriptStatus = '';
      let attemptTextLength = 0;
      if (fs.existsSync(transcriptPath)) {
        try {
          const transcript = JSON.parse(fs.readFileSync(transcriptPath, 'utf8'));
          attemptTranscriptStatus = transcript.status;
          attemptTextLength = clean(transcript.text).length;
        } catch {
          attemptTranscriptStatus = 'unreadable_transcript';
        }
      }
      candidateAttempts.push({
        keyPath: candidate.keyPath,
        exitCode: result.exitCode,
        transcriptStatus: attemptTranscriptStatus,
        textLength: attemptTextLength,
        stderr: result.stderr,
      });
      if (result.exitCode === 0 && attemptTranscriptStatus) break;
    }
    let transcriptStatus = '';
    let textLength = 0;
    const transcriptPath = path.join(result.videoDir, 'transcript.json');
    if (fs.existsSync(transcriptPath)) {
      try {
        const transcript = JSON.parse(fs.readFileSync(transcriptPath, 'utf8'));
        transcriptStatus = transcript.status;
        textLength = clean(transcript.text).length;
      } catch {
        transcriptStatus = 'unreadable_transcript';
      }
    }
    runLog.push({
      videoId: video.videoId,
      title: video.title,
      author: video.author,
      status: result.exitCode === 0 ? 'transcriber_finished' : 'transcriber_error',
      transcriptStatus,
      textLength,
      mediaKeyword: media.keyword,
      mediaKeyPath: media.candidate.keyPath,
      durationMs: media.durationMs,
      candidateAttempts,
      exitCode: result.exitCode,
      error: result.error,
      stderr: result.stderr,
      parsed: result.parsed,
    });
    fs.writeFileSync(path.join(outDir, 'transcription-run-log.json'), JSON.stringify(runLog, null, 2), 'utf8');
  }

  const summary = {
    status: 'ok',
    outDir,
    selectedCount: selected.length,
    excludedCount: excluded.length,
    okCount: runLog.filter(item => item.transcriptStatus === 'ok').length,
    pendingCount: runLog.filter(item => item.transcriptStatus === 'transcription_pending').length,
    failedCount: runLog.filter(item => item.status !== 'skipped_existing' && item.transcriptStatus !== 'ok').length,
    runLogPath: path.join(outDir, 'transcription-run-log.json'),
    poolPath: path.join(outDir, 'transcript-sample-pool.json'),
  };
  fs.writeFileSync(summary.runLogPath, JSON.stringify(runLog, null, 2), 'utf8');
  console.log(JSON.stringify(summary, null, 2));
}

main().catch(error => {
  console.error(JSON.stringify({ status: 'error', message: error.message }, null, 2));
  process.exit(1);
});