fmode-voc-projects/scripts/tools/voc-validation-analyze.js

// ==============================================================================
// VOC 验证报告 · 评论分析助手（合并双数据源：旧 extended.json + 新 _merged.json）
// 旧源：data/jiangzhong-liver-voc-extended.json（2456 条 XHS, 16 kw）
// 新源：docs/jiangzhong/raw/_merged.json（4719 条 XHS+抖音+Amazon, 32 kw, H1-H8 tags）
// 合计去重后 ≈ 6000+ 条
// ==============================================================================
const fs = require('fs');
const path = require('path');

const EXT_PATH = path.join('data', 'jiangzhong-liver-voc-extended.json');
const MERGED_PATH = path.join('docs', 'jiangzhong', 'raw', '_merged.json');
const FLAT_PATH = path.join('docs', 'jiangzhong', 'raw', 'comments-flat.jsonl');

// 关键词 → 模块映射（用于 _merged.json 集成到 by_module 结构）
const KW_TO_MODULE = {
    // positioning（藤茶 DMY 差异化）
    '肝纯片': 'positioning',
    '江中肝纯片': 'positioning',
    '藤茶': 'positioning',
    // scene（场景）
    '熬夜护肝': 'scene',
    '熬夜伤肝': 'scene',
    '脂肪肝': 'scene',
    '应酬解酒': 'scene',
    '送长辈保健品': 'scene',
    '酒局必备': 'scene',
    // competitor（竞品）
    '护肝片': 'competitor',
    '易善复': 'competitor',
    'swisse护肝片': 'competitor',
    '葵花护肝片': 'competitor',
    '汤臣倍健护肝': 'competitor',
    '海王金樽': 'competitor',
    '片仔癀护肝': 'competitor',
    '解酒神器': 'competitor',
    '水飞蓟': 'competitor',
    '奶蓟草': 'competitor',
    // 海外对标（amazon）单独一个 module
    'milk thistle': 'overseas',
    'silymarin': 'overseas',
    'liver support supplement': 'overseas',
    'dihydromyricetin': 'overseas',
    'liver detox': 'overseas',
    'hangover pills': 'overseas',
};

// 归一化单条评论结构，兼容 XHS / Douyin / Amazon / 旧 extended
function normalizeComment(raw, platform, keyword, moduleKey, noteId = '', hypotheses = []) {
    if (!raw) return null;
    let content = '', like_count = 0, user_nickname = '匿名', user_ip = '', time = null;
    if (platform === 'xhs') {
        content = String(raw.content || '').trim();
        like_count = raw.like_count || 0;
        user_nickname = raw.user?.nickname || raw.user?.user_id || '匿名';
        user_ip = raw.ip_location || '';
        time = raw.time || null;
    } else if (platform === 'douyin') {
        content = String(raw.text || raw.content || '').trim();
        like_count = raw.digg_count || raw.like_count || 0;
        user_nickname = raw.user?.nickname || raw.user_name || '匿名';
        user_ip = raw.ip_label || raw.ip_location || '';
        time = raw.create_time ? raw.create_time * 1000 : null;
    } else if (platform === 'amazon') {
        // Amazon review: Title + Content 合并
        const title = String(raw.Title || '').trim();
        const body = String(raw.Content || '').trim();
        content = title ? (title + (body ? (' — ' + body) : '')) : body;
        like_count = parseInt(String(raw.Helpful || '0').replace(/\D/g, '')) || 0;
        user_nickname = raw.ConsumerName || 'Amazon User';
        user_ip = raw.ReviewedCountry || '';
        time = raw.UpdateTime || raw.ReviewsDate || null;
    }
    if (!content) return null;
    return {
        content,
        like_count,
        time,
        user_nickname,
        user_ip,
        note_id: noteId,
        note_title: '',
        note_likes: 0,
        keyword,
        module: moduleKey,
        platform,
        hypotheses,  // H1-H8 tags（新源独有）
        star: raw.Star || null,  // Amazon 星级
    };
}

// 加载新源 _merged.json，转成 by_module 结构
function loadMerged() {
    if (!fs.existsSync(MERGED_PATH)) return null;
    let merged;
    try { merged = JSON.parse(fs.readFileSync(MERGED_PATH, 'utf-8')); } catch { return null; }

    const by_module = {};  // { moduleKey: { keyword: { notes, comments: {noteId: [...]}, _hypotheses, _platform } } }

    // XHS
    for (const kw of Object.keys(merged.xhs || {})) {
        const block = merged.xhs[kw];
        const moduleKey = KW_TO_MODULE[kw] || 'other';
        if (!by_module[moduleKey]) by_module[moduleKey] = {};
        const notes = block.top_notes || [];
        const commentsMap = {};
        for (const nid of Object.keys(block.comments_by_note_id || {})) {
            commentsMap[nid] = block.comments_by_note_id[nid] || [];
        }
        by_module[moduleKey][kw] = {
            notes,
            comments: commentsMap,
            _hypotheses: block.hypotheses || [],
            _platform: 'xhs',
        };
    }
    // Douyin
    for (const kw of Object.keys(merged.douyin || {})) {
        const block = merged.douyin[kw];
        // 抖音和 XHS 可能 keyword 重复（如"肝纯片"），合并进同一 module 但不同 key 避免冲突
        const moduleKey = KW_TO_MODULE[kw] || 'other';
        const dyKey = kw + ' (抖音)';
        if (!by_module[moduleKey]) by_module[moduleKey] = {};
        const commentsMap = {};
        for (const vid of Object.keys(block.comments_by_aweme_id || {})) {
            commentsMap[vid] = block.comments_by_aweme_id[vid] || [];
        }
        by_module[moduleKey][dyKey] = {
            notes: block.top_videos || [],
            comments: commentsMap,
            _hypotheses: block.hypotheses || [],
            _platform: 'douyin',
        };
    }
    // Amazon
    for (const kw of Object.keys(merged.amazon || {})) {
        const block = merged.amazon[kw];
        const moduleKey = KW_TO_MODULE[kw] || 'overseas';
        const amzKey = kw + ' (Amazon)';
        if (!by_module[moduleKey]) by_module[moduleKey] = {};
        const commentsMap = {};
        for (const asin of Object.keys(block.reviews_by_asin || {})) {
            const rev = block.reviews_by_asin[asin];
            const arr = Array.isArray(rev) ? rev : (rev?.Reviews || rev?.reviews || []);
            commentsMap[asin] = arr;
        }
        by_module[moduleKey][amzKey] = {
            notes: block.top_products || [],
            comments: commentsMap,
            _hypotheses: block.hypotheses || [],
            _platform: 'amazon',
        };
    }

    return { by_module, _source: 'merged', stats: merged.stats };
}

// 加载旧源 extended.json
function loadExtendedOnly() {
    if (!fs.existsSync(EXT_PATH)) return null;
    try { return JSON.parse(fs.readFileSync(EXT_PATH, 'utf-8')); } catch { return null; }
}

// 主入口：合并两源
function loadExtended() {
    const merged = loadMerged();
    const old = loadExtendedOnly();
    if (!merged && !old) return null;
    if (!merged) return old;
    if (!old) return merged;

    // 合并两源的 by_module（同模块同 keyword 时，新源优先但保留老源唯一 keyword）
    const combined = { by_module: {} };
    for (const src of [old, merged]) {
        if (!src.by_module) continue;
        for (const m of Object.keys(src.by_module)) {
            if (!combined.by_module[m]) combined.by_module[m] = {};
            Object.assign(combined.by_module[m], src.by_module[m]);
        }
    }
    combined._source = 'combined';
    combined.stats = merged.stats;
    return combined;
}

/** 返回某模块/关键词下的所有评论（扁平化），附 note 元信息。已按 commentId/content 去重 */
function getComments(data, moduleKey, keyword = null) {
    const out = [];
    const seen = new Set();
    if (!data?.by_module?.[moduleKey]) return out;
    const mod = data.by_module[moduleKey];
    const keywords = keyword ? [keyword] : Object.keys(mod);
    for (const kw of keywords) {
        const kwData = mod[kw];
        if (!kwData?.comments) continue;
        // 检测 platform：新源 _merged 块携带 _platform，老源无标记默认 xhs
        const platform = kwData._platform || 'xhs';
        const hypotheses = kwData._hypotheses || [];
        const noteMap = new Map();
        (kwData.notes || []).forEach((n) => {
            const nid = n?.id || n?.aweme_id || n?.asin || n?.ItemAsin;
            if (nid) noteMap.set(nid, n);
        });
        for (const noteId of Object.keys(kwData.comments)) {
            const note = noteMap.get(noteId);
            for (const raw of kwData.comments[noteId]) {
                const n = normalizeComment(raw, platform, kw, moduleKey, noteId, hypotheses);
                if (!n) continue;
                // 去重键
                const cid = raw.id || raw.cid || raw.Asin || '';
                const dedupKey = cid ? `${platform}|${cid}` : `${platform}|${n.content}|${n.user_nickname}|${noteId}`;
                if (seen.has(dedupKey)) continue;
                seen.add(dedupKey);
                // 补充 note 元信息
                if (note) {
                    n.note_title = note.title || note.desc?.slice(0, 30) || note.Title || '';
                    n.note_likes = note.liked_count || note.digg_count || 0;
                }
                out.push(n);
            }
        }
    }
    return out;
}

/** 所有模块所有评论（全量） */
function getAllComments(data) {
    const out = [];
    if (!data?.by_module) return out;
    for (const m of Object.keys(data.by_module)) {
        out.push(...getComments(data, m));
    }
    return out;
}

/** Top N 按 like_count 降序；可 filter */
function getTopComments(data, moduleKey, keyword = null, n = 10, filterFn = null) {
    let comments = getComments(data, moduleKey, keyword);
    if (filterFn) comments = comments.filter(filterFn);
    return comments
        .filter((c) => c.content.length >= 5 && c.content.length <= 300)  // 过滤过短/过长
        .sort((a, b) => b.like_count - a.like_count)
        .slice(0, n);
}

/** 基于关键词命中，抽取主题相关评论 */
function findByKeywords(data, moduleKey, keyword, matchWords, n = 10, includeSelf = true) {
    const comments = getComments(data, moduleKey, keyword);
    const matches = comments.filter((c) => {
        const txt = c.content;
        return matchWords.some((w) => txt.includes(w));
    });
    return matches.sort((a, b) => b.like_count - a.like_count).slice(0, n);
}

/** 跨模块搜索：在全部评论中搜索含特定关键词的评论 */
function searchAllComments(data, matchWords, n = 20) {
    const all = getAllComments(data);
    const matches = all.filter((c) => matchWords.some((w) => c.content.includes(w)));
    return matches
        .filter((c) => c.content.length >= 5 && c.content.length <= 400)
        .sort((a, b) => b.like_count - a.like_count)
        .slice(0, n);
}

/** 统计主题词频（在某模块内） */
function themeFreq(data, moduleKey, themes) {
    const comments = getComments(data, moduleKey);
    const stats = {};
    for (const theme of Object.keys(themes)) {
        const words = themes[theme];
        let count = 0;
        let totalLikes = 0;
        const hits = [];
        for (const c of comments) {
            if (words.some((w) => c.content.includes(w))) {
                count++;
                totalLikes += c.like_count;
                hits.push(c);
            }
        }
        stats[theme] = {
            count,
            totalLikes,
            avgLikes: count ? Math.round(totalLikes / count) : 0,
            samples: hits.sort((a, b) => b.like_count - a.like_count).slice(0, 3),
            pct: comments.length ? Math.round((count / comments.length) * 100) : 0,
        };
    }
    return { totalComments: comments.length, themes: stats };
}

/** 情感分布（简单规则分类） */
function sentimentSplit(data, moduleKey, keyword = null) {
    const comments = getComments(data, moduleKey, keyword);
    const positive = ['好', '有用', '有效', '推荐', '棒', '爱', '喜欢', '舒服', '舒坦', '真香', '绝', '神', '真的', '赞', '👍', '可以的', '不错'];
    const negative = ['难', '差', '无效', '没用', '骗', '智商税', '副作用', '别买', '踩雷', '拉', '垃圾', '假', '不好', '失望', '退', '后悔'];
    const doubt = ['吗', '吗？', '？', '怎么', '有没有', '求', '问一下', '会', '能', '真的假的', '到底'];
    const out = { positive: 0, negative: 0, neutral: 0, doubt: 0, samples: { positive: [], negative: [], doubt: [] } };
    for (const c of comments) {
        const t = c.content;
        const hasPos = positive.some((w) => t.includes(w));
        const hasNeg = negative.some((w) => t.includes(w));
        const hasDoubt = doubt.some((w) => t.includes(w));
        if (hasNeg) { out.negative++; out.samples.negative.push(c); }
        else if (hasDoubt) { out.doubt++; out.samples.doubt.push(c); }
        else if (hasPos) { out.positive++; out.samples.positive.push(c); }
        else out.neutral++;
    }
    out.total = comments.length;
    // 取 top 示例
    out.samples.positive = out.samples.positive.sort((a, b) => b.like_count - a.like_count).slice(0, 3);
    out.samples.negative = out.samples.negative.sort((a, b) => b.like_count - a.like_count).slice(0, 3);
    out.samples.doubt = out.samples.doubt.sort((a, b) => b.like_count - a.like_count).slice(0, 3);
    return out;
}

/** 元数据 · 全局统计 */
function getMeta(data) {
    if (!data?.by_module) return { modules: 0, keywords: 0, notes: 0, comments: 0, platforms: {} };
    let kws = 0, notes = 0, comments = 0;
    const platforms = { xhs: 0, douyin: 0, amazon: 0 };
    for (const m of Object.keys(data.by_module)) {
        for (const kw of Object.keys(data.by_module[m])) {
            kws++;
            const r = data.by_module[m][kw];
            const plat = r._platform || 'xhs';
            notes += r.notes?.length || 0;
            for (const nid of Object.keys(r.comments || {})) {
                const count = r.comments[nid]?.length || 0;
                comments += count;
                platforms[plat] = (platforms[plat] || 0) + count;
            }
        }
    }
    return { modules: Object.keys(data.by_module).length, keywords: kws, notes, comments, platforms };
}

/** 基于 H1-H8 假设 tag 搜索评论（新源独有） */
function getCommentsByHypothesis(data, hypothesis, n = 50) {
    const all = getAllComments(data);
    const matches = all.filter((c) => Array.isArray(c.hypotheses) && c.hypotheses.includes(hypothesis));
    return matches
        .filter((c) => c.content.length >= 5 && c.content.length <= 400)
        .sort((a, b) => b.like_count - a.like_count)
        .slice(0, n);
}

/** 按平台筛评论（xhs / douyin / amazon） */
function getCommentsByPlatform(data, platform, n = 50) {
    const all = getAllComments(data);
    return all
        .filter((c) => c.platform === platform && c.content.length >= 5 && c.content.length <= 400)
        .sort((a, b) => b.like_count - a.like_count)
        .slice(0, n);
}

/** Amazon 特有：按星级筛评论（1-5 星） */
function getAmazonByStar(data, targetStars = [1, 2], n = 20) {
    const all = getAllComments(data);
    return all
        .filter((c) => c.platform === 'amazon' && c.star)
        .filter((c) => {
            const s = parseInt(String(c.star).match(/\d/)?.[0] || '0');
            return targetStars.includes(s);
        })
        .sort((a, b) => b.like_count - a.like_count)
        .slice(0, n);
}

module.exports = {
    loadExtended,
    loadMerged,
    getComments,
    getAllComments,
    getTopComments,
    findByKeywords,
    searchAllComments,
    themeFreq,
    sentimentSplit,
    getMeta,
    getCommentsByHypothesis,
    getCommentsByPlatform,
    getAmazonByStar,
};