fmode-voc-projects/scripts/tools/liver-analyze.js

// ==============================================================================
// 江中肝纯片 · VOC 数据合并 + 分析模块（参考 houguyin-analyze.js 架构）
// ==============================================================================
// 数据源优先级：
//   1. docs/jiangzhong/raw/_merged.json  ← 真实多平台采集 · 4719 评论
//      (XHS 18kw × 80notes × 1515cmts + Douyin 8kw × 16videos × 627cmts + Amazon 6kw × 30prods × 2577reviews)
//   2. SKELETON                           ← 兜底骨架
//
// 与 houguyin-analyze.js 的区别：
//   - 肝纯片已有真实数据 ⇒ 不需要 seed/pattern-curated 回退
//   - items[] 在 loadMerged 时动态构造（从 _merged.json 的 xhs/douyin/amazon 三级嵌套结构展平）
//   - 字段标准化：likes/nickname/hypothesis 统一（源数据有 like/user/hypotheses）
//   - 8 假设为肝纯片专属（H1 DMY认知 / H2 海王金樽 / H3 副作用 / H4 场景 / H5 竞品 / H6 送礼 / H7 礼盒 / H8 Amazon）

const fs = require('fs');
const path = require('path');

const ROOT = path.resolve(__dirname, '..', '..');
const RAW_DIR = path.join(ROOT, 'docs', 'jiangzhong', 'raw');
const MERGED_PATH = path.join(RAW_DIR, '_merged.json');

// ------------------------------------------------------------
// 8 条核心假设定义（对应 docs/jiangzhong/2.数据采集矩阵.md）
// ------------------------------------------------------------
const HYPOTHESES = {
    H1: { title: '藤茶/DMY 认知空白',       desc: 'DMY 总黄酮在用户心智中几乎零认知，"藤茶"被识别为茶饮非护肝' },
    H2: { title: '海王金樽为什么没成',      desc: '硬广洗脑 + 体感弱 + 复购差，解酒饮料赛道负面教材' },
    H3: { title: '护肝片副作用恐惧',        desc: '易善复/水飞蓟高频"胃不舒服/拉肚子"吐槽，用户想吃又怕吃' },
    H4: { title: '脂肪肝+熬夜声量',         desc: '脂肪肝+熬夜伤肝是年轻化核心话题池，未被主流护肝品牌占据' },
    H5: { title: '五大竞品心智地图',        desc: 'Swisse高端吸收差/易善复医嘱/葵花药味/片仔癀贵/海王金樽广告' },
    H6: { title: '送长辈护肝空白',          desc: '父母孝心品类被钙片/阿胶/黄芪精占据，护肝几乎空白' },
    H7: { title: '礼盒心智 · "有面子"',    desc: '用户明确期待"送得出手/有面子"的高端包装，反感药盒感' },
    H8: { title: 'Amazon 国际对标',         desc: 'milk thistle 差评集中"no effect/hidden additives/allergy"，国际品类痛点' },
};

// 章节 → 假设映射（每章主要消费哪些假设的证据）
const CHAPTER_HYPOTHESIS_MAP = {
    challenge:   ['H1', 'H2', 'H8'],      // 诘问：认知空白 + 失败案例 + 国际差评
    dmy:         ['H1'],                   // DMY 归因学
    market:      ['H5', 'H4'],            // 市场全景：竞品 + 场景人群
    kano:        ['H3', 'H6', 'H7'],      // KANO×JTBD：副作用反向 + 兴奋需求
    scene:       ['H4', 'H2'],            // 场景地图：四大场景 + 解酒子场景
    competitor:  ['H5', 'H2'],            // 竞品三轴：精细对标 + 警示
    opportunity: ['H1', 'H6', 'H7'],      // 新机会：DMY教育 + 送礼 + 礼盒
    blueprint:   ['H4', 'H6'],            // 4P：场景人群 + 送礼
};

// ------------------------------------------------------------
// 平台标签（复用 houguyin-components 的结构）
// ------------------------------------------------------------
const PLATFORM_LABELS = {
    xhs:     { name: '小红书',   color: '#FF2442', short: '红' },
    douyin:  { name: '抖音',     color: '#1A1A1A', short: '抖' },
    amazon:  { name: 'Amazon',   color: '#FF9900', short: 'Az' },
    taobao:  { name: '淘宝',     color: '#FF5000', short: '淘' },
    jd:      { name: '京东',     color: '#E1251B', short: '京' },
    tmall:   { name: '天猫',     color: '#FF0036', short: '猫' },
    unknown: { name: '其他',     color: '#888',    short: '—' },
};

// ------------------------------------------------------------
// 关键词 → 产品名归一化
// ------------------------------------------------------------
const KEYWORD_TO_PRODUCT = {
    '肝纯片':      '江中肝纯片',
    '江中肝纯片':  '江中肝纯片',
    '护肝片':      '通用护肝片',
    'swisse护肝片':'Swisse 护肝片',
    '易善复':      '易善复',
    '葵花护肝片':  '葵花护肝片',
    '汤臣倍健护肝':'汤臣倍健护肝',
    '片仔癀护肝':  '片仔癀护肝',
    '海王金樽':    '海王金樽',
    '解酒神器':    '解酒品类',
    '应酬解酒':    '应酬解酒场景',
    '酒局必备':    '酒局必备场景',
    '熬夜护肝':    '熬夜护肝场景',
    '熬夜伤肝':    '熬夜伤肝场景',
    '脂肪肝':      '脂肪肝人群',
    '水飞蓟':      '水飞蓟成分',
    '奶蓟草':      '奶蓟草成分',
    '藤茶':        '藤茶 (本品核心)',
    '送长辈保健品':'送长辈品类',
    'milk thistle':         'Milk Thistle (国际)',
    'silymarin':            'Silymarin (国际)',
    'liver support supplement': 'Liver Support (国际)',
    'liver detox':          'Liver Detox (国际)',
    'dihydromyricetin':     'DMY 二氢杨梅素 (国际)',
    'hangover pills':       '解酒片 (国际)',
};

// ------------------------------------------------------------
// 情绪启发式判定（基于 rating/content 关键词）
// ------------------------------------------------------------
const NEG_KEYS = ['智商税', '没用', '没效', '没有效果', '骗人', '坑', '差评', '退货', '拉肚子', '胃不舒服', '副作用', '难吃', '难喝', '催吐', '进ICU', 'no effect', 'waste', 'terrible', 'horrible', 'garbage', 'scam', 'disappointed'];
const POS_KEYS = ['有效', '真香', '好用', '推荐', '回购', '神器', '绝了', '有感觉', 'amazing', 'works', 'effective', 'great', 'love', 'recommend'];
const CONFLICTED_KEYS = ['有点怕', '不敢', '纠结', '矛盾', '不知道', '真的有效吗', '是不是智商税', '感觉不太', '不确定'];

function inferSentiment(item) {
    if (typeof item.rating === 'number') {
        if (item.rating <= 2) return 'negative';
        if (item.rating >= 4) return 'positive';
        return 'neutral';
    }
    const c = String(item.content || '').toLowerCase();
    for (const k of NEG_KEYS) if (c.includes(k.toLowerCase())) return 'negative';
    for (const k of POS_KEYS) if (c.includes(k.toLowerCase())) return 'positive';
    for (const k of CONFLICTED_KEYS) if (c.includes(k.toLowerCase())) return 'conflicted';
    return 'neutral';
}

// ------------------------------------------------------------
// 标签启发式抽取（粗粒度，用于后续渲染 tag cloud）
// ------------------------------------------------------------
const TAG_PATTERNS = [
    { tag: '智商税',         keys: ['智商税', '骗人', 'waste of money', 'scam'] },
    { tag: '没有效果',       keys: ['没用', '没效', '没有效果', 'no effect', 'didn.?t work'] },
    { tag: '副作用',         keys: ['副作用', '拉肚子', '胃不舒服', '过敏', 'allergy', 'side effect'] },
    { tag: '推荐/真香',      keys: ['真香', '神器', '推荐', 'amazing', 'highly recommend'] },
    { tag: '送长辈',         keys: ['送长辈', '送父母', '送爸爸', '孝心', '给爸妈', '给我爸'] },
    { tag: '礼盒包装',       keys: ['礼盒', '包装好看', '有面子', '送得出手'] },
    { tag: '熬夜',           keys: ['熬夜', '通宵', '晚睡'] },
    { tag: '应酬解酒',       keys: ['应酬', '酒局', '喝酒', '解酒', '不上头'] },
    { tag: '脂肪肝',         keys: ['脂肪肝', '转氨酶', '肝功'] },
    { tag: '价格敏感',       keys: ['贵', '太贵', '便宜', '价格', 'expensive'] },
    { tag: '医生推荐',       keys: ['医生', '大夫', '医嘱', '开的'] },
    { tag: '怀疑',           keys: ['是不是', '真的有效', '靠谱吗', '智商税吗'] },
    { tag: '成分查询',       keys: ['成分', '配方', 'ingredient', '水飞蓟', '藤茶', 'DMY'] },
    { tag: '品牌认知',       keys: ['Swisse', '易善复', '葵花', '片仔癀', '海王金樽', '汤臣倍健', '江中'] },
    { tag: '国际好评',       keys: ['lifesaver', 'best', 'game changer'] },
    { tag: '长期服用焦虑',   keys: ['长期吃', '一直吃', 'long term'] },
];

function inferTags(item) {
    const c = String(item.content || '') + ' ' + String(item.title || '');
    const lo = c.toLowerCase();
    const out = new Set();
    for (const p of TAG_PATTERNS) {
        for (const k of p.keys) {
            const rx = new RegExp(k.toLowerCase());
            if (rx.test(lo)) { out.add(p.tag); break; }
        }
    }
    return [...out];
}

// ------------------------------------------------------------
// 数据归一化：从 _merged.json 的平台嵌套结构展平为标准 items[]
// ------------------------------------------------------------
function flattenMerged(merged) {
    const items = [];

    // ---- XHS: merged.xhs[keyword].comments_by_note_id[noteId][] + top_notes[] ----
    // 采集时分页重叠导致同一 cid 多次出现，按 cid 去重
    for (const [kw, kwData] of Object.entries(merged.xhs || {})) {
        const notes = kwData.top_notes || [];
        const noteMap = new Map(notes.map((n) => [n.id, n]));
        for (const [noteId, cmts] of Object.entries(kwData.comments_by_note_id || {})) {
            const note = noteMap.get(noteId) || {};
            const seenCids = new Set();
            for (const c of cmts) {
                const cid = c.id || `${(c.content || '').slice(0, 30)}|${c.like_count || 0}`;
                if (seenCids.has(cid)) continue;
                seenCids.add(cid);
                const nick = c.user?.nickname || note.user?.nickname || '小红书用户';
                items.push({
                    platform: 'xhs',
                    keyword: kw,
                    product: KEYWORD_TO_PRODUCT[kw] || kw,
                    type: 'comment',
                    id: `xhs_${noteId}_${c.id || Math.random().toString(36).slice(2, 8)}`,
                    nickname: nick,
                    ip: '',                                 // XHS 评论字段无 IP
                    content: c.content || '',
                    likes: c.like_count || 0,
                    subComments: c.sub_comment_count || 0,
                    noteId,
                    noteTitle: note.title || '',
                    noteLiked: note.liked_count || 0,
                    hypothesis: kwData.hypotheses || [],
                    source: 'real-collected',
                });
            }
        }
    }

    // ---- Douyin: merged.douyin[keyword].comments_by_aweme_id[awemeId][] ----
    // 同样按 cid 去重
    for (const [kw, kwData] of Object.entries(merged.douyin || {})) {
        const videos = kwData.top_videos || [];
        const videoMap = new Map(videos.map((v) => [v.aweme_id, v]));
        for (const [awemeId, cmts] of Object.entries(kwData.comments_by_aweme_id || {})) {
            const video = videoMap.get(awemeId) || {};
            const seenCids = new Set();
            for (const c of cmts) {
                const cid = c.cid || `${(c.text || '').slice(0, 30)}|${c.digg_count || 0}`;
                if (seenCids.has(cid)) continue;
                seenCids.add(cid);
                const nick = c.user?.nickname || '抖音用户';
                items.push({
                    platform: 'douyin',
                    keyword: kw,
                    product: KEYWORD_TO_PRODUCT[kw] || kw,
                    type: 'comment',
                    id: `dy_${awemeId}_${c.cid || Math.random().toString(36).slice(2, 8)}`,
                    nickname: nick,
                    ip: c.ip_label || '',
                    content: c.text || '',
                    likes: c.digg_count || 0,
                    replyTotal: c.reply_comment_total || 0,
                    awemeId,
                    videoDesc: (video.desc || '').slice(0, 80),
                    videoAuthor: video.author?.nickname || '',
                    videoDiggs: video.statistics?.digg_count || 0,
                    hypothesis: kwData.hypotheses || [],
                    source: 'real-collected',
                });
            }
        }
    }

    // ---- Amazon: merged.amazon[keyword].reviews_by_asin[asin][] ----
    // Amazon review 同样做 ReviewsLink + Content 组合去重
    for (const [kw, kwData] of Object.entries(merged.amazon || {})) {
        const prods = kwData.top_products || [];
        const prodMap = new Map(prods.map((p) => [p.Asin, p]));
        for (const [asin, reviews] of Object.entries(kwData.reviews_by_asin || {})) {
            const prod = prodMap.get(asin) || {};
            const arr = Array.isArray(reviews) ? reviews : (
                reviews?.Reviews || reviews?.reviews ||
                (typeof reviews === 'object' ? Object.values(reviews).filter((v) => v && typeof v === 'object' && (v.Content || v.Title)) : [])
            );
            const seenKeys = new Set();
            for (const r of arr) {
                const linkId = (r.ReviewsLink || '').split('/').pop();
                const dedupKey = linkId || `${r.ConsumerName || ''}|${(r.Title || '').slice(0, 40)}|${r.Star || 0}|${r.Helpful || 0}`;
                if (seenKeys.has(dedupKey)) continue;
                seenKeys.add(dedupKey);
                items.push({
                    platform: 'amazon',
                    keyword: kw,
                    product: KEYWORD_TO_PRODUCT[kw] || kw,
                    type: 'review',
                    id: `amz_${asin}_${linkId || Math.random().toString(36).slice(2, 8)}`,
                    nickname: r.ConsumerName || 'Amazon User',
                    ip: '',
                    content: r.Content || '',
                    title: r.Title || '',
                    rating: r.Star || null,
                    verified: r.IsVP ? 1 : 0,
                    likes: r.Helpful || 0,
                    date: r.ReviewsDate || '',
                    asin: r.Asin || asin,
                    parentAsin: asin,
                    productTitle: (prod.Title || '').slice(0, 80),
                    productBrand: prod.Brand || '',
                    productPrice: prod.Price || '',
                    variant: r.AsinProperty || '',
                    hypothesis: kwData.hypotheses || [],
                    source: 'real-collected',
                });
            }
        }
    }

    // 情绪 + 标签后处理
    for (const it of items) {
        it.sentiment = inferSentiment(it);
        it.tags = inferTags(it);
    }

    return items;
}

// ------------------------------------------------------------
// 数据加载（带回退骨架）
// ------------------------------------------------------------
const SKELETON = {
    meta: {
        collectedAt: '待采集',
        platforms: {},
        hypotheses: {},
        products: {},
        stage: 'skeleton',
        sourceNote: '暂无采集数据',
    },
    items: [],
    raw: null,
};

function loadMerged() {
    if (fs.existsSync(MERGED_PATH)) {
        try {
            const raw = JSON.parse(fs.readFileSync(MERGED_PATH, 'utf8'));
            const items = flattenMerged(raw);
            return {
                meta: {
                    sourceTier: 'real-collected',
                    stage: 'batch-real',
                    collectedAt: raw.collected_at || new Date().toISOString().slice(0, 10),
                    product: raw.product || '江中肝纯片',
                    stats: raw.stats || {},
                    sourceNote: 'docs/jiangzhong/raw/_merged.json · 真实多平台采集',
                },
                items,
                raw,
            };
        } catch (err) {
            console.warn(`⚠ _merged.json 解析失败：${err.message}`);
        }
    }
    return SKELETON;
}

// ------------------------------------------------------------
// 元数据汇总（给 cover / agenda 用）
// ------------------------------------------------------------
function getMeta(data) {
    const items = (data && data.items) || [];
    const platforms = {};
    const products = {};
    const hypotheses = {};
    const sources = {};
    const sentiments = {};
    const keywords = new Set();
    const tags = new Set();

    for (const it of items) {
        const pf = it.platform || 'unknown';
        platforms[pf] = (platforms[pf] || 0) + 1;
        const prod = it.product || 'unknown';
        products[prod] = (products[prod] || 0) + 1;
        const hs = Array.isArray(it.hypothesis) ? it.hypothesis : (it.hypothesis ? [it.hypothesis] : []);
        for (const h of hs) hypotheses[h] = (hypotheses[h] || 0) + 1;
        const src = it.source || 'unknown';
        sources[src] = (sources[src] || 0) + 1;
        sentiments[it.sentiment || 'unknown'] = (sentiments[it.sentiment || 'unknown'] || 0) + 1;
        if (it.keyword) keywords.add(it.keyword);
        if (Array.isArray(it.tags)) it.tags.forEach((t) => tags.add(t));
    }

    const stats = data?.raw?.stats || {};
    return {
        comments: items.length,
        notes: stats?.xhs?.notes || 0,
        videos: stats?.douyin?.videos || 0,
        products_count: stats?.amazon?.products || 0,
        keywords: keywords.size,
        tagsTotal: tags.size,
        platforms,
        products,
        productsCount: Object.keys(products).length,
        hypotheses,
        sources,
        sentiments,
        stage: data?.meta?.stage || 'unknown',
        sourceTier: data?.meta?.sourceTier || 'unknown',
        collectedAt: data?.meta?.collectedAt || '待采集',
        sourceNote: data?.meta?.sourceNote || '',
    };
}

// ------------------------------------------------------------
// 筛选接口
// ------------------------------------------------------------
function filterByHypothesis(items, h) {
    return items.filter((it) => {
        const hs = Array.isArray(it.hypothesis) ? it.hypothesis : (it.hypothesis ? [it.hypothesis] : []);
        return hs.includes(h);
    });
}
function filterByProduct(items, product) {
    return items.filter((it) => (it.product || '').includes(product) || (it.keyword || '').includes(product));
}
function filterByKeyword(items, kw) {
    return items.filter((it) => it.keyword === kw);
}
function filterByPlatform(items, platform) {
    return items.filter((it) => it.platform === platform);
}
function filterBySentiment(items, sentiment) {
    return items.filter((it) => it.sentiment === sentiment);
}
function filterByTag(items, tag) {
    return items.filter((it) => Array.isArray(it.tags) && it.tags.some((t) => t.includes(tag)));
}
function filterByContent(items, re) {
    const rx = re instanceof RegExp ? re : new RegExp(String(re), 'i');
    return items.filter((it) => rx.test(String(it.content || '')));
}
function filterByRating(items, { min, max } = {}) {
    return items.filter((it) => {
        if (typeof it.rating !== 'number') return false;
        if (min != null && it.rating < min) return false;
        if (max != null && it.rating > max) return false;
        return true;
    });
}
function filterByMinLikes(items, min = 1) {
    return items.filter((it) => (it.likes || 0) >= min);
}

// ------------------------------------------------------------
// 排序 / 抽样
// ------------------------------------------------------------
function topByLikes(items, n = 10) {
    return items.slice().sort((a, b) => (b.likes || 0) - (a.likes || 0)).slice(0, n);
}
function sample(items, n = 6, seed = 1) {
    const arr = items.slice();
    const result = [];
    let s = seed;
    while (result.length < n && arr.length) {
        s = (s * 9301 + 49297) % 233280;
        const idx = Math.floor((s / 233280) * arr.length);
        result.push(arr.splice(idx, 1)[0]);
    }
    return result;
}
function groupByTag(items) {
    const map = new Map();
    for (const it of items) {
        if (!Array.isArray(it.tags)) continue;
        for (const t of it.tags) {
            if (!map.has(t)) map.set(t, { tag: t, count: 0, items: [] });
            const g = map.get(t);
            g.count++;
            g.items.push(it);
        }
    }
    return Array.from(map.values()).sort((a, b) => b.count - a.count);
}

// ------------------------------------------------------------
// 主查询接口：getEvidence —— 章节渲染器直接用
//   新增默认去重 + 最小字符数 + 纯表情 / 纯短回复过滤
//   新增 requireContentHit 选项：强制评论里提到关键词或同义词片段
// ------------------------------------------------------------
const EMOJI_REPLY_REGEX = /^[\s\p{P}\p{Emoji_Presentation}\p{Extended_Pictographic}\[\]R]+$/u;

function isSubstantive(content, minChars) {
    const s = String(content || '').trim();
    if (s.length < minChars) return false;
    // 过滤纯表情 / 纯符号回复
    const stripped = s.replace(/\[[^\]]+\]/g, '').replace(/[\s\p{P}\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu, '');
    return stripped.length >= Math.max(4, Math.floor(minChars / 2));
}

function getEvidence(items, opts = {}) {
    const {
        hypothesis, product, keyword, platform, sentiment, tag,
        minLikes = 0, minChars = 10, contentMatch, requireContentHit = false,
        dedupByContent = true, dedupByNickname = false,
        top = 6, seed = 7, sortBy = 'likes',
    } = opts;
    let filtered = items.slice();
    if (hypothesis) filtered = filterByHypothesis(filtered, hypothesis);
    if (product)    filtered = filterByProduct(filtered, product);
    if (keyword)    filtered = filterByKeyword(filtered, keyword);
    if (platform)   filtered = filterByPlatform(filtered, platform);
    if (sentiment)  filtered = filterBySentiment(filtered, sentiment);
    if (tag)        filtered = filterByTag(filtered, tag);
    if (minLikes)   filtered = filterByMinLikes(filtered, minLikes);

    // 内容有效性过滤（默认开）
    filtered = filtered.filter((it) => isSubstantive(it.content, minChars));
    if (contentMatch) filtered = filterByContent(filtered, contentMatch);

    // requireContentHit: 评论必须提到 keyword 或其可识别片段
    if (requireContentHit) {
        filtered = filtered.filter((it) => {
            const c = String(it.content || '').toLowerCase();
            const kw = String(it.keyword || '').toLowerCase();
            if (!kw) return true;
            // keyword 拆分：去掉"护肝/片/解酒"等泛词，剩余碎片必须出现
            const fragments = kw.split(/护肝|片|解酒|保健品/).filter((x) => x.length >= 2);
            if (fragments.length === 0) return c.includes(kw);
            return fragments.some((f) => c.includes(f));
        });
    }

    // 排序
    if (sortBy === 'likes') {
        filtered = filtered.sort((a, b) => (b.likes || 0) - (a.likes || 0));
    }

    // 去重：content 前 40 字 + (可选) 昵称
    if (dedupByContent) {
        const seen = new Set();
        filtered = filtered.filter((it) => {
            const key = dedupByNickname
                ? `${(it.content || '').slice(0, 40)}|${it.nickname || ''}`
                : (it.content || '').slice(0, 40);
            if (seen.has(key)) return false;
            seen.add(key);
            return true;
        });
    }

    // 取 top N*2 然后稳定抽 N
    const pool = filtered.slice(0, Math.max(top * 2, top + 3));
    return sample(pool, Math.min(top, pool.length), seed);
}

// ------------------------------------------------------------
// 源标识辅助
// ------------------------------------------------------------
function isSeed(item) { return (item?.source || '').includes('pattern') || (item?.source || '').includes('seed'); }
function isReal(item) { return (item?.source || '') === 'real-collected'; }

function getGlobalSourceLabel(meta) {
    if (!meta) return '未加载';
    const tier = meta.sourceTier;
    if (tier === 'real-collected') return '真实采集';
    if (tier === 'pattern-curated') return '公开模式归纳 · 种子样本';
    return '骨架占位';
}

module.exports = {
    HYPOTHESES,
    CHAPTER_HYPOTHESIS_MAP,
    PLATFORM_LABELS,
    loadMerged,
    getMeta,
    flattenMerged,
    // filters
    filterByHypothesis, filterByProduct, filterByKeyword, filterByPlatform,
    filterBySentiment, filterByTag, filterByContent, filterByRating, filterByMinLikes,
    // sorting/sampling
    topByLikes, sample, groupByTag,
    // high-level
    getEvidence,
    isSeed, isReal, getGlobalSourceLabel,
    SKELETON,
};