kangshifu-board/scripts/review-five-topic-round3.py

"""Review preserved originals for sparse topics; this script makes no requests."""
from datetime import datetime, timezone
import hashlib
import importlib.util
import json
from pathlib import Path
import re
from lxml import html

ROOT = Path(__file__).resolve().parents[1]
WORK = ROOT / 'work/five-topic-collection-20261110/supplement-02'
spec = importlib.util.spec_from_file_location('public_originals', ROOT / 'scripts/collect-five-topic-public.py')
base = importlib.util.module_from_spec(spec)
spec.loader.exec_module(base)


def main():
    collection = json.loads((WORK / 'collection-result.json').read_bytes())
    rows = {row['sourceId']: row for row in collection['sources']}
    # These exact two bowls were missed by a strict keyword filter. Recover
    # only their saved canonical HTML, not snippets or recommended product text.
    recovered = []
    for pid in ['ag5rd99w', 'a944o2e7']:
        url = 'https://post.smzdm.com/talk/p/' + pid + '/'
        receipts = [json.loads(file.read_bytes()) for file in (WORK / 'sources').glob('*.http.json')]
        receipts = [r for r in receipts if r['requestedUrl'] == url]
        assert len(receipts) == 1
        receipt = receipts[0]
        assert receipt['httpStatus'] == 200 and receipt['finalUrl'] == url
        raw = (WORK / 'sources' / receipt['responseFile']).read_bytes()
        assert base.digest(raw) == receipt['sha256']
        tree = html.fromstring(raw.decode('utf8'))
        assert tree.xpath('//link[@rel="canonical"]/@href') == [url]
        nodes = tree.xpath('//article')
        assert len(nodes) == 1
        body = base.paragraphs(nodes[0])
        assert '当归' in body and ('碗面' in body or '鸡面' in body)
        dates = re.findall(r'\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}', nodes[0].text_content()[:1000])
        assert len(set(dates)) == 1
        row = {**collection['sources'][0], 'platform': 'smzdm', 'postId': pid, 'sourceId': 'smzdm:' + pid,
               'url': url, 'title': tree.xpath('string(//article//h1)').strip(), 'body': body,
               'bodySha256': base.digest(body.encode()), 'publishedAtRaw': dates[0],
               'collectedAt': receipt['collectedAt'], 'httpReceipt': receipt, 'topics': [],
               'searchDiscovery': {'kind': 'preserved-herbal-search', 'query': '当归鸡面 site:post.smzdm.com', 'receipt': 'search/herb-2.results.json'},
               'recoveryReason': 'Exact preserved bowl-noodle body has Chinese ingredient context; initial literal filter missed it.'}
        assert row['title']
        rows[row['sourceId']] = row
        recovered.append(row)
    base.save(WORK / 'review-recovered-originals.json', {'sources': recovered})

    selected = []

    def approve(rid, evidence, note, source_type='community_experience', scope='packaged_noodles'):
        row = rows[rid]
        topic_rows = []
        for topic, relation, passages in evidence:
            assert all(passage in row['body'] for passage in passages), 'PASSAGE_MISMATCH:' + rid
            topic_rows.append({'topic': topic, 'relation': relation, 'passages': passages})
        selected.append({**row, 'topics': topic_rows, 'reviewNote': note,
                         'sourceType': source_type, 'contentScope': scope})

    approve('smzdm:ag5rd99w', [
        ('wellness', 'chinese_ingredient_experience', ['当归的香味很浓郁，面汤也很鲜', '原价的话应该不会买，小贵'])
    ], '当归碗面香气与价格顾虑的短体验；原文未明确完整商品型号，不推定具体SKU或有效添加量。')
    approve('smzdm:a944o2e7', [
        ('wellness', 'ingredient_label_perception', ['统一当归花雕鸡面非油炸细面，93克一碗', '面饼虽是非油炸，但看配料表却是长长的一串'])
    ], '作者对包装、配料表和健康定位的主观感受；原文带激励计划标签。不能据此认定添加剂不安全或产品不健康，未独立核实配料及净含量。')
    approve('smzdm:a5o58q4k', [
        ('premium', 'price_convenience_tradeoff', ['首先面饼需要用开水煮一段时间，而泡面只需要开水泡开', '14元左右能给到真材实料的 牛肉 牛筋很不错了'])
    ], '真肉配料、独立辣酱与煮制时间的作者自述；非油炸健康判断不采信为功效结论，14元为历史作者报价。')
    approve('smzdm:a303ep3r', [
        ('premium', 'negative_product_experience', ['首先吐槽下面，因为是晒干的面，碎的一塌糊涂，煮好捞都捞不出来。', '牛肉牛筋一种2块，没有外包装上的多，但也还行，牛筋很软糯。', '日常价偏贵也不够方便和好吃'])
    ], '保留面碎、份量不足、包装图差异和价格便利性的负面体验；作者关于产地的强调不视为安全性证据。')
    approve('smzdm:akk0p3ok', [
        ('premium', 'consumer_experience', ['手工面需要上火煮', '肉酱包是辣的，可以视口味酌情添加', '19.9元一包面，牛肉可给的不算少'])
    ], '配料可调、需要煮制和肉量体验；非油炸更健康为作者观点，19.9元不写成当前价格。')
    approve('smzdm:a900qdv0', [
        ('premium', 'consumer_experience', ['里面有大块的半筋半肉，汤汁很浓郁'])
    ], '短体验仅支持肉块和浓汤的主观感受，不证明复购率、销量或真实熬汤工艺。')
    approve('smzdm:az6nqzvp', [
        ('premium', 'packaging_value_tradeoff', ['有点过分包装的嫌疑', '半筋半肉，的确都有，看得见，吃得到，量还不少', '入手价16.5元一盒'])
    ], '包装负担、肉量与露营后购买的自述；三种番茄是原文转述商家说法，价格保留历史口径。')
    approve('smzdm:aqm7vln2', [
        ('premium', 'portion_price_tradeoff', ['15元还可以接受', '成人的话多一份拉面才能吃饱，我经常去超市买点小包装的拉面一起煮'])
    ], '成人份量不足和家庭周末早餐体验；不把不会上火等说法当健康事实，也不把家庭用餐自动算成囤货证据。')
    approve('smzdm:aqm3qrvx', [
        ('stockpile', 'household_stock_experience', ['是我唯一会在家储备的杯面', '我个人家中最常备的泡面就是康师傅红烧牛肉面']),
        ('premium', 'price_convenience_tradeoff', ['好价我必囤，也愿意为它比起普通方便面更高的价格买单']),
        ('wellness', 'ingredient_taste_reference', ['虽然整体咸度偏高', '干料包里的枸杞'])
    ], '2022年居家十款面比较；枸杞鸡汤只是口味/可见配料参考，非功效证据。历史囤货低价不写入现价或净含量字段。')
    approve('smzdm:aqmgm7p7', [
        ('stockpile', 'historical_purchase_recommendation', ['又到了双11囤货季', '面条分量足，调料包分量足', '面条很扛煮，但分量有点少'])
    ], '2022年促销季多品牌试吃与采购建议，保留烹饪时间、咸度和份量差异；全文价格为历史作者报价。')
    approve('smzdm:a60rw3wg', [
        ('stockpile', 'negative_storage_experience', ['从柜子里翻出1包2年前的方便面，生产日期是2021年，闻了闻味道真的不能吃了', '这个生产日期很新鲜，5+1的口味'])
    ], '作者久置后异味与改为按需购买的反例；未核实保质期和储存条件，不认定具体产品法定保质期，也不推算普遍变质率。')
    approve('smzdm:andlvw9p', [
        ('stockpile', 'historical_editorial_purchase_guide', ['不过有这种多种口味包装的，可以试试扩充口粮，选出自己最喜欢的几个味道', '各种细碎的配料袋撕起来有点麻烦'])
    ], '2020年编辑推荐，区分编辑观点和消费者体验；保留组合装与拆袋负担参考，不采用榜单、辣度、卫生和销量声明为已核实事实。', 'editorial_purchase_guide')
    approve('smzdm:a5k6o293', [
        ('wellness', 'ingredient_taste_reference', ['康师傅蘑菇鲜蔬面除了面饼还同时配备三袋调料包']),
        ('stockpile', 'household_stock_experience', ['最近买了一箱康师傅五连包蘑菇鲜蔬面'])
    ], '2019年整箱五连包采购与个人二次烹饪；鸡蛋、火腿等为自加，不当作商品自带配料，不推断养生功效。')
    approve('smzdm:aklqxvnk', [
        ('wellness', 'ingredient_taste_reference', ['白象方便面，珍骨汤菌菇味，每包103g五连包', '量刚好，大晚上的，也不能吃太饱'])
    ], '菌菇猪骨汤风味与夜宵份量的作者自述；103g不直接认定面饼净含量，不据风味确认菌菇添加量。')
    approve('sspai:73701', [
        ('wellness', 'nutrition_process_reference', ['非油炸泡面与油炸泡面的差异能轻易地感受出来', '无法比较二者最终的人体摄入量']),
        ('premium', 'ingredient_packaging_tradeoff', ['翻看材料表，果然是素肉（植物蛋白），让我十分失望', '拌面泡熟倒热水时，手指接触盒底不会烫手'])
    ], '2022年作者多款测评：营养比较方法、口感复水、真肉预期与防烫包装参考。脂肪/钠数值未独立复算，健康判断和微膨化工艺猜测不是结论；不采用其总括性优劣判断。')
    approve('smzdm:akmvl8o9', [
        ('premium', 'ingredient_packaging_tradeoff', ['盒内有面饼一袋，牛肉一包，高汤一包，脱水蔬菜一包', '汤头整体偏淡，感觉不像是泡面，倒像是牛肉挂面'])
    ], '2020年速达面馆包装、真肉和偏淡汤头体验；二两出头/肉占一半是作者粗估，不入规格字段；热风干燥与原价未独立验证。')
    approve('foodaily:27103', [
        ('premium', 'historical_industry_opinion', ['对拉面说价格太贵，味道太差，还得二次加工的质疑'])
    ], '2022年壹览商业转载评论，投诉为媒体二手转述，不当独立用户评价；全文包含品牌否认传闻，不采信争议经营数据，也不把历史负面观点当当前经营事实。', 'industry_reporting_or_opinion')
    approve('foodaily:24347', [
        ('premium', 'packaged_frozen_meal_reference', ['向市场推出家庭装自煮乌冬面，正式布局零售业态', '零售价为22元左右，360克，适合一人食']),
        ('stockpile', 'frozen_storage_tradeoff_report', ['冷冻品，对产品生产、包装、运输、储存、配送等环节有着极高的要求'])
    ], '2021年预包装冷冻乌冬整餐报道，区分堂食、原材料乌冬与包装成品；采访对象的锁鲜/保质期比较未验证，销量、报价和渠道为历史口径。', 'industry_reporting_or_opinion', 'adjacent_noodle_food')

    ids, bodies = set(), set()
    for row in selected:
        assert row['sourceId'] not in ids and row['bodySha256'] not in bodies
        ids.add(row['sourceId']); bodies.add(row['bodySha256'])
        assert hashlib.sha256(row['body'].encode()).hexdigest() == row['bodySha256']
        raw = (WORK / row['collectionDirectory'] / row['httpReceipt']['responseFile']).read_bytes()
        assert base.digest(raw) == row['httpReceipt']['sha256']
    specific_exclusions = {
        'smzdm:ak3p84g8': '推广式种草，已有更具体的肉量、份量与包装正反体验。',
        'smzdm:a903en3p': '推广式内容较强，含大量未核实营养或功效话术，不为数量补入。',
        'smzdm:az6d4590': '推广式内容较强，缺少新增可交叉验证的具体体验。',
        'smzdm:aqn9gepv': '混合粉面与多类速食的长清单，本轮优先关联明确的面制品原文。',
        'smzdm:a4pmzlnw': '泛称纯素/无动物成分，缺少实际配料证据和具体体验。',
        'foodaily:24632': '多品牌混合商业文章，相关度与新增研发参考有限。',
        'foodaily:24499': '历史行业背景与已收录资料重复度高，优先保留一手具体体验。',
        'foodaily:24096': '混合新闻周报，只有一段方便面新品信息。',
    }
    excluded = [{'id': r['sourceId'], 'url': r['url'], 'reason': specific_exclusions[r['sourceId']]}
                for r in collection['sources'] if r['sourceId'] not in ids]
    excluded += [
        {'url': 'https://post.smzdm.com/talk/p/aqmkmpv7/', 'reason': '实际为螺蛳粉，本轮不将相邻米粉混充中式面制品。'},
        {'url': 'https://post.smzdm.com/talk/p/a7nd39kl/', 'reason': '实际为猴头菇米稀，不是面制品。'},
        {'url': 'https://post.smzdm.com/p/agow093m/', 'reason': '转至geetest验证入口，未取得原文，停止该站后续请求；不能视为无内容。'},
    ]
    counts = {t: sum(any(e['topic'] == t for e in row['topics']) for row in selected)
              for t in ['wellness', 'student', 'premium', 'stockpile', 'novelty']}
    result = {'schema': 'ksf.five-topic.reviewed-supplement.v1', 'reviewedAt': datetime.now(timezone.utc).isoformat(),
              'selected': selected, 'excluded': excluded, 'countsByTopic': counts,
              'acceptedUniqueArticles': len(selected),
              'limits': ['主题按正文逐字引文关联，不将历史作者说法、媒体引述或推广内容全部当成事实。',
                         '中式养生资料区分当归风味体验、菌菇/枸杞食材参考和营养工艺测评，不证明养生功效。',
                         '新增资料原日期为2019—2025年，适合研发对照，不代表当前市场趋势或当前报价。',
                         '独立URL不等于独立消费者或独立证据；保留编辑推荐、作者体验和二手报道的区别。',
                         '未新增电商购买评价、商品规格或价格；京东公开评价入口访问拒绝，未据此删除商品。']}
    base.save(WORK / 'reviewed-selection.json', result)
    print(json.dumps({'ok': True, 'articles': len(selected), 'countsByTopic': counts}, ensure_ascii=False))


if __name__ == '__main__': main()