صفحهٔ اصلی گشت‌و‌گذار راهنما
ورود
18307996893
/
kangshifu-board
1
0
انشعاب 0
پرونده‌ها مشکلات 0 درخواست واکشی 0 ویکی
درخت: 48c2134d2a
شاخه‌ها تگ‌ها
kangshifu-board
/
scripts
/
review-five-topic-multisource-20261111.py

review-five-topic-multisource-20261111.py 13 KB
تاريخچه خام

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160 
  1. """Curated selection with replayable text boundaries and exact topic quotations."""
    2. 

  2. import importlib.util
    3. 

  3. import json
    4. 

  4. from pathlib import Path
    5. 

  5. import re
    6. 

  6. import urllib.parse
    7. 

  7. from lxml import html
    8. 

  8. 

  9. ROOT=Path(__file__).resolve().parents[1]
    10. 

  10. WORK=ROOT/'work/five-topic-research-20261111'
    11. 

  11. spec=importlib.util.spec_from_file_location('base_review',ROOT/'scripts/collect-five-topic-public.py')
    12. 

  12. b=importlib.util.module_from_spec(spec);spec.loader.exec_module(b)
    13. 

  13. 

  14. def lines_select(row, start=None, stop=None, indexes=None):
    15. 

  15.  lines=row['body'].splitlines()
    16. 

  16.  if indexes is None:
    17. 

  17.   first=next((i for i,line in enumerate(lines) if start in line),None) if start else 0
    18. 

  18.   assert first is not None,('START_NOT_FOUND',start)
    19. 

  19.   last=next((i for i,line in enumerate(lines[first+1:],first+1) if stop in line),len(lines)) if stop else len(lines)
    20. 

  20.   indexes=list(range(first,last))
    21. 

  21.  row['paragraphIndexes']=indexes
    22. 

  22.  row['body']='\n'.join(lines[i] for i in indexes)
    23. 

  23.  row['bodySha256']=b.digest(row['body'].encode())
    24. 

  24.  return row
    25. 

  25. 

  26. def topic(row, key, phrase, relation):
    27. 

  27.  paragraph=next((line for line in row['body'].splitlines() if phrase in line),None)
    28. 

  28.  assert paragraph,('QUOTE_NOT_FOUND',row['url'],phrase)
    29. 

  29.  row['topics'].append({'topic':key,'relation':relation,'passages':[paragraph]})
    30. 

  30. 

  31. def main():
    32. 

  32.  rows=json.loads((WORK/'reextracted-sources.json').read_bytes())
    33. 

  33.  rows+=json.loads((WORK/'focused-collection-result.json').read_bytes())['sources']
    34. 

  34.  rows+=json.loads((WORK/'experience-collection-result.json').read_bytes())['sources']
    35. 

  35.  chosen=[]
    36. 

  36.  def choose(part, kind, note):
    37. 

  37.   original=next(x for x in rows if part in x['url']);r=json.loads(json.dumps(original))
    38. 

  38.   raw=(WORK/'sources'/r['httpReceipt']['responseFile']).read_bytes();assert b.digest(raw)==r['httpReceipt']['sha256']
    39. 

  39.   tree=html.fromstring(raw.decode(r['sourceEncoding']))
    40. 

  40.   if r['platform']=='smzdm':
    41. 

  41.    r['bodyExtractionSelector']='//article';assert len(tree.xpath('//article'))==1
    42. 

  42.    r['body']=b.paragraphs(tree.xpath('//article')[0])
    43. 

  43.    r['postId']=re.search(r'/(?:talk/)?p/([a-z0-9]+)/',r['url'])[1]
    44. 

  44.   if 'sohu.com' in r['url']:
    45. 

  45.    r['platform']='sohu';r['postId']=re.search(r'/a/([0-9_]+)',r['url'])[1]
    46. 

  46.    r['publishedAtRaw']=tree.xpath('//meta[@property="og:release_date"]/@content')[0]
    47. 

  47.   if 'sina.cn' in r['url']:
    48. 

  48.    r['platform']='sina';r['postId']=re.search(r'detail-([a-z0-9]+)\.d',r['url'])[1]
    49. 

  49.    r['url']=r['url'].split('?')[0]
    50. 

  50.   r.update(entity='social',id=r['platform']+':'+r['postId'],sourceType=kind,reviewNote=note,topics=[],contentScope='packaged_noodles')
    51. 

  51.   r['bodySha256']=b.digest(r['body'].encode());chosen.append(r);return r
    52. 

  52. 

  53.  r=choose('/a30dq7kd/','community_experience','宿舍冲泡耗时和藤椒、麻辣口味体验；身份和配方未独立核实。')
    54. 

  54.  topic(r,'student','泡个十多分钟','dormitory_preparation_time')
    55. 

  55.  topic(r,'novelty','藤椒味','pepper_flavor_trial')
    56. 

  56.  r=choose('/aov0k0on/','community_experience','5连包、面饼保护和分步加料体验；非油炸和重量仅为作者表述，不转为商品规格。')
    57. 

  57.  topic(r,'student','先泡面饼','hot_water_preparation')
    58. 

  58.  topic(r,'novelty','兰州拉面','regional_flavor_case')
    59. 

  59.  topic(r,'stockpile','5连包','multipack_protection')
    60. 

  60.  r=choose('/akko2xnr/','community_experience','保留短帖真实分量和口味偏好，不扩写为调查或购买评价。')
    61. 

  61.  topic(r,'student','一袋半','portion_preference')
    62. 

  62.  r=choose('/aevdzmom/','community_experience','宿舍饱腹与泡椒酸辣偏好；不据此推断学生总体偏好。')
    63. 

  63.  topic(r,'student','在宿舍','dormitory_satiety')
    64. 

  64.  topic(r,'novelty','泡椒','sour_spicy_flavor_case')
    65. 

  65.  r=choose('/a20o44kn/','community_experience','晚自习后进食和宿舍整箱储备；单帖不推算人数或市场份额。')
    66. 

  66.  topic(r,'student','晚自习','after_class_late_night')
    67. 

  67.  topic(r,'stockpile','各屯了一箱','dormitory_case_stocking')
    68. 

  68.  r=choose('/arqpdepg/','community_experience','重点保留汤味偏好和喝汤后口渴体验；历史与营养泛论不是研究结论。')
    69. 

  69.  lines_select(r,start='方便面已经成为')
    70. 

  70.  r['contentScope']='packaged_noodles_excerpt';r['title']+='（食用体验摘录）'
    71. 

  71.  topic(r,'student','夜宵','late_night_soup_experience')
    72. 

  72.  topic(r,'wellness','觉得渴','adjacent_health_taste_sodium_hypothesis')
    73. 

  73.  r=choose('/736876/','community_experience','2018年冻干牛肉面体验；工艺、健康宣传和历史报价均未独立核实，非当前售价或功效证据。')
    74. 

  74.  topic(r,'student','等待5分钟','late_night_hot_water_preparation')
    75. 

  75.  topic(r,'premium','大片蔬菜','freeze_dried_texture_experience')
    76. 

  76.  r=choose('/85734155_119756','editorial_comparative_review','2016年30款评测的评分框架与具体体验，作历史设计参考；非当前榜单，未核实每一款商品规格。')
    77. 

  77.  topic(r,'student','深夜','late_night_historical_comparison')
    78. 

  78.  topic(r,'novelty','辣白菜','regional_flavor_historical_comparison')
    79. 

  79.  topic(r,'premium','工具方便度','comparative_review_dimensions')
    80. 

  80.  r=choose('detail-imzzcerw1177484','industry_product_process_reporting','2023年上观新闻转载；熟面急冻、高汤和一体化是报道声明，专利与参数未逐一独立验证；需要冷链和加热，不是宿舍常温开水即食方案。')
    81. 

  81.  topic(r,'premium','4-6个小时','frozen_soup_process_reporting')
    82. 

  82.  topic(r,'novelty','全程不用加水','frozen_integrated_format_case')
    83. 

  83.  r=choose('/articles/41519','industry_reporting_or_opinion','保留DT商业观察署名；新品、财务、份额和消费者引语均是报道口径，不新增为电商商品或独立评论。')
    84. 

  84.  # Exclude publisher event promotions, retaining the reported article body.
    85. 

  85.  promo=next((line for line in r['body'].splitlines() if 'Foodaily创新博览会' in line),None)
    86. 

  86.  if promo:lines_select(r,stop=promo)
    87. 

  87.  topic(r,'premium','隔热包装','packaging_and_real_ingredient_design')
    88. 

  88.  topic(r,'novelty','地域风味上','regional_flavor_portfolio')
    89. 

  89.  topic(r,'student','年轻人的夜宵场景','late_night_scene_reporting')
    90. 

  90.  topic(r,'stockpile','家庭消费场景','family_channel_positioning')
    91. 

  91.  r=choose('/articles/25010','industry_reporting_or_opinion','2021年深燃历史报道，用于高端溢价与价值感取舍；所列售价、销量和市场规模均非本轮实时验证。')
    92. 

  92.  promo=next((line for line in r['body'].splitlines() if 'Foodaily创新博览会' in line),None)
    93. 

  93.  if promo:lines_select(r,stop=promo)
    94. 

  94.  topic(r,'premium','原材料上下功夫','historical_premium_value_tradeoff')
    95. 

  95.  r=choose('/articles/42324','industry_product_news_excerpt','只摘录盒马冻干方便面小节；其他饮料与招商文不入库。配方、温度、冻干和热风干燥均为报道声明，并非实测工艺参数。')
    96. 

  96.  lines=r['body'].splitlines();indexes=[i for i,line in enumerate(lines) if '据官方介绍，新品冻干方便面' in line]
    97. 

  97.  assert len(indexes)==1;lines_select(r,indexes=indexes)
    98. 

  98.  r['title']='盒马上新冻干方便面系列（Foodaily新品周报原文摘录）';r['contentScope']='packaged_noodles_excerpt'
    99. 

  99.  topic(r,'premium','汤菜一体','integrated_freeze_dried_format_reporting')
    100. 

  100.  topic(r,'novelty','番茄炒蛋','new_flavor_lineup_reporting')
    101. 

  101.  r=choose('/articles/41662','industry_product_news_excerpt','只摘录方便面新品两段；原文减卡、减脂和高纤为品牌经媒体转述的营养声称，不证明减重功效，未核对实物标签。')
    102. 

  102.  lines=r['body'].splitlines();indexes=[i for i,line in enumerate(lines) if line.startswith('4月27日，康师傅推出') or line.startswith('历经几十年发展，方便面行业')]
    103. 

  103.  assert len(indexes)==2;lines_select(r,indexes=indexes)
    104. 

  104.  r['title']='康师傅上新红烧牛肉面减脂高纤面（Foodaily热闻原文摘录）';r['contentScope']='packaged_noodles_excerpt'
    105. 

  105.  topic(r,'wellness','0油炸','adjacent_health_non_fried_fiber_claim_case')
    106. 

  106.  r=choose('/articles/42270','industry_interview_excerpt','只选居家煮面需求段，排除餐厅打卡段；报道人物均为化名，作者中产判断不是经验证的收入分层或总体需求。')
    107. 

  107.  lines=r['body'].splitlines();start=lines.index('“过日子”的最优解')
    108. 

  108.  end=next(i for i,line in enumerate(lines) if line.startswith('参考资料：'))
    109. 

  109.  lines_select(r,indexes=[0,1]+list(range(start,end)))
    110. 

  110.  r['title']='中产“过日子人”，爱上高阶泡面（居家需求段落摘录）';r['contentScope']='packaged_noodles_excerpt'
    111. 

  111.  topic(r,'premium','性价比最高','home_meal_convenience_value_tradeoff')
    112. 

  112. 

  113.  # Government publications are documents, not social posts or mandatory standards.
    114. 

  114.  for host,selector,docid,status,title in [
    115. 

  115.   ('ganxian.gov.cn',"//*[@id='zoomcon']",'gx-51706ef906154551b70187447738432f','official_explanation_republication','解读《关于党参等9种新增按照传统既是食品又是中药材的物质公告》（2023年第9号）'),
    116. 

  116.   ('baiyin.gov.cn',"//div[contains(concat(' ',normalize-space(@class),' '),' content ')]",'baiyin-0eece4641557434f8f67a48c85112ebc','consumer_guidance','解码食品科技——正确认识预包装食品保质期'),
    117. 

  117.   ('scjgj.km.gov.cn',"//*[contains(concat(' ',normalize-space(@class),' '),' articleContMain ')]",'kunming-4719685','consumer_guidance_republication','减盐食品不宜敞开吃')]:
    118. 

  118.   receipts=[(f,json.loads(f.read_bytes())) for f in (WORK/'sources').glob('*.http.json')]
    119. 

  119.   f,receipt=next((f,r) for f,r in receipts if host in r['requestedUrl'] and r['httpStatus']==200)
    120. 

  120.   raw=(f.parent/receipt['responseFile']).read_bytes();assert b.digest(raw)==receipt['sha256']
    121. 

  121.   tree=html.fromstring(raw.decode('utf8'));nodes=tree.xpath(selector);assert len(nodes)==1
    122. 

  122.   body=b.paragraphs(nodes[0]);date=tree.xpath('//meta[@name="PubDate"]/@content')
    123. 

  123.   assert date
    124. 

  124.   notes={'pageMetaPubDate':date[0]}
    125. 

  125.   if host=='baiyin.gov.cn':
    126. 

  126.    visible=re.search(r'发布时间:\s*(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})',tree.text_content())
    127. 

  127.    assert visible;date=[visible[1]];notes['visiblePublishedAt']=date[0];notes['dateDiscrepancyRetained']=True
    128. 

  128.   row={'entity':'documents','id':docid,'platform':'document','postId':docid,'title':title,'body':body,'bodySha256':b.digest(body.encode()),
    129. 

  129.        'url':receipt['finalUrl'],'publishedAtRaw':date[0],'sourceEncoding':'utf8','bodyExtractionSelector':selector,
    130. 

  130.        'collectionDirectory':'sources','httpReceipt':receipt,'declaredStatus':status,'sourceType':'official_explanation' if host=='ganxian.gov.cn' else 'official_consumer_guidance',
    131. 

  131.        'reviewNote':'政府网站正文；属于解读或消费科普，不是新增国家标准或产品认证。转载发布时间与原公告年份分开保留。',
    132. 

  132.        'dateEvidence':notes,'topics':[],'clauses':[],'limits':['非国家标准全文，不据科普单篇确认法规现行性；引用原标准时须结合研究库既有标准及生效日期复核。']}
    133. 

  133.   if host=='ganxian.gov.cn':
    134. 

  134.    topic(row,'wellness','上述物质作为食药物质','traditional_ingredient_application_boundary')
    135. 

  135.    row['clauses']=[{'name':'党参传统食用方式及同用限制','phrase':'党参。在','application':'中式汤底原料候选；需要核对植物基原、食用方式和产品标签，不视为疗效依据。'},
    136. 

  136.     {'name':'食药物质适量食用与特殊人群边界','phrase':'上述物质作为食药物质','application':'研发需明确适用人群；不得把食药目录等同普通食品功效许可。'}]
    137. 

  137.   elif host=='baiyin.gov.cn':
    138. 

  138.    topic(row,'stockpile','按需采购，尽快食用','storage_packaging_and_opening')
    139. 

  139.    row['clauses']=[{'name':'标签贮存条件与合理储存','phrase':'1.查看标签','application':'囤货规格与家庭储存方案需配套温湿度、避光等标签条件。'},
    140. 

  140.     {'name':'开封后的密封体系变化','phrase':'3.按需采购','application':'比较单份独立包装与大包装开封后的使用便利；保质期必须实测验证。'}]
    141. 

  141.   else:
    142. 

  142.    topic(row,'wellness','减盐食品也不一定是低钠食品','adjacent_health_sodium_claim_boundary')
    143. 

  143.    row['clauses']=[{'name':'减盐与低钠的差别','phrase':'阮光锋分析说','application':'是2023年媒体科普转载；相对减少与绝对低钠的阈值应回到适用标准验证，不能把减盐说成低钠。'},
    144. 

  144.     {'name':'钠含量的每100克与每份口径','phrase':'所以，科学减盐需要','application':'汤料研发比较须统一每份重量和营养表单位；不据主观咸淡判断钠含量。'}]
    145. 

  145.   for c in row['clauses']:
    146. 

  146.    c['excerpt']=next(line for line in body.splitlines() if c['phrase'] in line);assert c['excerpt'] in body
    147. 

  147.   chosen.append(row)
    148. 

  148.  assert len({r['id'] for r in chosen})==len(chosen)
    149. 

  149.  for r in chosen:
    150. 

  150.   assert r['topics'] and all(p in r['body'] for t in r['topics'] for p in t['passages'])
    151. 

  151.  result={'schema':'ksf.five-topic.reviewed-multisource.v1','selected':chosen,
    152. 

  152.   'countsByTopic':{t:sum(any(v['topic']==t for v in r['topics']) for r in chosen) for t in ['wellness','student','premium','stockpile','novelty']},
    153. 

  153.   'limits':['跨课题关联不重复计算总资料数；政策条款是同一文件的摘录，不额外算独立文件。',
    154. 

  154.    '非油炸、高纤、控钠是中式养生的相邻健康化研究，不等同中医功效或食药原料合法性。',
    155. 

  155.    '本轮未取得新增电商商品和购买评价；公开体验不冒称电商评价。',
    156. 

  156.    '所有历史资料保留日期，只作历史比较，不直接作为2026年市场趋势。']}
    157. 

  157.  b.save(WORK/'reviewed-selection.json',result)
    158. 

  158.  print(json.dumps({'selected':len(chosen),'social':sum(r['entity']=='social' for r in chosen),'documents':sum(r['entity']=='documents' for r in chosen),'clauses':sum(len(r.get('clauses',[])) for r in chosen),'topics':result['countsByTopic']},ensure_ascii=False))
    159. 

  159. 

  160. if __name__=='__main__':main()
    161.