kangshifu-board/scripts/package-voc-library.py

"""Prepare the reviewed VOC database + Agent contract as one complete handoff."""
import argparse
import csv
import hashlib
import html
import importlib.util
import json
from pathlib import Path
import shutil
import sqlite3
import zipfile

ROOT=Path(__file__).resolve().parents[1]
SOURCE=ROOT/'server/voc-library'
WORKSHOP=ROOT/'work/five-views-delivery-20261103/skills/ksf-workshop-runtime'
DIST=Path('F:/文档/康师傅skill')
NAME='ksf-voc-library-20261105-r1'


def sha(p):
    with p.open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()


def writejson(p,value):
    p.parent.mkdir(parents=True,exist_ok=True)
    p.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')


def copy(a,b):
    b.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(a,b)


def manifest(folder):
    files={p.relative_to(folder).as_posix():sha(p) for p in sorted(folder.rglob('*')) if p.is_file() and p!=folder/'FILES.sha256' and '__pycache__' not in p.parts}
    (folder/'FILES.sha256').write_text(''.join(f'{s}  {n}\n' for n,s in files.items()),encoding='utf-8',newline='\n')
    return files


def skill_manifest(folder):
    files={p.relative_to(folder).as_posix():sha(p) for p in sorted(folder.rglob('*')) if p.is_file() and p.name not in ('FILES.sha256','credential-binding.json') and '__pycache__' not in p.parts}
    (folder/'FILES.sha256').write_text(''.join(f'{s}  {n}\n' for n,s in files.items()),encoding='utf-8',newline='\n')
    return files


def package(db,build_receipt,validation):
    out=ROOT/'artifacts/康师傅VOC结构化入库-20261105'
    release=out/NAME
    if release.exists():raise ValueError('RELEASE_ALREADY_EXISTS')
    receipt=json.loads(build_receipt.read_text(encoding='utf-8'))
    verified=json.loads(validation.read_text(encoding='utf-8'))
    assert receipt['ok'] and verified['ok'] and receipt['generation']==verified['generation'] and sha(db)==verified['databaseSha256']
    baseline=DIST/'ksf-workshop-runtime'
    old_files={p.relative_to(baseline).as_posix():sha(p) for p in baseline.rglob('*') if p.is_file()}
    # Capture the actual distributed baseline before changing the current copy.
    history=ROOT/'work/voc-structure-20261105/workshop-baseline'
    if history.exists():raise ValueError('BASELINE_SNAPSHOT_ALREADY_EXISTS')
    shutil.copytree(baseline,history)
    changed=[]
    for p in WORKSHOP.rglob('*'):
        if p.is_file() and p.name!='FILES.sha256' and '__pycache__' not in p.parts:
            rel=p.relative_to(WORKSHOP).as_posix()
            if old_files.get(rel)!=sha(p):changed.append(rel)
    assert sorted(changed)==['SKILL.md','references/workshop-tasks.md'],changed
    workshopfiles=skill_manifest(WORKSHOP)
    skill=ROOT/'skills/ksf-voc-library'
    for filename in ('query_voc.py','voc_db.py','schema.sql','taxonomy.json'):
        copy(SOURCE/filename,skill/'scripts'/filename)
    skillfiles=skill_manifest(skill)
    shutil.copytree(skill,release/'skill',ignore=shutil.ignore_patterns('__pycache__'))
    copy(db,release/'library/library.sqlite')
    for p in SOURCE.iterdir():
        if p.is_file():copy(p,release/'ops'/p.name)
    shutil.copytree(WORKSHOP,release/'materials/skills/ksf-workshop-runtime',ignore=shutil.ignore_patterns('__pycache__','credential-binding.json'))
    copy(ROOT/'work/five-views-delivery-20261103/scripts/install-workshop-skill.mjs',release/'materials/scripts/install-workshop-skill.mjs')
    for p in (ROOT/'docs/voc-structure-20261105').iterdir():
        if p.is_file():copy(p,release/'docs'/p.name)
    copy(build_receipt,release/'receipts/build.json');copy(validation,release/'receipts/validation.json')
    examples={
        'jd-ksf-banmian-products':dict(entity='products',marketplace='京东',brand='康师傅',category='拌面',limit=20),
        'jd-ksf-banmian-reviews':dict(entity='reviews',marketplace='京东',brand='康师傅',category='拌面',reviewScope='sku_confirmed',includeDefault=False,limit=20),
        'tmall-ksf-banmian-products':dict(entity='products',marketplace='天猫',brand='康师傅',category='拌面',limit=20),
        'tmall-ksf-banmian-reviews':dict(entity='reviews',marketplace='天猫',brand='康师傅',category='拌面',reviewScope='sku_confirmed',includeDefault=False,limit=20),
        'tmall-ksf-banmian-listing-reviews':dict(entity='reviews',marketplace='tmall',brand='ksf',category='banmian',reviewScope='listing',includeDefault=False,limit=20),
        'jd-ksf-banmian-prices':dict(entity='prices',marketplace='jd',brand='ksf',category='banmian',limit=20)
    }
    for name,value in examples.items():writejson(release/'examples'/(name+'.json'),value)
    c=sqlite3.connect(db)
    with (release/'docs/待核对问题清单.csv').open('w',encoding='utf-8-sig',newline='') as f:
        writer=csv.writer(f);writer.writerow(['entity_type','entity_id','code','detail'])
        writer.writerows(c.execute('SELECT entity_type,entity_id,code,detail FROM quality_issues ORDER BY entity_type,entity_id,code'))
    c.close()
    installation={ 'version':NAME,'generation':receipt['generation'], 'files':{p.relative_to(release).as_posix():sha(p) for d in ['library','skill'] for p in (release/d).rglob('*') if p.is_file()} }
    writejson(release/'INSTALL.json',installation)
    delivery={**verified,'name':NAME,'archiveSha256':receipt['archiveSha256'],'mainWorkshopSkillChanged':['SKILL.md','references/workshop-tasks.md','FILES.sha256'],
              'mainWorkshopRuntimeScriptsChanged':False,'primaryProducts':4635,'recoveredProductReferences':4,'primaryEntities':10865,
              'workshopSkillManifestSha256':sha(WORKSHOP/'FILES.sha256'),'researchSkillManifestSha256':sha(skill/'FILES.sha256'),
              'localPrepared':True,'installedOnWanggang001':False,'realMessageVerified':False}
    writejson(release/'delivery-receipt.json',delivery)
    (release/'README.md').write_text('''# 王勇 VOC 结构化研究库与 Agent 规则 · 2026-11-05

本地已实际构建/查询/校验；尚未安装到王刚001，未写现网业务库或公开看板。

- 运维001：读 docs/发给运维001.md，必须转整包，先 --check 再 --apply。
- 王刚001：安装后读 docs/发给王刚001.md，用真实消息验范围、评价规格和来源。
- 结构与原则：docs/数据库与Agent规则.md。
- 真实数量、代际、数据库指纹、测试结果：delivery-receipt.json 与 receipts/。
- 待核对项：docs/待核对问题清单.csv。未分类不冒充已确认范围。

library 是私有研究数据库；skill 是新只读查询Skill；materials 是现有工作坊三文件说明升级；ops 是经审校的构建/安装/验收代码。原始采集ZIP不重复附带，继续留存原件。manifest校验后才使用，不执行原ZIP中的采集脚本。
''',encoding='utf-8')
    files=manifest(release)
    zip_path=out/(NAME+'.zip')
    with zipfile.ZipFile(zip_path,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=6) as z:
        for p in sorted(release.rglob('*')):
            if p.is_file():z.write(p,p.relative_to(release).as_posix())
    zip_info=dict(filename=zip_path.name,bytes=zip_path.stat().st_size,sha256=sha(zip_path),entries=len(files)+1)
    writejson(out/'整包校验.json',zip_info)
    # Keep the prior signed deployment record as history, don't relabel it new.
    previous=DIST/'历史归档/20261104-先查库'
    previous.mkdir(parents=True,exist_ok=True)
    for p in [DIST/'版本与校验.json',DIST/'ksf-workshop-runtime-20261104.zip']:
        if p.exists():copy(p,previous/p.name)
    for p in WORKSHOP.rglob('*'):
        if p.is_file():copy(p,baseline/p.relative_to(WORKSHOP))
    new_zip=DIST/'ksf-workshop-runtime-20261105.zip'
    with zipfile.ZipFile(new_zip,'w',compression=zipfile.ZIP_DEFLATED) as z:
        for p in sorted(baseline.rglob('*')):
            if p.is_file():z.write(p,p.relative_to(baseline).as_posix())
    researchdist=DIST/'ksf-voc-library'
    if researchdist.exists():raise ValueError('RESEARCH_SKILL_DIST_EXISTS')
    shutil.copytree(skill,researchdist,ignore=shutil.ignore_patterns('__pycache__'))
    writejson(DIST/'版本与校验.json',dict(name='ksf-workshop-runtime',preparedDate='2026-11-05',files=workshopfiles,
        manifestSha256=sha(baseline/'FILES.sha256'),manifestVerified='10/10',localPrepared=True,productionInstalled=False,realMessageValidated=False,
        runtimeScriptsChanged=False,baselineDeploymentRecord='历史归档/20261104-先查库/版本与校验.json',
        zip=dict(filename=new_zip.name,bytes=new_zip.stat().st_size,sha256=sha(new_zip)),
        companionSkill=dict(name='ksf-voc-library',files=skillfiles,manifestSha256=sha(researchdist/'FILES.sha256'),databaseIncludedInSkillDirectory=False,databasePackage=str(zip_path))))
    (DIST/'README.md').write_text('''# 康师傅工作坊 Skill

当前分发资料更新于2026-11-05，**本机已准备和测试，尚未安装到王刚001**。11月4日已报安装/真实查库验收的版本与回执保存在 历史归档/20261104-先查库，不能将其验收状态套到本版。

- ksf-workshop-runtime/ 与 ksf-workshop-runtime-20261105.zip：当前工作坊Skill，10内容文件＋清单。本轮只改SKILL.md、references/workshop-tasks.md和FILES.sha256，运行脚本/凭据不变。
- ksf-voc-library/：新配套只读研究库查询Skill，包含精确查询、来源回读和分类规则。数据库体积较大，不放Skill目录；整库与安装器在完整VOC结构化交付包内。
- 版本与校验.json：当前完整指纹与本地/远端状态。

新能力：平台×品牌×品类同时筛选；评价按平台＋商品ID精确关联；默认评价/规格不明/未知分类分开；价格观察与采集时间分开；可回读来源文件、行号、ZIP和记录SHA。研究资料先查ksf-voc-library，报告/证据/机会/项目另查业务库。未装结构库不能声称可查；仍不具备上游采集能力，不修改现网公开看板。

请使用完整交付包中的 docs/发给运维001.md 和 docs/发给王刚001.md；不要只转说明文件或只复制Skill而漏掉数据库。当前工作坊既有员工登记、昵称、S1—S5、成果归属与报告确认规则保留。原11月4日ZIP为历史冻结版，当前以11月5日版为准。
''',encoding='utf-8')
    # Leave the old new-agent handoff archive frozen; point at this complete add-on.
    handoff=Path('F:/文档/新数字人交接')
    if handoff.exists():
        (handoff/'09-VOC结构库增量-20261105.md').write_text(f'''# VOC 结构库增量交接 · 2026-11-05

旧00—08与原ZIP是11月4日冻结交接，不把旧manifest或验收状态改成新能力。新增数字人需要本次结构库时，先在实际授权范围内安装完整增量包：

`{zip_path}`

完整包包含数据库、ksf-voc-library、现有工作坊三文件说明升级、安装器、分类规则与双方验收单。新Agent不自动继承别的实例身份/会话/昵称或研究资料访问权限；是否分发这份私有研究库按该实例授权决定。当前仅本地准备，未在新实例安装。
''',encoding='utf-8')
    report=make_html(delivery,zip_info)
    (out/'结构化入库与Agent规则报告.html').write_text(report,encoding='utf-8')
    copy(ROOT/'docs/voc-structure-20261105/发给运维001.md',out/'发给运维001.md')
    copy(ROOT/'docs/voc-structure-20261105/发给王刚001.md',out/'发给王刚001.md')
    writejson(out/'交付状态.json',dict(package=zip_info,localPrepared=True,productionImported=False,agentInstalled=False,html='结构化入库与Agent规则报告.html'))
    print(json.dumps(dict(package=str(zip_path),**zip_info,report=str(out/'结构化入库与Agent规则报告.html')),ensure_ascii=False,indent=2))


def make_html(data,zip_info):
    esc=html.escape
    rows=''.join(f'<tr><td>{esc(label)}</td><td>{data["counts"][key]:,}</td><td>{esc(note)}</td></tr>' for key,label,note in [
        ('products','商品','4,635主导出＋4个历史目录恢复引用，不算新增采集'),('reviews','电商评价','789非默认；1,571默认文本，观点分析默认排除'),('social_posts','社媒内容','品牌提及与商品归属分开；评论量不等于已采评论'),('policy_documents','政策/标准','25份来源SHA已按文件/解码文本实际口径核对'),('policy_clauses','条款与应用提示','摘要/转写/应用建议不冒充逐字法规引文'),('prices','价格观察','分搜索展示与其他口径；遮挡价不补数'),('assets','原文件索引','逐文件SHA；重复副本不重复计实体')])
    examples=''.join(f'<tr><td>{e["filters"]["marketplace"]}</td><td>{e["filters"]["entity"]}</td><td>{e["totalMatched"]}</td><td>AND · 康师傅 · 拌面</td></tr>' for e in data['examples'])
    return '''<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>VOC结构化入库与Agent规则</title><style>
body{margin:0;background:#f4f5f2;color:#20332a;font:16px/1.8 system-ui,"Microsoft YaHei",sans-serif}main{max-width:1050px;margin:auto;padding:48px 24px}header{background:#173f31;color:#fff;border-radius:20px;padding:36px}h1{font-size:32px;line-height:1.3}h2{margin-top:34px;font-size:23px}section{background:white;padding:24px 30px;margin:22px 0;border-radius:16px}table{width:100%;border-collapse:collapse}th,td{text-align:left;border-bottom:1px solid #e1e8e1;padding:12px 9px;vertical-align:top}th{color:#496350}code,pre{background:#edf2ec;border-radius:7px;padding:4px 8px;overflow-wrap:anywhere}pre{padding:18px;white-space:pre-wrap}a{color:#14764b}.notice{background:#fff4d5;border-left:4px solid #c59631;padding:14px 18px}.tag{font-size:13px;color:#c9e4d3}small{overflow-wrap:anywhere}.table{overflow-x:auto}li{margin:7px 0}@media(max-width:600px){main{padding:20px 12px}header,section{padding:22px 17px}h1{font-size:26px}table{min-width:560px}}
</style><main><header><span class="tag">王勇采集资料 · 2026-11-05</span><h1>资料入库有结构，Agent查询有边界</h1><p>严格匹配平台 × 品牌 × 品类，商品、评价、价格与来源逐条关联。</p></header><p class="notice">本地研究数据库已构建并完成验证。正式数字人安装、真实消息验收尚待运维和王刚001执行；未写生产业务库或公开看板。</p><section><h2>实际入库内容</h2><div class="table"><table><tr><th>分类</th><th>数量</th><th>口径</th></tr>'''+rows+'''</table></div></section><section><h2>三个条件必须同时满足</h2><pre>{"entity":"reviews","marketplace":"京东","brand":"康师傅","category":"拌面","reviewScope":"sku_confirmed","includeDefault":false,"limit":20}</pre><p>天猫不混淘宝待核，康师傅不混统一，拌面不混汤面、面皮、炒面、挂面或拌面酱。标题关键词只作范围内的辅助检索。评价必须准确连接同平台族和商品ID，购买规格不清单列。</p><p>当前可支持严格分类 '''+str(data['statistics']['classification']['supported'])+''' 个商品，另外 '''+str(data['statistics']['classification']['pending'])+''' 个待核对。分类依据有平台属性与保守标题规则两档，规则命中不冒称人工核验。</p></section><section><h2>真实数据查询验证</h2><div class="table"><table><tr><th>平台</th><th>数据</th><th>命中</th><th>条件</th></tr>'''+examples+'''</table></div><p>商品命中组合索引，评价与价格使用商品关联索引；完整分页检查通过。本机查询20条并附来源，中位耗时 '''+str(data['queryBenchmark']['medianMs'])+''' ms；不代表线上网络或Agent总响应耗时。</p></section><section><h2>来源与分析规则</h2><ol><li>每条保留原文件、行号/历史表行ID、URL、实体ID、source_record_id及记录/文件/ZIP校验值，可从结论回溯原文。</li><li>来源快照、请求、入库三个时间分开；无时区不猜UTC，无日期不假装当天。</li><li>默认评价与非默认分别计数；链接下评价和规格明确评价分开，不按昵称计算独立消费者。</li><li>搜索展示价不是成交价，标题规格仅作候选，未经核对不计算每100克价格。</li><li>社媒评论量不是已下载评论，只有标题不能声称已读正文；多品牌提及不能按单品牌归属。</li><li>政策状态是截至2026-11-04的声明快照，未来标准、公告和征求意见分开，应用建议不是法规原话。</li></ol></section><section><h2>交付与下一步</h2><p><a href="'''+esc(zip_info['filename'])+'''">下载完整数据库＋Skill安装包</a> · <a href="发给运维001.md">运维安装单</a> · <a href="发给王刚001.md">王刚001验收单</a></p><p>运维先核验整包、安装独立私有研究库和查询Skill，再对现有工作坊说明做三文件升级；王刚001通过真实消息验精确条件、分页与出处。原业务报告、身份、项目仍走既有平台流程。</p><small>ZIP SHA256：'''+esc(zip_info['sha256'])+'''<br>数据库代际：'''+esc(data['generation'])+'''</small></section></main></html>'''


if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('--db',required=True);p.add_argument('--build-receipt',required=True);p.add_argument('--validation',required=True);a=p.parse_args()
    package(Path(a.db),Path(a.build_receipt),Path(a.validation))