kangshifu-board/scripts/package-voc-library-r2.py

"""Build the complete question-oriented VOC handoff; preserve the frozen r1 ZIP."""
import argparse
import csv
import hashlib
import html
import importlib.util
import json
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import zipfile

ROOT=Path(__file__).resolve().parents[1]
SOURCE=ROOT/'server/voc-library'
WORK=ROOT/'work/voc-structure-20261105'
OUT=ROOT/'artifacts/康师傅VOC结构化入库-20261105'
NAME='ksf-voc-library-20261105-r2'
RELEASE=OUT/NAME
WORKSHOP=ROOT/'work/five-views-delivery-20261103/skills/ksf-workshop-runtime'
SKILL=ROOT/'skills/ksf-voc-library'
DIST=Path('F:/文档/康师傅skill')


def sha(p):
    with p.open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()


def write(p,value):
    p.parent.mkdir(parents=True,exist_ok=True)
    p.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')


def read(p):return json.loads(p.read_text(encoding='utf-8'))


def copy(a,b):
    b.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(a,b)


def files(folder):
    return {p.relative_to(folder).as_posix():sha(p) for p in sorted(folder.rglob('*'))
            if p.is_file() and '__pycache__' not in p.parts and p.name!='credential-binding.json'}


def manifest(folder):
    data={n:s for n,s in files(folder).items() if n!='FILES.sha256'}
    (folder/'FILES.sha256').write_text(''.join(f'{s}  {n}\n' for n,s in data.items()),encoding='utf-8',newline='\n')
    return data


def load_cases():
    sys.path.insert(0,str(SOURCE))
    import verify_retrieval
    return verify_retrieval.CASES


EXAMPLE_NAMES=['01-brand-products','02-category-products','03-jd-brand-category','04-portion-reviews',
 '05-brand-feedback','06-product-related','07-price-range','08-spec-candidates','09-nutrition-policy',
 '10-document-clauses','11-ranking-signals','12-coverage','13-unknown-brand']


def prepare():
    if RELEASE.exists():raise ValueError('RELEASE_ALREADY_EXISTS')
    db=WORK/'library-r2-release.sqlite';build=read(WORK/'build-r2-release.json')
    validation=read(WORK/'retrieval-r2-validation.json')
    corpus=read(WORK/'corpus-r2-validation.json')
    assert build['ok'] and validation['ok'] and corpus['ok']
    assert build['generation']==validation['generation']==corpus['generation']
    assert sha(db)==validation['databaseSha256']==corpus['databaseSha256']
    # Check exactly which existing workshop content changes, before distribution.
    r1=OUT/'ksf-voc-library-20261105-r1/materials/skills/ksf-workshop-runtime'
    delta=sorted(n for n,s in files(WORKSHOP).items() if n!='FILES.sha256' and files(r1).get(n)!=s)
    assert delta==['SKILL.md','references/workshop-tasks.md'],delta
    for name in ['query_voc.py','voc_db.py','retrieval.py','schema.sql','taxonomy.json']:
        copy(SOURCE/name,SKILL/'scripts'/name)
    assert len(manifest(WORKSHOP))==10
    manifest(SKILL)
    copy(db,RELEASE/'library/library.sqlite')
    shutil.copytree(SKILL,RELEASE/'skill',ignore=shutil.ignore_patterns('__pycache__'))
    for p in SOURCE.iterdir():
        if p.is_file():copy(p,RELEASE/'ops'/p.name)
    shutil.copytree(WORKSHOP,RELEASE/'materials/skills/ksf-workshop-runtime',ignore=shutil.ignore_patterns('__pycache__','credential-binding.json'))
    copy(ROOT/'work/five-views-delivery-20261103/scripts/install-workshop-skill.mjs',RELEASE/'materials/scripts/install-workshop-skill.mjs')
    for p in (ROOT/'docs/voc-structure-20261105').iterdir():
        if p.is_file():copy(p,RELEASE/'docs'/p.name)
    for src,target in [('build-r2-release.json','build.json'),('retrieval-r2-validation.json','retrieval-validation.json'),('corpus-r2-validation.json','corpus-validation.json')]:
        copy(WORK/src,RELEASE/'receipts'/target)
    for name,(question,query) in zip(EXAMPLE_NAMES,load_cases()):write(RELEASE/'examples'/(name+'.json'),query)
    write(RELEASE/'examples/14-brand-facets.json',dict(entities=['products'],filters={'brand':'康师傅'},groupBy=['platform','category']))
    write(RELEASE/'examples/15-review-aggregate.json',dict(entities=['reviews'],filters={'brand':'康师傅'},groupBy=['platform','category']))
    with sqlite3.connect(db) as c:
        with (RELEASE/'docs/待核对问题清单.csv').open('w',encoding='utf-8-sig',newline='') as f:
            writer=csv.writer(f);writer.writerow(['entity_type','entity_id','code','detail'])
            writer.writerows(c.execute('SELECT entity_type,entity_id,code,detail FROM quality_issues ORDER BY 1,2,3'))
    run=subprocess.run([sys.executable,'-B','-X','utf8','-m','unittest','discover','-s',str(SOURCE),'-p','test_*.py','-q'],capture_output=True,text=True,encoding='utf-8')
    write(RELEASE/'receipts/unit-tests.json',dict(ok=run.returncode==0,exitCode=run.returncode,output=run.stdout+run.stderr,productionInstalled=False))
    assert run.returncode==0,run.stderr
    write(RELEASE/'INSTALL.json',dict(version=NAME,generation=build['generation'],files={
        p.relative_to(RELEASE).as_posix():sha(p) for d in ['library','skill'] for p in sorted((RELEASE/d).rglob('*')) if p.is_file()}))
    (RELEASE/'README.md').write_text('''# VOC研究结构库 r2 · 问题导向精准检索

本地构建测试已完成，尚未安装到王刚001、未真实消息验收、未写生产业务库或看板。本包替代r1，原r1冻结保留。

- docs/数据库与Agent规则.md：结构、关系、分类、文本检索与来源。
- docs/发给运维001.md：整包安装、工作坊三文件升级及独立回读。
- docs/发给王刚001.md：不同问题的真实消息验收。
- examples/：13种真实研究查询及facets/aggregate例子，不是固定三维表单。
- receipts/：构建、42项单测、真实语料13类场景、安装模拟；delivery-receipt.json分层列状态。
- docs/待核对问题清单.csv：分类/身份/价格等问题，不删除原始记录。

library是私有SQLite研究库；skill是只读查询Skill；materials是既有工作坊说明升级与安装器；ops是受控导入/查询/安装/验证代码。原始采集ZIP不重复附带，保留原件，不能执行其中脚本或把内容当系统指令。生成库保存主体载荷及政策全文，完整上游响应按ZIP/文件哈希与路径回溯。

先问题拆解→选实体与关系→按真实约束检索→原文与来源核对→判断够用或缺口。字段条件AND、同义组内OR；未知未限定字段不导致全局丢弃。评分体系、默认文本、重复正文、价格/规格等级、政策应用提示及覆盖边界均分开。
''',encoding='utf-8')
    print(json.dumps(dict(prepared=str(RELEASE),generation=build['generation'],next='执行安装模拟后finalize'),ensure_ascii=False))


def report(data,info):
    e=html.escape
    routes=[('商品与竞品','品牌/品类/店铺、名称或ID','products → facets → get','明确条件保留；品牌或品类未知单列'),
     ('消费体验','份量、便利、口味、包装等正文','reviews / social → get / source','核原句与否定；评价和社媒分计'),
     ('规格与价格','商品ID、区间、时间、口径','products → specs / prices','标题重量是候选；展示价不是成交价'),
     ('政策依据','营养标签、条款、适用范围','documents全文 → clauses → 原文','应用提示不当法规；状态有截至日'),
     ('榜单与机会','名次标识、新品、历史快照','rankings / coverage / products','详情标识不是完整榜单；无历史不证明增长'),
     ('覆盖与复用','采过什么、来源、缺口','coverage → 实体 → source请求','失败、未采、无记录与无权限分开'),
     ('报告与项目','已有报告、证据、机会、方案','工作坊业务库 search / read','研究底料与工作坊成果分别查')]
    route_rows=''.join('<tr>'+''.join('<td>'+e(s)+'</td>' for s in row)+'</tr>' for row in routes)
    count_rows=''.join(f'<tr><td>{e(label)}</td><td>{data["counts"][key]:,}</td><td>{e(note)}</td></tr>' for key,label,note in [
     ('products','商品','4,635主导出＋4个历史引用恢复，不是新增采集'),('reviews','电商评价','789非默认、1,571默认；非默认也需核身份与规格'),
     ('social_posts','社媒','正文/标题/元数据分开，品牌是提及关系'),('policy_documents','政策/标准','25份原文均入全文索引，哈希口径已核'),
     ('policy_clauses','条款/应用提示','原文、转写摘要和分析建议分开'),('prices','价格观察','3,968单数值；观察不等于独立商品'),
     ('specs','规格记录','未核定的候选不用于宣称精确单位价格'),('ranking_signals','榜单标识','详情页信号，不冒称Top100/400全榜'),
     ('coverage_observations','采集覆盖','保留可用页/声明数量，不保证历史全量'),('source_records','来源记录','可回读载荷、文件、行号、URL、请求与指纹')])
    scenario_rows=''.join(f'<tr><td>{e(x["question"])}</td><td>{x["totalMatched"]:,}</td><td>{e(" / ".join(x["countsByEntity"]))}</td></tr>' for x in data['scenarios'])
    return '''<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>VOC结构化研究库 · 问题导向检索r2</title><style>
:root{font:16px/1.75 system-ui,"Microsoft YaHei",sans-serif;color:#233c33;background:#f2f5f2}*{box-sizing:border-box}body{margin:0}main{max-width:1120px;margin:auto;padding:42px 24px}header{background:#153e30;color:#fff;padding:38px;border-radius:22px}h1{font-size:34px;line-height:1.35;margin:12px 0}h2{font-size:23px;margin:0 0 16px}section{background:#fff;padding:28px;margin:22px 0;border-radius:18px}p{margin:12px 0}.eyebrow{font-size:13px;color:#badcc6}.notice{background:#fff3d4;border-left:4px solid #b88b2d;padding:15px 20px;border-radius:5px}.flow{display:flex;flex-wrap:wrap;gap:9px;margin-top:20px}.flow span{background:#f0f6f0;color:#244e3b;border-radius:7px;padding:7px 12px;font-size:14px}.table{overflow-x:auto;width:100%}table{width:100%;border-collapse:collapse}td,th{padding:12px 10px;border-bottom:1px solid #e1e9e3;text-align:left;vertical-align:top}th{color:#577363;font-size:14px}td{font-size:15px}code,pre{background:#eef3ed;border-radius:8px;font-size:14px;overflow-wrap:anywhere}pre{padding:17px;white-space:pre-wrap}a{color:#137451}small{overflow-wrap:anywhere;color:#5b7166}li{margin:7px 0}.grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.panel{background:#f4f7f3;border-radius:12px;padding:20px}.panel strong{display:block;color:#254e39;margin-bottom:8px}@media(max-width:650px){main{padding:18px 12px}header,section{padding:23px 17px}h1{font-size:26px}.grid{grid-template-columns:1fr}table{min-width:640px}}
</style></head><body><main><header><div class="eyebrow">王勇已采集资料 · 2026-11-05 · r2</div><h1>让 Agent 根据问题<br>找到可核验的数据</h1><p>分类与字段帮助定位，正文与同义表达帮助找线索，关系和来源支持逐条核对。</p><div class="flow"><span>问题拆解</span><span>选数据与关系</span><span>保留真实约束</span><span>检索与统计</span><span>回读原文</span><span>判断够用或缺口</span></div></header><p class="notice">本地数据库与配套Skill已完成构建测试。正式数字人安装、规则实际加载与真实消息验收尚待执行；未改生产业务库或公开看板。</p><section><h2>按问题选路线</h2><p>“某平台、某品牌、拌面”是一个组合查询示例。只问某品牌就不硬加品类，问份量就查评价正文，问政策就读政策全文。明确限定的条件始终保留，未限定字段未知的相关记录继续展示并带质量标记。</p><div class="table"><table><tr><th>研究问题</th><th>输入与约束</th><th>查询路线</th><th>核对重点</th></tr>'''+route_rows+'''</table></div></section><section><h2>这批资料实际入库了什么</h2><div class="table"><table><tr><th>数据</th><th>数量</th><th>实际口径</th></tr>'''+count_rows+'''</table></div><p>另有4,556个原文件指纹、2,532个采集请求、8,288次商品观察、25,667条来源关联。检索投影21,108条，按实体保留字段、原文与文本指纹；重复观察不冒充新增商品。</p></section><section><h2>精准检索靠哪些机制</h2><div class="grid"><div class="panel"><strong>字段、全文与关系一起用</strong>参数化字段条件AND；同义组内部OR、组间AND；中文FTS先找候选，再核字面短语。按商品ID连规格/价格/评价，按文档ID连条款。明确条件不静默放宽。</div><div class="panel"><strong>数据目录与规则可发现</strong>contract返回九类数据、字段/单位、操作符、字典、关系和问题路线；search找记录，facets定位范围，aggregate分组，get/source核原文和出处。</div><div class="panel"><strong>来源和质量随结果返回</strong>实体编号、源文件/行号、URL、原始时间、记录/文件/ZIP指纹及请求追踪。默认评价、规格冲突、分类未知、标题候选与应用提示分开。</div><div class="panel"><strong>计数与边界清楚</strong>记录数、去重文本数、关联商品数分别提供，不能当独立消费者或销量。分页绑定代际与查询范围，失败和查无记录分别报告。</div></div><pre>{"entities":["reviews"],"text":{"concepts":["portion_size"]},"limit":20}</pre><p>上例在评价正文找份量相关线索，不要求先填品牌和平台。命中仍可能是正面、否定或比较，Agent必须回读上下文。有限同义表辅助找候选，不能保证任意语义全部命中。</p></section><section><h2>用真实资料验证13类问题</h2><div class="table"><table><tr><th>问题</th><th>命中记录</th><th>实体类型</th></tr>'''+scenario_rows+'''</table></div><p>42项测试通过；上述13类真实场景逐条校验条件、来源和回读，份量评价完整分页无重复，数据库查询前后SHA相同。52次本机查询中位 '''+str(data['benchmark']['medianMs'])+''' ms，最大 '''+str(data['benchmark']['maxMs'])+''' ms；这是已有SQLite连接的本地测量，包含匹配/来源统计，不含Agent推理与网络耗时。</p></section><section><h2>目前资料仍有哪些限制</h2><ul><li>2,730商品品牌/品类分类齐全，1,909项部分或未知；分类有保守标题规则，不冒称人工核验。未限定维度的未知值不全局排除。</li><li>价格是来源快照，标题克重是候选，不能直接宣称实付价或每100g精确比较；无连续记录不能证明趋势。</li><li>政策状态是截至2026-11-04的采集者声明；原文、摘要和分析建议分开，需核适用范围。</li><li>详情页榜单标识不等于完整榜单，社媒评论量不等于已采评论；本库不保证全行业覆盖，也不提供上游采集。</li></ul></section><section><h2>交给谁、怎么落地</h2><p><a href="'''+e(info['filename'])+'''">完整r2数据库＋Skill安装包</a> · <a href="发给运维001.md">运维001安装单</a> · <a href="发给王刚001.md">王刚001真实问题验收单</a></p><p>运维核验整包，安装私有研究库和只读Skill，再升级现有工作坊三份说明文件。王刚001用真实消息验证“只查品牌、评价主题、政策依据、明确组合约束”，回报实际调用与原文核对。当前报告/证据/项目仍走既有业务库。</p><small>r2替代本地r1准备包；r1原ZIP保留冻结。<br>整包SHA256：'''+e(info['sha256'])+'''<br>数据库代际：'''+e(data['generation'])+'''</small></section></main></body></html>'''


def finalize():
    build=read(RELEASE/'receipts/build.json');validation=read(RELEASE/'receipts/retrieval-validation.json')
    assert read(RELEASE/'receipts/install-validation.json')['ok']
    assert read(RELEASE/'receipts/workshop-install-validation.json')['ok']
    assert read(RELEASE/'receipts/skill-validation.json')['ok']
    assert sha(RELEASE/'library/library.sqlite')==validation['databaseSha256']
    data={**validation,'name':NAME,'archiveSha256':build['archiveSha256'],'schema':'ksf.voc-library.v2',
      'unitTests':42,'researchScenarios':13,'localPrepared':True,'productionInstalled':False,'realMessageVerified':False,
      'publicBoardConnected':False,'browserVerified':False,'mainWorkshopChanged':['SKILL.md','references/workshop-tasks.md','FILES.sha256'],
      'mainWorkshopRuntimeScriptsChanged':False,'primaryProducts':4635,'recoveredProductReferences':4,
      'workshopManifestSha256':sha(WORKSHOP/'FILES.sha256'),'researchManifestSha256':sha(SKILL/'FILES.sha256')}
    write(RELEASE/'delivery-receipt.json',data)
    entries=manifest(RELEASE)
    zpath=OUT/(NAME+'.zip')
    if zpath.exists():raise ValueError('FROZEN_ZIP_ALREADY_EXISTS')
    with zipfile.ZipFile(zpath,'w',compression=zipfile.ZIP_DEFLATED,compresslevel=6) as z:
        for n in sorted(files(RELEASE)):z.write(RELEASE/n,n)
    with zipfile.ZipFile(zpath) as z:
        assert z.testzip() is None
        for n,s in entries.items():assert hashlib.sha256(z.read(n)).hexdigest()==s,n
    info=dict(filename=zpath.name,bytes=zpath.stat().st_size,sha256=sha(zpath),entries=len(entries)+1,manifestEntriesVerified=len(entries),crcVerified=True)
    write(OUT/'整包校验.json',info)
    # Archive the prepared r1 distribution metadata; never modify its old ZIP.
    history=DIST/'历史归档/20261105-r1'
    for p in [DIST/'版本与校验.json',DIST/'README.md',DIST/'ksf-workshop-runtime-20261105.zip']:
        if p.exists():
            assert not (history/p.name).exists()
            copy(p,history/p.name)
    for src,dest in [(WORKSHOP,DIST/'ksf-workshop-runtime'),(SKILL,DIST/'ksf-voc-library')]:
        for n in files(src):copy(src/n,dest/n)
        assert files(src)==files(dest),'DISTRIBUTION_BYTE_MISMATCH'
    workshopzip=DIST/'ksf-workshop-runtime-20261105.zip'
    with zipfile.ZipFile(workshopzip,'w',compression=zipfile.ZIP_DEFLATED) as z:
        for n in sorted(files(WORKSHOP)):z.write(WORKSHOP/n,n)
    write(DIST/'版本与校验.json',dict(name='ksf-workshop-runtime',preparedDate='2026-11-05',revision='r2',files=manifest(WORKSHOP),
      manifestSha256=sha(WORKSHOP/'FILES.sha256'),manifestVerified='10/10',localPrepared=True,productionInstalled=False,
      realMessageValidated=False,runtimeScriptsChanged=False,zip=dict(filename=workshopzip.name,bytes=workshopzip.stat().st_size,sha256=sha(workshopzip)),
      companionSkill=dict(name='ksf-voc-library',files=files(SKILL),databasePackage=str(zpath)),
      baselineDeploymentRecord='历史归档/20261104-先查库/版本与校验.json',previousPreparedVersion='历史归档/20261105-r1'))
    (DIST/'README.md').write_text('''# 当前Skill分发 · 2026-11-05 r2

本机已准备/测试，尚未在王刚001安装或真实消息验收。11月4日生产回执保留在历史归档/20261104-先查库；本地r1准备资料保留在历史归档/20261105-r1，不套用旧验收状态。

- ksf-workshop-runtime/和同日ZIP：工作坊Skill，10个内容文件与清单；仅SKILL.md、references/workshop-tasks.md、FILES.sha256升级，运行脚本不变。
- ksf-voc-library/：新问题导向只读查询Skill，按问题选择商品/评价/价格/规格/社媒/政策/条款/榜单/覆盖，使用字段、中文全文、有限同义词、关联与来源。
- 数据库不放Skill目录；完整r2交付包包含数据库、配套Skill、升级器和分工验收单，路径见版本与校验.json。

平台/品牌/品类只是部分条件，不强制填写；用户明确约束保留，未限定维度未知的相关记录不丢弃。评价原句、默认文本、重复、规格和来源都需核，计数不当独立消费者。研究底料用新库，报告/机会/项目仍查原业务库。够用复用，核实缺口再补采；本次不增加采集入口、不修改现网看板、身份、昵称、S1—S5和报告确认规则。

请转发完整r2包及包内docs/发给运维001.md；安装后再把docs/发给王刚001.md转给本人真实消息验收。不要只转说明或只复制Skill漏掉数据库。
''',encoding='utf-8')
    handoff=Path('F:/文档/新数字人交接/09-VOC结构库增量-20261105.md')
    if handoff.parent.exists():handoff.write_text(f'''# VOC结构库增量 · 2026-11-05 r2

原00—08及11月4日ZIP保持冻结。当前新增研究资料能力使用完整包：

`{zpath}`

r2替代本地r1准备包。包括私有数据库、ksf-voc-library、工作坊三文件说明升级、安装器、规则与分工验收。按研究问题选择数据、字段/正文/同义词和关系并回读来源；不强制填平台/品牌/品类。当前仅本地构建测试，未在真实实例安装/加载/验收。新数字人需明确授权才可分发此库，不继承其他实例身份、昵称、会话或凭据。具体按包内两份验收单执行。
''',encoding='utf-8')
    (OUT/'结构化入库与Agent规则报告.html').write_text(report(data,info),encoding='utf-8')
    for name in ['发给运维001.md','发给王刚001.md']:copy(RELEASE/'docs'/name,OUT/name)
    write(OUT/'交付状态.json',dict(package=info,localPrepared=True,productionImported=False,agentInstalled=False,realMessageVerified=False,revision='r2',html='结构化入库与Agent规则报告.html'))
    print(json.dumps(dict(package=str(zpath),**info,report=str(OUT/'结构化入库与Agent规则报告.html')),ensure_ascii=False,indent=2))


if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('phase',choices=['prepare','finalize']);a=p.parse_args()
    prepare() if a.phase=='prepare' else finalize()