kangshifu-board/scripts/finalize-library-cleanup.py

"""Reconcile both sealed publications against the original inventory."""
from collections import Counter
from datetime import datetime,timezone
import hashlib
import json
from pathlib import Path

ROOT=Path(__file__).resolve().parents[1];WORK=ROOT/'work/library-cleanup-20261111'
def load(p):return json.loads(p.read_bytes())
def unwrap(v):return v['body']['result']
before=unwrap(load(WORK/'inventory-result.json'))
after=unwrap(load(WORK/'round-02/inventory-after-result.json'))
plans=[load(d/'sealed-plan.json')['plan'] for d in [WORK,WORK/'round-02']]
checks=[load(d/'verification-result.json') for d in [WORK,WORK/'round-02']]
browser=load(WORK/'browser-final/report.json')
assert all(c['ok'] for c in checks) and browser['ok']
assert checks[-1]['generation']==after['generation']
removals=[r for p in plans for r in p['removals']]
assert len({(r['entity'],r['id']) for r in removals})==len(removals)
def typed(row,entity,manifest):
    fields=next(t['columns'] for t in manifest['tables'] if t['name']==('social_posts' if entity=='social' else entity))
    result={k:v for k,v in row.items() if k not in ('_ksf_source_rowid','payload_sha256','source_payload')}
    for f in fields:
        if result.get(f['name']) is not None and 'INT' in f['type'].upper():result[f['name']]=int(result[f['name']])
    return result
retained={}
for entity in ['products','social']:
    removed={r['id'] for r in removals if r['entity']==entity}
    repairs={r['id']:r['changes'] for p in plans for r in p['repairs'] if r['entity']==entity and r['id'] not in removed}
    actual={r['id']:typed(r,entity,after['manifest']) for r in after[entity]}
    for raw in before[entity]:
        if raw['id'] in removed:assert raw['id'] not in actual;continue
        expected=typed(raw,entity,before['manifest']);expected.update(repairs.get(raw['id'],{}))
        assert expected==actual[raw['id']],raw['id']
    assert len(actual)==len(before[entity])-len(removed)
    retained[entity]=len(actual)
social_removed={r['id'] for r in removals if r['entity']=='social'}
retained_repairs={r['id'] for p in plans for r in p['repairs']}-social_removed
for table in ['assets','source_records','source_links','source_origins','collection_requests','prices','policy_documents','policy_clauses','ranking_signals']:
    a=next(t for t in after['manifest']['tables'] if t['name']==table)
    b=next(t for t in before['manifest']['tables'] if t['name']==table)
    assert a==b,table
receipt={'schema':'ksf.library-cleanup.final.v1','ok':True,'checkedAt':datetime.now(timezone.utc).isoformat(),
    'baseGeneration':before['generation'],'generation':after['generation'],
    'scannedProducts':len(before['products']),'scannedSocial':len(before['social']),
    'removedByEntity':dict(Counter(r['entity'] for r in removals)),
    'removedByReason':dict(Counter(r['reason'] for r in removals)),
    'retainedFulltextCorrections':len(retained_repairs),'correctionOperations':sum(len(p['repairs']) for p in plans),
    'correctedThenRemoved':sum(len(p['repairs']) for p in plans)-len(retained_repairs),
    'protectedArticles':63,'retainedRecordsVerified':retained,'allRowsVerified':True,'browserVerified':True,'browserScenes':len(browser['scenes']),
    'removedDependentReviews':150,'removedDependentSpecs':101,'sourceRecordsPreserved':20581,
    'historicalGenerationsPreserved':True,'publicCounts':checks[-1]['counts'],
    'limits':['本轮清理的是来源证据可确认的无关或无内容记录；未把访问失败、缺价格/图片/评论等同于无意义。',
              '有具体产品/配方/工艺参考价值但尚未取得正文的资料仍以仅标题线索保留，不能作为正文证据。']}
(WORK/'final-receipt.json').write_text(json.dumps(receipt,ensure_ascii=False,indent=2),encoding='utf8')
lines=['# 行业资料库清理验收','',f"核验时间：{receipt['checkedAt']}",'',
    '| 项目 | 清理前 | 清理后 | 本轮移除 |','|---|---:|---:|---:|',
    '| 商品与竞品 | 4,674 | 4,580 | 94 |','| 社媒内容 | 3,877 | 3,795 | 82 |',
    '| 关联电商评价 | 6,772 | 6,622 | 150 |','| 关联规格 | 5,737 | 5,636 | 101 |','',
    '商品清理：93条独立饮料、1条邮费链接。社媒清理：游戏/影视/歌曲/手工/明星片段及无有效内容的泛标题或挑战编号。',
    '当前保留资料中，1,258条正文标记已纠正为“仅标题线索”，不再把占位符、重复标题、纯链接和标签当正文。',
    '共执行1,272次正文标记纠正，其中14条在第二次复核后确认无意义并移除，以上统计不重复相加。','',
    '用户举例 bilibili:BV11ENX6BE6p 已移除；公开接口查询为0条，详情显示“未找到这条资料”。',
    '全量保留商品和社媒逐字段核对通过；两轮发布均全表SHA回读通过，关联外键无孤儿；63篇专项新资料完整保留。',
    '原始来源20,581条、来源文件7,450件、来源关联32,737条及历史代际保留；价格、政策和榜单原行未变。',
    '真实线上浏览器6场景通过：商品目录、移动社媒目录、删除社媒详情、删除商品详情、正文标记纠正、专项文章原文。未捕获异常0，横向溢出0。','',
    f"当前代际：`{after['generation']}`",'',*['- '+s for s in receipt['limits']]]
(WORK/'清理结果.md').write_text('\n'.join(lines)+'\n',encoding='utf8')
print(json.dumps({k:receipt[k] for k in ['ok','generation','removedByEntity','retainedFulltextCorrections','retainedRecordsVerified','browserScenes']},ensure_ascii=False))