kangshifu-board/scripts/review-industry-intake.py

"""Review a provided intake against a sealed release. No network, collection or writes to data."""
import argparse
from datetime import datetime, timezone
from decimal import Decimal
import hashlib
import json
from pathlib import Path, PurePosixPath
import sys
import zipfile


def sha(raw): return hashlib.sha256(raw).hexdigest()


def checked_zip(path, expected):
    raw=path.read_bytes()
    if sha(raw)!=expected: raise ValueError('INTAKE_SHA_MISMATCH')
    with zipfile.ZipFile(path) as z:
        names=z.namelist()
        if len(names)!=len(set(names)) or any(PurePosixPath(n).is_absolute() or '..' in PurePosixPath(n).parts for n in names):
            raise ValueError('UNSAFE_ARCHIVE_MEMBERS')
        files={n:z.read(n) for n in names}
        verified=set()
        for line in files['FILES.sha256'].decode('utf-8-sig').splitlines():
            digest,name=line.split('  ',1)
            if name in verified or name not in files or sha(files[name])!=digest: raise ValueError('INTAKE_MANIFEST_MISMATCH')
            verified.add(name)
        if verified!=set(files)-{'FILES.sha256'}: raise ValueError('INTAKE_MANIFEST_INCOMPLETE')
    return files


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('--intake',type=Path,required=True);p.add_argument('--sha256',required=True)
    p.add_argument('--release',type=Path,required=True);p.add_argument('--out',type=Path,required=True)
    a=p.parse_args();package=a.release.resolve()
    if a.out.exists(): raise ValueError('REVIEW_OUTPUT_ALREADY_EXISTS')
    files=checked_zip(a.intake,a.sha256)
    for line in (package/'FILES.sha256').read_text('utf-8').splitlines():
        digest,name=line.split('  ',1)
        if sha((package/name).read_bytes())!=digest:raise ValueError('SEALED_RELEASE_CHANGED')
    sys.path.insert(0,str(package/'server/voc-library'))
    from monitoring_policy import standard_facts
    from monitoring_dispatch import batch_preflight
    intake=json.loads(files['intake-report.json'])
    pricing=json.loads(files['pricing-voc-20261107.json'])
    price=[v for v in pricing['data']['pricing']['items'] if v['model']=='voc-e-commerce']
    if len(price)!=1 or Decimal(str(price[0]['unitPriceCny']))!=Decimal('0.30'):raise ValueError('UNIT_PRICE_CHANGED')
    scope=json.loads((package/'handoff/collector-scope.json').read_bytes());generation=scope['generation']
    old=package/'evidence/r9-policy-sources'
    assets=package/'data/monitoring-registry/assets'
    metadata=json.loads((old/'standard-page-receipt.json').read_bytes())['rows']
    pdfs=json.loads((old/'pdf-receipt.json').read_bytes())['rows']
    local=[]
    for row in pdfs:
        raw=(assets/row['rawFile']).read_bytes()
        page=next(x for x in metadata if x['documentId']==row['documentId'])
        html=(old/page['rawFile']).read_bytes()
        if sha(raw)!=row['sha256'] or not raw.startswith(b'%PDF-') or not row['matchesArchivedSha256']:
            raise ValueError('STANDARD_PDF_NOT_VERIFIED')
        if sha(html)!=page['sha256'] or page['sha256']!=row['sourcePageSha256']:raise ValueError('STANDARD_PAGE_CHANGED')
        facts=standard_facts({'record_id':row['documentId']},html)
        local.append({'documentId':row['documentId'],'pdfSha256':sha(raw),'pdfBytes':len(raw),
                      'pdfPathInRelease':'data/monitoring-registry/assets/'+row['rawFile'],
                      'metadataPathInRelease':'evidence/r9-policy-sources/'+page['rawFile'],
                      'metadataSha256':sha(html),'originalObservationTime':page['checkedAt'],
                      'requestMethod':page['method'],'officialStatusAtThatObservation':facts['officialStatus']})
    fetched=[json.loads(line) for line in files['policy-fetch/record.jsonl'].decode('utf-8-sig').splitlines() if line]
    shell=[v for v in fetched if str(v['httpCode'])=='200'];blocked=[v for v in fetched if str(v['httpCode'])=='412']
    if len(shell)!=9 or len(blocked)!=16 or len({(r['sha256'],r['bytes']) for r in shell})!=1:raise ValueError('POLICY_FETCH_COUNTS_CHANGED')
    shell_baseline=Path(__file__).resolve().parents[1]/'work/industry-policy-verification-20261107-r1'/(shell[0]['sha256']+'.raw')
    shell_raw=shell_baseline.read_bytes()
    if sha(shell_raw)!=shell[0]['sha256'] or len(shell_raw)!=625:raise ValueError('SHELL_BASELINE_CHANGED')
    if b'form1' not in shell_raw or b'task' not in shell_raw or b'index' not in shell_raw:raise ValueError('SHELL_CLASSIFICATION_UNVERIFIED')
    candidates=[{'endpointId':'jd.get_item_comments_v1','params':{'itemId':x['productId'].split(':',1)[1],'page':page},
                 'generation':generation,'window':'intake-quote-review-no-dispatch'} for x in scope['primaryProducts'] for page in [1,2]]
    before=sha((package/'data/monitoring-registry/monitoring.sqlite').read_bytes())
    preflight=batch_preflight(package/'data/monitoring-registry',candidates,generation)
    if sha((package/'data/monitoring-registry/monitoring.sqlite').read_bytes())!=before:raise ValueError('AUXILIARY_CHANGED')
    paid=sum('REQUEST_ALREADY_CHARGED_NO_AUTOMATIC_REPLAY' in r['blockingReasons'] for r in preflight['rows'])
    response=sum(bool(r['existingResponses']) for r in preflight['rows'])
    r11=package.parent/'ksf-industry-monitor-20261107-r11.zip'
    output={'schema':'ksf.industry-intake-review.v1','checkedAt':datetime.now(timezone.utc).isoformat(),
            'generation':generation,'inputSha256':a.sha256,'inputManifestFilesVerified':len(files)-1,
            'release':{'name':r11.name,'sha256':sha(r11.read_bytes()),'bytes':r11.stat().st_size,'manifestVerified':True,
                       'deploymentStillUnconfirmed':True},
            'policy':{'thisIntake':{'shell200':len(shell),'challenged412':len(blocked),'newBodies':0,
                                   'shellSha256':shell[0]['sha256'],'shellBytes':625},
                      'existingVerifiedPdfCount':len(local),'existingVerifiedPdfs':local,
                      'currentStatusAsOfThisIntake':None,'historicalEvidenceInvalidatedByShell':False,
                      'recommendation':'reuse_verified_release_assets_without_network; latest fetch failure remains a separate observation'},
            'pricing':{'unitPriceMinor':30,'currency':'CNY','billingMode':price[0]['billingMode'],
                       'cachePolicy':price[0]['cachePolicy'],'pricingVersion':price[0].get('pricingVersion'),
                       'accountFieldsOmitted':True,'worstCase7050Accepted':False,
                       'reason':'no per-product missing-page plan or verified current page limits; listing page estimate disagrees with archived receipts',
                       'failureBillingVerifiedFromPricingReceipt':False,'requestIdBillingIdempotenceVerified':False,
                       'ledgerStillRequiredForFailuresAndUnknownOutcomes':True},
            'scope':{'shops':[{k:x[k] for k in ['shopId','historicalProducts','historicalPages','currentLibraryProducts']} for x in scope['shops']],
                     'historicalListingPages':sum(x['historicalPages'] for x in scope['shops']),
                     'historicalListingProducts':sum(x['historicalProducts'] for x in scope['shops']),
                     'currentLibraryShopProducts':sum(x['currentLibraryProducts'] for x in scope['shops']),
                     'primaryProducts':165,'existingPrimaryDetailResponses':41,'existingPrimaryDetailTextProducts':12,
                     'primaryProductsWithoutDetailResponse':124,
                     'hypotheticalTwoPageReviewSlots':len(candidates),'hypotheticalSlotsWithKnownCharges':paid,
                     'hypotheticalSlotsWithRegisteredResponse':response,
                     'hypotheticalSlotsWithoutKnownCharge':len(candidates)-paid,
                     'twoPagesIsVerifiedCurrentLimit':False,'hypotheticalSlotsAreDispatchQueue':False},
            'mappingCorrections':{'appendObjectIsSupported':True,'correctAppendPointer':'/afterComment',
                                  'mainDatePointer':'/commentDate','mainContentPointer':'/commentData',
                                  'unknownDefaultAndRatingScaleRemainUnknown':True,
                                  'numericIdsNeedLosslessParsingOrExplicitSkipReason':True},
            'ranking':{'intakeCatalogSupport':False,'catalogSha256Provided':bool(intake['collector']['catalogSha256']),
                       'allProvidersUnsupported':False,'realAlternativeStillRequired':True},
            'dispatchAllowed':False,'paidCalls':0,'productionWrites':0,'chargedMinor':4620,'protectedMinor':380,
            'newBudgetAuthorizedMinor':0}
    a.out.parent.mkdir(parents=True,exist_ok=True);a.out.write_text(json.dumps(output,ensure_ascii=False,indent=2)+'\n','utf-8')
    print(json.dumps({'ok':True,'inputManifestVerified':True,'existingPdfVerified':len(local),
                     'historicalListingPages':output['scope']['historicalListingPages'],'knownPaidReviewSlots':paid,
                     'worstCase7050Accepted':False,'paidCalls':0,'productionWrites':0},ensure_ascii=False))


if __name__=='__main__':main()