kangshifu-board/scripts/package-industry-existing-data.py

"""Freeze the existing-data research release; never deploy or collect upstream data."""
import argparse
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import difflib
import hashlib
import json
from pathlib import Path
import re
import shutil
from urllib.error import HTTPError
from urllib.request import Request, urlopen
import zipfile

ROOT = Path(__file__).resolve().parents[1]
WORK = ROOT / 'work/industry-existing-data-release-20261108-r1'
BASELINE = WORK / 'public-baseline'
PRIOR = ROOT / 'artifacts/ksf-industry-monitor-20261107-r11/package'
RELEASE = 'ksf-industry-monitor-20261108-r12'
PACKAGE = ROOT / 'artifacts' / RELEASE / 'package'
POLICY = ROOT / 'artifacts/ksf-industry-policy-usage-20261108-r1/package'


def sha(raw):
    return hashlib.sha256(raw).hexdigest()


def normalized(raw):
    return raw.replace(b'\r\n', b'\n')


def save_json(path, data):
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')


def replace_one(text, before, after):
    assert text.count(before) == 1, 'PUBLIC_ANCHOR_CHANGED: ' + before[:80]
    return text.replace(before, after, 1)


def shared_payloads():
    """Use observed public code, adding only industry navigation integration."""
    app = (BASELINE / 'src/layout/kf-app.js').read_text(encoding='utf-8')
    changes = [
        ("import { CONFIG } from '../app.config.js';", "import { CONFIG } from '../app.config.js';\nimport { industrySection } from '../core/industry-navigation.js';"),
        ('      const previous = this._route;', '      const previous = this._route;\n      const previousQuery = this._query;'),
        ("      const samePool = ['pool', 'mode'].includes(previous.name) && ['pool', 'mode'].includes(name);", "      const samePool = ['pool', 'mode'].includes(previous.name) && ['pool', 'mode'].includes(name);\n      const industrySectionChanged = previous.name === 'industry' && name === 'industry' && (previousQuery.tab || 'insights') !== (query.tab || 'insights');"),
        ("navigation !== 'initial' && previous.path !== path && !samePool && name !== 'mode'", "navigation !== 'initial' && ((previous.path !== path && !samePool) || industrySectionChanged) && name !== 'mode'"),
        ("    const [title, sub] = reportType ? [reportType.name, '报告中心'] : TITLES[this._route.name] || TITLES.notfound;", "    const researchPage = this._route.name === 'industry' ? industrySection(this._query.tab || 'insights') : null;\n    const [title, sub] = researchPage ? [researchPage.label, '行业监测'] : reportType ? [reportType.name, '报告中心'] : TITLES[this._route.name] || TITLES.notfound;"),
        ('<kf-nav class="nav" .current=${this._route.path} .connection=', '<kf-nav class="nav" .current=${this._route.path} .query=${this._query} .connection='),
    ]
    for before, after in changes:
        app = replace_one(app, before, after)
    nav = (BASELINE / 'src/layout/kf-nav.js').read_text(encoding='utf-8')
    nav = replace_one(nav, "import { href } from '../utils/router.js';", "import { href } from '../utils/router.js';\nimport { INDUSTRY_SECTIONS, industrySectionPath } from '../core/industry-navigation.js';")
    nav = replace_one(nav, '    current: { type: String },', '    current: { type: String },\n    query: { type: Object },\n    _industryOpen: { state: true },')
    nav = replace_one(nav, "    this.current = '/';", "    this.current = '/';\n    this.query = {};\n    this._industryOpen = false;")
    local_nav = (ROOT / 'src/layout/kf-nav.js').read_text(encoding='utf-8')
    start = local_nav.index('  #industryLinks() {')
    end = local_nav.index('\n  static styles', start)
    method = local_nav[start:end]
    update = "  updated(changed) {\n    if (this.#cur() === '/industry' && (changed.has('current') || changed.has('query'))) this._industryOpen = true;\n  }\n\n"
    nav = replace_one(nav, '  static styles = css`', update + method + '\n  static styles = css`')
    css = '''
    .side>nav{flex:1;min-height:0;overflow-y:auto;align-content:start;scrollbar-width:thin}
    .brand,.foot{flex-shrink:0}
    .industry-group{min-width:0}.industry-group [hidden]{display:none!important}
    .industry-group button{font:inherit;cursor:pointer;touch-action:manipulation}
    .industry-group .nav-parent{display:flex;align-items:center;gap:12px;width:100%;min-height:44px;padding:5px 12px;border:0;border-radius:var(--r-md);background:transparent;color:var(--kf-ink-2);text-align:left;font-size:14px;font-weight:500}
    .industry-group.active>.nav-parent{color:var(--kf-accent-ink)}
    .industry-group .en{white-space:nowrap;font-size:12px}
    .industry-group .chevron{display:flex;flex:none}.industry-children{display:grid;gap:3px;margin:5px 0 7px 21px;padding-left:12px;border-left:1px solid var(--kf-line)}
    .industry-child{gap:8px;min-width:0;height:37px;padding:0 9px;font-size:12px;font-weight:400;line-height:1.6}
    .industry-child>svg{flex:none;width:15px;height:15px}.industry-child>span:not(.library-current-mark){flex:1}
    .industry-child[aria-current=page]{color:var(--kf-accent-ink);background:var(--kf-accent-soft);box-shadow:none;font-weight:600}.industry-child[aria-current=page]::before{display:none}.library-current-mark{display:flex;flex:none}
    .industry-group button:focus-visible,.industry-child:focus-visible{outline:2px solid var(--kf-accent);outline-offset:-2px}
'''
    nav = replace_one(nav, '    :host { display: block; height: 100%; }', '    :host { display: block; height: 100%; }' + css)
    branch = '''it.path === '/industry' ? html`<div class=${`nav-group industry-group ${active(it) ? 'active' : ''}`}><button class="nav-parent" data-industry-toggle aria-expanded=${String(this._industryOpen)} aria-controls="industry-subnav" @click=${() => this._industryOpen = !this._industryOpen}>${icon(it.icon, 18)}<span class="lbl">${it.label}<span class="en">${it.en}</span></span><span class="chevron">${icon(this._industryOpen ? 'down' : 'right', 13)}</span></button><div id="industry-subnav" class="industry-children" role="group" aria-label="行业监测分类" ?hidden=${!this._industryOpen}>${this.#industryLinks()}</div></div>` : '''
    nav = replace_one(nav, '${ITEMS.map((it) => html`<a', '${ITEMS.map((it) => ' + branch + 'html`<a')
    assert 'LIBRARY_GROUPS' not in nav and 'data-library-toggle' not in nav
    assert 'scrollRestore=${' not in app
    return {'src/layout/kf-app.js': app.encode(), 'src/layout/kf-nav.js': nav.encode()}


def industry_dependencies():
    found, dependencies = set(), set()
    queue = ['src/pages/view-industry.js', 'src/core/industry-navigation.js']
    while queue:
        name = queue.pop()
        if name in found:
            continue
        found.add(name)
        for rel in re.findall(r'(?:from\s*|import\s*)[\'\"]([.][^\'\"]+)[\'\"]', (ROOT / name).read_text(encoding='utf-8')):
            dest = (ROOT / name).parent.joinpath(rel).resolve().relative_to(ROOT).as_posix()
            if '/industry-' in dest or '/policy-reviewed-' in dest:
                queue.append(dest)
            else:
                dependencies.add(dest)
    return found, dependencies


def capture(name):
    target = BASELINE / name
    if target.is_file():
        raw = target.read_bytes()
        return dict(path=name, exists=True, sha256=sha(raw), normalizedSha256=sha(normalized(raw)))
    try:
        with urlopen(Request('https://kangshifu.fmode.cn/' + name, headers={'Cache-Control': 'no-cache'}), timeout=25) as response:
            raw, content_type = response.read(3_000_000), response.headers.get('Content-Type', '')
        if 'javascript' not in content_type:
            assert sha(raw) == sha((BASELINE / 'index.html').read_bytes()), 'UNEXPECTED_PUBLIC_RESPONSE:' + name
            return dict(path=name, exists=False, sha256=None, normalizedSha256=None)
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_bytes(raw)
        return dict(path=name, exists=True, sha256=sha(raw), normalizedSha256=sha(normalized(raw)))
    except HTTPError as error:
        assert error.code == 404, 'PUBLIC_READ_FAILED:' + name
        return dict(path=name, exists=False, sha256=None, normalizedSha256=None)


def build():
    assert not PACKAGE.exists(), 'RELEASE_DIRECTORY_MUST_BE_NEW'
    static, dependencies = industry_dependencies()
    shared = shared_payloads()
    with ThreadPoolExecutor(max_workers=8) as pool:
        captures = list(pool.map(capture, sorted(static | dependencies | set(shared))))
    by_path = {r['path']: r for r in captures}
    # Missing dependencies may be new pure modules. Existing shared modules stay public.
    for name in dependencies:
        if not by_path[name]['exists']:
            assert name.startswith('src/core/library-selection'), 'MISSING_PUBLIC_DEPENDENCY:' + name
            static.add(name)
    PACKAGE.mkdir(parents=True)
    save_json(PACKAGE / 'package.json', dict(name='ksf-industry-existing-data-r12', private=True, type='module'))
    frozen = {}

    def copy(name, destination=None):
        raw = (ROOT / name).read_bytes()
        frozen[name] = sha(raw)
        dest = PACKAGE / (destination or name)
        dest.parent.mkdir(parents=True, exist_ok=True)
        dest.write_bytes(raw)

    manifest, patch = [], []
    for name in sorted(static | set(shared)):
        if name in shared:
            raw = normalized(shared[name])
            dest = PACKAGE / 'payload' / name
            dest.parent.mkdir(parents=True, exist_ok=True)
            dest.write_bytes(raw)
        else:
            copy(name, 'payload/' + name)
            dest = PACKAGE / 'payload' / name
            raw = normalized(dest.read_bytes())
            dest.write_bytes(raw)
        prior = by_path[name]
        manifest.append(dict(path=name, sha256=sha(raw), allowMissing=not prior['exists'],
                             priorNormalizedSha256=[prior['normalizedSha256']] if prior['exists'] else [],
                             observedProductionSha256=prior['sha256']))
        old = (BASELINE / name).read_text(encoding='utf-8') if prior['exists'] else ''
        patch.extend(difflib.unified_diff(old.splitlines(True), raw.decode().splitlines(True),
                                        fromfile='public/' + name, tofile='release/' + name))
    for name in shared:
        assert by_path[name]['exists']
    backend = json.loads((PRIOR / 'RELEASE.json').read_bytes())['backendFiles']
    backend += ['server/voc-library/industry_workshop.py', 'server/voc-library/industry_decisions.py',
                'server/voc-library/policy_usage_review.py']
    for name in backend:
        copy(name)
    for p in (ROOT / 'server/voc-library').glob('test_industry*.py'):
        copy(p.relative_to(ROOT).as_posix())
    copy('server/voc-library/test_policy_usage_review.py')
    for p in (ROOT / 'test').glob('industry-*.test.js'):
        if p.name != 'industry-collection-guard.test.js':
            copy(p.relative_to(ROOT).as_posix())
            target = PACKAGE / 'test' / p.name
            target.write_text(target.read_text(encoding='utf-8').replace('../src/', '../payload/src/'), encoding='utf-8')
    copy('scripts/verify-industry-existing-data.py')
    copy('scripts/verify-industry-existing-data-public.py')
    copy('scripts/query-industry-policy-usage.py')
    for name in ('agent-queries.md', '19-已核政策原文与用法查询.md'):
        copy('docs/industry-monitoring-20261107/' + name, 'docs/' + name)
    shutil.copytree(POLICY / 'sources', PACKAGE / 'data/policy-review/sources')
    shutil.copyfile(POLICY / 'policy-usage.json', PACKAGE / 'data/policy-review/policy-usage.json')
    assert sha((PACKAGE / 'data/policy-review/policy-usage.json').read_bytes()) == 'd22574f1ce28e1487e0be2abb78c47b32d22de801dc6e54ae9a8e3b3d31e777f'
    publisher = (PRIOR / 'scripts/publish-industry-ui.mjs').read_text(encoding='utf-8')
    before = next(line for line in publisher.splitlines() if line.startswith('export const FILES='))
    static_files = sorted(static | set(shared))
    publisher = replace_one(publisher, before, 'export const FILES=Object.freeze(' + json.dumps(static_files) + ');')
    publisher = publisher.replace('ksf-industry-monitor-20261107-r11', RELEASE)
    (PACKAGE / 'scripts/publish-industry-ui.mjs').write_text(publisher, encoding='utf-8')
    save_json(PACKAGE / 'deploy/industry-ui-files.json', dict(version='ksf.industry-ui.publish.v1', release=RELEASE,
              homepageSha256=sha((BASELINE / 'index.html').read_bytes()), files=manifest))
    save_json(PACKAGE / 'evidence/production-baseline.json', dict(observedAt=datetime.now(timezone.utc).isoformat(), files=captures))
    (PACKAGE / 'evidence/ui.patch').write_text(''.join(patch), encoding='utf-8')
    save_json(PACKAGE / 'RELEASE.json', dict(release=RELEASE, createdAt=datetime.now(timezone.utc).isoformat(),
              scope='Existing archived data research release; collection expansion deferred',
              staticFiles=static_files, backendFiles=backend, sourceFilesSha256=frozen,
              curatedSharedFiles=list(shared), existingVocCoreModulesReplaced=0,
              activeResearchDatabaseIncluded=False, auxiliaryDatabaseIncluded=False, credentialsIncluded=False,
              primaryShop='1000084104', generation='82b104fc024b49a75d2f5376bf9778ced6f63a9433b5fc5a6daa3c458d05d5ea',
              policyReviewSha256='d22574f1ce28e1487e0be2abb78c47b32d22de801dc6e54ae9a8e3b3d31e777f',
              verifiedPolicyClauses=12, pendingPolicyClauses=19, policyOriginals=13,
              productionDeployed=False, nativeProductionRuntimeVerified=False, overallMainlineComplete=False,
              paidCalls=0, productionChanges=0, researchDatabaseWrites=0))
    print(json.dumps(dict(ok=True, package=str(PACKAGE), staticFiles=len(static_files), backendFiles=len(backend), sealed=False)))


def seal():
    receipt = json.loads((PACKAGE / 'evidence/acceptance.json').read_bytes())
    assert receipt['ok'] and receipt['browser']['ok'] and receipt['publisher']['ok']
    for name, digest in receipt['testedFilesSha256'].items():
        assert sha((PACKAGE / name).read_bytes()) == digest, 'TESTED_FILE_CHANGED:' + name
    assert not list(PACKAGE.rglob('*.sqlite')) and not list(PACKAGE.rglob('*.db'))
    files = sorted(p for p in PACKAGE.rglob('*') if p.is_file() and p.name != 'FILES.sha256')
    (PACKAGE / 'FILES.sha256').write_text(''.join(sha(p.read_bytes()) + '  ' + p.relative_to(PACKAGE).as_posix() + '\n' for p in files), encoding='utf-8')
    archive = PACKAGE.parent / (RELEASE + '.zip')
    assert not archive.exists(), 'SEALED_ARCHIVE_ALREADY_EXISTS'
    with zipfile.ZipFile(archive, 'x', zipfile.ZIP_DEFLATED) as z:
        for p in sorted(PACKAGE.rglob('*')):
            if p.is_file():
                z.write(p, p.relative_to(PACKAGE).as_posix())
    save_json(PACKAGE.parent / 'delivery.json', dict(ok=True, archive=str(archive), bytes=archive.stat().st_size,
              sha256=sha(archive.read_bytes()), manifestEntries=len(files), paidCalls=0, productionDeployed=False))
    print((PACKAGE.parent / 'delivery.json').read_text(encoding='utf-8'))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--seal', action='store_true')
    args = parser.parse_args()
    (seal if args.seal else build)()