kangshifu-board/scripts/prepare-industry-inbox.py

"""Wrap existing JD VOC response bytes for the passive inbox, without calling VOC or publishing."""
import argparse
import json
from pathlib import Path
import re
import sys

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'server/voc-library'))
from monitoring_store import exact_id, exact_int, need, pointer, sha, verify_manifest


def assert_no_credentials(value):
    if isinstance(value, dict):
        for key, item in value.items():
            need(not (key.lower() in ('authorization', 'cookie', 'password', 'token', 'apikey', 'api_key', 'secret', 'headers') and item), 'RAW_CONTAINS_PRIVATE_REQUEST_METADATA')
            assert_no_credentials(item)
    elif isinstance(value, list):
        for item in value:
            assert_no_credentials(item)


def prepare(raw_path, kind, window, generation, output):
    need(kind in ('listings', 'reviews'), 'UNVERIFIED_RESPONSE_PROFILE')
    raw = Path(raw_path).read_bytes()
    need(len(raw) <= 25_000_000, 'RAW_TOO_LARGE')
    doc = json.loads(raw)
    assert_no_credentials(doc)
    if kind == 'listings':
        need(doc.get('id') == 'jd.get_shop_item_list_v1', 'RESPONSE_ENDPOINT_NOT_PROVEN')
        base = '/result/data'
        checks = [{'pointer': '/httpStatus', 'equals': 200}, {'pointer': '/status', 'equals': 'ok'},
                  {'pointer': '/result/code', 'equals': 200}, {'pointer': base + '/code', 'equals': 0}]
        mapping = {'itemsPointer': base + '/data/ItemList', 'idPointer': '/itemId', 'titlePointer': '/title',
                   'scopePointers': {'shopId': base + '/data/shopId'}, 'totalPointer': base + '/data/totalResults',
                   'pageCountPointer': base + '/data/TotalPage'}
        scope = {'shopId': exact_id(pointer(doc, base + '/data/shopId'))}
        endpoint = doc['id']
        scope_name = pointer(doc, base + '/data/shopName')
    else:
        # The actually delivered g2/g3 raw response envelope. No guessing another provider's layout.
        base = '/data'
        checks = [{'pointer': '/code', 'equals': 200}, {'pointer': base + '/code', 'equals': 0}]
        mapping = {'itemsPointer': base + '/data/data', 'idPointer': '/commentId',
                   'scopePointers': {'productId': base + '/requestParams/query/itemId'},
                   'rowScopePointers': {'productId': '/productId'}, 'totalPointer': base + '/data/pageInfo/count',
                   'pageCountPointer': base + '/data/pageInfo/maxPage'}
        product = exact_id(pointer(doc, base + '/requestParams/query/itemId'))
        need(re.fullmatch(r'\d+', product), 'INVALID_JD_PRODUCT_ID')
        scope = {'productId': 'jd:' + product, 'mode': 'interface_default'}
        endpoint = 'jd.get_item_comments_v1'
        scope_name = scope['productId']
    mapping.update(pagePointer=base + '/requestParams/query/page', requestIdPointer=base + '/requestId',
                   observedAtPointer=base + '/recordTime', successChecks=checks)
    page = {'page': exact_int(pointer(doc, mapping['pagePointer'])),
            'requestKey': exact_id(pointer(doc, mapping['requestIdPointer'])), 'status': 'success',
            'rawFile': 'response.json', 'rawSha256': sha(raw), 'mapping': mapping}
    manifest = {'schema': 'ksf.monitoring-run.v1', 'kind': kind, 'platform': 'jd', 'scope': scope,
                'scopeName': scope_name, 'endpointId': endpoint, 'window': exact_id(window),
                'generation': exact_id(generation), 'termination': 'unknown', 'pages': [page]}
    output = Path(output)
    output.mkdir(parents=True, exist_ok=False)
    (output / 'response.json').write_bytes(raw)
    verified, _ = verify_manifest(manifest, output)
    # Write the manifest last, by atomic rename, so a running watcher never sees a half package.
    temp = output / 'manifest.pending'
    temp.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', 'utf-8')
    temp.rename(output / 'manifest.json')
    return {'ok': True, 'manifest': str(output / 'manifest.json'), 'rawSha256': sha(raw),
            'kind': kind, 'scope': scope, 'page': page['page'],
            'verification': verified['verification'], 'paidCalls': 0, 'productionWrites': 0}


if __name__ == '__main__':
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--raw', required=True, type=Path)
    p.add_argument('--kind', required=True, choices=['listings', 'reviews'])
    p.add_argument('--window', required=True)
    p.add_argument('--generation', required=True)
    p.add_argument('--out', required=True, type=Path)
    args = p.parse_args()
    try:
        print(json.dumps(prepare(args.raw, args.kind, args.window, args.generation, args.out), ensure_ascii=False))
    except Exception as exc:
        code = str(exc) if isinstance(exc, ValueError) and re.fullmatch('[A-Z][A-Z0-9_]+', str(exc)) else 'INBOX_PREPARE_FAILED'
        print(json.dumps({'ok': False, 'error': code, 'paidCalls': 0}))
        sys.exit(1)