kangshifu-board/server/voc-library/intake_contract.py

"""Typed source mappings for every writable research entity; support tables are generated."""
VERSION = 'ksf.voc-agent-batch.v2'
# field -> (normalized field, type); values always come from source JSON Pointers.
ENTITIES = {
 'products': ('products', {'itemId','title'}, {
  'itemId':('item_id','id'), 'title':('title','text'), 'url':('url','text'), 'shopId':('shop_id','id'),
  'shopName':('shop_name','text'), 'brandName':('brand_name','text'), 'price':('price','amount'), 'priceBasis':('price_type','text'), 'sourceTime':('source_time','text')}),
 'reviews': ('reviews', {'itemId','reviewId','content'}, {
  'itemId':('item_id','id'), 'reviewId':('review_id','id'), 'content':('content','text'), 'appendContent':('append_content','json'),
  'rating':('rating','amount'), 'ratingScale':('rating_scale','text'), 'reviewDate':('review_date','text'),
  'purchasedSku':('sku','text'), 'isDefault':('is_default_text','bool'), 'sourceTime':('source_time','text')}),
 'prices': ('prices', {'itemId','amount','priceBasis'}, {
  'itemId':('item_id','id'), 'amount':('amount','amount'), 'priceBasis':('price_type','text'),
  'currency':('currency','text'), 'sourceTime':('source_time','text')}),
 'specs': ('specs', {'itemId','specText'}, {
  'itemId':('item_id','id'), 'skuId':('upstream_sku_id','id'), 'specText':('spec_text','text'),
  'netWeightG':('net_weight_g','positive'), 'packCount':('pack_count','integer'),
  'totalWeightG':('total_weight_g','positive'), 'verification':('verification','text'), 'basis':('basis','text')}),
 'social': ('social_posts', {'postId'}, {
  'postId':('id','id'), 'title':('title','text'), 'content':('content','text'), 'url':('url','text'),
  'contentScope':('content_scope','text'), 'publishedAt':('published_at','text'),
  'likes':('likes','count'), 'commentCount':('comments','count'), 'collects':('collects','count'), 'shares':('shares','count')}),
 'documents': ('policy_documents', {'documentId','title'}, {
  'documentId':('document_id','id'), 'title':('title','text'), 'declaredStatus':('declared_status','text'),
  'statusAsOf':('status_as_of','date'), 'issuedDate':('issued_date','text'), 'effectiveDate':('effective_date','text'),
  'sourceUrl':('source_url','text'), 'fullText':('full_text','text'), 'expectedSha256':('sha256','text'), 'coverage':('coverage','text'),
  'pdfFile':('pdf_file','text'), 'textFile':('text_file','text')}),
 'clauses': ('policy_clauses', {'documentId','clause','excerpt'}, {
  'documentId':('document_id','id'), 'clause':('clause','text'), 'excerpt':('source_excerpt','text'),
  'pdfPage':('pdf_page','amount'), 'excerptMethod':('excerpt_method','text'), 'applicationNote':('application_note','text')}),
 'rankings': ('ranking_signals', {'itemId','scope'}, {
  'itemId':('item_id','id'), 'rankId':('rank_id','id'), 'rankNumber':('rank_number','integer'),
  'title':('title','text'), 'scope':('scope','text'), 'sourceTime':('source_time','text')}),
 'coverage': ('coverage_observations', {'itemId','status'}, {
  'itemId':('item_id','id'), 'status':('status','text'), 'availableReviews':('available_reviews','count'),
  'reportedTotal':('reported_total','amount'), 'historicalTotalComplete':('historical_total_complete','bool')}),
}

def contract():
    return {'ok':True, 'schema':VERSION, 'legacySchemas':['ksf.voc-agent-batch.v1'],
        'recordMapping':{'sourceId':'sources中的真实来源编号', 'pointer':'来源记录JSON Pointer', 'fields':'入库字段名→该记录内JSON Pointer，值由程序读取'},
        'entities':{name:{'table':table, 'requiredFields':sorted(required),
            'fields':{field:{'normalizedField':dest, 'type':kind, 'required':field in required,
                'unit':'CNY元原值，由程序转分' if field in ('price','amount') else 'g' if field in ('netWeightG','totalWeightG') else None}
                for field,(dest,kind) in fields.items()}} for name,(table,required,fields) in ENTITIES.items()},
        'sourceFields':['id','path','sha256','platform','marketplace','requestItemId','observedAt','collectedAt','url'],
        'attachments':{'fields':['path','sha256'], 'rule':'原始PDF/文本等附件保存在raw/下，真实字节SHA核对后随来源包保留；pdfFile/textFile映射值精确引用已登记path，不执行附件'},
        'sourceRequirements':['原始JSON必须保存在批次raw/下，SHA与真实字节一致', '电商来源声明jd或taobao接口族及jd/tmall/taobao_unverified渠道',
            '商品关联类须在同批提供真实商品上下文；条款须同批有对应文件', '未知字段值可空；必需主键/标题/关联不可空；不造0/日期/名字'],
        'generatedTables':['meta','batches','assets','source_records','source_links','collection_requests','brands','categories',
            'classification_assertions','product_observations','price_sources','social_brand_mentions','quality_issues',
            'source_origins','record_versions','retrieval_documents','retrieval_fts','sqlite_stat1'],
        'themes':{'table':'review_theme_annotations', 'fields':['reviewId','theme','quote','quoteField'], 'rule':'主题和逐字引文由Agent提交，原文指纹/归属/次数由程序生成'},
        'workflow':['validate','build-new-staging','verify-and-readback','publish-new-generation','query-current-generation'],
        'queryEntry':'query_voc.py', 'buildEntry':'agent_batch.py --batch ... --output 新暂存库 [--apply]',
        'publishEntry':'update_library.py --incoming 新暂存库 --registry 已绑定目录 [--apply]',
        'directSqlAllowed':False, 'activeLibraryModified':False, 'networkRequests':0}