kangshifu-board/scripts/finalize-five-topic-targets-20261111.mjs

/** Exact current record IDs, genre-separated counts and explicit missing facts. */
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {read as bridgeRead,publicApi,load,save,sha,unwrap} from './lib/research-enrichment.mjs';

async function read(...args){
 for(let attempt=0;;attempt++){
  try{return await bridgeRead(...args);}
  catch(error){
   const code=error.cause?.code||error.code||error.name;
   if(attempt>=2||!['ECONNRESET','ETIMEDOUT','UND_ERR_SOCKET','UND_ERR_CONNECT_TIMEOUT','TimeoutError'].includes(code))throw error;
   console.log(JSON.stringify({stage:'contract-read-only-retry',attempt:attempt+1,code}));
   await new Promise(resolve=>setTimeout(resolve,1000));
  }
 }
}

const work='work/five-topic-bulk-20261111';
const names=['five-topics-public-01','five-topics-public-02','five-topics-public-03',
 'five-topics-multisource-20261111-01','five-topics-bulk-20261111-01'];
const topics=['wellness','student','premium','stockpile','novelty'];
const labels=['中式养生','学生（宿舍与夜宵）','高端升级（中产）','居家囤货','潮流尝鲜'];
const records=new Map();
const sourceIntegrities=load(path.join(work,'local-source-integrity.json'));
for(const name of names){
 const directory=path.join('work/database-enrichment-20261110/batches',name);
 const applied=unwrap(load(path.join(directory,'apply-result.json'))),receipt=load(path.join(directory,'receipt.json'));
 assert(applied.ok&&applied.applied&&applied.allRowsVerified&&applied.archiveSha256===receipt.archiveSha256);
 assert.equal(sha(fs.readFileSync(path.join(directory,'transport.zip'))),receipt.archiveSha256);
 for(const source of load(path.join(directory,'batch.json')).sources){
  const file=path.join(directory,source.path),adapter=load(file),r=adapter.record;
  assert.equal(sha(fs.readFileSync(file)),source.sha256);
  assert.equal(sha(fs.readFileSync(path.join(directory,adapter.original.path))),adapter.original.sha256);
  const isDoc=!!r.documentId,id=isDoc?r.documentId:source.platform+':'+r.postId;
  const text=isDoc?r.fullText:r.content;
  assert(text?.trim()&&(isDoc?r.sourceUrl:r.url)&&(isDoc?r.issuedDate:r.publishedAt));
  const type=isDoc?'official':['smzdm','sspai'].includes(source.platform)?'community':'industry';
  assert(!records.has(id),'DUPLICATE_SOURCE_ID:'+id);
  records.set(id,{entity:isDoc?'documents':'social',id,title:r.title,url:isDoc?r.sourceUrl:r.url,
   publishedAtRaw:isDoc?r.issuedDate:r.publishedAt,body:text,bodySha256:sha(Buffer.from(text)),type,
   topics:adapter.researchTopics,limits:adapter.limits,sourceArchiveSha256:receipt.archiveSha256,
   adapterSha256:source.sha256,originalSha256:adapter.original.sha256,batch:name});
 }
}
// These are additional uses of existing full originals, not newly collected posts.
for(const [id,phrase] of [['smzdm:a6p37nn0','想换个口味'],['smzdm:a6p3l48e','另外一个味道试一试']]){
 const row=records.get(id),passage=row.body.split('\n').find(p=>p.includes(phrase));assert(passage);
 if(!row.topics.some(t=>t.topic==='novelty'))row.topics.push({topic:'novelty',relation:'首次换口味及菌菇风味兑现体验',passages:[passage]});
}
for(const row of records.values())for(const t of row.topics){
 assert(topics.includes(t.topic)&&t.passages.length);
 for(const q of t.passages)assert(row.body.includes(q),'QUOTE_NOT_IN_BODY:'+row.id);
}

const contract=await read('contract'),generation=contract.generation;
save(path.join(work,'target-contract-current.json'),contract);
const candidates=load(path.join(work,'product-candidates.json')).topics;
const oldAudit=load(path.join(work,'product-target-audit.json'));
const oldChecks=new Map(oldAudit.checks.map(r=>[r.id,r]));
const selected=Object.fromEntries(topics.map(t=>[t,candidates[t]
 .filter(r=>r.brandCode&&r.brandCode!=='unknown'&&r.shopId&&r.shopName)
 .sort((a,b)=>(b.existingReviewCount>0?20:0)+(oldChecks.get(b.id)?.essentialFieldsPresent?8:0)+b.score-
                 (a.existingReviewCount>0?20:0)-(oldChecks.get(a.id)?.essentialFieldsPresent?8:0)-a.score)
 .slice(0,100)]));
for(const t of topics)assert.equal(selected[t].length,100,'NOT_ENOUGH_REAL_PRODUCTS:'+t);
const products=[...new Map(Object.values(selected).flat().map(r=>[r.id,r])).values()];
const requests=[...products.map(r=>({entity:'products',id:r.id,row:r})),...Array.from(records.values()).map(r=>({entity:r.entity,id:r.id,row:r}))];
const cache=path.join(work,'target-current-details');fs.mkdirSync(cache,{recursive:true});
const checks=new Map();let next=0,done=0;
async function fetchDetail(entity,id){
 const filename=path.join(cache,sha(Buffer.from(entity+':'+id)).slice(0,24)+'.json');
 if(fs.existsSync(filename)){const d=load(filename);if(d.generation===generation&&d.ok)return d;}
 for(let attempt=0;;attempt++){
  try{const d=await publicApi({op:'detail',entity,id,generation});save(filename,d);return d;}
  catch(error){if(attempt>=2)throw error;await new Promise(resolve=>setTimeout(resolve,1000));}
 }
}
await Promise.all(Array.from({length:2},async()=>{
 while(next<requests.length){
  const item=requests[next++],data=await fetchDetail(item.entity,item.id),r=data.record;
  assert.equal(data.generation,generation);assert.equal(r.id,item.id);assert(r.source?.assetSha256&&r.source?.archiveSha256);
  if(item.entity==='products'){
   const c=item.row;assert.equal(r.title,c.title);assert.equal(r.url,c.url);assert.equal(r.shop_id,c.shopId);assert.equal(r.brand_code,c.brandCode);
   const missing=[];
   if(!r.specs?.some(s=>s.verification==='source_attribute'&&s.spec_text?.trim()))missing.push('sourceVerifiedSpec');
   if(!r.specs?.some(s=>s.verification==='source_attribute'&&s.net_weight_g>0))missing.push('verifiedNetWeight');
   const reviews=data.relations.find(x=>x.entity==='reviews')?.count??null;
   if(reviews===null)missing.push('reviewRelationUnavailable');else if(reviews===0)missing.push('reviewContent');
   if(!data.relations.some(x=>x.entity==='prices'&&x.count>0))missing.push('priceObservation');
   if(!c.imageMetadataPresent)missing.push('imageMetadata');
   checks.set(item.entity+':'+item.id,{id:r.id,title:r.title,url:r.url,brandCode:r.brand_code,shopId:r.shop_id,
    shopName:r.shop_name,sourceRecordId:r.source_record_id,sourceSha256:r.source.assetSha256,
    relevanceBasis:'标题中的明确汤底、食材、产品或场景词；不证明功效或购买者身份',titleTerms:c.titleTerms,
    basicIdentityAndSourceVerified:true,missing,reviewRecords:reviews,sourceSpecRows:r.specs.filter(s=>s.verification==='source_attribute').length,
    complete:missing.length===0});
  }else{
   const wanted=item.row,body=item.entity==='social'?r.content:r.full_text;
   assert.equal(body,wanted.body,'BODY_MISMATCH:'+item.id);
   assert.equal(r.title,wanted.title);assert.equal(item.entity==='social'?r.url:r.source_url,wanted.url);
   assert.equal(item.entity==='social'?r.published_at_raw:r.issued_date,wanted.publishedAtRaw);
   // Transport ZIP and the server-built source archive have distinct hashes.
   // Compare the actual normalized payload and source asset against staging.
   const expectedSource=sourceIntegrities[item.entity+':'+item.id];assert(expectedSource);
   assert.equal(r.source.payload_sha256,expectedSource.payloadSha256,'SOURCE_PAYLOAD_SHA_MISMATCH:'+item.id);
   assert.equal(r.source.assetSha256,expectedSource.assetSha256,'SOURCE_ASSET_SHA_MISMATCH:'+item.id);
   if(item.entity==='social')assert.equal(r.fulltext_available,1);
   else assert.equal(r.attachment_hash_state,'matched');
   checks.set(item.entity+':'+item.id,{id:item.id,entity:item.entity,bodyVerified:true,sourcePayloadAndAssetShaVerified:true,
    publishedSourceArchiveSha256:r.source.archiveSha256,
    originalBytesShaVerified:true,sourceRecordId:r.source_record_id,sourceSha256:r.source.assetSha256});
  }
  done++;if(done%25===0){save(path.join(work,'target-verification-progress.json'),{generation,done,target:requests.length});console.log(JSON.stringify({verified:done,target:requests.length}));}
 }
}));
const result={schema:'ksf.five-topic.minimum-targets.v2',generation,checkedAt:new Date().toISOString(),
 minimum:{products:100,social:10,other:5},quantityCriteria:'不同在库商品ID；社媒只计社区原文，其他计行业研究与官方资料，不跨体裁重复计数',
 sourceDocuments:records.size,uniqueProducts:products.length,
 newCollection:{products:0,community:5,industryResearch:20},topics:{},
 limits:['这是已经入库并逐条回读的课题资料清单，不是搜索结果数量。',
 '每课题100件商品为已有在库商品整理，跨课题关联允许；全局商品总数不因整理增加。',
 '商品缺规格、净含量、价格或评价时保留缺项，不以商品标题或运输重量补齐。',
 '社区体验不等于平台购买评价；行业营销数字、健康和工艺陈述不作为已验证结论。',
 '已保存图片元数据不等于本轮重新逐张解码验图。']};
for(let i=0;i<topics.length;i++){
 const t=topics[i],pr=selected[t].map(p=>checks.get('products:'+p.id));
 const material=Array.from(records.values()).filter(r=>r.topics.some(x=>x.topic===t)).map(r=>({...r,verification:checks.get(r.entity+':'+r.id)}));
 const social=material.filter(r=>r.type==='community'),other=material.filter(r=>r.type!=='community');
 const missingFieldCounts={};for(const p of pr)for(const f of p.missing)missingFieldCounts[f]=(missingFieldCounts[f]||0)+1;
 result.topics[t]={name:labels[i],counts:{products:pr.length,social:social.length,other:other.length,
  industryResearch:other.filter(r=>r.type==='industry').length,official:other.filter(r=>r.type==='official').length,
  productsWithReviews:pr.filter(r=>r.reviewRecords>0).length,completeProductDossiers:pr.filter(r=>r.complete).length},
  quantityMinimumMet:pr.length>=100&&social.length>=10&&other.length>=5,
  completeProductDossiers:pr.every(r=>r.complete),missingFieldCounts,products:pr,social,other};
}
result.allQuantityMinimumsMet=Object.values(result.topics).every(t=>t.quantityMinimumMet);
result.allProductDossiersComplete=Object.values(result.topics).every(t=>t.completeProductDossiers);
result.allSourceDocumentsReadbackVerified=true;
result.allRequestedWorkComplete=result.allQuantityMinimumsMet&&result.allProductDossiersComplete;
const after=await read('contract');assert.equal(after.generation,generation,'GENERATION_CHANGED_DURING_TARGET_AUDIT');
save(path.join(work,'topic-material-catalogue.json'),result);
save(path.join(work,'product-enrichment-remaining.json'),{
 schema:'ksf.five-topic.product-gaps.v1',generation,checkedAt:result.checkedAt,
 uniqueProducts:products.length,
 products:products.map(p=>({...checks.get('products:'+p.id),topics:topics.filter(t=>selected[t].some(r=>r.id===p.id))})).filter(p=>p.missing.length),
 sourceRules:['按原商品ID和原链接核对；主评、追评和精确评价ID分别保留。',
  '净含量必须来自所选SKU的明确属性或包装标示，不能使用运输重量和标题推定。',
  '平台验证或无有效返回属于采集受限，不自动判断商品不存在。'],
 currentLimit:'本轮电商接口未取得有效商品响应，京东公开评价接口返回非JSON；已保存原始回执。'});
const lines=['# 五课题资料核验（2026-11-11）','',
 '|课题|商品|社媒原文|其他资料|有评价商品|','|---|---:|---:|---:|---:|'];
for(const t of Object.values(result.topics))lines.push(`|${t.name}|${t.counts.products}|${t.counts.social}|${t.counts.other}|${t.counts.productsWithReviews}|`);
lines.push('',`全局去重：${products.length}件商品、${records.size}份独立资料。新增5篇社区体验、20篇行业研究，商品新增0。`,
 '',`数量门槛：${result.allQuantityMinimumsMet?'达到':'未全部达到'}；商品完整档案：${result.allProductDossiersComplete?'通过':'未全部通过'}。`,
 '全部资料正文、原始字节SHA与当前代际的具体记录和来源归档均通过核对。',
 '商品已核基础身份、店铺、品牌、原链接和来源；净含量等缺项另列，不据标题或运输重量推算。','',`研究代际：${generation}`);
fs.writeFileSync(path.join(work,'采集与完整性核验.md'),lines.join('\n'));
console.log(JSON.stringify({ok:true,generation,uniqueProducts:products.length,sourceDocuments:records.size,
 topics:Object.fromEntries(Object.entries(result.topics).map(([k,v])=>[k,v.counts])),allQuantityMinimumsMet:result.allQuantityMinimumsMet,
 allProductDossiersComplete:result.allProductDossiersComplete}));