kangshifu-board/scripts/audit-five-topic-targets-20261111.mjs

/** Exact product IDs and explicit title evidence; missing facts never become zero. */
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {load,save,publicApi,sha} from './lib/research-enrichment.mjs';
import {request} from '../server/scripts/platform-client.mjs';
const work='work/five-topic-bulk-20261111';
const minimumProductsPerTopic=Number(process.argv[2]||100);
assert(Number.isInteger(minimumProductsPerTopic)&&minimumProductsPerTopic>0);
fs.mkdirSync(path.join(work,'product-details'),{recursive:true});
fs.mkdirSync(path.join(work,'document-details'),{recursive:true});
const generation=load(path.join(work,'contract-before.json')).generation;
const inventory=load('work/library-cleanup-20261111/round-02/inventory-after-result.json').body.result;
const plan=load('work/five-topic-collection-20261110/collection-plan.json');
const definitions=plan.topics.map(t=>({...t,keywords:[...t.keywords]}));
// The stated scope includes Chinese soup/food ingredients. These are explicit
// recipe directions, not proof of health benefits or consumer income.
definitions.find(t=>t.id==='wellness').keywords.push('老母鸡','鸡汤','骨汤','黑芝麻','小米','杂粮','荞麦','燕麦','鲜蔬','番茄鸡蛋');
const excluded=new Set(['raw_noodle','sauce','other_food','dry_snack_noodle']);
const countMap=kind=>new Map(inventory.dependencies[kind].map(r=>[r.product_id,Number(r.count)]));
const reviewCounts=countMap('reviews'),priceCounts=countMap('prices');
const canonical=url=>{try{const u=new URL(url);return u.protocol.startsWith('http')&&u.hostname?u.href:null;}catch{return null;}};
let imageMetadata=[];
for(let skip=0;;skip+=1000){
 const result=await request('GET','/classes/KsfProductImage?'+new URLSearchParams({keys:'productId,width,height,sourceUrl,sha256',limit:'1000',skip:String(skip),order:'objectId'}));
 imageMetadata.push(...result.results);if(result.results.length<1000)break;
}
save(path.join(work,'product-image-metadata.json'),imageMetadata);
const imageIds=new Set(imageMetadata.filter(r=>r.width>0&&r.height>0).map(r=>r.productId));
const topicRows=Object.fromEntries(definitions.map(t=>[t.id,inventory.products.filter(r=>{
 const title=String(r.title||'').replace(/五谷道场/g,'');
 return !excluded.has(r.category_code)&&canonical(r.url)&&t.keywords.some(k=>title.includes(k));
}).map(r=>({id:r.id,title:r.title,url:r.url,shopId:r.shop_id,shopName:r.shop_name,brandCode:r.brand_code,category:r.category_code,
 basis:'商品标题明确出现研究要素；是研究相关性，不是功效、收入身份或已核规格',
 titleTerms:t.keywords.filter(k=>String(r.title||'').replace(/五谷道场/g,'').includes(k)),
 existingReviewCount:reviewCounts.get(r.id)||0,existingPriceCount:priceCounts.get(r.id)||0,imageMetadataPresent:imageIds.has(r.id),
 score:(reviewCounts.has(r.id)?20:0)+(priceCounts.has(r.id)?4:0)+(imageIds.has(r.id)?4:0)+(r.shop_id?2:0)+(r.brand_code!=='unknown'?2:0)
})).sort((a,b)=>b.score-a.score||a.id.localeCompare(b.id))]));
save(path.join(work,'product-candidates.json'),{generation,definitions,topics:topicRows,notice:'候选数量不是完整档案数量；品牌词五谷道场不作为五谷食材证据。'});
const selected=Object.values(topicRows).flatMap(rows=>rows.slice(0,minimumProductsPerTopic));
const unique=[...new Map(selected.map(r=>[r.id,r])).values()];
console.log(JSON.stringify({stage:'product-scope',candidates:Object.fromEntries(Object.entries(topicRows).map(([k,v])=>[k,v.length])),selectedMemberships:selected.length,uniqueProducts:unique.length,images:imageIds.size}));
let index=0;const checks=[];
await Promise.all(Array.from({length:2},async()=>{
 while(index<unique.length){
  const row=unique[index++],file=path.join(work,'product-details',row.id.replace(':','-')+'.json');
  let result=fs.existsSync(file)?load(file):null;
  if(result?.generation!==generation){
   try{result=await publicApi({op:'detail',entity:'products',id:row.id,generation});save(file,result);}
   catch(error){checks.push({id:row.id,status:'read_error',error:String(error.message)});continue;}
  }
  const r=result.record;assert.equal(r.id,row.id);
  const missing=[];
  if(!r.title?.trim())missing.push('title');if(!canonical(r.url))missing.push('sourceUrl');
  if(!r.shop_id||!r.shop_name)missing.push('shopIdentity');
  if(!r.brand_code||r.brand_code==='unknown')missing.push('brandIdentity');
  const sourceSpecs=(r.specs||[]).filter(s=>s.verification==='source_attribute'&&s.spec_text?.trim());
  if(!sourceSpecs.length)missing.push('sourceVerifiedSpec');
  if(!r.specs?.some(s=>s.verification==='source_attribute'&&s.net_weight_g>0))missing.push('verifiedNetWeight');
  if(!result.relations?.some(v=>v.entity==='prices'&&v.count>0))missing.push('priceObservation');
  if(!result.relations?.some(v=>v.entity==='reviews'&&v.count>0))missing.push('reviewContent');
  if(!imageIds.has(row.id))missing.push('productImage');
  if(!r.source?.assetSha256||!r.source?.archiveSha256)missing.push('sourceProvenance');
  checks.push({id:row.id,status:'read_verified',fieldsVerified:true,missing,complete:missing.length===0,
   essentialFieldsPresent:!missing.some(f=>['title','sourceUrl','shopIdentity','brandIdentity','sourceVerifiedSpec','sourceProvenance'].includes(f)),
   specCount:sourceSpecs.length,reviewCount:result.relations.find(v=>v.entity==='reviews')?.count,
   imageMetadataPresent:imageIds.has(row.id),sourceRecordId:r.source_record_id,sourceSha256:r.source?.assetSha256,
   detailFile:path.relative(work,file)});
  if(checks.length%25===0){save(path.join(work,'product-audit-progress.json'),{generation,checked:checks.length,target:unique.length,updatedAt:new Date().toISOString()});console.log(JSON.stringify({stage:'product-details',checked:checks.length,target:unique.length}));}
 }
}));
const byId=new Map(checks.map(r=>[r.id,r]));
const topics=Object.fromEntries(Object.entries(topicRows).map(([key,rows])=>{
 const picked=rows.slice(0,minimumProductsPerTopic),records=picked.map(r=>({...r,audit:byId.get(r.id)}));
 return [key,{candidateProducts:rows.length,selectedProducts:picked.length,completeProducts:records.filter(r=>r.audit?.complete).length,
 essentialFieldsPresent:records.filter(r=>r.audit?.essentialFieldsPresent).length,
 missingFieldCounts:records.reduce((o,r)=>{for(const f of r.audit?.missing||[])o[f]=(o[f]||0)+1;return o;},{}),records}];
}));
save(path.join(work,'product-target-audit.json'),{schema:'ksf.five-topic.product-target-audit.v1',generation,checkedAt:new Date().toISOString(),
 minimumProductsPerTopic,completeDefinition:['商品ID、平台、标题、链接、店铺和品牌','来源明确的规格及净含量','价格观察、评价正文、图片与来源归档'],
 titleTermsOnlyForTopicRelevance:true,uniqueProducts:unique.length,checks,topics,
 allProductTargetsComplete:Object.values(topics).every(t=>t.completeProducts>=minimumProductsPerTopic)});
const docs=await publicApi({op:'browse',entity:'documents',quality:'audit',limit:100,generation});
save(path.join(work,'documents-before.json'),docs);
const docChecks=[];
for(const row of docs.records){
 const file=path.join(work,'document-details',row.record_id+'.json');
 let data=fs.existsSync(file)?load(file):null;
 if(data?.generation!==generation){data=await publicApi({op:'detail',entity:'documents',id:row.record_id,generation});save(file,data);}
 const r=data.record,text=String(r.full_text||'');
 docChecks.push({id:r.id,title:r.title,sourceUrl:r.source_url,length:text.length,hasBody:text.length>300,
 sourceSha256:r.source?.assetSha256,attachmentHashMatched:r.attachment_hash_state==='matched',status:r.declared_status,
 complete:!!(text.length>300&&r.source_url&&r.source?.assetSha256&&r.attachment_hash_state==='matched')});
}
save(path.join(work,'document-target-audit.json'),{generation,documents:docChecks});
console.log(JSON.stringify({stage:'audit-complete',products:Object.fromEntries(Object.entries(topics).map(([k,v])=>[k,{selected:v.selectedProducts,complete:v.completeProducts,essential:v.essentialFieldsPresent}])),documents:docChecks.length}));