/** Exact current record IDs, genre-separated counts and explicit missing facts. */ import fs from 'node:fs'; import path from 'node:path'; import assert from 'node:assert/strict'; import {read as bridgeRead,publicApi,load,save,sha,unwrap} from './lib/research-enrichment.mjs'; async function read(...args){ for(let attempt=0;;attempt++){ try{return await bridgeRead(...args);} catch(error){ const code=error.cause?.code||error.code||error.name; if(attempt>=2||!['ECONNRESET','ETIMEDOUT','UND_ERR_SOCKET','UND_ERR_CONNECT_TIMEOUT','TimeoutError'].includes(code))throw error; console.log(JSON.stringify({stage:'contract-read-only-retry',attempt:attempt+1,code})); await new Promise(resolve=>setTimeout(resolve,1000)); } } } const work='work/five-topic-bulk-20261111'; const names=['five-topics-public-01','five-topics-public-02','five-topics-public-03', 'five-topics-multisource-20261111-01','five-topics-bulk-20261111-01']; const topics=['wellness','student','premium','stockpile','novelty']; const labels=['中式养生','学生(宿舍与夜宵)','高端升级(中产)','居家囤货','潮流尝鲜']; const records=new Map(); const sourceIntegrities=load(path.join(work,'local-source-integrity.json')); for(const name of names){ const directory=path.join('work/database-enrichment-20261110/batches',name); const applied=unwrap(load(path.join(directory,'apply-result.json'))),receipt=load(path.join(directory,'receipt.json')); assert(applied.ok&&applied.applied&&applied.allRowsVerified&&applied.archiveSha256===receipt.archiveSha256); assert.equal(sha(fs.readFileSync(path.join(directory,'transport.zip'))),receipt.archiveSha256); for(const source of load(path.join(directory,'batch.json')).sources){ const file=path.join(directory,source.path),adapter=load(file),r=adapter.record; assert.equal(sha(fs.readFileSync(file)),source.sha256); assert.equal(sha(fs.readFileSync(path.join(directory,adapter.original.path))),adapter.original.sha256); const isDoc=!!r.documentId,id=isDoc?r.documentId:source.platform+':'+r.postId; const text=isDoc?r.fullText:r.content; assert(text?.trim()&&(isDoc?r.sourceUrl:r.url)&&(isDoc?r.issuedDate:r.publishedAt)); const type=isDoc?'official':['smzdm','sspai'].includes(source.platform)?'community':'industry'; assert(!records.has(id),'DUPLICATE_SOURCE_ID:'+id); records.set(id,{entity:isDoc?'documents':'social',id,title:r.title,url:isDoc?r.sourceUrl:r.url, publishedAtRaw:isDoc?r.issuedDate:r.publishedAt,body:text,bodySha256:sha(Buffer.from(text)),type, topics:adapter.researchTopics,limits:adapter.limits,sourceArchiveSha256:receipt.archiveSha256, adapterSha256:source.sha256,originalSha256:adapter.original.sha256,batch:name}); } } // These are additional uses of existing full originals, not newly collected posts. for(const [id,phrase] of [['smzdm:a6p37nn0','想换个口味'],['smzdm:a6p3l48e','另外一个味道试一试']]){ const row=records.get(id),passage=row.body.split('\n').find(p=>p.includes(phrase));assert(passage); if(!row.topics.some(t=>t.topic==='novelty'))row.topics.push({topic:'novelty',relation:'首次换口味及菌菇风味兑现体验',passages:[passage]}); } for(const row of records.values())for(const t of row.topics){ assert(topics.includes(t.topic)&&t.passages.length); for(const q of t.passages)assert(row.body.includes(q),'QUOTE_NOT_IN_BODY:'+row.id); } const contract=await read('contract'),generation=contract.generation; save(path.join(work,'target-contract-current.json'),contract); const candidates=load(path.join(work,'product-candidates.json')).topics; const oldAudit=load(path.join(work,'product-target-audit.json')); const oldChecks=new Map(oldAudit.checks.map(r=>[r.id,r])); const selected=Object.fromEntries(topics.map(t=>[t,candidates[t] .filter(r=>r.brandCode&&r.brandCode!=='unknown'&&r.shopId&&r.shopName) .sort((a,b)=>(b.existingReviewCount>0?20:0)+(oldChecks.get(b.id)?.essentialFieldsPresent?8:0)+b.score- (a.existingReviewCount>0?20:0)-(oldChecks.get(a.id)?.essentialFieldsPresent?8:0)-a.score) .slice(0,100)])); for(const t of topics)assert.equal(selected[t].length,100,'NOT_ENOUGH_REAL_PRODUCTS:'+t); const products=[...new Map(Object.values(selected).flat().map(r=>[r.id,r])).values()]; const requests=[...products.map(r=>({entity:'products',id:r.id,row:r})),...Array.from(records.values()).map(r=>({entity:r.entity,id:r.id,row:r}))]; const cache=path.join(work,'target-current-details');fs.mkdirSync(cache,{recursive:true}); const checks=new Map();let next=0,done=0; async function fetchDetail(entity,id){ const filename=path.join(cache,sha(Buffer.from(entity+':'+id)).slice(0,24)+'.json'); if(fs.existsSync(filename)){const d=load(filename);if(d.generation===generation&&d.ok)return d;} for(let attempt=0;;attempt++){ try{const d=await publicApi({op:'detail',entity,id,generation});save(filename,d);return d;} catch(error){if(attempt>=2)throw error;await new Promise(resolve=>setTimeout(resolve,1000));} } } await Promise.all(Array.from({length:2},async()=>{ while(nexts.verification==='source_attribute'&&s.spec_text?.trim()))missing.push('sourceVerifiedSpec'); if(!r.specs?.some(s=>s.verification==='source_attribute'&&s.net_weight_g>0))missing.push('verifiedNetWeight'); const reviews=data.relations.find(x=>x.entity==='reviews')?.count??null; if(reviews===null)missing.push('reviewRelationUnavailable');else if(reviews===0)missing.push('reviewContent'); if(!data.relations.some(x=>x.entity==='prices'&&x.count>0))missing.push('priceObservation'); if(!c.imageMetadataPresent)missing.push('imageMetadata'); checks.set(item.entity+':'+item.id,{id:r.id,title:r.title,url:r.url,brandCode:r.brand_code,shopId:r.shop_id, shopName:r.shop_name,sourceRecordId:r.source_record_id,sourceSha256:r.source.assetSha256, relevanceBasis:'标题中的明确汤底、食材、产品或场景词;不证明功效或购买者身份',titleTerms:c.titleTerms, basicIdentityAndSourceVerified:true,missing,reviewRecords:reviews,sourceSpecRows:r.specs.filter(s=>s.verification==='source_attribute').length, complete:missing.length===0}); }else{ const wanted=item.row,body=item.entity==='social'?r.content:r.full_text; assert.equal(body,wanted.body,'BODY_MISMATCH:'+item.id); assert.equal(r.title,wanted.title);assert.equal(item.entity==='social'?r.url:r.source_url,wanted.url); assert.equal(item.entity==='social'?r.published_at_raw:r.issued_date,wanted.publishedAtRaw); // Transport ZIP and the server-built source archive have distinct hashes. // Compare the actual normalized payload and source asset against staging. const expectedSource=sourceIntegrities[item.entity+':'+item.id];assert(expectedSource); assert.equal(r.source.payload_sha256,expectedSource.payloadSha256,'SOURCE_PAYLOAD_SHA_MISMATCH:'+item.id); assert.equal(r.source.assetSha256,expectedSource.assetSha256,'SOURCE_ASSET_SHA_MISMATCH:'+item.id); if(item.entity==='social')assert.equal(r.fulltext_available,1); else assert.equal(r.attachment_hash_state,'matched'); checks.set(item.entity+':'+item.id,{id:item.id,entity:item.entity,bodyVerified:true,sourcePayloadAndAssetShaVerified:true, publishedSourceArchiveSha256:r.source.archiveSha256, originalBytesShaVerified:true,sourceRecordId:r.source_record_id,sourceSha256:r.source.assetSha256}); } done++;if(done%25===0){save(path.join(work,'target-verification-progress.json'),{generation,done,target:requests.length});console.log(JSON.stringify({verified:done,target:requests.length}));} } })); const result={schema:'ksf.five-topic.minimum-targets.v2',generation,checkedAt:new Date().toISOString(), minimum:{products:100,social:10,other:5},quantityCriteria:'不同在库商品ID;社媒只计社区原文,其他计行业研究与官方资料,不跨体裁重复计数', sourceDocuments:records.size,uniqueProducts:products.length, newCollection:{products:0,community:5,industryResearch:20},topics:{}, limits:['这是已经入库并逐条回读的课题资料清单,不是搜索结果数量。', '每课题100件商品为已有在库商品整理,跨课题关联允许;全局商品总数不因整理增加。', '商品缺规格、净含量、价格或评价时保留缺项,不以商品标题或运输重量补齐。', '社区体验不等于平台购买评价;行业营销数字、健康和工艺陈述不作为已验证结论。', '已保存图片元数据不等于本轮重新逐张解码验图。']}; for(let i=0;ichecks.get('products:'+p.id)); const material=Array.from(records.values()).filter(r=>r.topics.some(x=>x.topic===t)).map(r=>({...r,verification:checks.get(r.entity+':'+r.id)})); const social=material.filter(r=>r.type==='community'),other=material.filter(r=>r.type!=='community'); const missingFieldCounts={};for(const p of pr)for(const f of p.missing)missingFieldCounts[f]=(missingFieldCounts[f]||0)+1; result.topics[t]={name:labels[i],counts:{products:pr.length,social:social.length,other:other.length, industryResearch:other.filter(r=>r.type==='industry').length,official:other.filter(r=>r.type==='official').length, productsWithReviews:pr.filter(r=>r.reviewRecords>0).length,completeProductDossiers:pr.filter(r=>r.complete).length}, quantityMinimumMet:pr.length>=100&&social.length>=10&&other.length>=5, completeProductDossiers:pr.every(r=>r.complete),missingFieldCounts,products:pr,social,other}; } result.allQuantityMinimumsMet=Object.values(result.topics).every(t=>t.quantityMinimumMet); result.allProductDossiersComplete=Object.values(result.topics).every(t=>t.completeProductDossiers); result.allSourceDocumentsReadbackVerified=true; result.allRequestedWorkComplete=result.allQuantityMinimumsMet&&result.allProductDossiersComplete; const after=await read('contract');assert.equal(after.generation,generation,'GENERATION_CHANGED_DURING_TARGET_AUDIT'); save(path.join(work,'topic-material-catalogue.json'),result); save(path.join(work,'product-enrichment-remaining.json'),{ schema:'ksf.five-topic.product-gaps.v1',generation,checkedAt:result.checkedAt, uniqueProducts:products.length, products:products.map(p=>({...checks.get('products:'+p.id),topics:topics.filter(t=>selected[t].some(r=>r.id===p.id))})).filter(p=>p.missing.length), sourceRules:['按原商品ID和原链接核对;主评、追评和精确评价ID分别保留。', '净含量必须来自所选SKU的明确属性或包装标示,不能使用运输重量和标题推定。', '平台验证或无有效返回属于采集受限,不自动判断商品不存在。'], currentLimit:'本轮电商接口未取得有效商品响应,京东公开评价接口返回非JSON;已保存原始回执。'}); const lines=['# 五课题资料核验(2026-11-11)','', '|课题|商品|社媒原文|其他资料|有评价商品|','|---|---:|---:|---:|---:|']; for(const t of Object.values(result.topics))lines.push(`|${t.name}|${t.counts.products}|${t.counts.social}|${t.counts.other}|${t.counts.productsWithReviews}|`); lines.push('',`全局去重:${products.length}件商品、${records.size}份独立资料。新增5篇社区体验、20篇行业研究,商品新增0。`, '',`数量门槛:${result.allQuantityMinimumsMet?'达到':'未全部达到'};商品完整档案:${result.allProductDossiersComplete?'通过':'未全部通过'}。`, '全部资料正文、原始字节SHA与当前代际的具体记录和来源归档均通过核对。', '商品已核基础身份、店铺、品牌、原链接和来源;净含量等缺项另列,不据标题或运输重量推算。','',`研究代际:${generation}`); fs.writeFileSync(path.join(work,'采集与完整性核验.md'),lines.join('\n')); console.log(JSON.stringify({ok:true,generation,uniqueProducts:products.length,sourceDocuments:records.size, topics:Object.fromEntries(Object.entries(result.topics).map(([k,v])=>[k,v.counts])),allQuantityMinimumsMet:result.allQuantityMinimumsMet, allProductDossiersComplete:result.allProductDossiersComplete}));