kangshifu-board/scripts/verify-five-topic-multisource-20261111.mjs

/** Verify every new record, field, source hash and public board index. */
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {read as bridgeRead,save,load,sha,unwrap,checkPublication} from './lib/research-enrichment.mjs';
async function retryRead(operation){
 for(let attempt=0;;attempt++){
  try{return await operation();}
  catch(error){
   const code=error.cause?.code||error.code||error.name;
   if(attempt>=2||!['ECONNRESET','ETIMEDOUT','EAI_AGAIN','UND_ERR_SOCKET','UND_ERR_CONNECT_TIMEOUT','TimeoutError','PUBLIC_TRANSIENT_HTTP'].includes(code))throw error;
   console.log(JSON.stringify({stage:'read-only-network-retry',attempt:attempt+1,code}));
   await new Promise(resolve=>setTimeout(resolve,1200));
  }
 }
}
const read=(...args)=>retryRead(()=>bridgeRead(...args));
const publicApi=body=>retryRead(async()=>{
 const r=await fetch('https://kangshifu.fmode.cn/api/library',{method:'POST',redirect:'error',
  headers:{'Content-Type':'application/json'},body:JSON.stringify(body),signal:AbortSignal.timeout(150000)});
 const raw=await r.text();
 let value;try{value=JSON.parse(raw);}catch{
  const error=Error('PUBLIC_NON_JSON_RESPONSE:'+r.status);if(r.status>=500)error.code='PUBLIC_TRANSIENT_HTTP';throw error;
 }
 if(!r.ok||!value.ok){
  save(path.join(work,'public-error-'+body.op+'.json'),{request:body,httpStatus:r.status,response:value,checkedAt:new Date().toISOString()});
  const error=Error(JSON.stringify(value.error||'PUBLIC_READBACK_FAILED'));
  error.code=r.status>=500?'PUBLIC_TRANSIENT_HTTP':value.error?.code;throw error;
 }
 return value;
});
const work=process.argv[2]||'work/five-topic-research-20261111';
const directory=path.join('work/database-enrichment-20261110/batches',process.argv[3]||'five-topics-multisource-20261111-01');
const expected=load(path.join(directory,'expected-records.json'));
const receipt=load(path.join(directory,'receipt.json'));
const applied=unwrap(load(path.join(directory,'apply-result.json')));
const plan=unwrap(load(path.join(directory,'plan-result.json'))).mergePlan;
const status=unwrap(load(path.join(directory,'status-result.json')));
checkPublication(applied,receipt);
assert.equal(applied.generation,plan.plannedGeneration);
assert(status.ok&&status.publicationFound);
assert.equal(status.publishedGeneration,applied.generation);
assert.equal(sha(fs.readFileSync(path.join(directory,'transport.zip'))),receipt.archiveSha256);
for(const [entity,table] of [['social','social_posts'],['documents','policy_documents'],['clauses','policy_clauses']]){
 assert.equal(applied.delta.added[table]||0,expected[entity].length);
}
for(const table of ['products','reviews','specs','prices'])assert.equal(applied.delta.added[table]||0,0);
const before=unwrap(load(path.join(directory,'apply-contract-before.json')));
const contract=await read('contract');
assert.equal(contract.generation,applied.generation,'GENERATION_CHANGED_DURING_REVIEW');
for(const [table,count] of Object.entries(before.counts))assert.equal(contract.counts[table],count+(applied.delta.added[table]||0));
const generation=contract.generation,items=Object.values(expected).flat();
const checks=[];let next=0;
await Promise.all(Array.from({length:4},async()=>{
 while(next<items.length){
  const i=next++,row=items[i];
  const file=path.join(directory,'verified-record-'+i+'.json');
  const cached=fs.existsSync(file)?load(file):null;
  const response=cached?.record?.generation===generation?cached.record:await publicApi({op:'detail',entity:row.entity,id:row.id,generation});
  const actual=response.record;
  for(const [field,value] of Object.entries(row.record)){
   // Source IDs are recomputed under the published archive's namespace.
   if(field==='source_record_id'||field==='hash_basis')continue;
   assert.deepEqual(actual[field],value,'FIELD_MISMATCH:'+row.entity+':'+row.id+':'+field);
  }
  const source=cached?.source?.generation===generation?cached.source:await read('source',{id:actual.source_record_id,generation});
  assert(source.citation.upstreamSources.some(s=>s.sha256===row.adapterSha256),'ADAPTER_SOURCE_HASH_MISSING');
  const adapter=load(path.join(directory,row.adapterPath));
  assert.equal(sha(fs.readFileSync(path.join(directory,row.originalPath))),row.originalSha256);
  assert.equal(adapter.original.sha256,row.originalSha256);
  const text=row.entity==='social'?actual.content:row.entity==='documents'?actual.full_text:actual.excerpt;
  assert.equal(text,row.entity==='social'?source.sourcePayload.content:row.entity==='documents'?source.sourcePayload.full_text:source.sourcePayload.source_excerpt);
  if(row.entity!=='clauses')assert(row.topicEvidence.every(t=>t.passages.every(q=>text.includes(q))));
  if(row.entity==='documents')assert.equal(actual.attachment_hash_state,'matched');
  if(row.entity==='clauses')assert(adapter.record.fullText.includes(actual.excerpt));
  save(file,{record:response,source,originalShaVerified:true});
  checks.push({entity:row.entity,id:row.id,fieldsVerified:true,sourceVerified:true,originalSha256:row.originalSha256});
  console.log(JSON.stringify({checked:checks.length,target:items.length,entity:row.entity,id:row.id}));
 }
}));
const recentProbeFile=path.join(work,'public-summary-probe.json');
const recentProbe=fs.existsSync(recentProbeFile)?load(recentProbeFile):null;
const summary=recentProbe?.ok&&recentProbe.generation===generation?recentProbe:await publicApi({op:'summary',generation});
assert.equal(summary.generation,generation);
for(const [table,count] of Object.entries(contract.counts))assert.equal(summary.counts[table],count);
const publicChecks=[];
for(const [entity,rows] of Object.entries(expected)){
 if(!rows.length)continue;
 const page=await publicApi({op:'browse',entity,quality:'audit',filters:{recordId:rows.map(r=>r.id)},limit:100,generation});
 assert.equal(page.totalMatched,rows.length);
 assert.deepEqual(new Set(page.records.map(r=>r.record_id)),new Set(rows.map(r=>r.id)));
 save(path.join(directory,'public-browse-'+entity+'.json'),page);
 for(const r of page.records)assert(r.preview||r.match?.text||r.title);
 publicChecks.push({entity,records:page.totalMatched,allIdsFound:true});
}
// The board's actual detail endpoint must deliver the new content too.
for(const entity of ['social','documents']){
 const row=expected[entity][0];
 if(!row)continue;
 const detail=await publicApi({op:'detail',entity,id:row.id,generation});
 assert.equal(detail.record.id,row.id);
 assert.equal(entity==='social'?detail.record.content:detail.record.full_text,entity==='social'?row.record.content:row.record.full_text);
 save(path.join(directory,'public-detail-'+entity+'.json'),detail);
}
const reviewed=load(path.join(directory,'reviewed-source-index.json'));
const sourceTypes={};for(const row of reviewed)sourceTypes[row.sourceType]=(sourceTypes[row.sourceType]||0)+1;
const topicTypes=Object.fromEntries(['wellness','student','premium','stockpile','novelty'].map(t=>[t,{
 community:reviewed.filter(r=>r.topics.some(x=>x.topic===t)&&r.sourceType==='community_experience').length,
 industry:reviewed.filter(r=>r.topics.some(x=>x.topic===t)&&r.entity==='social'&&r.sourceType!=='community_experience').length,
 official:reviewed.filter(r=>r.topics.some(x=>x.topic===t)&&r.entity==='documents').length}]));
const verified={schema:'ksf.five-topic.multisource-final.v1',ok:true,checkedAt:new Date().toISOString(),generation,
 allRowsVerified:applied.allRowsVerified,allExactRecordsVerified:checks.length===items.length,
 publicReadbackPassed:true,sourceCitationReadbackPassed:true,
 sourceDocuments:reviewed.length,added:{communityExperiences:sourceTypes.community_experience||0,
  industryArticles:reviewed.filter(r=>r.entity==='social'&&r.sourceType!=='community_experience').length,
  officialDocuments:expected.documents.length,documentExcerpts:expected.clauses.length,products:0,purchaseReviews:0},
 topicTypes,sourceTypes,checks,publicChecks,counts:contract.counts,
 limits:load(path.join(work,'reviewed-selection.json')).limits};
save(path.join(directory,'pipeline-readback.json'),verified);
save(path.join(work,'final-receipt.json'),verified);
save(path.join(work,'contract-after.json'),contract);
save(path.join(work,'summary-after.json'),summary);
console.log(JSON.stringify({ok:true,generation,added:verified.added,topicTypes,publicReadbackPassed:true}));