kangshifu-board/scripts/dedup-five-topic-multisource-20261111.mjs

import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {read, publicApi, save, load} from './lib/research-enrichment.mjs';
const work=process.argv[2]||'work/five-topic-research-20261111';
const review=load(path.join(work,'reviewed-selection.json'));
const before=load(path.join(work,'contract-before.json'));
const contract=await read('contract');
assert.equal(contract.generation,before.generation,'BASELINE_CHANGED_REAUDIT_REQUIRED');
const inventory=load('work/library-cleanup-20261111/round-02/inventory-after-result.json').body.result;
const documents=load(path.join(work,'documents-before.json')).records;
const normalizeUrl=u=>{try{const p=new URL(u);return p.hostname.replace(/^www\./,'')+p.pathname.replace(/\/$/,'');}catch{return '';}};
const normalizeText=t=>String(t||'').replace(/[\s\p{P}\p{S}]/gu,'').toLowerCase();
const existingSocial=inventory.social.map(r=>({...r,urlKey:normalizeUrl(r.url),normalized:normalizeText(r.content)}));
// Include later verified batches that are newer than the inventory snapshot.
for(const name of fs.readdirSync('work/database-enrichment-20261110/batches')){
 const directory=path.join('work/database-enrichment-20261110/batches',name);
 if(!fs.existsSync(path.join(directory,'apply-result.json')))continue;
 const file=path.join(directory,'reviewed-source-index.json');
 if(!fs.existsSync(file))continue;
 for(const row of load(file).filter(r=>r.entity==='social'))existingSocial.push({...row,urlKey:normalizeUrl(row.url),normalized:normalizeText(row.body)});
}
const remotelyFound=new Map();
for(const entity of ['social','documents']){
 const ids=review.selected.filter(r=>r.entity===entity).map(r=>r.id);
 if(!ids.length)continue;
 const page=await publicApi({op:'browse',entity,quality:'audit',filters:{recordId:ids},limit:100,generation:contract.generation});
 save(path.join(work,'dedup-remote-'+entity+'.json'),page);
 for(const r of page.records)remotelyFound.set(entity+':'+r.record_id,r);
}
const rejected=[],accepted=[];
for(const row of review.selected){
 let existing=remotelyFound.get(row.entity+':'+row.id),reason=existing?'existing_exact_id':null;
 if(row.entity==='social'&&!existing){
  existing=existingSocial.find(r=>r.urlKey===normalizeUrl(row.url));
  if(existing)reason='existing_original_url';
  else {
   const text=normalizeText(row.body);
   existing=existingSocial.find(r=>text.length>80&&r.normalized.length>80&&
      (r.normalized===text||r.normalized.includes(text)||(text.includes(r.normalized)&&r.normalized.length/text.length>0.85)));
   if(existing)reason='same_original_text';
  }
 }
 if(row.entity==='documents'&&!existing){
  // Full source URL is confirmed by get because document browse is an index.
  for(const doc of documents){
   if(normalizeText(doc.title)===normalizeText(row.title)){
    existing=await read('get',{entity:'documents',id:doc.record_id,generation:contract.generation});reason='existing_document_title_requires_reuse';break;
   }
  }
 }
 if(existing)rejected.push({id:row.id,url:row.url,reason,existingId:existing.id||existing.record_id||existing.record?.id});
 else {
  const prior=accepted.find(r=>normalizeUrl(r.url)===normalizeUrl(row.url)||normalizeText(r.body)===normalizeText(row.body));
  if(prior)rejected.push({id:row.id,url:row.url,reason:'duplicate_within_batch',existingId:prior.id});
  else accepted.push(row);
 }
}
assert.equal(new Set(accepted.map(r=>r.entity+':'+r.id)).size,accepted.length);
const result={schema:'ksf.five-topic.dedup.v1',ok:true,generation:contract.generation,checkedAt:new Date().toISOString(),
 acceptedIds:accepted.map(r=>({entity:r.entity,id:r.id})),rejected,selected:accepted,existingRecords:rejected.length,
 countsByTopic:Object.fromEntries(['wellness','student','premium','stockpile','novelty'].map(t=>[t,accepted.filter(r=>r.topics.some(x=>x.topic===t)).length]))};
save(path.join(work,'dedup-result.json'),result);
console.log(JSON.stringify({ok:true,accepted:accepted.length,rejected,topics:result.countsByTopic}));