kangshifu-board/scripts/verify-five-topic-collection.mjs

/** Verify each new topic source and report new collection separately from existing matches. */
import fs from 'node:fs';
import path from 'node:path';
import assert from 'node:assert/strict';
import {read, publicApi, save, load, sha, unwrap, checkPublication} from './lib/research-enrichment.mjs';
const work = path.resolve('work/five-topic-collection-20261110');
const directory = path.resolve('work/database-enrichment-20261110/batches/five-topics-public-01');
const plan = load(path.join(work, 'collection-plan.json'));
const receipt = load(path.join(directory, 'receipt.json'));
const expected = load(path.join(directory, 'expected-records.json'));
const applied = unwrap(load(path.join(directory, 'apply-result.json')));
const status = unwrap(load(path.join(directory, 'status-result.json')));
const planned = unwrap(load(path.join(directory, 'plan-result.json'))).mergePlan;
checkPublication(applied, receipt);
assert.equal(applied.generation, planned.plannedGeneration);
assert(status.ok && status.publicationFound);
assert.equal(status.publishedGeneration, applied.generation);
assert.equal(sha(fs.readFileSync(path.join(directory, 'transport.zip'))), receipt.archiveSha256);
const contract = await read('contract');
const before = load(path.join(work, 'contract-before.json'));
assert.equal(contract.generation, applied.generation);
for (const [table, count] of Object.entries(before.counts)) {
  assert.equal(contract.counts[table], count + (applied.delta.added[table] || 0), 'COUNT_DRIFT:' + table);
}
const checks = [];
let next = 0;
await Promise.all(Array.from({length: 2}, async () => {
  while (next < expected.length) {
    const index = next++, row = expected[index];
    const response = await read('get', {entity: 'social', id: row.id, generation: contract.generation});
    const actual = response.record;
    assert.equal(actual.id, row.id); assert.equal(actual.post_id, row.postId);
    assert.equal(actual.platform, 'smzdm'); assert.equal(actual.content_scope, 'packaged_noodles');
    assert.equal(actual.fulltext_available, 1);
    for (const field of ['title', 'content', 'url', 'published_at_raw']) assert.equal(actual[field], row[field]);
    const source = await read('source', {id: actual.source_record_id, generation: contract.generation});
    assert.equal(source.sourcePayload.content, row.content);
    assert.equal(source.sourcePayload.id, row.postId);
    assert(source.citation.upstreamSources.some(item => item.sha256 === row.adapterSha256));
    const adapter = load(path.join(directory, row.adapterPath));
    assert.equal(adapter.original.sha256, row.originalSha256);
    assert.equal(sha(fs.readFileSync(path.join(directory, row.originalPath))), row.originalSha256);
    assert.equal(adapter.record.content, row.content);
    save(path.join(directory, 'verified-source-' + index + '.json'), {record: response, source,
      original: {...adapter.original, shaVerified: true}});
    checks.push({id: row.id, fieldsVerified: true, sourceVerified: true,
      originalSha256: row.originalSha256, topics: row.topics});
    save(path.join(work, 'verification-state.json'), {stage: 'get-source', completed: checks.length,
      target: expected.length, generation: contract.generation, updatedAt: new Date().toISOString()});
    console.log(JSON.stringify({verified: checks.length, target: expected.length, id: row.id}));
  }
}));
const summary = await publicApi({op: 'summary', generation: contract.generation});
for (const [table, count] of Object.entries(contract.counts)) assert.equal(summary.counts[table], count);
const page = await publicApi({op: 'browse', entity: 'social', quality: 'audit', limit: 100,
  filters: {recordId: expected.map(row => row.id)}, generation: contract.generation});
assert.equal(page.totalMatched, expected.length);
assert.equal(page.records.length, expected.length);
assert(page.records.every(row => row.fulltext_available === 1));
const union = new Map(), topics = [];
for (const topic of plan.topics) {
  const library = load(path.join(work, 'library-' + topic.id + '.json'));
  assert(library.allPagesRetrieved && library.sourceIdsPresent);
  assert.equal(library.generation, before.generation);
  const byEntity = {}, reviewFingerprints = new Set();
  for (const row of library.records) {
    byEntity[row.entity] = (byEntity[row.entity] || 0) + 1;
    union.set(row.entity + ':' + row.record_id, row);
    if (row.entity === 'reviews') {
      assert(row.match.matchedTerms.length > 0, 'REVIEW_TITLE_ONLY_MATCH');
      reviewFingerprints.add(row.content_fingerprint);
    }
  }
  assert.deepEqual(byEntity, library.counts);
  const newRows = expected.filter(row => row.topics.includes(topic.id));
  topics.push({id: topic.id, name: topic.name, newArticleOriginalsCollected: newRows.length,
    newArticlesImported: newRows.length, newProductsCollected: 0, newReviewsCollected: 0,
    existingProductCandidates: byEntity.products || 0, existingReviewBodyMatches: byEntity.reviews || 0,
    existingDistinctReviewTexts: reviewFingerprints.size,
    existingSocialCandidates: byEntity.social || 0, newRecordIds: newRows.map(row => row.id)});
}
const unionCounts = {};
for (const row of union.values()) unionCounts[row.entity] = (unionCounts[row.entity] || 0) + 1;
const collection = load(path.join(work, 'collection-result.json'));
assert.equal(collection.stopReason.code, 'UPSTREAM_HTTP_402');
const publication = {ok: true, checkedAt: new Date().toISOString(), batchGeneration: applied.generation,
  generation: contract.generation, allRowsVerified: true, publicReadbackPassed: true,
  sourceCitationReadbackPassed: true, checks, summary};
save(path.join(directory, 'pipeline-readback.json'), publication);
const result = {schema: 'ksf.five-topic.verified-results.v1', checkedAt: new Date().toISOString(),
  status: 'public-source-batch-verified-platform-collection-still-blocked',
  generation: contract.generation, existingLibrarySnapshotGeneration: before.generation,
  topics, newCollectionTotals: {articleOriginals: expected.length, importedSocialArticles: expected.length,
    products: 0, ecommerceReviews: 0}, existingLibraryUniqueRecords: unionCounts,
  topicRecordMemberships: topics.reduce((sum, row) => sum + row.existingProductCandidates + row.existingReviewBodyMatches + row.existingSocialCandidates, 0),
  allRowsVerified: true, publicReadbackPassed: true, sourceCitationReadbackPassed: true,
  productionCounts: contract.counts, upstreamStop: collection.stopReason,
  limitations: ['商品、评价和原有社媒数量为库内关键词候选，不是本轮外部新增量',
    '库内评价仅按正文或追评关键词匹配，尚未逐条判读实质关联，不作为已确认需求或analysis资格',
    '课题间可重叠，按精确记录ID计算总体去重数；同文也不等于独立消费者',
    '公开文章为保留原日期的历史/当时体验，不代表2026年趋势、现价或功效实证',
    '标题中的人群/场景营销词不证明实际消费者身份；评价列要求正文或追评命中',
    '新增公开社区文章可按精确recordId查询；未冒充京东/淘宝购买评价'],
  batchDirectory: path.relative(process.cwd(), directory)};
save(path.join(work, 'verified-results.json'), result);
const table = ['| 课题 | 本轮新采并入库文章 | 库内商品线索 | 库内评价正文关键词命中 | 库内社媒线索 |',
  '|---|---:|---:|---:|---:|', ...topics.map(row => `| ${row.name} | ${row.newArticlesImported} | ${row.existingProductCandidates} | ${row.existingReviewBodyMatches} | ${row.existingSocialCandidates} |`)];
fs.writeFileSync(path.join(work, '五课题采集量.md'), `# 五课题采集与入库结果\n\n核验时间：${result.checkedAt}\n\n${table.join('\n')}\n\n本轮新增公开社区原文去重${expected.length}篇，正式入库${expected.length}篇；表内跨课题会重复。商品和电商评价新增均为0：现有VOC采集探测返回HTTP402，已停止排队请求。\n\n库内课题线索按记录ID总体去重：${JSON.stringify(unionCounts)}。库内数字来自版本${before.generation}的逐页查询；当前正式版本${contract.generation}新增上述6篇原文。\n\n6/6具体记录、6/6来源及原件SHA、看板接口和逐表计数均通过；未新增商品、评价、价格或规格。\n\n公开网页候选122件，读到可定位正文12篇，复核选入6篇产品体验原文；其余包括验证页、搜索跳转页、无有效正文、健康泛谈、无来源引语或食谱等，未计入有效入库量。\n\n${result.limitations.map(text => '- ' + text).join('\n')}\n\n具体来源链接、原文、日期与精确数据库ID见reviewed-public-sources.json和verified-results.json；原件和发布回执保留在${result.batchDirectory}。\n`, 'utf8');
console.log(JSON.stringify({ok: true, generation: result.generation, topics,
  newCollectionTotals: result.newCollectionTotals, existingLibraryUniqueRecords: unionCounts}));