kangshifu-board/scripts/collect-review-coverage.cjs

// A bounded continuation of the existing direct VOC MCP collection workflow.
// Reserve before each call, reuse actual credentials, never repeat a request.
require('../server/industry-collection-guard.cjs').assertPaidCollectionAllowed();
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const { createRequire } = require('node:module');
const plugin = 'C:/Users/JSHY/AppData/Roaming/npm/node_modules/@vocmarket/voc-skill';
const req = createRequire(path.join(plugin, 'mcp/src/server.js'));
const { Client } = req('@modelcontextprotocol/sdk/client/index.js');
const { StdioClientTransport } = req('@modelcontextprotocol/sdk/client/stdio.js');
const { readCredentialContext } = req('./core/credentials.js');
const credentials = readCredentialContext({});
const root = path.resolve('work/product-reviews/round2');
const live = path.resolve('work/product-reviews/live-r2');
const plan = JSON.parse(fs.readFileSync(path.join(root, 'plan-expanded.json'), 'utf8'));
const limit = Number(process.argv[2] || plan.candidateCount);
if (!Number.isInteger(limit) || limit < 1 || limit > 60) throw Error('INVALID_LIMIT');
function save(file, value) {
  fs.mkdirSync(path.dirname(file), { recursive: true });
  let raw = JSON.stringify(value, null, 2);
  for (const secret of [credentials.newApiToken, credentials.sessionToken]) {
    if (secret) raw = raw.split(secret).join('[REDACTED]');
  }
  fs.writeFileSync(file, raw + '\n', { flag: 'wx' });
}
async function library(query) {
  const response = await fetch('https://kangshifu.fmode.cn/api/library', {
    method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(query),
    signal: AbortSignal.timeout(35000),
  });
  const data = await response.json();
  if (!response.ok || !data.ok) throw Error(data.error?.code || 'LIBRARY_READ_FAILED');
  return data;
}
const transport = new StdioClientTransport({ command: process.execPath, args: [path.join(plugin, 'mcp/src/server.js')],
  cwd: plugin, stderr: 'pipe', env: { ...process.env, VOC_SKILL_NO_UPDATE_CHECK: '1' } });
transport.stderr?.on('data', () => {});
const client = new Client({ name: 'ksf-expand-review-coverage', version: '1.0.0' });
async function tool(name, args, file, timeout = 30000) {
  const response = await client.callTool({ name, arguments: args }, undefined, { timeout });
  save(file, { request: { name, arguments: args }, response });
  return response.structuredContent || {};
}
(async () => {
  try {
    if (!credentials.newApiToken || credentials.sessionToken) throw Error('EXPECTED_SINGLE_EXISTING_AUTH_WITHOUT_FALLBACK');
    if (plan.worstCaseOccupiedCny > 50 || plan.maxNewCostCny > 18) throw Error('BUDGET_LIMIT');
    const summary = await library({ op: 'summary' });
    if (summary.generation !== plan.generation) throw Error('LIBRARY_GENERATION_CHANGED_REVIEW_PLAN');
    await client.connect(transport, { timeout: 15000 });
    const pricing = await tool('voc_billing_query', { operations: [{ model: 'voc-e-commerce',
      path: 'jd/get-item-comments/v1', count: 1 }], createPaymentQr: false }, path.join(root, 'pricing-' + Date.now() + '.json'));
    if (pricing.status !== 'ok' || pricing.data?.estimate?.operations?.[0]?.priceCnyPerCall !== 0.30) throw Error('PRICE_CHANGED');
    const queue = plan.candidates.filter(c => !fs.existsSync(path.join(live, c.productId.replace(':', '-') + '-p1', 'reservation.json'))).slice(0, limit);
    let next = 0, started = 0, finished = 0, failures = 0, stop = false;
    async function worker() {
      while (!stop) {
        const index = next++;
        if (index >= queue.length) return;
        const item = queue[index];
        const label = item.productId.replace(':', '-') + '-p1';
        const out = path.join(live, label);
        try {
          const context = await library({ op: 'detail', entity: 'products', id: item.productId, generation: plan.generation });
          const count = context.relations.find(r => r.entity === 'reviews')?.count;
          if (context.generation !== plan.generation || count !== 0) throw Error('PRODUCT_ALREADY_UPDATED');
          const params = { itemId: item.productId.split(':')[1], page: item.page };
          if (item.productId.startsWith('taobao:')) params.orderType = 'general';
          // Exact endpoint/product/page has already been excluded from the old
          // round by the source-verified plan; this directory prevents replay.
          if (fs.existsSync(path.join(out, 'reservation.json'))) throw Error('REQUEST_ALREADY_RESERVED');
          save(path.join(out, 'product-context.json'), context);
          const usageTraceId = 'usage_ksf_reviews_r2_' + crypto.randomUUID();
          const reservation = { endpoint: item.endpoint, params, usageTraceId, maxNewCostCny: 0.30,
            maxGatewayRequests: 1, retries: 0, automaticFallback: false, timestamp: new Date().toISOString(),
            catalogGeneration: plan.generation, historicalCountDisplay: item.volumeEvidence.countText,
            scope: 'expand_missing_high_volume_product_coverage', sharedTaskBudgetCny: 50 };
          save(path.join(out, 'reservation.json'), reservation);
          started++;
          console.log(JSON.stringify({ event: 'started', productId: item.productId, number: started, maxCostCny: 0.30 }));
          const result = await tool('voc_api_call', { id: item.endpoint, params, retries: 0, usageTraceId,
            skillId: 'ksf-review-coverage-r2', outputDir: path.join(out, 'tool-output') }, path.join(out, 'mcp-result.json'), 600000);
          const rows = item.productId.startsWith('jd:') ? result.data?.result?.data : result.data?.result?.comments;
          finished++;
          if (result.status !== 'ok' || !Array.isArray(rows) || !rows.length) failures++;
          console.log(JSON.stringify({ event: 'finished', productId: item.productId, status: result.status,
            rows: Array.isArray(rows) ? rows.length : 0, finished, failures }));
          try {
            const usage = await tool('voc_usage_report', { usageTraceId, groupBy: ['usageTraceId', 'module', 'endpoint'],
              limit: 1000, pageSize: 200, refresh: true }, path.join(out, 'usage-' + Date.now() + '.json'), 60000);
            console.log(JSON.stringify({ event: 'usage', productId: item.productId, status: usage.status,
              costCny: usage.summary?.costCny, billedCalls: usage.summary?.billedCalls }));
          } catch (error) {
            // The collection response is already durable. An unavailable
            // billing query keeps its full reservation; never repeat data.
            save(path.join(out, 'usage-pending-' + Date.now() + '.json'),
              { usageTraceId, errorType: error.name, unsettledReservationCny: 0.30 });
            console.log(JSON.stringify({ event: 'usage-pending', productId: item.productId, reservationCny: 0.30 }));
          }
          if (['needs_valid_token', 'needs_recharge', 'permission_denied'].includes(result.status)
              || (finished >= 8 && failures / finished > 0.65)) {
            stop = true;
            console.log(JSON.stringify({ stopped: true, reason: 'ACCESS_OR_FAILURE_RATE', finished, failures }));
          }
        } catch (error) {
          stop = true;
          console.log(JSON.stringify({ stopped: true, productId: item.productId, errorType: error.name,
            reason: /GENERATION|UPDATED|RESERVED/.test(error.message) ? error.message : 'RESULT_UNCERTAIN_OR_READ_FAILED' }));
        }
      }
    }
    await Promise.allSettled([worker(), worker()]);
    console.log(JSON.stringify({ runFinished: true, started, finished, failures, stopped: stop,
      newReservationUpperCny: Number((started * .30).toFixed(2)) }));
  } finally {
    await client.close();
    await transport.close();
  }
})().catch(error => {
  console.log(JSON.stringify({ ok: false, errorType: error.name,
    reason: /GENERATION|BUDGET|PRICE|AUTH/.test(error.message) ? error.message : 'ACCESS_FAILED_NO_RETRY' }));
  process.exitCode = 1;
});