QIWEI-skill/scripts/group-operations-quality-eval.js

#!/usr/bin/env node
const fs = require('fs');
const os = require('os');
const path = require('path');
const { GroupOperationsStore } = require('../mcp/src/core/group-operations-store');

const dataset = [
  { id: 'risk-1', messages: [{ messageId: 'r1', senderRole: 'customer', content: '我要投诉你们的服务', sentAt: '2026-07-18T08:00:00Z' }], expected: ['service_risk'] },
  { id: 'neutral-1', messages: [{ messageId: 'n1', senderRole: 'customer', content: '谢谢，已经收到了', sentAt: '2026-07-18T08:00:00Z' }], expected: [] },
  { id: 'question-1', messages: [{ messageId: 'q1', senderRole: 'customer', content: '什么时候可以处理？', sentAt: '2026-07-18T08:00:00Z' }], expected: ['unanswered_question'] },
  { id: 'answered-1', messages: [
    { messageId: 'a1', senderRole: 'customer', content: '请问什么时候可以处理？', sentAt: '2026-07-18T08:00:00Z' },
    { messageId: 'a2', senderRole: 'staff', content: '今天下午为您处理。', sentAt: '2026-07-18T08:10:00Z' }
  ], expected: [] },
  { id: 'unrelated-answer-1', messages: [
    { messageId: 'u1', senderRole: 'customer', content: '退款什么时候到账？', sentAt: '2026-07-18T08:00:00Z' },
    { messageId: 'u2', senderRole: 'staff', content: '今天天气不错', sentAt: '2026-07-18T08:01:00Z' }
  ], expected: ['service_risk', 'unanswered_question'] },
  { id: 'relevant-answer-1', messages: [
    { messageId: 'v1', senderRole: 'customer', content: '退款什么时候到账？', sentAt: '2026-07-18T08:00:00Z' },
    { messageId: 'v2', senderRole: 'staff', content: '退款已提交，预计今天下午原路到账。', sentAt: '2026-07-18T08:10:00Z' }
  ], expected: ['service_risk'] }
];

const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'qiwei-group-ops-eval-'));
const store = new GroupOperationsStore(path.join(tempDir, 'eval.db'));
let tp = 0; let fp = 0; let fn = 0;
try {
  for (const sample of dataset) {
    store.upsertGroup('eval', { roomId: sample.id, roomName: sample.id }, 'eval');
    const result = store.reviewMessages('eval', { roomId: sample.id, messages: sample.messages, slaMinutes: 120 }, 'eval');
    const predicted = new Set(result.findings.map(item => item.type));
    const expected = new Set(sample.expected);
    for (const type of predicted) expected.has(type) ? tp++ : fp++;
    for (const type of expected) if (!predicted.has(type)) fn++;
  }
  const precision = tp + fp ? tp / (tp + fp) : 1;
  const recall = tp + fn ? tp / (tp + fn) : 1;
  const report = { status: precision >= 0.9 && recall >= 0.9 ? 'ok' : 'failed', samples: dataset.length, truePositive: tp, falsePositive: fp, falseNegative: fn, precision, recall };
  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
  if (report.status !== 'ok') process.exitCode = 1;
} finally {
  store.close();
  fs.rmSync(tempDir, { recursive: true, force: true });
}