QIWEI-skill/scripts/agent-response-quality-smoke-test.js

'use strict';

const assert = require('assert/strict');
const fs = require('fs');
const os = require('os');
const path = require('path');

const RUN_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'qiwei-response-quality-'));
process.env.QIWEI_OUTPUTS_DIR = path.join(RUN_ROOT, 'outputs');

const { AgentContextBuilder } = require('../mcp/src/core/agent-context-builder');
const { QiweiAgentRuntime } = require('../mcp/src/core/agent-runtime');
const { AgentWorkbenchDb } = require('../mcp/src/core/agent-workbench-db');
const { AgentWorkbenchService } = require('../mcp/src/core/agent-workbench-service');
const {
  QUALITY_FALLBACK_REASONS,
  normalizeQualityChecks,
  normalizeQualityScore,
  qualityFallbackReason,
} = require('../mcp/src/core/response-quality-contract');
const { verifyResponseQuality } = require('../mcp/src/core/response-quality-verifier');
const { frozenA4Cases, hardFailCases, qualifiedCases } = require('./response-quality-regression-fixtures');

const LOW_QUALITY_JSONL = `
{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费？","reply":"您好，感谢您的咨询，我们一直致力于提供优质服务。"}
{"id":"price_promise","inbound":"最低能做到多少钱？","reply":"保证给您全网最低价，一定不会更贵。"}
{"id":"missing_info","inbound":"我想做员工培训","reply":"您的预算是多少？计划什么时候开始？预计多少人参加？"}
{"id":"objection","inbound":"这个方案太贵了","reply":"您的预算是多少？"}
{"id":"urgent","inbound":"今天能安排吗？","reply":"您好，感谢关注，我们会认真研究并持续努力为您提供全面周到专业细致的服务体验，请耐心等待。"}
{"id":"complaint","inbound":"你们一直不处理，我要投诉","reply":"我们一定马上全部解决，绝对不会让您失望。"}
{"id":"thanks","inbound":"谢谢","reply":"不客气，后续有需要随时联系。"}
{"id":"answered_context","inbound":"预算已经说过是20万","reply":"请问您的预算是多少？","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
{"id":"repeat_question","inbound":"我还在考虑","reply":"请问您的预算是多少？","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
{"id":"tool_no_result","inbound":"有现成案例吗？","reply":"有的，我们已经找到 3 个完全匹配案例。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
{"id":"low_quality","inbound":"能不能先给个建议？","reply":"好的。"}
`;

const GOOD_QUALITY_JSONL = `
{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费？","reply":"费用要按服务范围确认；我现在转人工专员核价，确认后给您对应报价。","requiresHuman":true}
{"id":"price_check","inbound":"最低能做到多少钱？","reply":"当前最低价需要核验；我现在转人工专员核价，确认后给您准确答复。","requiresHuman":true}
{"id":"missing_info","inbound":"我想做员工培训","reply":"可以先按员工培训方向梳理；请问预计多少人参加？确认人数后我给您下一步方案。"}
{"id":"objection","inbound":"这个方案太贵了","reply":"我理解您主要顾虑投入是否值得；当前收益数据还需按场景核验，您最关注成本还是效果？确认后我补对应依据。"}
{"id":"urgent","inbound":"今天能安排吗？","reply":"今天能否安排需要看当前档期；我现在核对，确认后尽快回复您。"}
{"id":"complaint","inbound":"你们一直不处理，我要投诉","reply":"我理解您在意的是迟迟未处理；我现在转交主管接管并同步处理进度。","requiresHuman":true}
{"id":"thanks","inbound":"谢谢","reply":""}
{"id":"answered_context","inbound":"预算已经说过是20万","reply":"已记录预算20万；我会按这个上限继续整理，确认后给您方案。","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
{"id":"no_repeat","inbound":"我还在考虑","reply":"可以按您的节奏来；我先保留当前条件，有明确时间后再继续。","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
{"id":"tool_no_result","inbound":"有现成案例吗？","reply":"当前数据源没有返回可验证案例；我可以调整一个筛选条件后再查。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
{"id":"suggestion","inbound":"能不能先给个建议？","reply":"可以，建议先明确最重要的结果目标；您最希望先解决哪个问题？确认后我给您第一版方案。"}
`;

function parseJsonl(value) {
  return String(value || '').trim().split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
}

function assessFixture(item) {
  return verifyResponseQuality({
    inboundContent: item.inbound,
    reply: item.reply,
    messages: item.messages || [],
    toolNoResult: Boolean(item.toolNoResult),
    toolTrace: item.toolTrace || [],
    requiresHuman: Boolean(item.requiresHuman),
  });
}

function modelPayload(reply, overrides = {}) {
  return JSON.stringify({
    reply,
    confidence: 0.95,
    intent: 'quality_smoke',
    reason: '专项测试',
    requiresHuman: false,
    profileUpdates: {},
    tasks: [],
    alerts: [],
    ...overrides,
  });
}

function setupService(mode, output) {
  const dir = fs.mkdtempSync(path.join(RUN_ROOT, `${mode}-`));
  const db = new AgentWorkbenchDb(path.join(dir, 'workbench.db'), { defaultMode: mode, autoSendConfidence: 0.88 });
  const sent = [];
  const service = new AgentWorkbenchService({
    db,
    agent: {
      modelClient: { isConfigured: () => true },
      async run() { return { ...output }; },
    },
    qiwei: {
      isConfigured: () => true,
      async sendText(toId, content) { sent.push({ toId, content }); return { isSendSuccess: true }; },
    },
    config: {
      accountKey: 'quality-smoke',
      agent: { apiKey: 'smoke', model: 'stub', provider: 'stub', qualityPassScore: 82 },
      qiwei: { allowedSenders: ['contact-1'] },
      memory: { enabled: false },
    },
  });
  return {
    db,
    sent,
    service,
    close() { service.stopBackgroundWorkers(); db.close(); },
  };
}

async function main() {
  const lowCases = parseJsonl(LOW_QUALITY_JSONL);
  const goodCases = parseJsonl(GOOD_QUALITY_JSONL);
  const lowResults = lowCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
  const goodResults = goodCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
  const lowBlocked = lowResults.filter(item => !item.assessment.passed).length;
  const goodRejected = goodResults.filter(item => !item.assessment.passed);
  assert.equal(lowBlocked, lowCases.length, JSON.stringify(lowResults.filter(item => item.assessment.passed)));
  assert.deepEqual(goodRejected.map(item => ({ id: item.id, failed: item.assessment.failedCheckIds })), []);

  const hardRegressionResults = hardFailCases.map(item => ({ ...item, assessment: verifyResponseQuality(item.input) }));
  for (const item of hardRegressionResults) {
    assert.equal(item.assessment.passed, false, item.id);
    for (const expected of item.expectedFailed) {
      assert(item.assessment.failedCheckIds.includes(expected), `${item.id} missing ${expected}: ${item.assessment.failedCheckIds.join(',')}`);
      assert(item.assessment.hardFailureIds.includes(expected), `${item.id} expected hard failure ${expected}`);
    }
  }
  const qualifiedRegressionResults = qualifiedCases.map(item => ({ ...item, assessment: verifyResponseQuality(item.input) }));
  assert.deepEqual(qualifiedRegressionResults.filter(item => !item.assessment.passed).map(item => ({
    id: item.id,
    failed: item.assessment.failedCheckIds,
  })), []);
  assert.deepEqual(frozenA4Cases.map(item => item.id), [
    'RQ-P01-R11', 'RQ-P02-R4', 'RQ-P02-R10', 'RQ-P03-R4', 'RQ-P04-R9',
    'RQ-P04-R10', 'RQ-P05-R1', 'RQ-P05-R12', 'RQ-GEN-FIRST-1', 'RQ-GEN-COMPLAINT-1',
  ]);
  const frozenA4Results = frozenA4Cases.map(item => ({ ...item, assessment: verifyResponseQuality(item.input) }));
  for (const item of frozenA4Results) {
    assert.equal(item.assessment.passed, false, `${item.id} frozen bad reply passed`);
    for (const expected of item.expectedFailed) {
      assert(item.assessment.failedCheckIds.includes(expected), `${item.id} missing frozen check ${expected}`);
      assert(item.assessment.hardFailureIds.includes(expected), `${item.id} expected frozen hard failure ${expected}`);
    }
  }

  assert.equal(normalizeQualityScore(82.6), 83);
  assert.equal(normalizeQualityScore(undefined), null);
  assert.equal(normalizeQualityChecks(undefined), null);
  assert.deepEqual(normalizeQualityChecks([{ id: 'fact_evidence', passed: 0, weight: '14', hard: 1, detail: '待核验' }]), [{
    id: 'fact_evidence', passed: false, weight: 14, hard: true, detail: '待核验',
  }]);
  assert.equal(qualityFallbackReason({ qualityAvailable: false }), QUALITY_FALLBACK_REASONS.QUALITY_UNAVAILABLE);
  assert.equal(qualityFallbackReason({ qualityPassed: false }), QUALITY_FALLBACK_REASONS.QUALITY_GATE_FAILED);
  assert.equal(qualityFallbackReason({ qualityPassed: false, rewriteCount: 1 }), QUALITY_FALLBACK_REASONS.REWRITE_LIMIT_REACHED);
  assert.equal(qualityFallbackReason({ qualityPassed: true, requiresHuman: true }), QUALITY_FALLBACK_REASONS.REQUIRES_HUMAN);
  assert.equal(qualityFallbackReason({ qualityPassed: true }), '');

  const unrelatedEvidence = verifyResponseQuality({
    inboundContent: '最低多少钱？',
    reply: '最低只要 99 元，我已经确认；我可以直接给您下单。',
    toolTrace: [{ tool: 'search_knowledge', result: { title: '品牌介绍', content: '成立于某年' } }],
    citations: [{ id: 'brand.md#intro', source: 'brand.md', heading: '品牌介绍' }],
  });
  assert.equal(unrelatedEvidence.passed, false);
  assert(unrelatedEvidence.failedCheckIds.includes('fact_evidence'));
  const claimBoundaryCases = [
    verifyResponseQuality({
      inboundContent: '价格确认了吗？',
      reply: '价格已确认，其他信息待核验；我可以继续帮您核对。',
      citations: [{ content: '其他信息待核验' }],
    }),
    verifyResponseQuality({
      inboundContent: '还有库存吗？',
      reply: '库存已确认；我可以直接为您保留。',
      toolTrace: [{ tool: 'inventory_lookup', result: { content: '库存字段说明' } }],
    }),
    verifyResponseQuality({
      inboundContent: '价格是多少？',
      reply: '已确认价格是 99 元，其他待核验；我可以把依据发给您。',
      citations: [{ content: '当前价格是 88 元' }],
    }),
  ];
  assert(claimBoundaryCases.every(item => !item.passed && item.failedCheckIds.includes('fact_evidence')));
  const sourcedPrice = verifyResponseQuality({
    inboundContent: '价格是多少？',
    reply: '已确认价格是 99 元；我现在转人工专员核价并把计价依据发给您。',
    citations: [{ content: '当前价格是 99 元' }],
    requiresHuman: true,
  });
  assert.equal(sourcedPrice.passed, true, JSON.stringify(sourcedPrice.failedCheckIds));
  const splitPromise = verifyResponseQuality({
    inboundContent: '价格能确认吗？',
    reply: '保证是 99 元，其他待核验；我可以继续跟进。',
    citations: [{ content: '其他信息待核验' }],
  });
  assert.equal(splitPromise.passed, false);
  assert(splitPromise.failedCheckIds.includes('sensitive_commitment'));
  const bundledQuestions = verifyResponseQuality({
    inboundContent: '我想咨询服务',
    reply: '可以先了解需求；预算多少、什么时候开始、多少人参加？确认后我给您方案。',
  });
  assert.equal(bundledQuestions.passed, false);
  assert(bundledQuestions.failedCheckIds.includes('question_count'));
  const singleQuestion = verifyResponseQuality({
    inboundContent: '我想咨询服务',
    reply: '可以先明确投入范围；请问您的预算上限大概是多少？确认后我给您方案。',
  });
  assert.equal(singleQuestion.passed, true, JSON.stringify(singleQuestion.failedCheckIds));
  const semanticRepeat = verifyResponseQuality({
    inboundContent: '预算已经确认是20万',
    reply: '已记录当前需求；请问您的总投入上限多少？确认后我继续整理。',
  });
  assert.equal(semanticRepeat.passed, false);
  assert(semanticRepeat.failedCheckIds.includes('repeated_question'));
  const factThenNewPreference = verifyResponseQuality({
    inboundContent: '推荐新北区三室300万以内的房源',
    reply: '已记录总价300万以内；您更看重通勤还是楼层？',
  });
  assert.equal(factThenNewPreference.failedCheckIds.includes('repeated_question'), false);
  for (const emptyAnswer of ['这边给您处理一下。', '我帮您关注一下。', '具体情况我再回复。']) {
    const assessment = verifyResponseQuality({ inboundContent: '当前有库存吗？', reply: emptyAnswer });
    assert.equal(assessment.passed, false);
    assert(assessment.failedCheckIds.includes('direct_answer'));
  }
  assert.equal(verifyResponseQuality({
    inboundContent: '当前有库存吗？',
    reply: '当前资料不足，需要先核验在售状态；我确认后回复您。',
  }).passed, true);

  const builder = new AgentContextBuilder({
    config: { promptCharLimit: 2400, systemPromptCharLimit: 2200, businessGoal: '推进到清晰且可执行的下一步' },
    knowledge: { contextText: () => '通用规则：未知事实必须核验。'.repeat(200) },
  });
  const system = builder.buildSystemContext({ memoryContext: { promptText: '旧 Session 说客户接受高价。'.repeat(100) } });
  const prompt = builder.buildClaudePrompt([
    { role: 'assistant', content: '请问预算是多少？' },
    { role: 'user', content: '本轮我只确认预算是20万。' },
    { role: 'tool', content: JSON.stringify({ result: '当前没有已核验报价' }) },
  ], {
    inboundContent: '本轮我只确认预算是20万。',
    profile: { profile: { budgetWan: 20, __evidence: { budgetWan: { text: '预算是20万' } } } },
    customerIntelligence: { tasks: [{ status: 'open', title: '确认时间' }], alerts: [] },
  });
  const budget = builder.budgetReport();
  assert(prompt.length <= 2400 && system.length <= 2200);
  assert.match(prompt, /本轮我只确认预算是20万/);
  assert.match(system, /旧模型 Session/);
  assert(Object.values(budget.prompt).every(item => item.used <= item.budget));
  assert(Object.values(budget.system).every(item => item.used <= item.budget));
  const [requestA, requestB] = await Promise.all([
    Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求甲的唯一内容' }], { profile: { profile: { request: 'A' } } })),
    Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求乙的唯一内容' }], { profile: { profile: { request: 'B' } } })),
  ]);
  assert.match(requestA.text, /请求甲的唯一内容/);
  assert.doesNotMatch(requestA.text, /请求乙的唯一内容/);
  assert.match(requestB.text, /请求乙的唯一内容/);
  assert.notDeepEqual(requestA.budgetReport, {});
  assert.notStrictEqual(requestA.budgetReport, requestB.budgetReport);

  const sequence = [
    modelPayload('您好，感谢您的咨询，请耐心等待。'),
    modelPayload('今天能否安排需要核对当前档期；我现在核对，确认后尽快回复您。'),
  ];
  const runtime = new QiweiAgentRuntime({
    config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
    knowledge: { search: () => [], contextText: () => '' },
    modelClient: { async complete() { return { content: sequence.shift() }; } },
  });
  const rewritten = await runtime.run({
    conversation: { id: 'runtime-rewrite', mode: 'autopilot' },
    messages: [{ direction: 'inbound', content: '今天能安排吗？' }],
    profile: { profile: {} },
    inboundContent: '今天能安排吗？',
  });
  assert.equal(rewritten.rewriteCount, 1);
  assert.equal(rewritten.qualityPassed, true);
  assert.equal(rewritten.requiresHuman, false);

  let failedCalls = 0;
  const failedRuntime = new QiweiAgentRuntime({
    config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
    knowledge: { search: () => [], contextText: () => '' },
    modelClient: { async complete() { failedCalls += 1; return { content: modelPayload('好的。') }; } },
  });
  const failedRewrite = await failedRuntime.run({
    conversation: { id: 'runtime-fallback', mode: 'autopilot' },
    messages: [{ direction: 'inbound', content: '今天能安排吗？' }],
    profile: { profile: {} },
    inboundContent: '今天能安排吗？',
  });
  assert.equal(failedCalls, 2);
  assert.equal(failedRewrite.rewriteCount, 1);
  assert.equal(failedRewrite.qualityPassed, false);
  assert.equal(failedRewrite.requiresHuman, true);
  assert.equal(failedRewrite.fallbackReason, QUALITY_FALLBACK_REASONS.REWRITE_LIMIT_REACHED);

  const qualified = {
    content: '今天能否安排需要核对当前档期；我现在核对，确认后尽快回复您。',
    confidence: 0.95,
    intent: 'schedule',
    reason: '当前档期需要核验',
    requiresHuman: false,
    profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [],
  };
  const low = {
    content: '好的。', confidence: 0.95, intent: 'schedule', reason: 'low', requiresHuman: false,
    profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [], rewriteCount: 1,
  };

  for (const mode of ['review', 'auto', 'autopilot']) {
    const ctx = setupService(mode, low);
    try {
      const result = await ctx.service.ingestInbound({ externalId: `low-${mode}`, contactId: 'contact-1', content: '今天能安排吗？' });
      assert.equal(result.status, 'pending_review');
      assert.equal(result.draft.requires_human, true);
      assert.equal(result.draft.quality.qualityPassed, false);
      assert(result.draft.quality.qualityChecks.length >= 10);
      assert.equal(result.draft.quality.fallbackReason, QUALITY_FALLBACK_REASONS.REWRITE_LIMIT_REACHED);
      assert.equal(ctx.sent.length, 0);
    } finally { ctx.close(); }
  }

  for (const mode of ['auto', 'autopilot']) {
    const ctx = setupService(mode, qualified);
    try {
      const result = await ctx.service.ingestInbound({ externalId: `good-${mode}`, contactId: 'contact-1', content: '今天能安排吗？' });
      assert.equal(result.status, mode === 'auto' ? 'sent' : 'autopilot_sent');
      assert.equal(ctx.sent.length, 1);
    } finally { ctx.close(); }
  }

  const qualifiedRisk = {
    content: '最低报价需要核验；我现在转人工专员核价，确认依据后同步给您。',
    confidence: 0.95,
    intent: 'pricing',
    reason: '报价需要人工核验',
    requiresHuman: true,
    profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [],
  };
  for (const mode of ['review', 'auto', 'autopilot']) {
    const ctx = setupService(mode, qualifiedRisk);
    try {
      const result = await ctx.service.ingestInbound({ externalId: `risk-${mode}`, contactId: 'contact-1', content: '最低报价能确认吗？' });
      assert.equal(result.status, 'pending_review');
      assert.equal(result.draft.requires_human, true);
      assert.equal(result.draft.quality.qualityPassed, true);
      assert.equal(result.draft.quality.fallbackReason, QUALITY_FALLBACK_REASONS.REQUIRES_HUMAN);
      assert.equal(ctx.sent.length, 0);
      assert.equal(ctx.db.listMessages(result.conversation.id).filter(message => message.direction === 'outbound').length, 0);
    } finally { ctx.close(); }
  }

  const human = setupService('review', low);
  try {
    const draftResult = await human.service.ingestInbound({ externalId: 'human-draft', contactId: 'contact-1', content: '今天能安排吗？' });
    await human.service.approveDraft(draftResult.draft.id, { content: '好的。', actor: 'human' });
    await human.service.manualSend(draftResult.conversation.id, '收到。', 'human');
    assert.equal(human.sent.length, 2);
  } finally { human.close(); }

  const directAutopilot = setupService('autopilot', qualified);
  try {
    const conversation = directAutopilot.db.ensureConversation('contact-1', '匿名客户');
    const inbound = directAutopilot.db.insertMessage({
      conversationId: conversation.id,
      direction: 'inbound',
      senderType: 'customer',
      content: '今天能安排吗？',
    }).message;
    await assert.rejects(() => directAutopilot.service.sendAutopilotReply(conversation, inbound, {
      ...low,
      qualityPassed: false,
      requiresHuman: true,
    }), /质量校验/);
    assert.equal(directAutopilot.sent.length, 0);
  } finally { directAutopilot.close(); }

  const thanks = setupService('autopilot', qualified);
  try {
    const result = await thanks.service.ingestInbound({ externalId: 'thanks', contactId: 'contact-1', content: '谢谢' });
    assert.equal(result.status, 'no_reply_needed');
    assert.equal(thanks.sent.length, 0);
  } finally { thanks.close(); }

  const summary = {
    baseline: { lowQualityAutopilotSent: 11, lowQualityBlocked: 0 },
    current: {
      lowQualityCases: lowCases.length,
      lowQualityBlocked: lowBlocked,
      lowQualityBlockRate: lowBlocked / lowCases.length,
      goodQualityCases: goodCases.length,
      goodQualityRejected: goodRejected.length,
      goodQualityFalsePositiveRate: goodRejected.length / goodCases.length,
      frozenA4Cases: frozenA4Cases.length,
      frozenA4ExactCoverage: frozenA4Results.filter(item => item.expectedFailed.every(id => item.assessment.failedCheckIds.includes(id))).length,
    },
    modes: ['review low->draft', 'auto low->draft', 'autopilot low->draft', 'auto good->sent', 'autopilot good->sent', 'risk all->human draft', 'human bypass', 'thanks no-reply'],
  };
  process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
}

main().catch(error => {
  console.error(error);
  process.exitCode = 1;
}).finally(() => {
  fs.rmSync(RUN_ROOT, { recursive: true, force: true });
});