|
|
@@ -0,0 +1,332 @@
|
|
|
+'use strict';
|
|
|
+
|
|
|
+const assert = require('assert/strict');
|
|
|
+const fs = require('fs');
|
|
|
+const os = require('os');
|
|
|
+const path = require('path');
|
|
|
+
|
|
|
+const RUN_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'qiwei-response-quality-'));
|
|
|
+process.env.QIWEI_OUTPUTS_DIR = path.join(RUN_ROOT, 'outputs');
|
|
|
+
|
|
|
+const { AgentContextBuilder } = require('../mcp/src/core/agent-context-builder');
|
|
|
+const { QiweiAgentRuntime } = require('../mcp/src/core/agent-runtime');
|
|
|
+const { AgentWorkbenchDb } = require('../mcp/src/core/agent-workbench-db');
|
|
|
+const { AgentWorkbenchService } = require('../mcp/src/core/agent-workbench-service');
|
|
|
+const { verifyResponseQuality } = require('../mcp/src/core/response-quality-verifier');
|
|
|
+
|
|
|
+const LOW_QUALITY_JSONL = `
|
|
|
+{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费？","reply":"您好，感谢您的咨询，我们一直致力于提供优质服务。"}
|
|
|
+{"id":"price_promise","inbound":"最低能做到多少钱？","reply":"保证给您全网最低价，一定不会更贵。"}
|
|
|
+{"id":"missing_info","inbound":"我想做员工培训","reply":"您的预算是多少？计划什么时候开始？预计多少人参加？"}
|
|
|
+{"id":"objection","inbound":"这个方案太贵了","reply":"您的预算是多少？"}
|
|
|
+{"id":"urgent","inbound":"今天能安排吗？","reply":"您好，感谢关注，我们会认真研究并持续努力为您提供全面周到专业细致的服务体验，请耐心等待。"}
|
|
|
+{"id":"complaint","inbound":"你们一直不处理，我要投诉","reply":"我们一定马上全部解决，绝对不会让您失望。"}
|
|
|
+{"id":"thanks","inbound":"谢谢","reply":"不客气，后续有需要随时联系。"}
|
|
|
+{"id":"answered_context","inbound":"预算已经说过是20万","reply":"请问您的预算是多少？","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
|
|
|
+{"id":"repeat_question","inbound":"我还在考虑","reply":"请问您的预算是多少？","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
|
|
|
+{"id":"tool_no_result","inbound":"有现成案例吗？","reply":"有的，我们已经找到 3 个完全匹配案例。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
|
|
|
+{"id":"low_quality","inbound":"能不能先给个建议？","reply":"好的。"}
|
|
|
+`;
|
|
|
+
|
|
|
+const GOOD_QUALITY_JSONL = `
|
|
|
+{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费？","reply":"费用要按服务范围确认；您方便提供预计人数吗？确认后我给您对应报价。"}
|
|
|
+{"id":"price_check","inbound":"最低能做到多少钱？","reply":"当前最低价需要核验；我先核对适用方案，确认后给您准确答复。"}
|
|
|
+{"id":"missing_info","inbound":"我想做员工培训","reply":"可以先按员工培训方向梳理；请问预计多少人参加？确认人数后我给您下一步方案。"}
|
|
|
+{"id":"objection","inbound":"这个方案太贵了","reply":"我理解您主要顾虑投入是否值得；当前收益数据还需按场景核验，您最关注成本还是效果？确认后我补对应依据。"}
|
|
|
+{"id":"urgent","inbound":"今天能安排吗？","reply":"今天能否安排需要看当前档期；我现在核对，确认后尽快回复您。"}
|
|
|
+{"id":"complaint","inbound":"你们一直不处理，我要投诉","reply":"我理解您在意的是迟迟未处理；当前责任和处理结果需要人工核验，我现在转人工跟进并同步进度。"}
|
|
|
+{"id":"thanks","inbound":"谢谢","reply":""}
|
|
|
+{"id":"answered_context","inbound":"预算已经说过是20万","reply":"已记录预算20万；我会按这个上限继续整理，确认后给您方案。","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
|
|
|
+{"id":"no_repeat","inbound":"我还在考虑","reply":"可以按您的节奏来；我先保留当前条件，有明确时间后再继续。","messages":[{"role":"assistant","content":"请问您的预算是多少？"}]}
|
|
|
+{"id":"tool_no_result","inbound":"有现成案例吗？","reply":"当前数据源没有返回可验证案例；我可以调整一个筛选条件后再查。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
|
|
|
+{"id":"suggestion","inbound":"能不能先给个建议？","reply":"可以，建议先明确最重要的结果目标；您最希望先解决哪个问题？确认后我给您第一版方案。"}
|
|
|
+`;
|
|
|
+
|
|
|
+function parseJsonl(value) {
|
|
|
+ return String(value || '').trim().split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
|
|
|
+}
|
|
|
+
|
|
|
+function assessFixture(item) {
|
|
|
+ return verifyResponseQuality({
|
|
|
+ inboundContent: item.inbound,
|
|
|
+ reply: item.reply,
|
|
|
+ messages: item.messages || [],
|
|
|
+ toolNoResult: Boolean(item.toolNoResult),
|
|
|
+ toolTrace: item.toolTrace || [],
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+function modelPayload(reply, overrides = {}) {
|
|
|
+ return JSON.stringify({
|
|
|
+ reply,
|
|
|
+ confidence: 0.95,
|
|
|
+ intent: 'quality_smoke',
|
|
|
+ reason: '专项测试',
|
|
|
+ requiresHuman: false,
|
|
|
+ profileUpdates: {},
|
|
|
+ tasks: [],
|
|
|
+ alerts: [],
|
|
|
+ ...overrides,
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+function setupService(mode, output) {
|
|
|
+ const dir = fs.mkdtempSync(path.join(RUN_ROOT, `${mode}-`));
|
|
|
+ const db = new AgentWorkbenchDb(path.join(dir, 'workbench.db'), { defaultMode: mode, autoSendConfidence: 0.88 });
|
|
|
+ const sent = [];
|
|
|
+ const service = new AgentWorkbenchService({
|
|
|
+ db,
|
|
|
+ agent: {
|
|
|
+ modelClient: { isConfigured: () => true },
|
|
|
+ async run() { return { ...output }; },
|
|
|
+ },
|
|
|
+ qiwei: {
|
|
|
+ isConfigured: () => true,
|
|
|
+ async sendText(toId, content) { sent.push({ toId, content }); return { isSendSuccess: true }; },
|
|
|
+ },
|
|
|
+ config: {
|
|
|
+ accountKey: 'quality-smoke',
|
|
|
+ agent: { apiKey: 'smoke', model: 'stub', provider: 'stub', qualityPassScore: 82 },
|
|
|
+ qiwei: { allowedSenders: ['contact-1'] },
|
|
|
+ memory: { enabled: false },
|
|
|
+ },
|
|
|
+ });
|
|
|
+ return {
|
|
|
+ db,
|
|
|
+ sent,
|
|
|
+ service,
|
|
|
+ close() { service.stopBackgroundWorkers(); db.close(); },
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+async function main() {
|
|
|
+ const lowCases = parseJsonl(LOW_QUALITY_JSONL);
|
|
|
+ const goodCases = parseJsonl(GOOD_QUALITY_JSONL);
|
|
|
+ const lowResults = lowCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
|
|
|
+ const goodResults = goodCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
|
|
|
+ const lowBlocked = lowResults.filter(item => !item.assessment.passed).length;
|
|
|
+ const goodRejected = goodResults.filter(item => !item.assessment.passed);
|
|
|
+ assert.equal(lowBlocked, lowCases.length, JSON.stringify(lowResults.filter(item => item.assessment.passed)));
|
|
|
+ assert.deepEqual(goodRejected.map(item => ({ id: item.id, failed: item.assessment.failedCheckIds })), []);
|
|
|
+
|
|
|
+ const unrelatedEvidence = verifyResponseQuality({
|
|
|
+ inboundContent: '最低多少钱？',
|
|
|
+ reply: '最低只要 99 元，我已经确认；我可以直接给您下单。',
|
|
|
+ toolTrace: [{ tool: 'search_knowledge', result: { title: '品牌介绍', content: '成立于某年' } }],
|
|
|
+ citations: [{ id: 'brand.md#intro', source: 'brand.md', heading: '品牌介绍' }],
|
|
|
+ });
|
|
|
+ assert.equal(unrelatedEvidence.passed, false);
|
|
|
+ assert(unrelatedEvidence.failedCheckIds.includes('fact_evidence'));
|
|
|
+ const claimBoundaryCases = [
|
|
|
+ verifyResponseQuality({
|
|
|
+ inboundContent: '价格确认了吗？',
|
|
|
+ reply: '价格已确认，其他信息待核验；我可以继续帮您核对。',
|
|
|
+ citations: [{ content: '其他信息待核验' }],
|
|
|
+ }),
|
|
|
+ verifyResponseQuality({
|
|
|
+ inboundContent: '还有库存吗？',
|
|
|
+ reply: '库存已确认；我可以直接为您保留。',
|
|
|
+ toolTrace: [{ tool: 'inventory_lookup', result: { content: '库存字段说明' } }],
|
|
|
+ }),
|
|
|
+ verifyResponseQuality({
|
|
|
+ inboundContent: '价格是多少？',
|
|
|
+ reply: '已确认价格是 99 元，其他待核验；我可以把依据发给您。',
|
|
|
+ citations: [{ content: '当前价格是 88 元' }],
|
|
|
+ }),
|
|
|
+ ];
|
|
|
+ assert(claimBoundaryCases.every(item => !item.passed && item.failedCheckIds.includes('fact_evidence')));
|
|
|
+ const sourcedPrice = verifyResponseQuality({
|
|
|
+ inboundContent: '价格是多少？',
|
|
|
+ reply: '已确认价格是 99 元；我可以把计价依据发给您。',
|
|
|
+ citations: [{ content: '当前价格是 99 元' }],
|
|
|
+ });
|
|
|
+ assert.equal(sourcedPrice.passed, true, JSON.stringify(sourcedPrice.failedCheckIds));
|
|
|
+ const splitPromise = verifyResponseQuality({
|
|
|
+ inboundContent: '价格能确认吗？',
|
|
|
+ reply: '保证是 99 元，其他待核验；我可以继续跟进。',
|
|
|
+ citations: [{ content: '其他信息待核验' }],
|
|
|
+ });
|
|
|
+ assert.equal(splitPromise.passed, false);
|
|
|
+ assert(splitPromise.failedCheckIds.includes('sensitive_commitment'));
|
|
|
+ const bundledQuestions = verifyResponseQuality({
|
|
|
+ inboundContent: '我想咨询服务',
|
|
|
+ reply: '可以先了解需求；预算多少、什么时候开始、多少人参加？确认后我给您方案。',
|
|
|
+ });
|
|
|
+ assert.equal(bundledQuestions.passed, false);
|
|
|
+ assert(bundledQuestions.failedCheckIds.includes('question_count'));
|
|
|
+ const singleQuestion = verifyResponseQuality({
|
|
|
+ inboundContent: '我想咨询服务',
|
|
|
+ reply: '可以先明确投入范围；请问您的预算上限大概是多少？确认后我给您方案。',
|
|
|
+ });
|
|
|
+ assert.equal(singleQuestion.passed, true, JSON.stringify(singleQuestion.failedCheckIds));
|
|
|
+ const semanticRepeat = verifyResponseQuality({
|
|
|
+ inboundContent: '预算已经确认是20万',
|
|
|
+ reply: '已记录当前需求；请问您的总投入上限多少？确认后我继续整理。',
|
|
|
+ });
|
|
|
+ assert.equal(semanticRepeat.passed, false);
|
|
|
+ assert(semanticRepeat.failedCheckIds.includes('repeated_question'));
|
|
|
+ for (const emptyAnswer of ['这边给您处理一下。', '我帮您关注一下。', '具体情况我再回复。']) {
|
|
|
+ const assessment = verifyResponseQuality({ inboundContent: '当前有库存吗？', reply: emptyAnswer });
|
|
|
+ assert.equal(assessment.passed, false);
|
|
|
+ assert(assessment.failedCheckIds.includes('direct_answer'));
|
|
|
+ }
|
|
|
+ assert.equal(verifyResponseQuality({
|
|
|
+ inboundContent: '当前有库存吗？',
|
|
|
+ reply: '当前资料不足，需要先核验在售状态；我确认后回复您。',
|
|
|
+ }).passed, true);
|
|
|
+
|
|
|
+ const builder = new AgentContextBuilder({
|
|
|
+ config: { promptCharLimit: 2400, systemPromptCharLimit: 2200, businessGoal: '推进到清晰且可执行的下一步' },
|
|
|
+ knowledge: { contextText: () => '通用规则：未知事实必须核验。'.repeat(200) },
|
|
|
+ });
|
|
|
+ const system = builder.buildSystemContext({ memoryContext: { promptText: '旧 Session 说客户接受高价。'.repeat(100) } });
|
|
|
+ const prompt = builder.buildClaudePrompt([
|
|
|
+ { role: 'assistant', content: '请问预算是多少？' },
|
|
|
+ { role: 'user', content: '本轮我只确认预算是20万。' },
|
|
|
+ { role: 'tool', content: JSON.stringify({ result: '当前没有已核验报价' }) },
|
|
|
+ ], {
|
|
|
+ inboundContent: '本轮我只确认预算是20万。',
|
|
|
+ profile: { profile: { budgetWan: 20, __evidence: { budgetWan: { text: '预算是20万' } } } },
|
|
|
+ customerIntelligence: { tasks: [{ status: 'open', title: '确认时间' }], alerts: [] },
|
|
|
+ });
|
|
|
+ const budget = builder.budgetReport();
|
|
|
+ assert(prompt.length <= 2400 && system.length <= 2200);
|
|
|
+ assert.match(prompt, /本轮我只确认预算是20万/);
|
|
|
+ assert.match(system, /旧模型 Session/);
|
|
|
+ assert(Object.values(budget.prompt).every(item => item.used <= item.budget));
|
|
|
+ assert(Object.values(budget.system).every(item => item.used <= item.budget));
|
|
|
+ const [requestA, requestB] = await Promise.all([
|
|
|
+ Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求甲的唯一内容' }], { profile: { profile: { request: 'A' } } })),
|
|
|
+ Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求乙的唯一内容' }], { profile: { profile: { request: 'B' } } })),
|
|
|
+ ]);
|
|
|
+ assert.match(requestA.text, /请求甲的唯一内容/);
|
|
|
+ assert.doesNotMatch(requestA.text, /请求乙的唯一内容/);
|
|
|
+ assert.match(requestB.text, /请求乙的唯一内容/);
|
|
|
+ assert.notDeepEqual(requestA.budgetReport, {});
|
|
|
+ assert.notStrictEqual(requestA.budgetReport, requestB.budgetReport);
|
|
|
+
|
|
|
+ const sequence = [
|
|
|
+ modelPayload('您好，感谢您的咨询，请耐心等待。'),
|
|
|
+ modelPayload('今天能否安排需要核对当前档期；我现在核对，确认后尽快回复您。'),
|
|
|
+ ];
|
|
|
+ const runtime = new QiweiAgentRuntime({
|
|
|
+ config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
|
|
|
+ knowledge: { search: () => [], contextText: () => '' },
|
|
|
+ modelClient: { async complete() { return { content: sequence.shift() }; } },
|
|
|
+ });
|
|
|
+ const rewritten = await runtime.run({
|
|
|
+ conversation: { id: 'runtime-rewrite', mode: 'autopilot' },
|
|
|
+ messages: [{ direction: 'inbound', content: '今天能安排吗？' }],
|
|
|
+ profile: { profile: {} },
|
|
|
+ inboundContent: '今天能安排吗？',
|
|
|
+ });
|
|
|
+ assert.equal(rewritten.rewriteCount, 1);
|
|
|
+ assert.equal(rewritten.qualityPassed, true);
|
|
|
+ assert.equal(rewritten.requiresHuman, false);
|
|
|
+
|
|
|
+ let failedCalls = 0;
|
|
|
+ const failedRuntime = new QiweiAgentRuntime({
|
|
|
+ config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
|
|
|
+ knowledge: { search: () => [], contextText: () => '' },
|
|
|
+ modelClient: { async complete() { failedCalls += 1; return { content: modelPayload('好的。') }; } },
|
|
|
+ });
|
|
|
+ const failedRewrite = await failedRuntime.run({
|
|
|
+ conversation: { id: 'runtime-fallback', mode: 'autopilot' },
|
|
|
+ messages: [{ direction: 'inbound', content: '今天能安排吗？' }],
|
|
|
+ profile: { profile: {} },
|
|
|
+ inboundContent: '今天能安排吗？',
|
|
|
+ });
|
|
|
+ assert.equal(failedCalls, 2);
|
|
|
+ assert.equal(failedRewrite.rewriteCount, 1);
|
|
|
+ assert.equal(failedRewrite.qualityPassed, false);
|
|
|
+ assert.equal(failedRewrite.requiresHuman, true);
|
|
|
+ assert(failedRewrite.fallbackReason);
|
|
|
+
|
|
|
+ const qualified = {
|
|
|
+ content: '今天能否安排需要核对当前档期；我现在核对，确认后尽快回复您。',
|
|
|
+ confidence: 0.95,
|
|
|
+ intent: 'schedule',
|
|
|
+ reason: '当前档期需要核验',
|
|
|
+ requiresHuman: false,
|
|
|
+ profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [],
|
|
|
+ };
|
|
|
+ const low = {
|
|
|
+ content: '好的。', confidence: 0.95, intent: 'schedule', reason: 'low', requiresHuman: false,
|
|
|
+ profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [], rewriteCount: 1,
|
|
|
+ };
|
|
|
+
|
|
|
+ for (const mode of ['review', 'auto', 'autopilot']) {
|
|
|
+ const ctx = setupService(mode, low);
|
|
|
+ try {
|
|
|
+ const result = await ctx.service.ingestInbound({ externalId: `low-${mode}`, contactId: 'contact-1', content: '今天能安排吗？' });
|
|
|
+ assert.equal(result.status, 'pending_review');
|
|
|
+ assert.equal(result.draft.requires_human, true);
|
|
|
+ assert.equal(result.draft.quality.qualityPassed, false);
|
|
|
+ assert(result.draft.quality.qualityChecks.length >= 10);
|
|
|
+ assert(result.draft.quality.fallbackReason);
|
|
|
+ assert.equal(ctx.sent.length, 0);
|
|
|
+ } finally { ctx.close(); }
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const mode of ['auto', 'autopilot']) {
|
|
|
+ const ctx = setupService(mode, qualified);
|
|
|
+ try {
|
|
|
+ const result = await ctx.service.ingestInbound({ externalId: `good-${mode}`, contactId: 'contact-1', content: '今天能安排吗？' });
|
|
|
+ assert.equal(result.status, mode === 'auto' ? 'sent' : 'autopilot_sent');
|
|
|
+ assert.equal(ctx.sent.length, 1);
|
|
|
+ } finally { ctx.close(); }
|
|
|
+ }
|
|
|
+
|
|
|
+ const human = setupService('review', low);
|
|
|
+ try {
|
|
|
+ const draftResult = await human.service.ingestInbound({ externalId: 'human-draft', contactId: 'contact-1', content: '今天能安排吗？' });
|
|
|
+ await human.service.approveDraft(draftResult.draft.id, { content: '好的。', actor: 'human' });
|
|
|
+ await human.service.manualSend(draftResult.conversation.id, '收到。', 'human');
|
|
|
+ assert.equal(human.sent.length, 2);
|
|
|
+ } finally { human.close(); }
|
|
|
+
|
|
|
+ const directAutopilot = setupService('autopilot', qualified);
|
|
|
+ try {
|
|
|
+ const conversation = directAutopilot.db.ensureConversation('contact-1', '匿名客户');
|
|
|
+ const inbound = directAutopilot.db.insertMessage({
|
|
|
+ conversationId: conversation.id,
|
|
|
+ direction: 'inbound',
|
|
|
+ senderType: 'customer',
|
|
|
+ content: '今天能安排吗？',
|
|
|
+ }).message;
|
|
|
+ await assert.rejects(() => directAutopilot.service.sendAutopilotReply(conversation, inbound, {
|
|
|
+ ...low,
|
|
|
+ qualityPassed: false,
|
|
|
+ requiresHuman: true,
|
|
|
+ }), /质量校验/);
|
|
|
+ assert.equal(directAutopilot.sent.length, 0);
|
|
|
+ } finally { directAutopilot.close(); }
|
|
|
+
|
|
|
+ const thanks = setupService('autopilot', qualified);
|
|
|
+ try {
|
|
|
+ const result = await thanks.service.ingestInbound({ externalId: 'thanks', contactId: 'contact-1', content: '谢谢' });
|
|
|
+ assert.equal(result.status, 'no_reply_needed');
|
|
|
+ assert.equal(thanks.sent.length, 0);
|
|
|
+ } finally { thanks.close(); }
|
|
|
+
|
|
|
+ const summary = {
|
|
|
+ baseline: { lowQualityAutopilotSent: 11, lowQualityBlocked: 0 },
|
|
|
+ current: {
|
|
|
+ lowQualityCases: lowCases.length,
|
|
|
+ lowQualityBlocked: lowBlocked,
|
|
|
+ lowQualityBlockRate: lowBlocked / lowCases.length,
|
|
|
+ goodQualityCases: goodCases.length,
|
|
|
+ goodQualityRejected: goodRejected.length,
|
|
|
+ goodQualityFalsePositiveRate: goodRejected.length / goodCases.length,
|
|
|
+ },
|
|
|
+ modes: ['review low->draft', 'auto low->draft', 'autopilot low->draft', 'auto good->sent', 'autopilot good->sent', 'human bypass', 'thanks no-reply'],
|
|
|
+ };
|
|
|
+ process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
|
|
+}
|
|
|
+
|
|
|
+main().catch(error => {
|
|
|
+ console.error(error);
|
|
|
+ process.exitCode = 1;
|
|
|
+}).finally(() => {
|
|
|
+ fs.rmSync(RUN_ROOT, { recursive: true, force: true });
|
|
|
+});
|
gangvy