|
@@ -0,0 +1,332 @@
|
|
|
|
|
+'use strict';
|
|
|
|
|
+
|
|
|
|
|
+const assert = require('assert/strict');
|
|
|
|
|
+const fs = require('fs');
|
|
|
|
|
+const os = require('os');
|
|
|
|
|
+const path = require('path');
|
|
|
|
|
+
|
|
|
|
|
+const RUN_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'qiwei-response-quality-'));
|
|
|
|
|
+process.env.QIWEI_OUTPUTS_DIR = path.join(RUN_ROOT, 'outputs');
|
|
|
|
|
+
|
|
|
|
|
+const { AgentContextBuilder } = require('../mcp/src/core/agent-context-builder');
|
|
|
|
|
+const { QiweiAgentRuntime } = require('../mcp/src/core/agent-runtime');
|
|
|
|
|
+const { AgentWorkbenchDb } = require('../mcp/src/core/agent-workbench-db');
|
|
|
|
|
+const { AgentWorkbenchService } = require('../mcp/src/core/agent-workbench-service');
|
|
|
|
|
+const { verifyResponseQuality } = require('../mcp/src/core/response-quality-verifier');
|
|
|
|
|
+
|
|
|
|
|
+const LOW_QUALITY_JSONL = `
|
|
|
|
|
+{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费?","reply":"您好,感谢您的咨询,我们一直致力于提供优质服务。"}
|
|
|
|
|
+{"id":"price_promise","inbound":"最低能做到多少钱?","reply":"保证给您全网最低价,一定不会更贵。"}
|
|
|
|
|
+{"id":"missing_info","inbound":"我想做员工培训","reply":"您的预算是多少?计划什么时候开始?预计多少人参加?"}
|
|
|
|
|
+{"id":"objection","inbound":"这个方案太贵了","reply":"您的预算是多少?"}
|
|
|
|
|
+{"id":"urgent","inbound":"今天能安排吗?","reply":"您好,感谢关注,我们会认真研究并持续努力为您提供全面周到专业细致的服务体验,请耐心等待。"}
|
|
|
|
|
+{"id":"complaint","inbound":"你们一直不处理,我要投诉","reply":"我们一定马上全部解决,绝对不会让您失望。"}
|
|
|
|
|
+{"id":"thanks","inbound":"谢谢","reply":"不客气,后续有需要随时联系。"}
|
|
|
|
|
+{"id":"answered_context","inbound":"预算已经说过是20万","reply":"请问您的预算是多少?","messages":[{"role":"assistant","content":"请问您的预算是多少?"}]}
|
|
|
|
|
+{"id":"repeat_question","inbound":"我还在考虑","reply":"请问您的预算是多少?","messages":[{"role":"assistant","content":"请问您的预算是多少?"}]}
|
|
|
|
|
+{"id":"tool_no_result","inbound":"有现成案例吗?","reply":"有的,我们已经找到 3 个完全匹配案例。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
|
|
|
|
|
+{"id":"low_quality","inbound":"能不能先给个建议?","reply":"好的。"}
|
|
|
|
|
+`;
|
|
|
|
|
+
|
|
|
|
|
+const GOOD_QUALITY_JSONL = `
|
|
|
|
|
+{"id":"first_consult","inbound":"想了解你们的企业服务怎么收费?","reply":"费用要按服务范围确认;您方便提供预计人数吗?确认后我给您对应报价。"}
|
|
|
|
|
+{"id":"price_check","inbound":"最低能做到多少钱?","reply":"当前最低价需要核验;我先核对适用方案,确认后给您准确答复。"}
|
|
|
|
|
+{"id":"missing_info","inbound":"我想做员工培训","reply":"可以先按员工培训方向梳理;请问预计多少人参加?确认人数后我给您下一步方案。"}
|
|
|
|
|
+{"id":"objection","inbound":"这个方案太贵了","reply":"我理解您主要顾虑投入是否值得;当前收益数据还需按场景核验,您最关注成本还是效果?确认后我补对应依据。"}
|
|
|
|
|
+{"id":"urgent","inbound":"今天能安排吗?","reply":"今天能否安排需要看当前档期;我现在核对,确认后尽快回复您。"}
|
|
|
|
|
+{"id":"complaint","inbound":"你们一直不处理,我要投诉","reply":"我理解您在意的是迟迟未处理;当前责任和处理结果需要人工核验,我现在转人工跟进并同步进度。"}
|
|
|
|
|
+{"id":"thanks","inbound":"谢谢","reply":""}
|
|
|
|
|
+{"id":"answered_context","inbound":"预算已经说过是20万","reply":"已记录预算20万;我会按这个上限继续整理,确认后给您方案。","messages":[{"role":"assistant","content":"请问您的预算是多少?"}]}
|
|
|
|
|
+{"id":"no_repeat","inbound":"我还在考虑","reply":"可以按您的节奏来;我先保留当前条件,有明确时间后再继续。","messages":[{"role":"assistant","content":"请问您的预算是多少?"}]}
|
|
|
|
|
+{"id":"tool_no_result","inbound":"有现成案例吗?","reply":"当前数据源没有返回可验证案例;我可以调整一个筛选条件后再查。","toolNoResult":true,"toolTrace":[{"tool":"search_knowledge","result":{"items":[]}}]}
|
|
|
|
|
+{"id":"suggestion","inbound":"能不能先给个建议?","reply":"可以,建议先明确最重要的结果目标;您最希望先解决哪个问题?确认后我给您第一版方案。"}
|
|
|
|
|
+`;
|
|
|
|
|
+
|
|
|
|
|
+function parseJsonl(value) {
|
|
|
|
|
+ return String(value || '').trim().split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+function assessFixture(item) {
|
|
|
|
|
+ return verifyResponseQuality({
|
|
|
|
|
+ inboundContent: item.inbound,
|
|
|
|
|
+ reply: item.reply,
|
|
|
|
|
+ messages: item.messages || [],
|
|
|
|
|
+ toolNoResult: Boolean(item.toolNoResult),
|
|
|
|
|
+ toolTrace: item.toolTrace || [],
|
|
|
|
|
+ });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+function modelPayload(reply, overrides = {}) {
|
|
|
|
|
+ return JSON.stringify({
|
|
|
|
|
+ reply,
|
|
|
|
|
+ confidence: 0.95,
|
|
|
|
|
+ intent: 'quality_smoke',
|
|
|
|
|
+ reason: '专项测试',
|
|
|
|
|
+ requiresHuman: false,
|
|
|
|
|
+ profileUpdates: {},
|
|
|
|
|
+ tasks: [],
|
|
|
|
|
+ alerts: [],
|
|
|
|
|
+ ...overrides,
|
|
|
|
|
+ });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+function setupService(mode, output) {
|
|
|
|
|
+ const dir = fs.mkdtempSync(path.join(RUN_ROOT, `${mode}-`));
|
|
|
|
|
+ const db = new AgentWorkbenchDb(path.join(dir, 'workbench.db'), { defaultMode: mode, autoSendConfidence: 0.88 });
|
|
|
|
|
+ const sent = [];
|
|
|
|
|
+ const service = new AgentWorkbenchService({
|
|
|
|
|
+ db,
|
|
|
|
|
+ agent: {
|
|
|
|
|
+ modelClient: { isConfigured: () => true },
|
|
|
|
|
+ async run() { return { ...output }; },
|
|
|
|
|
+ },
|
|
|
|
|
+ qiwei: {
|
|
|
|
|
+ isConfigured: () => true,
|
|
|
|
|
+ async sendText(toId, content) { sent.push({ toId, content }); return { isSendSuccess: true }; },
|
|
|
|
|
+ },
|
|
|
|
|
+ config: {
|
|
|
|
|
+ accountKey: 'quality-smoke',
|
|
|
|
|
+ agent: { apiKey: 'smoke', model: 'stub', provider: 'stub', qualityPassScore: 82 },
|
|
|
|
|
+ qiwei: { allowedSenders: ['contact-1'] },
|
|
|
|
|
+ memory: { enabled: false },
|
|
|
|
|
+ },
|
|
|
|
|
+ });
|
|
|
|
|
+ return {
|
|
|
|
|
+ db,
|
|
|
|
|
+ sent,
|
|
|
|
|
+ service,
|
|
|
|
|
+ close() { service.stopBackgroundWorkers(); db.close(); },
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+async function main() {
|
|
|
|
|
+ const lowCases = parseJsonl(LOW_QUALITY_JSONL);
|
|
|
|
|
+ const goodCases = parseJsonl(GOOD_QUALITY_JSONL);
|
|
|
|
|
+ const lowResults = lowCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
|
|
|
|
|
+ const goodResults = goodCases.map(item => ({ id: item.id, assessment: assessFixture(item) }));
|
|
|
|
|
+ const lowBlocked = lowResults.filter(item => !item.assessment.passed).length;
|
|
|
|
|
+ const goodRejected = goodResults.filter(item => !item.assessment.passed);
|
|
|
|
|
+ assert.equal(lowBlocked, lowCases.length, JSON.stringify(lowResults.filter(item => item.assessment.passed)));
|
|
|
|
|
+ assert.deepEqual(goodRejected.map(item => ({ id: item.id, failed: item.assessment.failedCheckIds })), []);
|
|
|
|
|
+
|
|
|
|
|
+ const unrelatedEvidence = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '最低多少钱?',
|
|
|
|
|
+ reply: '最低只要 99 元,我已经确认;我可以直接给您下单。',
|
|
|
|
|
+ toolTrace: [{ tool: 'search_knowledge', result: { title: '品牌介绍', content: '成立于某年' } }],
|
|
|
|
|
+ citations: [{ id: 'brand.md#intro', source: 'brand.md', heading: '品牌介绍' }],
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(unrelatedEvidence.passed, false);
|
|
|
|
|
+ assert(unrelatedEvidence.failedCheckIds.includes('fact_evidence'));
|
|
|
|
|
+ const claimBoundaryCases = [
|
|
|
|
|
+ verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '价格确认了吗?',
|
|
|
|
|
+ reply: '价格已确认,其他信息待核验;我可以继续帮您核对。',
|
|
|
|
|
+ citations: [{ content: '其他信息待核验' }],
|
|
|
|
|
+ }),
|
|
|
|
|
+ verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '还有库存吗?',
|
|
|
|
|
+ reply: '库存已确认;我可以直接为您保留。',
|
|
|
|
|
+ toolTrace: [{ tool: 'inventory_lookup', result: { content: '库存字段说明' } }],
|
|
|
|
|
+ }),
|
|
|
|
|
+ verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '价格是多少?',
|
|
|
|
|
+ reply: '已确认价格是 99 元,其他待核验;我可以把依据发给您。',
|
|
|
|
|
+ citations: [{ content: '当前价格是 88 元' }],
|
|
|
|
|
+ }),
|
|
|
|
|
+ ];
|
|
|
|
|
+ assert(claimBoundaryCases.every(item => !item.passed && item.failedCheckIds.includes('fact_evidence')));
|
|
|
|
|
+ const sourcedPrice = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '价格是多少?',
|
|
|
|
|
+ reply: '已确认价格是 99 元;我可以把计价依据发给您。',
|
|
|
|
|
+ citations: [{ content: '当前价格是 99 元' }],
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(sourcedPrice.passed, true, JSON.stringify(sourcedPrice.failedCheckIds));
|
|
|
|
|
+ const splitPromise = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '价格能确认吗?',
|
|
|
|
|
+ reply: '保证是 99 元,其他待核验;我可以继续跟进。',
|
|
|
|
|
+ citations: [{ content: '其他信息待核验' }],
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(splitPromise.passed, false);
|
|
|
|
|
+ assert(splitPromise.failedCheckIds.includes('sensitive_commitment'));
|
|
|
|
|
+ const bundledQuestions = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '我想咨询服务',
|
|
|
|
|
+ reply: '可以先了解需求;预算多少、什么时候开始、多少人参加?确认后我给您方案。',
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(bundledQuestions.passed, false);
|
|
|
|
|
+ assert(bundledQuestions.failedCheckIds.includes('question_count'));
|
|
|
|
|
+ const singleQuestion = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '我想咨询服务',
|
|
|
|
|
+ reply: '可以先明确投入范围;请问您的预算上限大概是多少?确认后我给您方案。',
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(singleQuestion.passed, true, JSON.stringify(singleQuestion.failedCheckIds));
|
|
|
|
|
+ const semanticRepeat = verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '预算已经确认是20万',
|
|
|
|
|
+ reply: '已记录当前需求;请问您的总投入上限多少?确认后我继续整理。',
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(semanticRepeat.passed, false);
|
|
|
|
|
+ assert(semanticRepeat.failedCheckIds.includes('repeated_question'));
|
|
|
|
|
+ for (const emptyAnswer of ['这边给您处理一下。', '我帮您关注一下。', '具体情况我再回复。']) {
|
|
|
|
|
+ const assessment = verifyResponseQuality({ inboundContent: '当前有库存吗?', reply: emptyAnswer });
|
|
|
|
|
+ assert.equal(assessment.passed, false);
|
|
|
|
|
+ assert(assessment.failedCheckIds.includes('direct_answer'));
|
|
|
|
|
+ }
|
|
|
|
|
+ assert.equal(verifyResponseQuality({
|
|
|
|
|
+ inboundContent: '当前有库存吗?',
|
|
|
|
|
+ reply: '当前资料不足,需要先核验在售状态;我确认后回复您。',
|
|
|
|
|
+ }).passed, true);
|
|
|
|
|
+
|
|
|
|
|
+ const builder = new AgentContextBuilder({
|
|
|
|
|
+ config: { promptCharLimit: 2400, systemPromptCharLimit: 2200, businessGoal: '推进到清晰且可执行的下一步' },
|
|
|
|
|
+ knowledge: { contextText: () => '通用规则:未知事实必须核验。'.repeat(200) },
|
|
|
|
|
+ });
|
|
|
|
|
+ const system = builder.buildSystemContext({ memoryContext: { promptText: '旧 Session 说客户接受高价。'.repeat(100) } });
|
|
|
|
|
+ const prompt = builder.buildClaudePrompt([
|
|
|
|
|
+ { role: 'assistant', content: '请问预算是多少?' },
|
|
|
|
|
+ { role: 'user', content: '本轮我只确认预算是20万。' },
|
|
|
|
|
+ { role: 'tool', content: JSON.stringify({ result: '当前没有已核验报价' }) },
|
|
|
|
|
+ ], {
|
|
|
|
|
+ inboundContent: '本轮我只确认预算是20万。',
|
|
|
|
|
+ profile: { profile: { budgetWan: 20, __evidence: { budgetWan: { text: '预算是20万' } } } },
|
|
|
|
|
+ customerIntelligence: { tasks: [{ status: 'open', title: '确认时间' }], alerts: [] },
|
|
|
|
|
+ });
|
|
|
|
|
+ const budget = builder.budgetReport();
|
|
|
|
|
+ assert(prompt.length <= 2400 && system.length <= 2200);
|
|
|
|
|
+ assert.match(prompt, /本轮我只确认预算是20万/);
|
|
|
|
|
+ assert.match(system, /旧模型 Session/);
|
|
|
|
|
+ assert(Object.values(budget.prompt).every(item => item.used <= item.budget));
|
|
|
|
|
+ assert(Object.values(budget.system).every(item => item.used <= item.budget));
|
|
|
|
|
+ const [requestA, requestB] = await Promise.all([
|
|
|
|
|
+ Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求甲的唯一内容' }], { profile: { profile: { request: 'A' } } })),
|
|
|
|
|
+ Promise.resolve(builder.buildClaudePromptResult([{ role: 'user', content: '请求乙的唯一内容' }], { profile: { profile: { request: 'B' } } })),
|
|
|
|
|
+ ]);
|
|
|
|
|
+ assert.match(requestA.text, /请求甲的唯一内容/);
|
|
|
|
|
+ assert.doesNotMatch(requestA.text, /请求乙的唯一内容/);
|
|
|
|
|
+ assert.match(requestB.text, /请求乙的唯一内容/);
|
|
|
|
|
+ assert.notDeepEqual(requestA.budgetReport, {});
|
|
|
|
|
+ assert.notStrictEqual(requestA.budgetReport, requestB.budgetReport);
|
|
|
|
|
+
|
|
|
|
|
+ const sequence = [
|
|
|
|
|
+ modelPayload('您好,感谢您的咨询,请耐心等待。'),
|
|
|
|
|
+ modelPayload('今天能否安排需要核对当前档期;我现在核对,确认后尽快回复您。'),
|
|
|
|
|
+ ];
|
|
|
|
|
+ const runtime = new QiweiAgentRuntime({
|
|
|
|
|
+ config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
|
|
|
|
|
+ knowledge: { search: () => [], contextText: () => '' },
|
|
|
|
|
+ modelClient: { async complete() { return { content: sequence.shift() }; } },
|
|
|
|
|
+ });
|
|
|
|
|
+ const rewritten = await runtime.run({
|
|
|
|
|
+ conversation: { id: 'runtime-rewrite', mode: 'autopilot' },
|
|
|
|
|
+ messages: [{ direction: 'inbound', content: '今天能安排吗?' }],
|
|
|
|
|
+ profile: { profile: {} },
|
|
|
|
|
+ inboundContent: '今天能安排吗?',
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(rewritten.rewriteCount, 1);
|
|
|
|
|
+ assert.equal(rewritten.qualityPassed, true);
|
|
|
|
|
+ assert.equal(rewritten.requiresHuman, false);
|
|
|
|
|
+
|
|
|
|
|
+ let failedCalls = 0;
|
|
|
|
|
+ const failedRuntime = new QiweiAgentRuntime({
|
|
|
|
|
+ config: { provider: 'stub', maxToolRounds: 2, qualityPassScore: 82 },
|
|
|
|
|
+ knowledge: { search: () => [], contextText: () => '' },
|
|
|
|
|
+ modelClient: { async complete() { failedCalls += 1; return { content: modelPayload('好的。') }; } },
|
|
|
|
|
+ });
|
|
|
|
|
+ const failedRewrite = await failedRuntime.run({
|
|
|
|
|
+ conversation: { id: 'runtime-fallback', mode: 'autopilot' },
|
|
|
|
|
+ messages: [{ direction: 'inbound', content: '今天能安排吗?' }],
|
|
|
|
|
+ profile: { profile: {} },
|
|
|
|
|
+ inboundContent: '今天能安排吗?',
|
|
|
|
|
+ });
|
|
|
|
|
+ assert.equal(failedCalls, 2);
|
|
|
|
|
+ assert.equal(failedRewrite.rewriteCount, 1);
|
|
|
|
|
+ assert.equal(failedRewrite.qualityPassed, false);
|
|
|
|
|
+ assert.equal(failedRewrite.requiresHuman, true);
|
|
|
|
|
+ assert(failedRewrite.fallbackReason);
|
|
|
|
|
+
|
|
|
|
|
+ const qualified = {
|
|
|
|
|
+ content: '今天能否安排需要核对当前档期;我现在核对,确认后尽快回复您。',
|
|
|
|
|
+ confidence: 0.95,
|
|
|
|
|
+ intent: 'schedule',
|
|
|
|
|
+ reason: '当前档期需要核验',
|
|
|
|
|
+ requiresHuman: false,
|
|
|
|
|
+ profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [],
|
|
|
|
|
+ };
|
|
|
|
|
+ const low = {
|
|
|
|
|
+ content: '好的。', confidence: 0.95, intent: 'schedule', reason: 'low', requiresHuman: false,
|
|
|
|
|
+ profileUpdates: {}, tasks: [], alerts: [], citations: [], toolTrace: [], rewriteCount: 1,
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ for (const mode of ['review', 'auto', 'autopilot']) {
|
|
|
|
|
+ const ctx = setupService(mode, low);
|
|
|
|
|
+ try {
|
|
|
|
|
+ const result = await ctx.service.ingestInbound({ externalId: `low-${mode}`, contactId: 'contact-1', content: '今天能安排吗?' });
|
|
|
|
|
+ assert.equal(result.status, 'pending_review');
|
|
|
|
|
+ assert.equal(result.draft.requires_human, true);
|
|
|
|
|
+ assert.equal(result.draft.quality.qualityPassed, false);
|
|
|
|
|
+ assert(result.draft.quality.qualityChecks.length >= 10);
|
|
|
|
|
+ assert(result.draft.quality.fallbackReason);
|
|
|
|
|
+ assert.equal(ctx.sent.length, 0);
|
|
|
|
|
+ } finally { ctx.close(); }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for (const mode of ['auto', 'autopilot']) {
|
|
|
|
|
+ const ctx = setupService(mode, qualified);
|
|
|
|
|
+ try {
|
|
|
|
|
+ const result = await ctx.service.ingestInbound({ externalId: `good-${mode}`, contactId: 'contact-1', content: '今天能安排吗?' });
|
|
|
|
|
+ assert.equal(result.status, mode === 'auto' ? 'sent' : 'autopilot_sent');
|
|
|
|
|
+ assert.equal(ctx.sent.length, 1);
|
|
|
|
|
+ } finally { ctx.close(); }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const human = setupService('review', low);
|
|
|
|
|
+ try {
|
|
|
|
|
+ const draftResult = await human.service.ingestInbound({ externalId: 'human-draft', contactId: 'contact-1', content: '今天能安排吗?' });
|
|
|
|
|
+ await human.service.approveDraft(draftResult.draft.id, { content: '好的。', actor: 'human' });
|
|
|
|
|
+ await human.service.manualSend(draftResult.conversation.id, '收到。', 'human');
|
|
|
|
|
+ assert.equal(human.sent.length, 2);
|
|
|
|
|
+ } finally { human.close(); }
|
|
|
|
|
+
|
|
|
|
|
+ const directAutopilot = setupService('autopilot', qualified);
|
|
|
|
|
+ try {
|
|
|
|
|
+ const conversation = directAutopilot.db.ensureConversation('contact-1', '匿名客户');
|
|
|
|
|
+ const inbound = directAutopilot.db.insertMessage({
|
|
|
|
|
+ conversationId: conversation.id,
|
|
|
|
|
+ direction: 'inbound',
|
|
|
|
|
+ senderType: 'customer',
|
|
|
|
|
+ content: '今天能安排吗?',
|
|
|
|
|
+ }).message;
|
|
|
|
|
+ await assert.rejects(() => directAutopilot.service.sendAutopilotReply(conversation, inbound, {
|
|
|
|
|
+ ...low,
|
|
|
|
|
+ qualityPassed: false,
|
|
|
|
|
+ requiresHuman: true,
|
|
|
|
|
+ }), /质量校验/);
|
|
|
|
|
+ assert.equal(directAutopilot.sent.length, 0);
|
|
|
|
|
+ } finally { directAutopilot.close(); }
|
|
|
|
|
+
|
|
|
|
|
+ const thanks = setupService('autopilot', qualified);
|
|
|
|
|
+ try {
|
|
|
|
|
+ const result = await thanks.service.ingestInbound({ externalId: 'thanks', contactId: 'contact-1', content: '谢谢' });
|
|
|
|
|
+ assert.equal(result.status, 'no_reply_needed');
|
|
|
|
|
+ assert.equal(thanks.sent.length, 0);
|
|
|
|
|
+ } finally { thanks.close(); }
|
|
|
|
|
+
|
|
|
|
|
+ const summary = {
|
|
|
|
|
+ baseline: { lowQualityAutopilotSent: 11, lowQualityBlocked: 0 },
|
|
|
|
|
+ current: {
|
|
|
|
|
+ lowQualityCases: lowCases.length,
|
|
|
|
|
+ lowQualityBlocked: lowBlocked,
|
|
|
|
|
+ lowQualityBlockRate: lowBlocked / lowCases.length,
|
|
|
|
|
+ goodQualityCases: goodCases.length,
|
|
|
|
|
+ goodQualityRejected: goodRejected.length,
|
|
|
|
|
+ goodQualityFalsePositiveRate: goodRejected.length / goodCases.length,
|
|
|
|
|
+ },
|
|
|
|
|
+ modes: ['review low->draft', 'auto low->draft', 'autopilot low->draft', 'auto good->sent', 'autopilot good->sent', 'human bypass', 'thanks no-reply'],
|
|
|
|
|
+ };
|
|
|
|
|
+ process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+main().catch(error => {
|
|
|
|
|
+ console.error(error);
|
|
|
|
|
+ process.exitCode = 1;
|
|
|
|
|
+}).finally(() => {
|
|
|
|
|
+ fs.rmSync(RUN_ROOT, { recursive: true, force: true });
|
|
|
|
|
+});
|