#!/usr/bin/env node /** * 验证 ✅ reaction 作为结束信号的覆盖率与边界(2026-11-11) * * 关心三件事: * ① 普通回合:✅ 是否稳定出现、是否晚于正文 * ② 多步/长工具回合:✅ 是否仍在**最后**才出现(不会被中间步骤骗) * ③ 并发两个会话:各自的 ✅ 是否只出现在各自房间 * * 用法:node scripts/verify-reaction-coverage.mjs */ import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; const F = 'https://server.fmode.cn/api/functions'; const TARGET = 'agent-node-lj-test'; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const tok = JSON.parse(readFileSync(join(homedir(), '.fmode/config/user.json'), 'utf8')).sessionToken; async function prov(id) { const r = await fetch(F, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: 'RP5Ht55NeX', token: tok, params: { agentId: id } }), }); const j = await r.json(); const d = j.result ?? j; if (!d?.accessToken) throw new Error('provision 失败'); return d; } const p = await prov(TARGET); const a = await prov(TARGET); const mx = await import('matrix-js-sdk'); const c = mx.createClient({ baseUrl: p.homeserver, accessToken: p.accessToken, userId: p.userMxId }); await c.startClient({ initialSyncLimit: 1 }); await sleep(3000); const hs = p.homeserver.replace(/\/$/, ''); const auth = { Authorization: `Bearer ${p.accessToken}` }; async function makeRoom(tag) { const room = ( await c.createRoom({ invite: [a.agentMxId], name: `${tag}-${Date.now()}`, preset: mx.Preset.TrustedPrivateChat, }) ).room_id; for (let i = 0; i < 30; i++) { await sleep(2000); const r = await fetch(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(room)}/joined_members`, { headers: auth, }).catch(() => null); const j = r ? await r.json().catch(() => ({})) : {}; if ((j.joined ?? {})[a.agentMxId]) return room; } throw new Error('容器未进房'); } const log = new Map(); // roomId -> [{t, kind, value}] const T0 = Date.now(); const rel = (t) => ((t - T0) / 1000).toFixed(1); c.on('Room.timeline', (ev, rm) => { const rid = rm?.roomId; if (!log.has(rid)) return; if (ev.getSender() !== a.agentMxId) return; const type = ev.getType(); const arr = log.get(rid); if (type === 'm.reaction') { arr.push({ t: Date.now(), kind: 'reaction', value: ev.getContent()?.['m.relates_to']?.key ?? '' }); } else if (type === 'm.room.message') { arr.push({ t: Date.now(), kind: 'msg', value: String(ev.getContent()?.body ?? '').slice(0, 35) }); } }); console.log('建 3 个房间:普通 / 多步长任务 / 并发B\n'); const rNormal = await makeRoom('rc-normal'); const rMulti = await makeRoom('rc-multi'); const rB = await makeRoom('rc-b'); log.set(rNormal, []); log.set(rMulti, []); log.set(rB, []); const send = (rid, body) => c.sendEvent(rid, 'm.room.message', { msgtype: 'm.text', body, 'm.mentions': { user_ids: [a.agentMxId] }, }); // ① 普通一问一答 await send(rNormal, '用一句话回答:一年有多少个月?不要调用工具。'); await sleep(1000); // ② 多步长任务(要求调用工具多次) await send(rMulti, '帮我依次算:12*12、13*13、14*14、15*15、16*16,每算一个都要用工具实际计算,最后汇总。'); await sleep(1000); // ③ 并发第二个会话 await send(rB, '用一句话回答:中国的首都是哪里?不要调用工具。'); console.log('三条已发送,监听 180 秒…'); await sleep(180_000); const names = [ ['普通一问一答', rNormal], ['多步长任务', rMulti], ['并发B', rB], ]; let allOk = true; for (const [name, rid] of names) { const arr = log.get(rid); console.log(`\n########## ${name} ##########`); arr.forEach((e) => { const tag = e.kind === 'reaction' ? `reaction ${e.value}` : `消息「${e.value}」`; console.log(` +${rel(e.t)}s ${tag}`); }); const msgs = arr.filter((e) => e.kind === 'msg'); const ok = arr.filter((e) => e.kind === 'reaction' && e.value === '✅'); const bad = arr.filter((e) => e.kind === 'reaction' && e.value === '❌'); const eye = arr.filter((e) => e.kind === 'reaction' && e.value === '👀'); console.log( ` → 消息 ${msgs.length} 条 / 👀 ${eye.length} / ✅ ${ok.length} / ❌ ${bad.length}`, ); if (ok.length === 0) { console.log(' ⚠️ 没有 ✅ —— 需要兜底'); allOk = false; } else { const lastMsg = msgs.at(-1); const lastOk = ok.at(-1); if (lastMsg && lastOk) { const d = (lastOk.t - lastMsg.t) / 1000; console.log( d >= 0 ? ` ✅ ✅ 晚于末条消息 ${d.toFixed(1)}s` : ` ❌ ✅ 早于末条消息 ${(-d).toFixed(1)}s —— 会导致提前收尾`, ); if (d < 0) allOk = false; } // 多步回合:中间是否出现多个 ✅(若多个,取最后一个才是真结束) if (ok.length > 1) console.log(` ℹ️ 出现 ${ok.length} 个 ✅ —— App 只认最后一个,需确认不会提前收尾`); } } console.log(`\n===== 总结 =====`); console.log(allOk ? '✅ ✅ reaction 可作为可靠结束信号' : '❌ 存在覆盖缺口,App 侧必须保留静默兜底'); process.exit(0);