fmode-app/scripts/verify-reaction-coverage.mjs

#!/usr/bin/env node
/**
 * 验证 ✅ reaction 作为结束信号的覆盖率与边界（2026-11-11）
 *
 * 关心三件事：
 *   ① 普通回合：✅ 是否稳定出现、是否晚于正文
 *   ② 多步/长工具回合：✅ 是否仍在**最后**才出现（不会被中间步骤骗）
 *   ③ 并发两个会话：各自的 ✅ 是否只出现在各自房间
 *
 * 用法：node scripts/verify-reaction-coverage.mjs
 */
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';

const F = 'https://server.fmode.cn/api/functions';
const TARGET = 'agent-node-lj-test';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const tok = JSON.parse(readFileSync(join(homedir(), '.fmode/config/user.json'), 'utf8')).sessionToken;

async function prov(id) {
  const r = await fetch(F, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ id: 'RP5Ht55NeX', token: tok, params: { agentId: id } }),
  });
  const j = await r.json();
  const d = j.result ?? j;
  if (!d?.accessToken) throw new Error('provision 失败');
  return d;
}

const p = await prov(TARGET);
const a = await prov(TARGET);
const mx = await import('matrix-js-sdk');
const c = mx.createClient({ baseUrl: p.homeserver, accessToken: p.accessToken, userId: p.userMxId });
await c.startClient({ initialSyncLimit: 1 });
await sleep(3000);

const hs = p.homeserver.replace(/\/$/, '');
const auth = { Authorization: `Bearer ${p.accessToken}` };

async function makeRoom(tag) {
  const room = (
    await c.createRoom({
      invite: [a.agentMxId],
      name: `${tag}-${Date.now()}`,
      preset: mx.Preset.TrustedPrivateChat,
    })
  ).room_id;
  for (let i = 0; i < 30; i++) {
    await sleep(2000);
    const r = await fetch(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(room)}/joined_members`, {
      headers: auth,
    }).catch(() => null);
    const j = r ? await r.json().catch(() => ({})) : {};
    if ((j.joined ?? {})[a.agentMxId]) return room;
  }
  throw new Error('容器未进房');
}

const log = new Map(); // roomId -> [{t, kind, value}]
const T0 = Date.now();
const rel = (t) => ((t - T0) / 1000).toFixed(1);

c.on('Room.timeline', (ev, rm) => {
  const rid = rm?.roomId;
  if (!log.has(rid)) return;
  if (ev.getSender() !== a.agentMxId) return;
  const type = ev.getType();
  const arr = log.get(rid);
  if (type === 'm.reaction') {
    arr.push({ t: Date.now(), kind: 'reaction', value: ev.getContent()?.['m.relates_to']?.key ?? '' });
  } else if (type === 'm.room.message') {
    arr.push({ t: Date.now(), kind: 'msg', value: String(ev.getContent()?.body ?? '').slice(0, 35) });
  }
});

console.log('建 3 个房间：普通 / 多步长任务 / 并发B\n');
const rNormal = await makeRoom('rc-normal');
const rMulti = await makeRoom('rc-multi');
const rB = await makeRoom('rc-b');
log.set(rNormal, []);
log.set(rMulti, []);
log.set(rB, []);

const send = (rid, body) =>
  c.sendEvent(rid, 'm.room.message', {
    msgtype: 'm.text',
    body,
    'm.mentions': { user_ids: [a.agentMxId] },
  });

// ① 普通一问一答
await send(rNormal, '用一句话回答：一年有多少个月？不要调用工具。');
await sleep(1000);
// ② 多步长任务（要求调用工具多次）
await send(rMulti, '帮我依次算：12*12、13*13、14*14、15*15、16*16，每算一个都要用工具实际计算，最后汇总。');
await sleep(1000);
// ③ 并发第二个会话
await send(rB, '用一句话回答：中国的首都是哪里？不要调用工具。');

console.log('三条已发送，监听 180 秒…');
await sleep(180_000);

const names = [
  ['普通一问一答', rNormal],
  ['多步长任务', rMulti],
  ['并发B', rB],
];

let allOk = true;
for (const [name, rid] of names) {
  const arr = log.get(rid);
  console.log(`\n########## ${name} ##########`);
  arr.forEach((e) => {
    const tag = e.kind === 'reaction' ? `reaction ${e.value}` : `消息「${e.value}」`;
    console.log(`  +${rel(e.t)}s  ${tag}`);
  });
  const msgs = arr.filter((e) => e.kind === 'msg');
  const ok = arr.filter((e) => e.kind === 'reaction' && e.value === '✅');
  const bad = arr.filter((e) => e.kind === 'reaction' && e.value === '❌');
  const eye = arr.filter((e) => e.kind === 'reaction' && e.value === '👀');
  console.log(
    `  → 消息 ${msgs.length} 条 / 👀 ${eye.length} / ✅ ${ok.length} / ❌ ${bad.length}`,
  );

  if (ok.length === 0) {
    console.log('  ⚠️ 没有 ✅ —— 需要兜底');
    allOk = false;
  } else {
    const lastMsg = msgs.at(-1);
    const lastOk = ok.at(-1);
    if (lastMsg && lastOk) {
      const d = (lastOk.t - lastMsg.t) / 1000;
      console.log(
        d >= 0
          ? `  ✅ ✅ 晚于末条消息 ${d.toFixed(1)}s`
          : `  ❌ ✅ 早于末条消息 ${(-d).toFixed(1)}s —— 会导致提前收尾`,
      );
      if (d < 0) allOk = false;
    }
    // 多步回合：中间是否出现多个 ✅（若多个，取最后一个才是真结束）
    if (ok.length > 1) console.log(`  ℹ️ 出现 ${ok.length} 个 ✅ —— App 只认最后一个，需确认不会提前收尾`);
  }
}

console.log(`\n===== 总结 =====`);
console.log(allOk ? '✅ ✅ reaction 可作为可靠结束信号' : '❌ 存在覆盖缺口，App 侧必须保留静默兜底');
process.exit(0);