loadtest-baseline.mjs 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108
  1. #!/usr/bin/env node
  2. /**
  3. * 单人基线响应时间(基准测试)
  4. *
  5. * 目的:给出「一个人用容器,正常要等多久」的基准值,
  6. * 用于和并发档位对比,量化「人多带来的额外等待」。
  7. *
  8. * 做法:串行跑 R 轮,每轮 1 个新会话、1 个请求,记录端到端耗时。
  9. * 串行是刻意的 —— 基准必须没有任何并发干扰。
  10. *
  11. * 用法:node scripts/loadtest-baseline.mjs --rounds 8 --mode both
  12. */
  13. import { readFileSync, writeFileSync } from 'node:fs';
  14. import { homedir } from 'node:os';
  15. import { join } from 'node:path';
  16. const FUNCTIONS = 'https://server.fmode.cn/api/functions';
  17. const FN_PROVISION = 'RP5Ht55NeX';
  18. const AGENT = 'agent-node-lj-test';
  19. const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
  20. const arg = (n, d) => {
  21. const i = process.argv.indexOf(`--${n}`);
  22. return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : d;
  23. };
  24. const ROUNDS = Number(arg('rounds', '8'));
  25. const MODE = arg('mode', 'text'); // text | file
  26. const TIMEOUT_S = Number(arg('timeout', '300'));
  27. const RUN = `base${Date.now().toString().slice(-6)}`;
  28. const TOKEN = JSON.parse(readFileSync(join(homedir(), '.fmode/config/user.json'), 'utf8')).sessionToken;
  29. const pct = (a, p) => (a.length ? [...a].sort((x, y) => x - y)[Math.min(a.length - 1, Math.floor((p / 100) * a.length))] : 0);
  30. const r1 = (n) => Math.round(n * 10) / 10;
  31. async function main() {
  32. const j = await (await fetch(FUNCTIONS, {
  33. method: 'POST',
  34. headers: { 'Content-Type': 'application/json', 'X-Parse-Application-Id': 'ncloudmaster' },
  35. body: JSON.stringify({ id: FN_PROVISION, token: TOKEN, params: { agentId: AGENT } }),
  36. })).json();
  37. const p = j.result ?? j;
  38. const mx = await import('matrix-js-sdk');
  39. const c = mx.createClient({ baseUrl: p.homeserver, accessToken: p.accessToken, userId: p.userMxId });
  40. const hs = p.homeserver.replace(/\/$/, '');
  41. const auth = { Authorization: `Bearer ${p.accessToken}` };
  42. console.log(`基线测试:${ROUNDS} 轮串行,模式 ${MODE},容器 ${p.agentMxId}\n`);
  43. const results = [];
  44. for (let k = 1; k <= ROUNDS; k++) {
  45. const room = (await c.createRoom({ invite: [p.agentMxId], name: `${RUN}-${k}`, preset: mx.Preset.TrustedPrivateChat })).room_id;
  46. // 等进房
  47. for (let i = 0; i < 40; i++) {
  48. await sleep(2000);
  49. const jm = await (await fetch(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(room)}/joined_members`, { headers: auth })).json().catch(() => ({}));
  50. if ((jm.joined ?? {})[p.agentMxId]) break;
  51. }
  52. const tok = `B${k}${Math.random().toString(36).slice(2, 7).toUpperCase()}`;
  53. const prompt =
  54. MODE === 'text'
  55. ? `只回复一行:${tok} 已收到`
  56. : `在 /opt/data/${RUN}/${k}/ 下生成 1 个文件 r.txt(三行文字,内容含 ${tok}),并作为附件发给我。完成后一句话总结。`;
  57. const sent = await c.sendEvent(room, 'm.room.message', { msgtype: 'm.text', body: prompt, 'm.mentions': { user_ids: [p.agentMxId] } });
  58. const t0 = Date.now();
  59. let firstProcess = 0, firstBody = 0, lastEv = 0;
  60. /*
  61. * 🔴 附件必须按 event_id 去重计数:轮询是「每次拉最近 N 条」,
  62. * 同一个 m.file 会被反复读到。之前用累加,1 个附件被数成了 13 个。
  63. */
  64. const fileIds = new Set();
  65. const seenEv = new Set();
  66. const deadline = t0 + TIMEOUT_S * 1000;
  67. const isProg = (b) => /^[*·•\s]*(🐍|💻|🔎|📖|✍️|🧠|📝|⚙|🔧|Writing|Reading|Running|terminal|write_file|execute_code)/i.test(b.trim());
  68. while (Date.now() < deadline) {
  69. await sleep(2500);
  70. const jj = await (await fetch(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(room)}/messages?dir=b&limit=40`, { headers: auth })).json().catch(() => ({}));
  71. for (const ev of (jj.chunk ?? []).reverse()) {
  72. if (ev.event_id === sent.event_id) continue;
  73. if (ev.type !== 'm.room.message' || ev.sender !== p.agentMxId) continue;
  74. const cc = ev.content ?? {}, body = String(cc.body ?? '');
  75. if (seenEv.has(ev.event_id)) continue;
  76. seenEv.add(ev.event_id);
  77. lastEv = ev.origin_server_ts;
  78. if (cc['fmode.kind'] === 'process' || isProg(body)) { if (!firstProcess) firstProcess = ev.origin_server_ts; continue; }
  79. if (!firstBody && body.trim()) firstBody = ev.origin_server_ts;
  80. if (cc.url && (cc.msgtype === 'm.file' || cc.msgtype === 'm.image')) fileIds.add(ev.event_id);
  81. }
  82. if (firstBody && Date.now() - lastEv > 30000) break;
  83. }
  84. const files = fileIds.size;
  85. const e2e = firstBody ? (lastEv - t0) / 1000 : null;
  86. const queue = firstProcess ? (firstProcess - t0) / 1000 : null;
  87. results.push({ k, room, tok, e2e, queue, files });
  88. console.log(` 第 ${k} 轮:端到端 ${e2e ?? '超时'}s | 首条过程 ${queue ?? '-'}s | 文件 ${files}`);
  89. }
  90. const e = results.map((r) => r.e2e).filter((v) => v != null);
  91. const q = results.map((r) => r.queue).filter((v) => v != null);
  92. const stats = {
  93. run: RUN, mode: MODE, rounds: ROUNDS,
  94. success: e.length,
  95. p50: r1(pct(e, 50)), p90: r1(pct(e, 90)), min: r1(Math.min(...e)), max: r1(Math.max(...e)),
  96. queueP50: r1(pct(q, 50)),
  97. results,
  98. };
  99. console.log(`\n基线(${MODE}):成功 ${stats.success}/${ROUNDS} | p50 ${stats.p50}s | p90 ${stats.p90}s | 区间 ${stats.min}–${stats.max}s | 首条响应 p50 ${stats.queueP50}s`);
  100. writeFileSync(`docs/03-interface/baseline-ljtest-${MODE}-${RUN}.json`, JSON.stringify(stats, null, 2));
  101. console.log(`落盘:docs/03-interface/baseline-ljtest-${MODE}-${RUN}.json`);
  102. c.stopClient();
  103. }
  104. main().catch((e) => { console.error('失败:', e.message ?? e); process.exit(1); });