loadtest-ljtest.mjs 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. #!/usr/bin/env node
  2. /**
  3. * 单一容器多人并发压测(逐级加压找拐点)
  4. *
  5. * 用法:
  6. * node scripts/loadtest-ljtest.mjs --levels 20,30,40,50 --timeout 900
  7. * node scripts/loadtest-ljtest.mjs --levels 20 --mode text # 只跑文本轮
  8. * node scripts/loadtest-ljtest.mjs --levels 20 --mode file # 只跑文件轮
  9. *
  10. * ## 与 loadtest-agent.mjs 的区别
  11. *
  12. * 那个只回答「N 条里几条回了、有没有串台」。本次要回答的是
  13. * **「多人同时用会不会卡、单一容器上限在哪」**,所以补了:
  14. * · 优先级:**延迟分位**(p50/p90/p95/max)—— 「卡」的可操作定义
  15. * · **排队延迟** vs **处理延迟** 拆开(首条 process 事件 - 发出 = 排队)
  16. * · **两种负载**:纯文本(测调度)/ 文件(测生成+上云+投递)
  17. * · **逐级加压**,每档独立、冷却后进下一档,满足判停条件即停止
  18. * · 每档落盘 JSON,便于事后逐会话取证
  19. *
  20. * ## 判停条件(满足任一 → 当前档即拐点,不再加压)
  21. *
  22. * ① 成功率 < 90%
  23. * ② p95 端到端耗时 > 首档基线的 3 倍
  24. * ③ 出现串台(回复里出现别的会话 token)
  25. * ④ 文件轮:请求了文件但没收到 m.file
  26. *
  27. * ## 为什么要等「静默」而不是等 ✅
  28. *
  29. * 实测该容器**经常不打 ✅**(2026-11-17 四轮实测 0/4)。
  30. * 若只等 ✅,每档都要耗到超时。所以:
  31. * 收到本会话的实质正文 且 该会话 45 秒无新事件 → 视为完成
  32. * ✅ 到了就立刻算完成(更快)
  33. */
  34. import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
  35. import { homedir } from 'node:os';
  36. import { join } from 'node:path';
  37. const FUNCTIONS = 'https://server.fmode.cn/api/functions';
  38. const FN_PROVISION = 'RP5Ht55NeX';
  39. const AGENT = 'agent-node-lj-test';
  40. const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
  41. function arg(name, fallback) {
  42. const i = process.argv.indexOf(`--${name}`);
  43. return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
  44. }
  45. const LEVELS = arg('levels', '20,30,40,50').split(',').map(Number);
  46. const TIMEOUT_S = Number(arg('timeout', '900'));
  47. const MODE = arg('mode', 'both'); // both | text | file
  48. const POLL_MS = Number(arg('poll', '3000'));
  49. const QUIET_MS = Number(arg('quiet', '45000'));
  50. const COOLDOWN_S = Number(arg('cooldown', '60'));
  51. const RUN = `lt${Date.now().toString().slice(-6)}`;
  52. const OUT_DIR = 'docs/03-interface';
  53. const TOKEN = JSON.parse(readFileSync(join(homedir(), '.fmode/config/user.json'), 'utf8'))?.sessionToken;
  54. /** 带重试的 fetch(压测期间高并发请求会被限流/瞬时失败,不能一次失败就整轮崩掉) */
  55. async function fetchRetry(url, opts = {}, tries = 4) {
  56. let lastErr;
  57. for (let k = 0; k < tries; k++) {
  58. try {
  59. const r = await fetch(url, opts);
  60. if (r.status < 500) return r;
  61. lastErr = new Error(`HTTP ${r.status}`);
  62. } catch (e) {
  63. lastErr = e;
  64. }
  65. await sleep(1000 * (k + 1));
  66. }
  67. throw lastErr;
  68. }
  69. async function provision() {
  70. const r = await fetchRetry(FUNCTIONS, {
  71. method: 'POST',
  72. headers: { 'Content-Type': 'application/json', 'X-Parse-Application-Id': 'ncloudmaster' },
  73. body: JSON.stringify({ id: FN_PROVISION, token: TOKEN, params: { agentId: AGENT } }),
  74. });
  75. const j = await r.json();
  76. const d = j.result ?? j;
  77. if (!d?.accessToken) throw new Error('provision 失败 ' + JSON.stringify(j).slice(0, 200));
  78. return d;
  79. }
  80. const pct = (arr, p) => {
  81. if (!arr.length) return 0;
  82. const s = [...arr].sort((a, b) => a - b);
  83. return s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
  84. };
  85. const r1 = (n) => Math.round(n * 10) / 10;
  86. /** 一轮(一档 + 一种负载)的完整执行与统计 */
  87. async function runWave(level, mode, ctx) {
  88. const { mx, client, hs, auth, agentMxId, userMxId } = ctx;
  89. const tag = `L${level}-${mode}`;
  90. console.log(`\n${'='.repeat(60)}\n档位 ${tag}:${level} 个并发会话(${mode === 'text' ? '纯文本' : '生成文件'})\n${'='.repeat(60)}`);
  91. // 1) 建 N 个独立房间(串行 + 重试:并发建房会被限流)
  92. const peers = [];
  93. for (let i = 1; i <= level; i++) {
  94. let room;
  95. for (let k = 0; k < 4 && !room; k++) {
  96. try {
  97. room = (
  98. await client.createRoom({
  99. invite: [agentMxId],
  100. name: `${RUN}-${tag}-${i}`,
  101. preset: mx.Preset.TrustedPrivateChat,
  102. })
  103. ).room_id;
  104. } catch (e) {
  105. console.log(` 建房 ${i} 第 ${k + 1} 次失败:${String(e.message ?? e).slice(0, 80)}`);
  106. await sleep(1500 * (k + 1));
  107. }
  108. }
  109. if (!room) {
  110. console.log(` ⚠️ 建房 ${i} 彻底失败,跳过`);
  111. continue;
  112. }
  113. peers.push({ i, room, token: `T${i}${Math.random().toString(36).slice(2, 8).toUpperCase()}`, sentAt: 0, firstProcessAt: 0, firstBodyAt: 0, ackedAt: 0, lastEventAt: 0, files: [], bodies: [], seen: new Set(), joined: false });
  114. }
  115. console.log(` 建房 ${peers.length}/${level}`);
  116. // 2) 等容器进房(并发)
  117. const joinDeadline = Date.now() + 180_000;
  118. while (Date.now() < joinDeadline && peers.some((p) => !p.joined)) {
  119. await Promise.all(
  120. peers.filter((p) => !p.joined).map(async (p) => {
  121. const j = await (await fetchRetry(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(p.room)}/joined_members`, { headers: auth }).catch(() => null))?.json().catch(() => ({})) ?? {};
  122. if ((j.joined ?? {})[agentMxId]) p.joined = true;
  123. }),
  124. );
  125. if (peers.some((p) => !p.joined)) await sleep(2000);
  126. }
  127. const bound = peers.filter((p) => p.joined);
  128. console.log(` 容器进房 ${bound.length}/${level}`);
  129. if (!bound.length) return { level, mode, error: '容器未进任何房间', peers: [] };
  130. // 3) 同时发送
  131. const t0 = Date.now();
  132. const promptFor = (p) =>
  133. mode === 'text'
  134. ? `只回复一行:${p.token} 已收到`
  135. : `在 /opt/data/${RUN}/${tag}/${p.i}/ 下生成 1 个文件 r.txt(三行文字,内容含 ${p.token}),并作为附件发给我。完成后一句话总结。`;
  136. await Promise.all(
  137. bound.map(async (p) => {
  138. const sent = await client.sendEvent(p.room, 'm.room.message', {
  139. msgtype: 'm.text',
  140. body: promptFor(p),
  141. 'm.mentions': { user_ids: [agentMxId] },
  142. });
  143. p.sentId = sent.event_id;
  144. p.sentAt = Date.now();
  145. }),
  146. );
  147. console.log(` ${bound.length} 条已同时发出,开始观测…`);
  148. // 4) 轮询收集直到全部完成或超时
  149. const deadline = t0 + TIMEOUT_S * 1000;
  150. let lastPrint = 0;
  151. const isProgress = (b) => /^[*·•\s]*(🐍|💻|🔎|📖|✍️|🧠|📝|⚙|🔧|Writing|Reading|Running|terminal|write_file|execute_code)/i.test(b.trim());
  152. while (Date.now() < deadline) {
  153. await sleep(POLL_MS);
  154. await Promise.all(
  155. bound.map(async (p) => {
  156. if (p.ackedAt) return;
  157. const j = await (await fetchRetry(`${hs}/_matrix/client/v3/rooms/${encodeURIComponent(p.room)}/messages?dir=b&limit=80`, { headers: auth }).catch(() => null))?.json().catch(() => ({})) ?? {};
  158. for (const ev of (j.chunk ?? []).reverse()) {
  159. if (ev.event_id === p.sentId) continue;
  160. if (ev.origin_server_ts < p.sentAt) continue;
  161. if (ev.type === 'm.reaction') {
  162. const rel = ev.content?.['m.relates_to'];
  163. if (rel?.event_id === p.sentId && String(rel.key ?? '').includes('✅')) p.ackedAt = ev.origin_server_ts;
  164. continue;
  165. }
  166. if (ev.type !== 'm.room.message' || ev.sender === userMxId) continue;
  167. /* 🔴 按 event_id 去重:轮询会反复读到同一事件(附件曾被数成 13 个) */
  168. if (p.seen.has(ev.event_id)) continue;
  169. p.seen.add(ev.event_id);
  170. const c = ev.content ?? {};
  171. const body = String(c.body ?? '');
  172. p.lastEventAt = ev.origin_server_ts;
  173. if (c['fmode.kind'] === 'process' || isProgress(body)) {
  174. if (!p.firstProcessAt) p.firstProcessAt = ev.origin_server_ts;
  175. continue;
  176. }
  177. if (!p.firstBodyAt && body.trim()) p.firstBodyAt = ev.origin_server_ts;
  178. if (body.trim()) p.bodies.push(body);
  179. if (c.url && (c.msgtype === 'm.file' || c.msgtype === 'm.image')) p.files.push({ name: body, url: c.url, size: c.info?.size });
  180. }
  181. }),
  182. );
  183. const now = Date.now();
  184. const doneCount = bound.filter((p) => p.ackedAt || (p.firstBodyAt && now - p.lastEventAt > QUIET_MS)).length;
  185. if (now - lastPrint > 20000) {
  186. lastPrint = now;
  187. console.log(` [${Math.round((now - t0) / 1000)}s] 完成 ${doneCount}/${bound.length}`);
  188. }
  189. if (doneCount === bound.length) break;
  190. }
  191. // 5) 统计
  192. const rows = bound.map((p) => {
  193. const all = p.bodies.join('\n');
  194. const leaked = bound.filter((q) => q !== p && all.includes(q.token)).map((q) => q.token);
  195. const own = all.includes(p.token);
  196. const finishedAt = p.ackedAt || (p.firstBodyAt ? Math.max(p.firstBodyAt, p.lastEventAt) : 0);
  197. return {
  198. i: p.i,
  199. room: p.room,
  200. ok: !!p.firstBodyAt && own,
  201. ownToken: own,
  202. leaked,
  203. e2e: finishedAt && p.sentAt ? (finishedAt - p.sentAt) / 1000 : null,
  204. queue: p.firstProcessAt && p.sentAt ? (p.firstProcessAt - p.sentAt) / 1000 : null,
  205. files: p.files.length,
  206. acked: !!p.ackedAt,
  207. };
  208. });
  209. const okRows = rows.filter((r) => r.ok);
  210. const e2e = okRows.map((r) => r.e2e).filter((v) => v != null);
  211. const queue = okRows.map((r) => r.queue).filter((v) => v != null);
  212. const fileOk = mode === 'file' ? rows.filter((r) => r.files > 0).length : null;
  213. const stats = {
  214. level,
  215. mode,
  216. run: RUN,
  217. total: bound.length,
  218. success: okRows.length,
  219. successRate: r1((okRows.length / bound.length) * 100),
  220. leakedSessions: rows.filter((r) => r.leaked.length).length,
  221. p50: r1(pct(e2e, 50)),
  222. p90: r1(pct(e2e, 90)),
  223. p95: r1(pct(e2e, 95)),
  224. max: r1(Math.max(0, ...e2e)),
  225. queueP50: r1(pct(queue, 50)),
  226. queueP95: r1(pct(queue, 95)),
  227. elapsedS: Math.round((Date.now() - t0) / 1000),
  228. throughputPerMin: r1((okRows.length / Math.max(1, (Date.now() - t0) / 60000))),
  229. fileOk,
  230. rows,
  231. };
  232. console.log(` 结果:成功 ${stats.success}/${stats.total} (${stats.successRate}%) | p50 ${stats.p50}s p90 ${stats.p90}s p95 ${stats.p95}s max ${stats.max}s | 排队 p50 ${stats.queueP50}s | 串台 ${stats.leakedSessions}${mode === 'file' ? ` | 带文件 ${stats.fileOk}` : ''}`);
  233. return stats;
  234. }
  235. async function main() {
  236. console.log(`\n=== lj-test 单容器并发压测 ===\n档位: ${LEVELS.join(' → ')}\n模式: ${MODE}\n单档超时: ${TIMEOUT_S}s\n运行号: ${RUN}\n`);
  237. const prov = await provision();
  238. const mx = await import('matrix-js-sdk');
  239. const client = mx.createClient({ baseUrl: prov.homeserver, accessToken: prov.accessToken, userId: prov.userMxId });
  240. const hs = prov.homeserver.replace(/\/$/, '');
  241. const ctx = { mx, client, hs, auth: { Authorization: `Bearer ${prov.accessToken}` }, agentMxId: prov.agentMxId, userMxId: prov.userMxId };
  242. console.log(`用户 ${prov.userMxId} | 容器 ${prov.agentMxId}`);
  243. const modes = MODE === 'both' ? ['text', 'file'] : [MODE];
  244. const all = [];
  245. let baselineP95 = null;
  246. let stop = false;
  247. for (const level of LEVELS) {
  248. for (const mode of modes) {
  249. if (stop) break;
  250. const s = await runWave(level, mode, ctx);
  251. all.push(s);
  252. if (s.error) { stop = true; break; }
  253. if (mode === 'text' && baselineP95 == null) baselineP95 = s.p95;
  254. const reasons = [];
  255. if (s.successRate < 90) reasons.push(`成功率 ${s.successRate}% < 90%`);
  256. if (baselineP95 && mode === 'text' && s.p95 > baselineP95 * 3) reasons.push(`p95 ${s.p95}s > 基线3倍(${r1(baselineP95 * 3)}s)`);
  257. if (s.leakedSessions > 0) reasons.push(`串台 ${s.leakedSessions} 个会话`);
  258. if (mode === 'file' && s.fileOk < s.total * 0.9) reasons.push(`文件缺失 ${s.total - s.fileOk} 个`);
  259. if (reasons.length) {
  260. console.log(`\n⛔ 拐点:${tag(level, mode)} —— ${reasons.join(';')}`);
  261. s.breakReason = reasons.join(';');
  262. stop = true;
  263. }
  264. }
  265. if (stop) break;
  266. if (level !== LEVELS[LEVELS.length - 1]) {
  267. console.log(`\n冷却 ${COOLDOWN_S}s 后进入下一档…`);
  268. await sleep(COOLDOWN_S * 1000);
  269. }
  270. }
  271. mkdirSync(OUT_DIR, { recursive: true });
  272. const file = join(OUT_DIR, `loadtest-ljtest-${RUN}.json`);
  273. writeFileSync(file, JSON.stringify({ run: RUN, agent: AGENT, levels: LEVELS, mode: MODE, waves: all }, null, 2));
  274. console.log(`\n===== 汇总 =====`);
  275. for (const w of all) {
  276. if (w.error) console.log(`${tag(w.level, w.mode)}: 失败 ${w.error}`);
  277. else console.log(`${tag(w.level, w.mode)}: 成功 ${w.successRate}% | p50 ${w.p50}s p95 ${w.p95}s max ${w.max}s | 吞吐 ${w.throughputPerMin}/分${w.breakReason ? ` | ⛔ ${w.breakReason}` : ''}`);
  278. }
  279. console.log(`\n落盘: ${file}`);
  280. client.stopClient();
  281. }
  282. const tag = (l, m) => `L${l}-${m}`;
  283. main().catch((e) => { console.error('失败:', e.message ?? e); process.exit(1); });