base.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. import { BaseCallbackHandler, } from "../callbacks/base.js";
  2. // eslint-disable-next-line @typescript-eslint/no-explicit-any
  3. function _coerceToDict(value, defaultKey) {
  4. return value && !Array.isArray(value) && typeof value === "object"
  5. ? value
  6. : { [defaultKey]: value };
  7. }
  8. function stripNonAlphanumeric(input) {
  9. return input.replace(/[-:.]/g, "");
  10. }
  11. function convertToDottedOrderFormat(epoch, runId, executionOrder) {
  12. const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
  13. return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
  14. }
  15. export function isBaseTracer(x) {
  16. return typeof x._addRunToRunMap === "function";
  17. }
  18. export class BaseTracer extends BaseCallbackHandler {
  19. constructor(_fields) {
  20. super(...arguments);
  21. Object.defineProperty(this, "runMap", {
  22. enumerable: true,
  23. configurable: true,
  24. writable: true,
  25. value: new Map()
  26. });
  27. }
  28. copy() {
  29. return this;
  30. }
  31. stringifyError(error) {
  32. // eslint-disable-next-line no-instanceof/no-instanceof
  33. if (error instanceof Error) {
  34. return error.message + (error?.stack ? `\n\n${error.stack}` : "");
  35. }
  36. if (typeof error === "string") {
  37. return error;
  38. }
  39. return `${error}`;
  40. }
  41. _addChildRun(parentRun, childRun) {
  42. parentRun.child_runs.push(childRun);
  43. }
  44. _addRunToRunMap(run) {
  45. const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id, run.execution_order);
  46. const storedRun = { ...run };
  47. if (storedRun.parent_run_id !== undefined) {
  48. const parentRun = this.runMap.get(storedRun.parent_run_id);
  49. if (parentRun) {
  50. this._addChildRun(parentRun, storedRun);
  51. parentRun.child_execution_order = Math.max(parentRun.child_execution_order, storedRun.child_execution_order);
  52. storedRun.trace_id = parentRun.trace_id;
  53. if (parentRun.dotted_order !== undefined) {
  54. storedRun.dotted_order = [
  55. parentRun.dotted_order,
  56. currentDottedOrder,
  57. ].join(".");
  58. }
  59. else {
  60. // This can happen naturally for callbacks added within a run
  61. // console.debug(`Parent run with UUID ${storedRun.parent_run_id} has no dotted order.`);
  62. }
  63. }
  64. else {
  65. // This can happen naturally for callbacks added within a run
  66. // console.debug(
  67. // `Parent run with UUID ${storedRun.parent_run_id} not found.`
  68. // );
  69. }
  70. }
  71. else {
  72. storedRun.trace_id = storedRun.id;
  73. storedRun.dotted_order = currentDottedOrder;
  74. }
  75. this.runMap.set(storedRun.id, storedRun);
  76. return storedRun;
  77. }
  78. async _endTrace(run) {
  79. const parentRun = run.parent_run_id !== undefined && this.runMap.get(run.parent_run_id);
  80. if (parentRun) {
  81. parentRun.child_execution_order = Math.max(parentRun.child_execution_order, run.child_execution_order);
  82. }
  83. else {
  84. await this.persistRun(run);
  85. }
  86. this.runMap.delete(run.id);
  87. await this.onRunUpdate?.(run);
  88. }
  89. _getExecutionOrder(parentRunId) {
  90. const parentRun = parentRunId !== undefined && this.runMap.get(parentRunId);
  91. // If a run has no parent then execution order is 1
  92. if (!parentRun) {
  93. return 1;
  94. }
  95. return parentRun.child_execution_order + 1;
  96. }
  97. /**
  98. * Create and add a run to the run map for LLM start events.
  99. * This must sometimes be done synchronously to avoid race conditions
  100. * when callbacks are backgrounded, so we expose it as a separate method here.
  101. */
  102. _createRunForLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name) {
  103. const execution_order = this._getExecutionOrder(parentRunId);
  104. const start_time = Date.now();
  105. const finalExtraParams = metadata
  106. ? { ...extraParams, metadata }
  107. : extraParams;
  108. const run = {
  109. id: runId,
  110. name: name ?? llm.id[llm.id.length - 1],
  111. parent_run_id: parentRunId,
  112. start_time,
  113. serialized: llm,
  114. events: [
  115. {
  116. name: "start",
  117. time: new Date(start_time).toISOString(),
  118. },
  119. ],
  120. inputs: { prompts },
  121. execution_order,
  122. child_runs: [],
  123. child_execution_order: execution_order,
  124. run_type: "llm",
  125. extra: finalExtraParams ?? {},
  126. tags: tags || [],
  127. };
  128. return this._addRunToRunMap(run);
  129. }
  130. async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name) {
  131. const run = this.runMap.get(runId) ??
  132. this._createRunForLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name);
  133. await this.onRunCreate?.(run);
  134. await this.onLLMStart?.(run);
  135. return run;
  136. }
  137. /**
  138. * Create and add a run to the run map for chat model start events.
  139. * This must sometimes be done synchronously to avoid race conditions
  140. * when callbacks are backgrounded, so we expose it as a separate method here.
  141. */
  142. _createRunForChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
  143. const execution_order = this._getExecutionOrder(parentRunId);
  144. const start_time = Date.now();
  145. const finalExtraParams = metadata
  146. ? { ...extraParams, metadata }
  147. : extraParams;
  148. const run = {
  149. id: runId,
  150. name: name ?? llm.id[llm.id.length - 1],
  151. parent_run_id: parentRunId,
  152. start_time,
  153. serialized: llm,
  154. events: [
  155. {
  156. name: "start",
  157. time: new Date(start_time).toISOString(),
  158. },
  159. ],
  160. inputs: { messages },
  161. execution_order,
  162. child_runs: [],
  163. child_execution_order: execution_order,
  164. run_type: "llm",
  165. extra: finalExtraParams ?? {},
  166. tags: tags || [],
  167. };
  168. return this._addRunToRunMap(run);
  169. }
  170. async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
  171. const run = this.runMap.get(runId) ??
  172. this._createRunForChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name);
  173. await this.onRunCreate?.(run);
  174. await this.onLLMStart?.(run);
  175. return run;
  176. }
  177. async handleLLMEnd(output, runId, _parentRunId, _tags, extraParams) {
  178. const run = this.runMap.get(runId);
  179. if (!run || run?.run_type !== "llm") {
  180. throw new Error("No LLM run to end.");
  181. }
  182. run.end_time = Date.now();
  183. run.outputs = output;
  184. run.events.push({
  185. name: "end",
  186. time: new Date(run.end_time).toISOString(),
  187. });
  188. run.extra = { ...run.extra, ...extraParams };
  189. await this.onLLMEnd?.(run);
  190. await this._endTrace(run);
  191. return run;
  192. }
  193. async handleLLMError(error, runId, _parentRunId, _tags, extraParams) {
  194. const run = this.runMap.get(runId);
  195. if (!run || run?.run_type !== "llm") {
  196. throw new Error("No LLM run to end.");
  197. }
  198. run.end_time = Date.now();
  199. run.error = this.stringifyError(error);
  200. run.events.push({
  201. name: "error",
  202. time: new Date(run.end_time).toISOString(),
  203. });
  204. run.extra = { ...run.extra, ...extraParams };
  205. await this.onLLMError?.(run);
  206. await this._endTrace(run);
  207. return run;
  208. }
  209. /**
  210. * Create and add a run to the run map for chain start events.
  211. * This must sometimes be done synchronously to avoid race conditions
  212. * when callbacks are backgrounded, so we expose it as a separate method here.
  213. */
  214. _createRunForChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType, name) {
  215. const execution_order = this._getExecutionOrder(parentRunId);
  216. const start_time = Date.now();
  217. const run = {
  218. id: runId,
  219. name: name ?? chain.id[chain.id.length - 1],
  220. parent_run_id: parentRunId,
  221. start_time,
  222. serialized: chain,
  223. events: [
  224. {
  225. name: "start",
  226. time: new Date(start_time).toISOString(),
  227. },
  228. ],
  229. inputs,
  230. execution_order,
  231. child_execution_order: execution_order,
  232. run_type: runType ?? "chain",
  233. child_runs: [],
  234. extra: metadata ? { metadata } : {},
  235. tags: tags || [],
  236. };
  237. return this._addRunToRunMap(run);
  238. }
  239. async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType, name) {
  240. const run = this.runMap.get(runId) ??
  241. this._createRunForChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType, name);
  242. await this.onRunCreate?.(run);
  243. await this.onChainStart?.(run);
  244. return run;
  245. }
  246. async handleChainEnd(outputs, runId, _parentRunId, _tags, kwargs) {
  247. const run = this.runMap.get(runId);
  248. if (!run) {
  249. throw new Error("No chain run to end.");
  250. }
  251. run.end_time = Date.now();
  252. run.outputs = _coerceToDict(outputs, "output");
  253. run.events.push({
  254. name: "end",
  255. time: new Date(run.end_time).toISOString(),
  256. });
  257. if (kwargs?.inputs !== undefined) {
  258. run.inputs = _coerceToDict(kwargs.inputs, "input");
  259. }
  260. await this.onChainEnd?.(run);
  261. await this._endTrace(run);
  262. return run;
  263. }
  264. async handleChainError(error, runId, _parentRunId, _tags, kwargs) {
  265. const run = this.runMap.get(runId);
  266. if (!run) {
  267. throw new Error("No chain run to end.");
  268. }
  269. run.end_time = Date.now();
  270. run.error = this.stringifyError(error);
  271. run.events.push({
  272. name: "error",
  273. time: new Date(run.end_time).toISOString(),
  274. });
  275. if (kwargs?.inputs !== undefined) {
  276. run.inputs = _coerceToDict(kwargs.inputs, "input");
  277. }
  278. await this.onChainError?.(run);
  279. await this._endTrace(run);
  280. return run;
  281. }
  282. /**
  283. * Create and add a run to the run map for tool start events.
  284. * This must sometimes be done synchronously to avoid race conditions
  285. * when callbacks are backgrounded, so we expose it as a separate method here.
  286. */
  287. _createRunForToolStart(tool, input, runId, parentRunId, tags, metadata, name) {
  288. const execution_order = this._getExecutionOrder(parentRunId);
  289. const start_time = Date.now();
  290. const run = {
  291. id: runId,
  292. name: name ?? tool.id[tool.id.length - 1],
  293. parent_run_id: parentRunId,
  294. start_time,
  295. serialized: tool,
  296. events: [
  297. {
  298. name: "start",
  299. time: new Date(start_time).toISOString(),
  300. },
  301. ],
  302. inputs: { input },
  303. execution_order,
  304. child_execution_order: execution_order,
  305. run_type: "tool",
  306. child_runs: [],
  307. extra: metadata ? { metadata } : {},
  308. tags: tags || [],
  309. };
  310. return this._addRunToRunMap(run);
  311. }
  312. async handleToolStart(tool, input, runId, parentRunId, tags, metadata, name) {
  313. const run = this.runMap.get(runId) ??
  314. this._createRunForToolStart(tool, input, runId, parentRunId, tags, metadata, name);
  315. await this.onRunCreate?.(run);
  316. await this.onToolStart?.(run);
  317. return run;
  318. }
  319. // eslint-disable-next-line @typescript-eslint/no-explicit-any
  320. async handleToolEnd(output, runId) {
  321. const run = this.runMap.get(runId);
  322. if (!run || run?.run_type !== "tool") {
  323. throw new Error("No tool run to end");
  324. }
  325. run.end_time = Date.now();
  326. run.outputs = { output };
  327. run.events.push({
  328. name: "end",
  329. time: new Date(run.end_time).toISOString(),
  330. });
  331. await this.onToolEnd?.(run);
  332. await this._endTrace(run);
  333. return run;
  334. }
  335. async handleToolError(error, runId) {
  336. const run = this.runMap.get(runId);
  337. if (!run || run?.run_type !== "tool") {
  338. throw new Error("No tool run to end");
  339. }
  340. run.end_time = Date.now();
  341. run.error = this.stringifyError(error);
  342. run.events.push({
  343. name: "error",
  344. time: new Date(run.end_time).toISOString(),
  345. });
  346. await this.onToolError?.(run);
  347. await this._endTrace(run);
  348. return run;
  349. }
  350. async handleAgentAction(action, runId) {
  351. const run = this.runMap.get(runId);
  352. if (!run || run?.run_type !== "chain") {
  353. return;
  354. }
  355. const agentRun = run;
  356. agentRun.actions = agentRun.actions || [];
  357. agentRun.actions.push(action);
  358. agentRun.events.push({
  359. name: "agent_action",
  360. time: new Date().toISOString(),
  361. kwargs: { action },
  362. });
  363. await this.onAgentAction?.(run);
  364. }
  365. async handleAgentEnd(action, runId) {
  366. const run = this.runMap.get(runId);
  367. if (!run || run?.run_type !== "chain") {
  368. return;
  369. }
  370. run.events.push({
  371. name: "agent_end",
  372. time: new Date().toISOString(),
  373. kwargs: { action },
  374. });
  375. await this.onAgentEnd?.(run);
  376. }
  377. /**
  378. * Create and add a run to the run map for retriever start events.
  379. * This must sometimes be done synchronously to avoid race conditions
  380. * when callbacks are backgrounded, so we expose it as a separate method here.
  381. */
  382. _createRunForRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name) {
  383. const execution_order = this._getExecutionOrder(parentRunId);
  384. const start_time = Date.now();
  385. const run = {
  386. id: runId,
  387. name: name ?? retriever.id[retriever.id.length - 1],
  388. parent_run_id: parentRunId,
  389. start_time,
  390. serialized: retriever,
  391. events: [
  392. {
  393. name: "start",
  394. time: new Date(start_time).toISOString(),
  395. },
  396. ],
  397. inputs: { query },
  398. execution_order,
  399. child_execution_order: execution_order,
  400. run_type: "retriever",
  401. child_runs: [],
  402. extra: metadata ? { metadata } : {},
  403. tags: tags || [],
  404. };
  405. return this._addRunToRunMap(run);
  406. }
  407. async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name) {
  408. const run = this.runMap.get(runId) ??
  409. this._createRunForRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name);
  410. await this.onRunCreate?.(run);
  411. await this.onRetrieverStart?.(run);
  412. return run;
  413. }
  414. async handleRetrieverEnd(documents, runId) {
  415. const run = this.runMap.get(runId);
  416. if (!run || run?.run_type !== "retriever") {
  417. throw new Error("No retriever run to end");
  418. }
  419. run.end_time = Date.now();
  420. run.outputs = { documents };
  421. run.events.push({
  422. name: "end",
  423. time: new Date(run.end_time).toISOString(),
  424. });
  425. await this.onRetrieverEnd?.(run);
  426. await this._endTrace(run);
  427. return run;
  428. }
  429. async handleRetrieverError(error, runId) {
  430. const run = this.runMap.get(runId);
  431. if (!run || run?.run_type !== "retriever") {
  432. throw new Error("No retriever run to end");
  433. }
  434. run.end_time = Date.now();
  435. run.error = this.stringifyError(error);
  436. run.events.push({
  437. name: "error",
  438. time: new Date(run.end_time).toISOString(),
  439. });
  440. await this.onRetrieverError?.(run);
  441. await this._endTrace(run);
  442. return run;
  443. }
  444. async handleText(text, runId) {
  445. const run = this.runMap.get(runId);
  446. if (!run || run?.run_type !== "chain") {
  447. return;
  448. }
  449. run.events.push({
  450. name: "text",
  451. time: new Date().toISOString(),
  452. kwargs: { text },
  453. });
  454. await this.onText?.(run);
  455. }
  456. async handleLLMNewToken(token, idx, runId, _parentRunId, _tags, fields) {
  457. const run = this.runMap.get(runId);
  458. if (!run || run?.run_type !== "llm") {
  459. throw new Error(`Invalid "runId" provided to "handleLLMNewToken" callback.`);
  460. }
  461. run.events.push({
  462. name: "new_token",
  463. time: new Date().toISOString(),
  464. kwargs: { token, idx, chunk: fields?.chunk },
  465. });
  466. await this.onLLMNewToken?.(run, token, { chunk: fields?.chunk });
  467. return run;
  468. }
  469. }