vercel.js 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815
  1. import { Client, RunTree } from "./index.js";
  2. import { v5 as uuid5 } from "uuid";
  3. import { getCurrentRunTree } from "./singletons/traceable.js";
  4. import { getLangSmithEnvironmentVariable, getEnvironmentVariable, } from "./utils/env.js";
  5. import { isTracingEnabled } from "./env.js";
  6. // Attempt to convert CoreMessage to a LangChain-compatible format
  7. // which allows us to render messages more nicely in LangSmith
  8. function convertCoreToSmith(message) {
  9. if (message.role === "assistant") {
  10. const data = { content: message.content };
  11. if (Array.isArray(message.content)) {
  12. data.content = message.content.map((part) => {
  13. if (part.type === "text") {
  14. return {
  15. type: "text",
  16. text: part.text,
  17. ...part.experimental_providerMetadata,
  18. };
  19. }
  20. if (part.type === "tool-call") {
  21. return {
  22. type: "tool_use",
  23. name: part.toolName,
  24. id: part.toolCallId,
  25. input: part.args,
  26. ...part.experimental_providerMetadata,
  27. };
  28. }
  29. return part;
  30. });
  31. const toolCalls = message.content.filter((part) => part.type === "tool-call");
  32. if (toolCalls.length > 0) {
  33. data.additional_kwargs ??= {};
  34. data.additional_kwargs.tool_calls = toolCalls.map((part) => {
  35. return {
  36. id: part.toolCallId,
  37. type: "function",
  38. function: {
  39. name: part.toolName,
  40. id: part.toolCallId,
  41. arguments: JSON.stringify(part.args),
  42. },
  43. };
  44. });
  45. }
  46. }
  47. return { type: "ai", data };
  48. }
  49. if (message.role === "user") {
  50. const data = { content: message.content };
  51. if (Array.isArray(message.content)) {
  52. data.content = message.content.map((part) => {
  53. if (part.type === "text") {
  54. return {
  55. type: "text",
  56. text: part.text,
  57. ...part.experimental_providerMetadata,
  58. };
  59. }
  60. if (part.type === "image") {
  61. let imageUrl = part.image;
  62. if (typeof imageUrl !== "string") {
  63. let uint8Array;
  64. if (imageUrl != null &&
  65. typeof imageUrl === "object" &&
  66. "type" in imageUrl &&
  67. "data" in imageUrl) {
  68. // Typing is wrong here if a buffer is passed in
  69. uint8Array = new Uint8Array(imageUrl.data);
  70. }
  71. else if (imageUrl != null &&
  72. typeof imageUrl === "object" &&
  73. Object.keys(imageUrl).every((key) => !isNaN(Number(key)))) {
  74. // ArrayBuffers get turned into objects with numeric keys for some reason
  75. uint8Array = new Uint8Array(Array.from({
  76. ...imageUrl,
  77. length: Object.keys(imageUrl).length,
  78. }));
  79. }
  80. if (uint8Array) {
  81. let binary = "";
  82. for (let i = 0; i < uint8Array.length; i++) {
  83. binary += String.fromCharCode(uint8Array[i]);
  84. }
  85. imageUrl = btoa(binary);
  86. }
  87. }
  88. return {
  89. type: "image_url",
  90. image_url: imageUrl,
  91. ...part.experimental_providerMetadata,
  92. };
  93. }
  94. return part;
  95. });
  96. }
  97. return { type: "human", data };
  98. }
  99. if (message.role === "system") {
  100. return { type: "system", data: { content: message.content } };
  101. }
  102. if (message.role === "tool") {
  103. const res = message.content.map((toolCall) => {
  104. return {
  105. type: "tool",
  106. data: {
  107. content: JSON.stringify(toolCall.result),
  108. name: toolCall.toolName,
  109. tool_call_id: toolCall.toolCallId,
  110. },
  111. };
  112. });
  113. if (res.length === 1)
  114. return res[0];
  115. return res;
  116. }
  117. return message;
  118. }
  119. const tryJson = (str) => {
  120. try {
  121. if (!str)
  122. return str;
  123. if (typeof str !== "string")
  124. return str;
  125. return JSON.parse(str);
  126. }
  127. catch {
  128. return str;
  129. }
  130. };
  131. function stripNonAlphanumeric(input) {
  132. return input.replace(/[-:.]/g, "");
  133. }
  134. function getDotOrder(item) {
  135. const { startTime: [seconds, nanoseconds], id: runId, executionOrder, } = item;
  136. // Date only has millisecond precision, so we use the microseconds to break
  137. // possible ties, avoiding incorrect run order
  138. const nanosecondString = String(nanoseconds).padStart(9, "0");
  139. const msFull = Number(nanosecondString.slice(0, 6)) + executionOrder;
  140. const msString = String(msFull).padStart(6, "0");
  141. const ms = Number(msString.slice(0, -3));
  142. const ns = msString.slice(-3);
  143. return (stripNonAlphanumeric(`${new Date(seconds * 1000 + ms).toISOString().slice(0, -1)}${ns}Z`) + runId);
  144. }
  145. function joinDotOrder(...segments) {
  146. return segments.filter(Boolean).join(".");
  147. }
  148. function removeDotOrder(dotOrder, ...ids) {
  149. return dotOrder
  150. .split(".")
  151. .filter((i) => !ids.some((id) => i.includes(id)))
  152. .join(".");
  153. }
  154. function reparentDotOrder(dotOrder, sourceRunId, parentDotOrder) {
  155. const segments = dotOrder.split(".");
  156. const sourceIndex = segments.findIndex((i) => i.includes(sourceRunId));
  157. if (sourceIndex === -1)
  158. return dotOrder;
  159. return joinDotOrder(...parentDotOrder.split("."), ...segments.slice(sourceIndex));
  160. }
  161. function getMutableRunCreate(dotOrder) {
  162. const segments = dotOrder.split(".").map((i) => {
  163. const [startTime, runId] = i.split("Z");
  164. return { startTime, runId };
  165. });
  166. const traceId = segments[0].runId;
  167. const parentRunId = segments.at(-2)?.runId;
  168. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
  169. const runId = segments.at(-1).runId;
  170. return {
  171. id: runId,
  172. trace_id: traceId,
  173. dotted_order: dotOrder,
  174. parent_run_id: parentRunId,
  175. };
  176. }
  177. function convertToTimestamp([seconds, nanoseconds]) {
  178. const ms = String(nanoseconds).slice(0, 3);
  179. return Number(String(seconds) + ms);
  180. }
  181. function sortByHr(a, b) {
  182. if (a.startTime[0] !== b.startTime[0]) {
  183. return Math.sign(a.startTime[0] - b.startTime[0]);
  184. }
  185. else if (a.startTime[1] !== b.startTime[1]) {
  186. return Math.sign(a.startTime[1] - b.startTime[1]);
  187. }
  188. else if (getParentSpanId(a) === b.spanContext().spanId) {
  189. return -1;
  190. }
  191. else if (getParentSpanId(b) === a.spanContext().spanId) {
  192. return 1;
  193. }
  194. else {
  195. return 0;
  196. }
  197. }
  198. const ROOT = "$";
  199. const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47";
  200. const RUN_ID_METADATA_KEY = {
  201. input: "langsmith:runId",
  202. output: "ai.telemetry.metadata.langsmith:runId",
  203. };
  204. const RUN_NAME_METADATA_KEY = {
  205. input: "langsmith:runName",
  206. output: "ai.telemetry.metadata.langsmith:runName",
  207. };
  208. const TRACE_METADATA_KEY = {
  209. input: "langsmith:trace",
  210. output: "ai.telemetry.metadata.langsmith:trace",
  211. };
  212. const BAGGAGE_METADATA_KEY = {
  213. input: "langsmith:baggage",
  214. output: "ai.telemetry.metadata.langsmith:baggage",
  215. };
  216. const RESERVED_METADATA_KEYS = [
  217. RUN_ID_METADATA_KEY.output,
  218. RUN_NAME_METADATA_KEY.output,
  219. TRACE_METADATA_KEY.output,
  220. BAGGAGE_METADATA_KEY.output,
  221. ];
  222. function getParentSpanId(span) {
  223. // Backcompat shim to support OTEL 1.x and 2.x
  224. // eslint-disable-next-line @typescript-eslint/no-explicit-any
  225. return (span.parentSpanId ?? span.parentSpanContext?.spanId ?? undefined);
  226. }
  227. /**
  228. * OpenTelemetry trace exporter for Vercel AI SDK.
  229. *
  230. * @example
  231. * ```ts
  232. * import { AISDKExporter } from "langsmith/vercel";
  233. * import { Client } from "langsmith";
  234. *
  235. * import { generateText } from "ai";
  236. * import { openai } from "@ai-sdk/openai";
  237. *
  238. * import { NodeSDK } from "@opentelemetry/sdk-node";
  239. * import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
  240. *
  241. * const client = new Client();
  242. *
  243. * const sdk = new NodeSDK({
  244. * traceExporter: new AISDKExporter({ client }),
  245. * instrumentations: [getNodeAutoInstrumentations()],
  246. * });
  247. *
  248. * sdk.start();
  249. *
  250. * const res = await generateText({
  251. * model: openai("gpt-4o-mini"),
  252. * messages: [
  253. * {
  254. * role: "user",
  255. * content: "What color is the sky?",
  256. * },
  257. * ],
  258. * experimental_telemetry: AISDKExporter.getSettings({
  259. * runName: "langsmith_traced_call",
  260. * metadata: { userId: "123", language: "english" },
  261. * }),
  262. * });
  263. *
  264. * await sdk.shutdown();
  265. * ```
  266. */
  267. export class AISDKExporter {
  268. constructor(args) {
  269. Object.defineProperty(this, "client", {
  270. enumerable: true,
  271. configurable: true,
  272. writable: true,
  273. value: void 0
  274. });
  275. Object.defineProperty(this, "traceByMap", {
  276. enumerable: true,
  277. configurable: true,
  278. writable: true,
  279. value: {}
  280. });
  281. Object.defineProperty(this, "seenSpanInfo", {
  282. enumerable: true,
  283. configurable: true,
  284. writable: true,
  285. value: {}
  286. });
  287. Object.defineProperty(this, "pendingSpans", {
  288. enumerable: true,
  289. configurable: true,
  290. writable: true,
  291. value: {}
  292. });
  293. Object.defineProperty(this, "debug", {
  294. enumerable: true,
  295. configurable: true,
  296. writable: true,
  297. value: void 0
  298. });
  299. Object.defineProperty(this, "projectName", {
  300. enumerable: true,
  301. configurable: true,
  302. writable: true,
  303. value: void 0
  304. });
  305. /** @internal */
  306. Object.defineProperty(this, "getSpanAttributeKey", {
  307. enumerable: true,
  308. configurable: true,
  309. writable: true,
  310. value: (span, key) => {
  311. const attributes = span.attributes;
  312. return key in attributes && typeof attributes[key] === "string"
  313. ? attributes[key]
  314. : undefined;
  315. }
  316. });
  317. this.client = args?.client ?? new Client();
  318. this.debug =
  319. args?.debug ?? getEnvironmentVariable("OTEL_LOG_LEVEL") === "DEBUG";
  320. this.projectName = args?.projectName;
  321. this.logDebug("creating exporter", { tracingEnabled: isTracingEnabled() });
  322. }
  323. static getSettings(settings) {
  324. const { runId, runName, ...rest } = settings ?? {};
  325. const metadata = { ...rest?.metadata };
  326. if (runId != null)
  327. metadata[RUN_ID_METADATA_KEY.input] = runId;
  328. if (runName != null)
  329. metadata[RUN_NAME_METADATA_KEY.input] = runName;
  330. // attempt to obtain the run tree if used within a traceable function
  331. let defaultEnabled = settings?.isEnabled ?? isTracingEnabled();
  332. try {
  333. const runTree = getCurrentRunTree();
  334. const headers = runTree.toHeaders();
  335. metadata[TRACE_METADATA_KEY.input] = headers["langsmith-trace"];
  336. metadata[BAGGAGE_METADATA_KEY.input] = headers["baggage"];
  337. // honor the tracingEnabled flag if coming from traceable
  338. if (runTree.tracingEnabled != null) {
  339. defaultEnabled = runTree.tracingEnabled;
  340. }
  341. }
  342. catch {
  343. // pass
  344. }
  345. if (metadata[RUN_ID_METADATA_KEY.input] &&
  346. metadata[TRACE_METADATA_KEY.input]) {
  347. throw new Error("Cannot provide `runId` when used within traceable function.");
  348. }
  349. return { ...rest, isEnabled: rest.isEnabled ?? defaultEnabled, metadata };
  350. }
  351. /** @internal */
  352. parseInteropFromMetadata(span, parentSpan) {
  353. if (!this.isRootRun(span))
  354. return undefined;
  355. if (parentSpan?.name === "ai.toolCall") {
  356. return undefined;
  357. }
  358. const userTraceId = this.getSpanAttributeKey(span, RUN_ID_METADATA_KEY.output);
  359. const parentTrace = this.getSpanAttributeKey(span, TRACE_METADATA_KEY.output);
  360. if (parentTrace && userTraceId) {
  361. throw new Error(`Cannot provide both "${RUN_ID_METADATA_KEY.input}" and "${TRACE_METADATA_KEY.input}" metadata keys.`);
  362. }
  363. if (parentTrace) {
  364. const parentRunTree = RunTree.fromHeaders({
  365. "langsmith-trace": parentTrace,
  366. baggage: this.getSpanAttributeKey(span, BAGGAGE_METADATA_KEY.output) || "",
  367. });
  368. if (!parentRunTree)
  369. throw new Error("Unreachable code: empty parent run tree");
  370. return { type: "traceable", parentRunTree };
  371. }
  372. if (userTraceId)
  373. return { type: "user", userRunId: userTraceId };
  374. return undefined;
  375. }
  376. /** @internal */
  377. getRunCreate(span, projectName) {
  378. const asRunCreate = (rawConfig) => {
  379. const aiMetadata = Object.keys(span.attributes)
  380. .filter((key) => key.startsWith("ai.telemetry.metadata.") &&
  381. !RESERVED_METADATA_KEYS.includes(key))
  382. .reduce((acc, key) => {
  383. acc[key.slice("ai.telemetry.metadata.".length)] =
  384. span.attributes[key];
  385. return acc;
  386. }, {});
  387. if (("ai.telemetry.functionId" in span.attributes &&
  388. span.attributes["ai.telemetry.functionId"]) ||
  389. ("resource.name" in span.attributes && span.attributes["resource.name"])) {
  390. aiMetadata["functionId"] =
  391. span.attributes["ai.telemetry.functionId"] ||
  392. span.attributes["resource.name"];
  393. }
  394. const parsedStart = convertToTimestamp(span.startTime);
  395. const parsedEnd = convertToTimestamp(span.endTime);
  396. let name = rawConfig.name;
  397. // if user provided a custom name, only use it if it's the root
  398. if (this.isRootRun(span)) {
  399. name =
  400. this.getSpanAttributeKey(span, RUN_NAME_METADATA_KEY.output) || name;
  401. }
  402. const config = {
  403. ...rawConfig,
  404. name,
  405. extra: {
  406. ...rawConfig.extra,
  407. metadata: {
  408. ...rawConfig.extra?.metadata,
  409. ...aiMetadata,
  410. "ai.operationId": span.attributes["ai.operationId"],
  411. },
  412. },
  413. session_name: projectName ??
  414. this.projectName ??
  415. getLangSmithEnvironmentVariable("PROJECT") ??
  416. getLangSmithEnvironmentVariable("SESSION"),
  417. start_time: Math.min(parsedStart, parsedEnd),
  418. end_time: Math.max(parsedStart, parsedEnd),
  419. };
  420. return config;
  421. };
  422. switch (span.name) {
  423. case "ai.generateText.doGenerate":
  424. case "ai.generateText":
  425. case "ai.streamText.doStream":
  426. case "ai.streamText": {
  427. const inputs = (() => {
  428. if ("ai.prompt.messages" in span.attributes) {
  429. return {
  430. messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
  431. };
  432. }
  433. if ("ai.prompt" in span.attributes) {
  434. const input = tryJson(span.attributes["ai.prompt"]);
  435. if (typeof input === "object" &&
  436. input != null &&
  437. "messages" in input &&
  438. Array.isArray(input.messages)) {
  439. return {
  440. messages: input.messages.flatMap((i) => convertCoreToSmith(i)),
  441. };
  442. }
  443. return { input };
  444. }
  445. return {};
  446. })();
  447. const outputs = (() => {
  448. let result = undefined;
  449. if (span.attributes["ai.response.toolCalls"]) {
  450. let content = tryJson(span.attributes["ai.response.toolCalls"]);
  451. if (Array.isArray(content)) {
  452. content = content.map((i) => ({
  453. type: "tool-call",
  454. ...i,
  455. args: tryJson(i.args),
  456. }));
  457. }
  458. result = {
  459. llm_output: convertCoreToSmith({
  460. role: "assistant",
  461. content,
  462. }),
  463. };
  464. }
  465. else if (span.attributes["ai.response.text"]) {
  466. result = {
  467. llm_output: convertCoreToSmith({
  468. role: "assistant",
  469. content: span.attributes["ai.response.text"],
  470. }),
  471. };
  472. }
  473. if (span.attributes["ai.usage.completionTokens"]) {
  474. result ??= {};
  475. result.llm_output ??= {};
  476. result.llm_output.token_usage ??= {};
  477. result.llm_output.token_usage["completion_tokens"] =
  478. span.attributes["ai.usage.completionTokens"];
  479. }
  480. if (span.attributes["ai.usage.promptTokens"]) {
  481. result ??= {};
  482. result.llm_output ??= {};
  483. result.llm_output.token_usage ??= {};
  484. result.llm_output.token_usage["prompt_tokens"] =
  485. span.attributes["ai.usage.promptTokens"];
  486. }
  487. return result;
  488. })();
  489. const invocationParams = (() => {
  490. if ("ai.prompt.tools" in span.attributes) {
  491. return {
  492. tools: span.attributes["ai.prompt.tools"].flatMap((tool) => {
  493. try {
  494. return JSON.parse(tool);
  495. }
  496. catch {
  497. // pass
  498. }
  499. return [];
  500. }),
  501. };
  502. }
  503. return {};
  504. })();
  505. const events = [];
  506. const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
  507. if (firstChunkEvent) {
  508. events.push({
  509. name: "new_token",
  510. time: convertToTimestamp(firstChunkEvent.time),
  511. });
  512. }
  513. // TODO: add first_token_time
  514. return asRunCreate({
  515. run_type: "llm",
  516. name: span.attributes["ai.model.provider"],
  517. inputs,
  518. outputs,
  519. events,
  520. extra: {
  521. invocation_params: invocationParams,
  522. batch_size: 1,
  523. metadata: {
  524. ls_provider: span.attributes["ai.model.provider"]
  525. .split(".")
  526. .at(0),
  527. ls_model_type: span.attributes["ai.model.provider"]
  528. .split(".")
  529. .at(1),
  530. ls_model_name: span.attributes["ai.model.id"],
  531. },
  532. },
  533. });
  534. }
  535. case "ai.toolCall": {
  536. const args = tryJson(span.attributes["ai.toolCall.args"]);
  537. let inputs = { args };
  538. if (typeof args === "object" && args != null) {
  539. inputs = args;
  540. }
  541. const output = tryJson(span.attributes["ai.toolCall.result"]);
  542. let outputs = { output };
  543. if (typeof output === "object" && output != null) {
  544. outputs = output;
  545. }
  546. return asRunCreate({
  547. run_type: "tool",
  548. name: span.attributes["ai.toolCall.name"],
  549. inputs,
  550. outputs,
  551. });
  552. }
  553. case "ai.streamObject":
  554. case "ai.streamObject.doStream":
  555. case "ai.generateObject":
  556. case "ai.generateObject.doGenerate": {
  557. const inputs = (() => {
  558. if ("ai.prompt.messages" in span.attributes) {
  559. return {
  560. messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap((i) => convertCoreToSmith(i)),
  561. };
  562. }
  563. if ("ai.prompt" in span.attributes) {
  564. return { input: tryJson(span.attributes["ai.prompt"]) };
  565. }
  566. return {};
  567. })();
  568. const outputs = (() => {
  569. let result = undefined;
  570. if (span.attributes["ai.response.object"]) {
  571. result = {
  572. output: tryJson(span.attributes["ai.response.object"]),
  573. };
  574. }
  575. if (span.attributes["ai.usage.completionTokens"]) {
  576. result ??= {};
  577. result.llm_output ??= {};
  578. result.llm_output.token_usage ??= {};
  579. result.llm_output.token_usage["completion_tokens"] =
  580. span.attributes["ai.usage.completionTokens"];
  581. }
  582. if (span.attributes["ai.usage.promptTokens"]) {
  583. result ??= {};
  584. result.llm_output ??= {};
  585. result.llm_output.token_usage ??= {};
  586. result.llm_output.token_usage["prompt_tokens"] =
  587. +span.attributes["ai.usage.promptTokens"];
  588. }
  589. return result;
  590. })();
  591. const events = [];
  592. const firstChunkEvent = span.events.find((i) => i.name === "ai.stream.firstChunk");
  593. if (firstChunkEvent) {
  594. events.push({
  595. name: "new_token",
  596. time: convertToTimestamp(firstChunkEvent.time),
  597. });
  598. }
  599. return asRunCreate({
  600. run_type: "llm",
  601. name: span.attributes["ai.model.provider"],
  602. inputs,
  603. outputs,
  604. events,
  605. extra: {
  606. batch_size: 1,
  607. metadata: {
  608. ls_provider: span.attributes["ai.model.provider"]
  609. .split(".")
  610. .at(0),
  611. ls_model_type: span.attributes["ai.model.provider"]
  612. .split(".")
  613. .at(1),
  614. ls_model_name: span.attributes["ai.model.id"],
  615. },
  616. },
  617. });
  618. }
  619. case "ai.embed":
  620. case "ai.embed.doEmbed":
  621. case "ai.embedMany":
  622. case "ai.embedMany.doEmbed":
  623. default:
  624. return undefined;
  625. }
  626. }
  627. /** @internal */
  628. isRootRun(span) {
  629. switch (span.name) {
  630. case "ai.generateText":
  631. case "ai.streamText":
  632. case "ai.generateObject":
  633. case "ai.streamObject":
  634. case "ai.embed":
  635. case "ai.embedMany":
  636. return true;
  637. default:
  638. return false;
  639. }
  640. }
  641. _export(spans, resultCallback) {
  642. this.logDebug("exporting spans", spans);
  643. const typedSpans = spans
  644. .concat(Object.values(this.pendingSpans))
  645. .slice()
  646. // Parent spans should go before child spans in the final order,
  647. // but may have the same exact start time as their children.
  648. // They will end earlier, so break ties by end time.
  649. // TODO: Figure out why this happens.
  650. .sort((a, b) => sortByHr(a, b));
  651. for (const span of typedSpans) {
  652. const { traceId, spanId } = span.spanContext();
  653. const runId = uuid5(spanId, RUN_ID_NAMESPACE);
  654. let parentId = getParentSpanId(span);
  655. let parentRunId = parentId
  656. ? uuid5(parentId, RUN_ID_NAMESPACE)
  657. : undefined;
  658. let parentSpanInfo = parentRunId
  659. ? this.seenSpanInfo[parentRunId]
  660. : undefined;
  661. // Unrelated, untraced spans should behave as passthroughs from LangSmith's perspective.
  662. while (parentSpanInfo != null &&
  663. this.getRunCreate(parentSpanInfo.span) == null) {
  664. parentId = getParentSpanId(parentSpanInfo.span);
  665. if (parentId == null) {
  666. break;
  667. }
  668. parentRunId = parentId ? uuid5(parentId, RUN_ID_NAMESPACE) : undefined;
  669. parentSpanInfo = parentRunId
  670. ? this.seenSpanInfo[parentRunId]
  671. : undefined;
  672. }
  673. // Export may be called in any order, so we need to queue any spans with missing parents
  674. // for retry later in order to determine whether their parents are tool calls
  675. // and should not be reparented below.
  676. if (parentRunId !== undefined && parentSpanInfo === undefined) {
  677. this.pendingSpans[spanId] = span;
  678. continue;
  679. }
  680. else {
  681. delete this.pendingSpans[spanId];
  682. }
  683. this.traceByMap[traceId] ??= {
  684. childMap: {},
  685. nodeMap: {},
  686. relativeExecutionOrder: {},
  687. };
  688. const traceMap = this.traceByMap[traceId];
  689. traceMap.relativeExecutionOrder[parentRunId ?? ROOT] ??= -1;
  690. traceMap.relativeExecutionOrder[parentRunId ?? ROOT] += 1;
  691. const interop = this.parseInteropFromMetadata(span, parentSpanInfo?.span);
  692. const projectName = (interop?.type === "traceable"
  693. ? interop.parentRunTree.project_name
  694. : undefined) ?? parentSpanInfo?.projectName;
  695. const run = this.getRunCreate(span, projectName);
  696. traceMap.nodeMap[runId] ??= {
  697. id: runId,
  698. startTime: span.startTime,
  699. run,
  700. sent: false,
  701. interop,
  702. executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? ROOT],
  703. };
  704. if (this.seenSpanInfo[runId] == null) {
  705. this.seenSpanInfo[runId] = {
  706. span,
  707. dotOrder: joinDotOrder(parentSpanInfo?.dotOrder, getDotOrder(traceMap.nodeMap[runId])),
  708. projectName,
  709. sent: false,
  710. };
  711. }
  712. if (this.debug)
  713. console.log(`[${span.name}] ${runId}`, run);
  714. traceMap.childMap[parentRunId ?? ROOT] ??= [];
  715. traceMap.childMap[parentRunId ?? ROOT].push(traceMap.nodeMap[runId]);
  716. }
  717. const sampled = [];
  718. const actions = [];
  719. for (const traceId of Object.keys(this.traceByMap)) {
  720. const traceMap = this.traceByMap[traceId];
  721. const queue = Object.keys(traceMap.childMap)
  722. .map((runId) => {
  723. if (runId === ROOT) {
  724. return traceMap.childMap[runId];
  725. }
  726. return [];
  727. })
  728. .flat();
  729. const seen = new Set();
  730. while (queue.length) {
  731. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
  732. const task = queue.shift();
  733. if (seen.has(task.id))
  734. continue;
  735. let taskDotOrder = this.seenSpanInfo[task.id].dotOrder;
  736. if (!task.sent) {
  737. if (task.run != null) {
  738. if (task.interop?.type === "user") {
  739. actions.push({
  740. type: "rename",
  741. sourceRunId: task.id,
  742. targetRunId: task.interop.userRunId,
  743. });
  744. }
  745. if (task.interop?.type === "traceable") {
  746. actions.push({
  747. type: "reparent",
  748. runId: task.id,
  749. parentDotOrder: task.interop.parentRunTree.dotted_order,
  750. });
  751. }
  752. for (const action of actions) {
  753. if (action.type === "delete") {
  754. taskDotOrder = removeDotOrder(taskDotOrder, action.runId);
  755. }
  756. if (action.type === "reparent") {
  757. taskDotOrder = reparentDotOrder(taskDotOrder, action.runId, action.parentDotOrder);
  758. }
  759. if (action.type === "rename") {
  760. taskDotOrder = taskDotOrder.replace(action.sourceRunId, action.targetRunId);
  761. }
  762. }
  763. this.seenSpanInfo[task.id].dotOrder = taskDotOrder;
  764. if (!this.seenSpanInfo[task.id].sent) {
  765. sampled.push({
  766. ...task.run,
  767. ...getMutableRunCreate(taskDotOrder),
  768. });
  769. }
  770. this.seenSpanInfo[task.id].sent = true;
  771. }
  772. else {
  773. actions.push({ type: "delete", runId: task.id });
  774. }
  775. task.sent = true;
  776. }
  777. const children = traceMap.childMap[task.id] ?? [];
  778. queue.push(...children);
  779. }
  780. }
  781. this.logDebug(`sampled runs to be sent to LangSmith`, sampled);
  782. Promise.all(sampled.map((run) => this.client.createRun(run))).then(() => resultCallback({ code: 0 }), (error) => resultCallback({ code: 1, error }));
  783. }
  784. export(spans, resultCallback) {
  785. this._export(spans, (result) => {
  786. if (result.code === 0) {
  787. // Empty export to try flushing pending spans to rule out any trace order shenanigans
  788. this._export([], resultCallback);
  789. }
  790. else {
  791. resultCallback(result);
  792. }
  793. });
  794. }
  795. async shutdown() {
  796. // find nodes which are incomplete
  797. const incompleteNodes = Object.values(this.traceByMap).flatMap((trace) => Object.values(trace.nodeMap).filter((i) => !i.sent && i.run != null));
  798. this.logDebug("shutting down", { incompleteNodes });
  799. if (incompleteNodes.length > 0) {
  800. console.warn("Some incomplete nodes were found before shutdown and not sent to LangSmith.");
  801. }
  802. await this.forceFlush();
  803. }
  804. async forceFlush() {
  805. await new Promise((resolve) => {
  806. this.export([], resolve);
  807. });
  808. await this.client.awaitPendingTraceBatches();
  809. }
  810. logDebug(...args) {
  811. if (!this.debug)
  812. return;
  813. console.debug(`[${new Date().toISOString()}] [LangSmith]`, ...args);
  814. }
  815. }