index.cjs 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651
  1. "use strict";
  2. /* eslint-disable import/no-extraneous-dependencies */
  3. /* eslint-disable @typescript-eslint/no-namespace */
  4. var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
  5. if (k2 === undefined) k2 = k;
  6. var desc = Object.getOwnPropertyDescriptor(m, k);
  7. if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
  8. desc = { enumerable: true, get: function() { return m[k]; } };
  9. }
  10. Object.defineProperty(o, k2, desc);
  11. }) : (function(o, m, k, k2) {
  12. if (k2 === undefined) k2 = k;
  13. o[k2] = m[k];
  14. }));
  15. var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
  16. Object.defineProperty(o, "default", { enumerable: true, value: v });
  17. }) : function(o, v) {
  18. o["default"] = v;
  19. });
  20. var __importStar = (this && this.__importStar) || (function () {
  21. var ownKeys = function(o) {
  22. ownKeys = Object.getOwnPropertyNames || function (o) {
  23. var ar = [];
  24. for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
  25. return ar;
  26. };
  27. return ownKeys(o);
  28. };
  29. return function (mod) {
  30. if (mod && mod.__esModule) return mod;
  31. var result = {};
  32. if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
  33. __setModuleDefault(result, mod);
  34. return result;
  35. };
  36. })();
  37. var __exportStar = (this && this.__exportStar) || function(m, exports) {
  38. for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
  39. };
  40. var __importDefault = (this && this.__importDefault) || function (mod) {
  41. return (mod && mod.__esModule) ? mod : { "default": mod };
  42. };
  43. Object.defineProperty(exports, "__esModule", { value: true });
  44. exports.wrapEvaluator = void 0;
  45. exports.logFeedback = logFeedback;
  46. exports.logOutputs = logOutputs;
  47. exports._objectHash = _objectHash;
  48. exports.generateWrapperFromJestlikeMethods = generateWrapperFromJestlikeMethods;
  49. exports.isInTestContext = isInTestContext;
  50. const crypto_1 = __importDefault(require("crypto"));
  51. const uuid_1 = require("uuid");
  52. const os = __importStar(require("node:os"));
  53. const path = __importStar(require("node:path"));
  54. const fs = __importStar(require("node:fs/promises"));
  55. const child_process_1 = require("child_process");
  56. const traceable_js_1 = require("../../traceable.cjs");
  57. const _random_name_js_1 = require("../../evaluation/_random_name.cjs");
  58. const matchers_js_1 = require("./matchers.cjs");
  59. const globals_js_1 = require("./globals.cjs");
  60. const chain_js_1 = require("./vendor/chain.cjs");
  61. const env_js_1 = require("../env.cjs");
  62. const constants_js_1 = require("./constants.cjs");
  63. function logFeedback(feedback, config) {
  64. const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  65. if (context === undefined) {
  66. throw new Error([
  67. `Could not retrieve test context. Make sure your logFeedback call is nested within a "ls.describe()" block.`,
  68. `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
  69. ].join("\n"));
  70. }
  71. if (context.currentExample === undefined) {
  72. throw new Error([
  73. `Could not retrieve current example. Make sure your logFeedback call is nested within a "ls.test()" block.`,
  74. `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
  75. ].join("\n"));
  76. }
  77. (0, globals_js_1._logTestFeedback)({
  78. ...config,
  79. exampleId: context.currentExample.id,
  80. feedback: feedback,
  81. context,
  82. runTree: context.testRootRunTree,
  83. client: context.client,
  84. });
  85. }
  86. function logOutputs(output) {
  87. const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  88. if (context === undefined) {
  89. throw new Error(`Could not retrieve test context. Make sure your logFeedback call is nested within a "ls.describe()" block.`);
  90. }
  91. if (context.currentExample === undefined ||
  92. context.setLoggedOutput === undefined) {
  93. throw new Error([
  94. `Could not retrieve current example. Make sure your logFeedback call is nested within a "ls.test()" block.`,
  95. `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
  96. ].join("\n"));
  97. }
  98. context.setLoggedOutput(output);
  99. }
  100. function _objectHash(obj, depth = 0) {
  101. // Prevent infinite recursion
  102. if (depth > 50) {
  103. throw new Error("Object is too deep to check equality for serialization. Please use a simpler example.");
  104. }
  105. if (Array.isArray(obj)) {
  106. const arrayHash = obj.map((item) => _objectHash(item, depth + 1)).join(",");
  107. return crypto_1.default.createHash("sha256").update(arrayHash).digest("hex");
  108. }
  109. if (obj && typeof obj === "object") {
  110. const sortedHash = Object.keys(obj)
  111. .sort()
  112. .map((key) => `${key}:${_objectHash(obj[key], depth + 1)}`)
  113. .join(",");
  114. return crypto_1.default.createHash("sha256").update(sortedHash).digest("hex");
  115. }
  116. return (crypto_1.default
  117. .createHash("sha256")
  118. // Treat null and undefined as equal for serialization purposes
  119. .update(JSON.stringify(obj ?? null))
  120. .digest("hex"));
  121. }
  122. function generateWrapperFromJestlikeMethods(methods, testRunnerName) {
  123. const { expect, test, describe, beforeAll, afterAll } = methods;
  124. async function _createProject(client, datasetId, projectConfig) {
  125. // Create the project, updating the experimentName until we find a unique one.
  126. let project;
  127. let experimentName = (0, _random_name_js_1.randomName)();
  128. for (let i = 0; i < 10; i++) {
  129. try {
  130. project = await client.createProject({
  131. projectName: experimentName,
  132. ...projectConfig,
  133. referenceDatasetId: datasetId,
  134. });
  135. return project;
  136. }
  137. catch (e) {
  138. // Naming collision
  139. if (e?.name === "LangSmithConflictError") {
  140. const ent = (0, uuid_1.v4)().slice(0, 6);
  141. experimentName = `${experimentName}-${ent}`;
  142. }
  143. else {
  144. throw e;
  145. }
  146. }
  147. }
  148. throw new Error("Could not generate a unique experiment name within 10 attempts." +
  149. " Please try again.");
  150. }
  151. const datasetSetupInfo = new Map();
  152. function getExampleId(datasetId, inputs, outputs) {
  153. const identifier = JSON.stringify({
  154. datasetId,
  155. inputsHash: _objectHash(inputs),
  156. outputsHash: _objectHash(outputs ?? {}),
  157. });
  158. return (0, uuid_1.v5)(identifier, constants_js_1.UUID5_NAMESPACE);
  159. }
  160. async function syncExample(params) {
  161. const { client, exampleId, inputs, outputs, metadata, createdAt, datasetId, } = params;
  162. let example;
  163. try {
  164. example = await client.readExample(exampleId);
  165. if (_objectHash(example.inputs) !== _objectHash(inputs) ||
  166. _objectHash(example.outputs ?? {}) !== _objectHash(outputs ?? {}) ||
  167. example.dataset_id !== datasetId) {
  168. await client.updateExample(exampleId, {
  169. inputs,
  170. outputs,
  171. metadata,
  172. dataset_id: datasetId,
  173. });
  174. }
  175. }
  176. catch (e) {
  177. if (e.message.includes("not found")) {
  178. example = await client.createExample(inputs, outputs, {
  179. exampleId,
  180. datasetId,
  181. createdAt: new Date(createdAt ?? new Date()),
  182. metadata,
  183. });
  184. }
  185. else {
  186. throw e;
  187. }
  188. }
  189. return example;
  190. }
  191. async function runDatasetSetup(context) {
  192. const { client: testClient, suiteName: datasetName, projectConfig, } = context;
  193. let storageValue;
  194. if (!(0, globals_js_1.trackingEnabled)(context)) {
  195. storageValue = {
  196. createdAt: new Date().toISOString(),
  197. };
  198. }
  199. else {
  200. let dataset;
  201. try {
  202. dataset = await testClient.readDataset({
  203. datasetName,
  204. });
  205. }
  206. catch (e) {
  207. if (e.message.includes("not found")) {
  208. dataset = await testClient.createDataset(datasetName, {
  209. description: `Dataset for unit tests created on ${new Date().toISOString()}`,
  210. metadata: { __ls_runner: testRunnerName },
  211. });
  212. }
  213. else {
  214. throw e;
  215. }
  216. }
  217. const project = await _createProject(testClient, dataset.id, projectConfig);
  218. const datasetUrl = await testClient.getDatasetUrl({
  219. datasetId: dataset.id,
  220. });
  221. const experimentUrl = `${datasetUrl}/compare?selectedSessions=${project.id}`;
  222. console.log(`[LANGSMITH]: Experiment starting for dataset "${datasetName}"!\n[LANGSMITH]: View results at ${experimentUrl}`);
  223. storageValue = {
  224. dataset,
  225. project,
  226. client: testClient,
  227. experimentUrl,
  228. };
  229. }
  230. return storageValue;
  231. }
  232. function wrapDescribeMethod(method, methodName) {
  233. if ((0, env_js_1.isJsDom)()) {
  234. console.error(`[LANGSMITH]: You seem to be using a jsdom environment. This is not supported and you may experience unexpected behavior. Please set the "environment" or "testEnvironment" field in your test config file to "node".`);
  235. }
  236. return function (testSuiteName, fn, experimentConfig) {
  237. if (typeof method !== "function") {
  238. throw new Error(`"${methodName}" is not supported by your test runner.`);
  239. }
  240. if (globals_js_1.testWrapperAsyncLocalStorageInstance.getStore() !== undefined) {
  241. throw new Error([
  242. `You seem to be nesting an ls.describe block named "${testSuiteName}" inside another ls.describe block.`,
  243. "This is not supported because each ls.describe block corresponds to a LangSmith dataset.",
  244. "To logically group tests, nest the native Jest or Vitest describe methods instead.",
  245. ].join("\n"));
  246. }
  247. const client = experimentConfig?.client ?? globals_js_1.DEFAULT_TEST_CLIENT;
  248. const suiteName = experimentConfig?.testSuiteName ?? testSuiteName;
  249. let setupPromiseResolver;
  250. const setupPromise = new Promise((resolve) => {
  251. setupPromiseResolver = resolve;
  252. });
  253. return method(suiteName, () => {
  254. const startTime = new Date();
  255. const suiteUuid = (0, uuid_1.v4)();
  256. const environment = experimentConfig?.metadata?.ENVIRONMENT ??
  257. (0, env_js_1.getEnvironmentVariable)("ENVIRONMENT");
  258. const nodeEnv = experimentConfig?.metadata?.NODE_ENV ??
  259. (0, env_js_1.getEnvironmentVariable)("NODE_ENV");
  260. const langsmithEnvironment = experimentConfig?.metadata?.LANGSMITH_ENVIRONMENT ??
  261. (0, env_js_1.getEnvironmentVariable)("LANGSMITH_ENVIRONMENT");
  262. // eslint-disable-next-line @typescript-eslint/no-explicit-any
  263. const suiteMetadata = {
  264. ...experimentConfig?.metadata,
  265. __ls_runner: testRunnerName,
  266. };
  267. if (environment !== undefined) {
  268. suiteMetadata.ENVIRONMENT = environment;
  269. }
  270. if (nodeEnv !== undefined) {
  271. suiteMetadata.NODE_ENV = nodeEnv;
  272. }
  273. if (langsmithEnvironment !== undefined) {
  274. suiteMetadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
  275. }
  276. const context = {
  277. suiteUuid,
  278. suiteName,
  279. client,
  280. createdAt: new Date().toISOString(),
  281. projectConfig: {
  282. ...experimentConfig,
  283. metadata: suiteMetadata,
  284. },
  285. enableTestTracking: experimentConfig?.enableTestTracking,
  286. setupPromise,
  287. };
  288. beforeAll(async () => {
  289. const storageValue = await runDatasetSetup(context);
  290. datasetSetupInfo.set(suiteUuid, storageValue);
  291. setupPromiseResolver();
  292. });
  293. afterAll(async () => {
  294. await Promise.all([
  295. client.awaitPendingTraceBatches(),
  296. ...globals_js_1.syncExamplePromises.values(),
  297. ...globals_js_1.evaluatorLogFeedbackPromises.values(),
  298. ]);
  299. if (!(0, globals_js_1.trackingEnabled)(context)) {
  300. return;
  301. }
  302. const examples = [...globals_js_1.syncExamplePromises.values()];
  303. if (examples.length === 0) {
  304. return;
  305. }
  306. const endTime = new Date();
  307. let branch;
  308. let commit;
  309. let dirty;
  310. try {
  311. branch = (0, child_process_1.execSync)("git rev-parse --abbrev-ref HEAD")
  312. .toString()
  313. .trim();
  314. commit = (0, child_process_1.execSync)("git rev-parse HEAD").toString().trim();
  315. dirty = (0, child_process_1.execSync)("git status --porcelain").toString().trim() !== "";
  316. }
  317. catch {
  318. return;
  319. }
  320. if (branch === undefined || commit === undefined) {
  321. return;
  322. }
  323. try {
  324. let finalModifiedAt = examples.reduce((latestModifiedAt, example) => {
  325. if (new Date(latestModifiedAt).getTime() >
  326. new Date(example.modified_at).getTime()) {
  327. return latestModifiedAt;
  328. }
  329. else {
  330. return example.modified_at;
  331. }
  332. }, examples[0].modified_at);
  333. if (new Date(finalModifiedAt).getTime() < startTime.getTime()) {
  334. finalModifiedAt = endTime.toISOString();
  335. }
  336. const datasetInfo = datasetSetupInfo.get(suiteUuid);
  337. await client.updateProject(datasetInfo.project.id, {
  338. metadata: {
  339. ...suiteMetadata,
  340. commit,
  341. branch,
  342. dirty,
  343. },
  344. });
  345. await client.updateDatasetTag({
  346. datasetId: datasetInfo.dataset.id,
  347. asOf: finalModifiedAt,
  348. tag: `git:commit:${commit}`,
  349. });
  350. }
  351. catch (e) {
  352. console.error(e);
  353. return;
  354. }
  355. });
  356. /**
  357. * We cannot rely on setting AsyncLocalStorage in beforeAll or beforeEach,
  358. * due to https://github.com/jestjs/jest/issues/13653 and needing to use
  359. * the janky .enterWith.
  360. *
  361. * We also cannot do async setup in describe due to Jest restrictions.
  362. * However, .run without asynchronous logic works.
  363. *
  364. * We really just need a way to pass suiteUuid as global state to inner tests
  365. * that can handle concurrently running test suites. If we drop the
  366. * concurrency requirement, we can remove this hack.
  367. */
  368. void globals_js_1.testWrapperAsyncLocalStorageInstance.run(context, fn);
  369. });
  370. };
  371. }
  372. const lsDescribe = Object.assign(wrapDescribeMethod(describe, "describe"), {
  373. only: wrapDescribeMethod(describe.only, "describe.only"),
  374. skip: wrapDescribeMethod(describe.skip, "describe.skip"),
  375. concurrent: wrapDescribeMethod(describe.concurrent, "describe.concurrent"),
  376. });
  377. function wrapTestMethod(method) {
  378. return function (name, lsParams, testFn, timeout) {
  379. // Due to https://github.com/jestjs/jest/issues/13653,
  380. // we must access the local store value here before
  381. // doing anything async.
  382. const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  383. if (context !== undefined &&
  384. lsParams.config?.enableTestTracking !== undefined) {
  385. context.enableTestTracking = lsParams.config.enableTestTracking;
  386. }
  387. const { config, inputs, referenceOutputs, ...rest } = lsParams;
  388. const totalRuns = config?.iterations ?? 1;
  389. for (let i = 0; i < totalRuns; i += 1) {
  390. const testUuid = (0, uuid_1.v4)().replace(/-/g, "").slice(0, 13);
  391. // Jest will not group tests under the same "describe" group if you await the test and
  392. // total runs is greater than 1.
  393. const resultsPath = path.join(os.tmpdir(), "langsmith_test_results", `${testUuid}.json`);
  394. void method(`${name}${totalRuns > 1 ? `, run ${i}` : ""}${constants_js_1.TEST_ID_DELIMITER}${testUuid}`, async (...args) => {
  395. // Jest will magically introspect args and pass a "done" callback if
  396. // we use a non-spread parameter. To obtain and pass Vitest test context
  397. // through into the test function, we must therefore refer to Vitest
  398. // args using this signature
  399. const jestlikeArgs = args[0];
  400. if (context === undefined) {
  401. throw new Error([
  402. `Could not retrieve test context.`,
  403. `Please make sure you have tracing enabled and you are wrapping all of your test cases in an "ls.describe()" function.`,
  404. `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
  405. ].join("\n"));
  406. }
  407. // Jest .concurrent is super buggy and doesn't wait for beforeAll to complete
  408. // before running test functions, so we need to wait for the setup promise
  409. // to resolve before we can continue.
  410. // Seee https://github.com/jestjs/jest/issues/4281
  411. await context.setupPromise;
  412. if (!datasetSetupInfo.get(context.suiteUuid)) {
  413. throw new Error("Dataset failed to initialize. Please check your LangSmith environment variables.");
  414. }
  415. const { dataset, createdAt, project, client, experimentUrl } = datasetSetupInfo.get(context.suiteUuid);
  416. const testInput = inputs;
  417. const testOutput = referenceOutputs ?? {};
  418. const testFeedback = [];
  419. const onFeedbackLogged = (feedback) => testFeedback.push(feedback);
  420. let loggedOutput;
  421. const setLoggedOutput = (value) => {
  422. if (loggedOutput !== undefined) {
  423. console.warn(`[WARN]: New "logOutputs()" call will override output set by previous "logOutputs()" call.`);
  424. }
  425. loggedOutput = value;
  426. };
  427. let exampleId;
  428. const runTestFn = async () => {
  429. let testContext = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  430. if (testContext === undefined) {
  431. throw new Error("Could not identify test context. Please contact us for help.");
  432. }
  433. return globals_js_1.testWrapperAsyncLocalStorageInstance.run({
  434. ...testContext,
  435. testRootRunTree: (0, globals_js_1.trackingEnabled)(testContext)
  436. ? (0, traceable_js_1.getCurrentRunTree)()
  437. : undefined,
  438. }, async () => {
  439. testContext = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  440. if (testContext === undefined) {
  441. throw new Error("Could not identify test context after setting test root run tree. Please contact us for help.");
  442. }
  443. try {
  444. const res = await testFn(Object.assign(typeof jestlikeArgs === "object" && jestlikeArgs != null
  445. ? jestlikeArgs
  446. : {}, {
  447. ...rest,
  448. inputs: testInput,
  449. referenceOutputs: testOutput,
  450. }));
  451. (0, globals_js_1._logTestFeedback)({
  452. exampleId,
  453. feedback: { key: "pass", score: true },
  454. context: testContext,
  455. runTree: testContext.testRootRunTree,
  456. client: testContext.client,
  457. });
  458. if (res != null) {
  459. if (loggedOutput !== undefined) {
  460. console.warn(`[WARN]: Returned value from test function will override output set by previous "logOutputs()" call.`);
  461. }
  462. loggedOutput =
  463. typeof res === "object"
  464. ? res
  465. : { result: res };
  466. }
  467. return loggedOutput;
  468. }
  469. catch (e) {
  470. (0, globals_js_1._logTestFeedback)({
  471. exampleId,
  472. feedback: { key: "pass", score: false },
  473. context: testContext,
  474. runTree: testContext.testRootRunTree,
  475. client: testContext.client,
  476. });
  477. const rawError = e;
  478. const strippedErrorMessage = e.message.replace(constants_js_1.STRIP_ANSI_REGEX, "");
  479. const langsmithFriendlyError = new Error(strippedErrorMessage);
  480. langsmithFriendlyError.rawJestError = rawError;
  481. throw langsmithFriendlyError;
  482. }
  483. });
  484. };
  485. try {
  486. if ((0, globals_js_1.trackingEnabled)(context)) {
  487. const missingFields = [];
  488. if (dataset === undefined) {
  489. missingFields.push("dataset");
  490. }
  491. if (project === undefined) {
  492. missingFields.push("project");
  493. }
  494. if (client === undefined) {
  495. missingFields.push("client");
  496. }
  497. if (missingFields.length > 0) {
  498. throw new Error(`Failed to initialize test tracking: Could not identify ${missingFields
  499. .map((field) => `"${field}"`)
  500. .join(", ")} while syncing to LangSmith. Please contact us for help.`);
  501. }
  502. exampleId = getExampleId(dataset.id, inputs, referenceOutputs);
  503. // TODO: Create or update the example in the background
  504. // Currently run end time has to be after example modified time
  505. // for examples to render properly, so we must modify the example
  506. // first before running the test.
  507. if (globals_js_1.syncExamplePromises.get(exampleId) === undefined) {
  508. globals_js_1.syncExamplePromises.set(exampleId, await syncExample({
  509. client,
  510. exampleId,
  511. datasetId: dataset.id,
  512. inputs,
  513. outputs: referenceOutputs ?? {},
  514. metadata: {},
  515. createdAt,
  516. }));
  517. }
  518. const traceableOptions = {
  519. reference_example_id: exampleId,
  520. project_name: project.name,
  521. metadata: {
  522. ...config?.metadata,
  523. },
  524. client,
  525. tracingEnabled: true,
  526. name,
  527. };
  528. // Pass inputs into traceable so tracing works correctly but
  529. // provide both to the user-defined test function
  530. const tracedFunction = (0, traceable_js_1.traceable)(async () => {
  531. return globals_js_1.testWrapperAsyncLocalStorageInstance.run({
  532. ...context,
  533. currentExample: {
  534. inputs,
  535. outputs: referenceOutputs,
  536. id: exampleId,
  537. },
  538. setLoggedOutput,
  539. onFeedbackLogged,
  540. }, runTestFn);
  541. }, {
  542. ...traceableOptions,
  543. ...config,
  544. });
  545. try {
  546. await tracedFunction(testInput);
  547. }
  548. catch (e) {
  549. // Extract raw Jest error from LangSmith formatted one and throw
  550. if (e.rawJestError !== undefined) {
  551. throw e.rawJestError;
  552. }
  553. throw e;
  554. }
  555. }
  556. else {
  557. try {
  558. await globals_js_1.testWrapperAsyncLocalStorageInstance.run({
  559. ...context,
  560. currentExample: {
  561. inputs: testInput,
  562. outputs: testOutput,
  563. },
  564. setLoggedOutput,
  565. onFeedbackLogged,
  566. }, runTestFn);
  567. }
  568. catch (e) {
  569. // Extract raw Jest error from LangSmith formatted one and throw
  570. if (e.rawJestError !== undefined) {
  571. throw e.rawJestError;
  572. }
  573. throw e;
  574. }
  575. }
  576. }
  577. finally {
  578. await fs.mkdir(path.dirname(resultsPath), { recursive: true });
  579. await fs.writeFile(resultsPath, JSON.stringify({
  580. inputs,
  581. referenceOutputs,
  582. outputs: loggedOutput,
  583. feedback: testFeedback,
  584. experimentUrl,
  585. }));
  586. }
  587. }, timeout ?? constants_js_1.DEFAULT_TEST_TIMEOUT);
  588. }
  589. };
  590. }
  591. function createEachMethod(method) {
  592. function eachMethod(table, config) {
  593. const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  594. if (context === undefined) {
  595. throw new Error([
  596. `Could not retrieve test context. Make sure your test is nested within a "ls.describe()" block.`,
  597. `See this page for more information: https://docs.smith.langchain.com/evaluation/how_to_guides/vitest_jest`,
  598. ].join("\n"));
  599. }
  600. return function (name, fn, timeout) {
  601. for (let i = 0; i < table.length; i += 1) {
  602. const example = table[i];
  603. wrapTestMethod(method)(`${name}, item ${i}`, {
  604. ...example,
  605. inputs: example.inputs,
  606. referenceOutputs: example.referenceOutputs,
  607. config,
  608. }, fn, timeout);
  609. }
  610. };
  611. }
  612. return eachMethod;
  613. }
  614. // Roughly mirrors: https://jestjs.io/docs/api#methods
  615. const concurrentMethod = Object.assign(wrapTestMethod(test.concurrent), {
  616. each: createEachMethod(test.concurrent),
  617. only: Object.assign(wrapTestMethod(test.concurrent.only), {
  618. each: createEachMethod(test.concurrent.only),
  619. }),
  620. skip: Object.assign(wrapTestMethod(test.concurrent.skip), {
  621. each: createEachMethod(test.concurrent.skip),
  622. }),
  623. });
  624. const lsTest = Object.assign(wrapTestMethod(test), {
  625. only: Object.assign(wrapTestMethod(test.only), {
  626. each: createEachMethod(test.only),
  627. }),
  628. skip: Object.assign(wrapTestMethod(test.skip), {
  629. each: createEachMethod(test.skip),
  630. }),
  631. concurrent: concurrentMethod,
  632. each: createEachMethod(test),
  633. });
  634. const wrappedExpect = (0, chain_js_1.wrapExpect)(expect);
  635. return {
  636. test: lsTest,
  637. it: lsTest,
  638. describe: lsDescribe,
  639. expect: wrappedExpect,
  640. toBeRelativeCloseTo: matchers_js_1.toBeRelativeCloseTo,
  641. toBeAbsoluteCloseTo: matchers_js_1.toBeAbsoluteCloseTo,
  642. toBeSemanticCloseTo: matchers_js_1.toBeSemanticCloseTo,
  643. };
  644. }
  645. function isInTestContext() {
  646. const context = globals_js_1.testWrapperAsyncLocalStorageInstance.getStore();
  647. return context !== undefined;
  648. }
  649. var evaluatedBy_js_1 = require("./vendor/evaluatedBy.cjs");
  650. Object.defineProperty(exports, "wrapEvaluator", { enumerable: true, get: function () { return evaluatedBy_js_1.wrapEvaluator; } });
  651. __exportStar(require("./types.cjs"), exports);