reporter.js 1.6 KB

1234567891011121314151617181920212223242526272829303132333435
  1. /* eslint-disable import/no-extraneous-dependencies */
  2. import { DefaultReporter } from "@jest/reporters";
  3. import { printReporterTable } from "../utils/jestlike/reporter.js";
  4. class LangSmithEvalReporter extends DefaultReporter {
  5. async onTestResult(test, testResult, aggregatedResults) {
  6. if (testResult.failureMessage) {
  7. console.log(testResult.failureMessage);
  8. }
  9. const groupedTestResults = testResult.testResults.reduce((groups, testResult) => {
  10. const ancestorTitle = testResult.ancestorTitles.join(" > ");
  11. if (groups[ancestorTitle] === undefined) {
  12. groups[ancestorTitle] = [];
  13. }
  14. groups[ancestorTitle].push(testResult);
  15. return groups;
  16. }, {});
  17. try {
  18. for (const testGroupName of Object.keys(groupedTestResults)) {
  19. const resultGroup = groupedTestResults[testGroupName];
  20. const unskippedTests = resultGroup.filter((result) => result.status !== "pending");
  21. const overallResult = unskippedTests.length === 0
  22. ? "skip"
  23. : unskippedTests.every((result) => result.status === "passed")
  24. ? "pass"
  25. : "fail";
  26. await printReporterTable(testGroupName, resultGroup, overallResult);
  27. }
  28. }
  29. catch (e) {
  30. console.log("Failed to display LangSmith eval results:", e.message);
  31. super.onTestResult(test, testResult, aggregatedResults);
  32. }
  33. }
  34. }
  35. export default LangSmithEvalReporter;