tiktoken.cjs 920 B

12345678910111213141516171819202122232425
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. exports.encodingForModel = exports.getEncoding = void 0;
  4. const lite_1 = require("js-tiktoken/lite");
  5. const async_caller_js_1 = require("./async_caller.cjs");
  6. const cache = {};
  7. const caller = /* #__PURE__ */ new async_caller_js_1.AsyncCaller({});
  8. async function getEncoding(encoding) {
  9. if (!(encoding in cache)) {
  10. cache[encoding] = caller
  11. .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
  12. .then((res) => res.json())
  13. .then((data) => new lite_1.Tiktoken(data))
  14. .catch((e) => {
  15. delete cache[encoding];
  16. throw e;
  17. });
  18. }
  19. return await cache[encoding];
  20. }
  21. exports.getEncoding = getEncoding;
  22. async function encodingForModel(model) {
  23. return getEncoding((0, lite_1.getEncodingNameForModel)(model));
  24. }
  25. exports.encodingForModel = encodingForModel;