tiktoken.js 702 B

1234567891011121314151617181920
  1. import { Tiktoken, getEncodingNameForModel, } from "js-tiktoken/lite";
  2. import { AsyncCaller } from "./async_caller.js";
  3. const cache = {};
  4. const caller = /* #__PURE__ */ new AsyncCaller({});
  5. export async function getEncoding(encoding) {
  6. if (!(encoding in cache)) {
  7. cache[encoding] = caller
  8. .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
  9. .then((res) => res.json())
  10. .then((data) => new Tiktoken(data))
  11. .catch((e) => {
  12. delete cache[encoding];
  13. throw e;
  14. });
  15. }
  16. return await cache[encoding];
  17. }
  18. export async function encodingForModel(model) {
  19. return getEncoding(getEncodingNameForModel(model));
  20. }