lexer.d.ts 1.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445
  1. import { Token } from './ast';
  2. import type { Source } from './source';
  3. import { TokenKind } from './tokenKind';
  4. /**
  5. * Given a Source object, creates a Lexer for that source.
  6. * A Lexer is a stateful stream generator in that every time
  7. * it is advanced, it returns the next token in the Source. Assuming the
  8. * source lexes, the final Token emitted by the lexer will be of kind
  9. * EOF, after which the lexer will repeatedly return the same EOF token
  10. * whenever called.
  11. */
  12. export declare class Lexer {
  13. source: Source;
  14. /**
  15. * The previously focused non-ignored token.
  16. */
  17. lastToken: Token;
  18. /**
  19. * The currently focused non-ignored token.
  20. */
  21. token: Token;
  22. /**
  23. * The (1-indexed) line containing the current token.
  24. */
  25. line: number;
  26. /**
  27. * The character offset at which the current line begins.
  28. */
  29. lineStart: number;
  30. constructor(source: Source);
  31. get [Symbol.toStringTag](): string;
  32. /**
  33. * Advances the token stream to the next non-ignored token.
  34. */
  35. advance(): Token;
  36. /**
  37. * Looks ahead and returns the next non-ignored token, but does not change
  38. * the state of Lexer.
  39. */
  40. lookahead(): Token;
  41. }
  42. /**
  43. * @internal
  44. */
  45. export declare function isPunctuatorTokenKind(kind: TokenKind): boolean;