- # Generated content DO NOT EDIT
- from .. import pre_tokenizers
- PreTokenizer = pre_tokenizers.PreTokenizer
- BertPreTokenizer = pre_tokenizers.BertPreTokenizer
- ByteLevel = pre_tokenizers.ByteLevel
- CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
- Digits = pre_tokenizers.Digits
- Metaspace = pre_tokenizers.Metaspace
- Punctuation = pre_tokenizers.Punctuation
- Sequence = pre_tokenizers.Sequence
- Split = pre_tokenizers.Split
- UnicodeScripts = pre_tokenizers.UnicodeScripts
- Whitespace = pre_tokenizers.Whitespace
- WhitespaceSplit = pre_tokenizers.WhitespaceSplit
|
