__init__.py 1.8 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859
  1. # Copyright 2020 The HuggingFace Team. All rights reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from typing import TYPE_CHECKING
  15. from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available
  16. _import_structure = {}
  17. try:
  18. if not is_sentencepiece_available():
  19. raise OptionalDependencyNotAvailable()
  20. except OptionalDependencyNotAvailable:
  21. pass
  22. else:
  23. _import_structure["tokenization_cpm"] = ["CpmTokenizer"]
  24. try:
  25. if not is_tokenizers_available():
  26. raise OptionalDependencyNotAvailable()
  27. except OptionalDependencyNotAvailable:
  28. pass
  29. else:
  30. _import_structure["tokenization_cpm_fast"] = ["CpmTokenizerFast"]
  31. if TYPE_CHECKING:
  32. try:
  33. if not is_sentencepiece_available():
  34. raise OptionalDependencyNotAvailable()
  35. except OptionalDependencyNotAvailable:
  36. pass
  37. else:
  38. from .tokenization_cpm import CpmTokenizer
  39. try:
  40. if not is_tokenizers_available():
  41. raise OptionalDependencyNotAvailable()
  42. except OptionalDependencyNotAvailable:
  43. pass
  44. else:
  45. from .tokenization_cpm_fast import CpmTokenizerFast
  46. else:
  47. import sys
  48. sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)