_known_annotated_metadata.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. from __future__ import annotations
  2. from collections import defaultdict
  3. from copy import copy
  4. from functools import lru_cache, partial
  5. from typing import TYPE_CHECKING, Any, Iterable
  6. from pydantic_core import CoreSchema, PydanticCustomError, ValidationError, to_jsonable_python
  7. from pydantic_core import core_schema as cs
  8. from ._fields import PydanticMetadata
  9. from ._import_utils import import_cached_field_info
  10. if TYPE_CHECKING:
  11. pass
  12. STRICT = {'strict'}
  13. FAIL_FAST = {'fail_fast'}
  14. LENGTH_CONSTRAINTS = {'min_length', 'max_length'}
  15. INEQUALITY = {'le', 'ge', 'lt', 'gt'}
  16. NUMERIC_CONSTRAINTS = {'multiple_of', *INEQUALITY}
  17. ALLOW_INF_NAN = {'allow_inf_nan'}
  18. STR_CONSTRAINTS = {
  19. *LENGTH_CONSTRAINTS,
  20. *STRICT,
  21. 'strip_whitespace',
  22. 'to_lower',
  23. 'to_upper',
  24. 'pattern',
  25. 'coerce_numbers_to_str',
  26. }
  27. BYTES_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  28. LIST_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  29. TUPLE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  30. SET_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  31. DICT_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  32. GENERATOR_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  33. SEQUENCE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *FAIL_FAST}
  34. FLOAT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT}
  35. DECIMAL_CONSTRAINTS = {'max_digits', 'decimal_places', *FLOAT_CONSTRAINTS}
  36. INT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT}
  37. BOOL_CONSTRAINTS = STRICT
  38. UUID_CONSTRAINTS = STRICT
  39. DATE_TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  40. TIMEDELTA_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  41. TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  42. LAX_OR_STRICT_CONSTRAINTS = STRICT
  43. ENUM_CONSTRAINTS = STRICT
  44. COMPLEX_CONSTRAINTS = STRICT
  45. UNION_CONSTRAINTS = {'union_mode'}
  46. URL_CONSTRAINTS = {
  47. 'max_length',
  48. 'allowed_schemes',
  49. 'host_required',
  50. 'default_host',
  51. 'default_port',
  52. 'default_path',
  53. }
  54. TEXT_SCHEMA_TYPES = ('str', 'bytes', 'url', 'multi-host-url')
  55. SEQUENCE_SCHEMA_TYPES = ('list', 'tuple', 'set', 'frozenset', 'generator', *TEXT_SCHEMA_TYPES)
  56. NUMERIC_SCHEMA_TYPES = ('float', 'int', 'date', 'time', 'timedelta', 'datetime')
  57. CONSTRAINTS_TO_ALLOWED_SCHEMAS: dict[str, set[str]] = defaultdict(set)
  58. constraint_schema_pairings: list[tuple[set[str], tuple[str, ...]]] = [
  59. (STR_CONSTRAINTS, TEXT_SCHEMA_TYPES),
  60. (BYTES_CONSTRAINTS, ('bytes',)),
  61. (LIST_CONSTRAINTS, ('list',)),
  62. (TUPLE_CONSTRAINTS, ('tuple',)),
  63. (SET_CONSTRAINTS, ('set', 'frozenset')),
  64. (DICT_CONSTRAINTS, ('dict',)),
  65. (GENERATOR_CONSTRAINTS, ('generator',)),
  66. (FLOAT_CONSTRAINTS, ('float',)),
  67. (INT_CONSTRAINTS, ('int',)),
  68. (DATE_TIME_CONSTRAINTS, ('date', 'time', 'datetime', 'timedelta')),
  69. # TODO: this is a bit redundant, we could probably avoid some of these
  70. (STRICT, (*TEXT_SCHEMA_TYPES, *SEQUENCE_SCHEMA_TYPES, *NUMERIC_SCHEMA_TYPES, 'typed-dict', 'model')),
  71. (UNION_CONSTRAINTS, ('union',)),
  72. (URL_CONSTRAINTS, ('url', 'multi-host-url')),
  73. (BOOL_CONSTRAINTS, ('bool',)),
  74. (UUID_CONSTRAINTS, ('uuid',)),
  75. (LAX_OR_STRICT_CONSTRAINTS, ('lax-or-strict',)),
  76. (ENUM_CONSTRAINTS, ('enum',)),
  77. (DECIMAL_CONSTRAINTS, ('decimal',)),
  78. (COMPLEX_CONSTRAINTS, ('complex',)),
  79. ]
  80. for constraints, schemas in constraint_schema_pairings:
  81. for c in constraints:
  82. CONSTRAINTS_TO_ALLOWED_SCHEMAS[c].update(schemas)
  83. def as_jsonable_value(v: Any) -> Any:
  84. if type(v) not in (int, str, float, bytes, bool, type(None)):
  85. return to_jsonable_python(v)
  86. return v
  87. def expand_grouped_metadata(annotations: Iterable[Any]) -> Iterable[Any]:
  88. """Expand the annotations.
  89. Args:
  90. annotations: An iterable of annotations.
  91. Returns:
  92. An iterable of expanded annotations.
  93. Example:
  94. ```python
  95. from annotated_types import Ge, Len
  96. from pydantic._internal._known_annotated_metadata import expand_grouped_metadata
  97. print(list(expand_grouped_metadata([Ge(4), Len(5)])))
  98. #> [Ge(ge=4), MinLen(min_length=5)]
  99. ```
  100. """
  101. import annotated_types as at
  102. FieldInfo = import_cached_field_info()
  103. for annotation in annotations:
  104. if isinstance(annotation, at.GroupedMetadata):
  105. yield from annotation
  106. elif isinstance(annotation, FieldInfo):
  107. yield from annotation.metadata
  108. # this is a bit problematic in that it results in duplicate metadata
  109. # all of our "consumers" can handle it, but it is not ideal
  110. # we probably should split up FieldInfo into:
  111. # - annotated types metadata
  112. # - individual metadata known only to Pydantic
  113. annotation = copy(annotation)
  114. annotation.metadata = []
  115. yield annotation
  116. else:
  117. yield annotation
  118. @lru_cache
  119. def _get_at_to_constraint_map() -> dict[type, str]:
  120. """Return a mapping of annotated types to constraints.
  121. Normally, we would define a mapping like this in the module scope, but we can't do that
  122. because we don't permit module level imports of `annotated_types`, in an attempt to speed up
  123. the import time of `pydantic`. We still only want to have this dictionary defined in one place,
  124. so we use this function to cache the result.
  125. """
  126. import annotated_types as at
  127. return {
  128. at.Gt: 'gt',
  129. at.Ge: 'ge',
  130. at.Lt: 'lt',
  131. at.Le: 'le',
  132. at.MultipleOf: 'multiple_of',
  133. at.MinLen: 'min_length',
  134. at.MaxLen: 'max_length',
  135. }
  136. def apply_known_metadata(annotation: Any, schema: CoreSchema) -> CoreSchema | None: # noqa: C901
  137. """Apply `annotation` to `schema` if it is an annotation we know about (Gt, Le, etc.).
  138. Otherwise return `None`.
  139. This does not handle all known annotations. If / when it does, it can always
  140. return a CoreSchema and return the unmodified schema if the annotation should be ignored.
  141. Assumes that GroupedMetadata has already been expanded via `expand_grouped_metadata`.
  142. Args:
  143. annotation: The annotation.
  144. schema: The schema.
  145. Returns:
  146. An updated schema with annotation if it is an annotation we know about, `None` otherwise.
  147. Raises:
  148. PydanticCustomError: If `Predicate` fails.
  149. """
  150. import annotated_types as at
  151. from ._validators import NUMERIC_VALIDATOR_LOOKUP, forbid_inf_nan_check
  152. schema = schema.copy()
  153. schema_update, other_metadata = collect_known_metadata([annotation])
  154. schema_type = schema['type']
  155. chain_schema_constraints: set[str] = {
  156. 'pattern',
  157. 'strip_whitespace',
  158. 'to_lower',
  159. 'to_upper',
  160. 'coerce_numbers_to_str',
  161. }
  162. chain_schema_steps: list[CoreSchema] = []
  163. for constraint, value in schema_update.items():
  164. if constraint not in CONSTRAINTS_TO_ALLOWED_SCHEMAS:
  165. raise ValueError(f'Unknown constraint {constraint}')
  166. allowed_schemas = CONSTRAINTS_TO_ALLOWED_SCHEMAS[constraint]
  167. # if it becomes necessary to handle more than one constraint
  168. # in this recursive case with function-after or function-wrap, we should refactor
  169. # this is a bit challenging because we sometimes want to apply constraints to the inner schema,
  170. # whereas other times we want to wrap the existing schema with a new one that enforces a new constraint.
  171. if schema_type in {'function-before', 'function-wrap', 'function-after'} and constraint == 'strict':
  172. schema['schema'] = apply_known_metadata(annotation, schema['schema']) # type: ignore # schema is function schema
  173. return schema
  174. # if we're allowed to apply constraint directly to the schema, like le to int, do that
  175. if schema_type in allowed_schemas:
  176. if constraint == 'union_mode' and schema_type == 'union':
  177. schema['mode'] = value # type: ignore # schema is UnionSchema
  178. else:
  179. schema[constraint] = value
  180. continue
  181. # else, apply a function after validator to the schema to enforce the corresponding constraint
  182. if constraint in chain_schema_constraints:
  183. def _apply_constraint_with_incompatibility_info(
  184. value: Any, handler: cs.ValidatorFunctionWrapHandler
  185. ) -> Any:
  186. try:
  187. x = handler(value)
  188. except ValidationError as ve:
  189. # if the error is about the type, it's likely that the constraint is incompatible the type of the field
  190. # for example, the following invalid schema wouldn't be caught during schema build, but rather at this point
  191. # with a cryptic 'string_type' error coming from the string validator,
  192. # that we'd rather express as a constraint incompatibility error (TypeError)
  193. # Annotated[list[int], Field(pattern='abc')]
  194. if 'type' in ve.errors()[0]['type']:
  195. raise TypeError(
  196. f"Unable to apply constraint '{constraint}' to supplied value {value} for schema of type '{schema_type}'" # noqa: B023
  197. )
  198. raise ve
  199. return x
  200. chain_schema_steps.append(
  201. cs.no_info_wrap_validator_function(
  202. _apply_constraint_with_incompatibility_info, cs.str_schema(**{constraint: value})
  203. )
  204. )
  205. elif constraint in NUMERIC_VALIDATOR_LOOKUP:
  206. if constraint in LENGTH_CONSTRAINTS:
  207. inner_schema = schema
  208. while inner_schema['type'] in {'function-before', 'function-wrap', 'function-after'}:
  209. inner_schema = inner_schema['schema'] # type: ignore
  210. inner_schema_type = inner_schema['type']
  211. if inner_schema_type == 'list' or (
  212. inner_schema_type == 'json-or-python' and inner_schema['json_schema']['type'] == 'list' # type: ignore
  213. ):
  214. js_constraint_key = 'minItems' if constraint == 'min_length' else 'maxItems'
  215. else:
  216. js_constraint_key = 'minLength' if constraint == 'min_length' else 'maxLength'
  217. else:
  218. js_constraint_key = constraint
  219. schema = cs.no_info_after_validator_function(
  220. partial(NUMERIC_VALIDATOR_LOOKUP[constraint], **{constraint: value}), schema
  221. )
  222. metadata = schema.get('metadata', {})
  223. if (existing_json_schema_updates := metadata.get('pydantic_js_updates')) is not None:
  224. metadata['pydantic_js_updates'] = {
  225. **existing_json_schema_updates,
  226. **{js_constraint_key: as_jsonable_value(value)},
  227. }
  228. else:
  229. metadata['pydantic_js_updates'] = {js_constraint_key: as_jsonable_value(value)}
  230. schema['metadata'] = metadata
  231. elif constraint == 'allow_inf_nan' and value is False:
  232. schema = cs.no_info_after_validator_function(
  233. forbid_inf_nan_check,
  234. schema,
  235. )
  236. else:
  237. # It's rare that we'd get here, but it's possible if we add a new constraint and forget to handle it
  238. # Most constraint errors are caught at runtime during attempted application
  239. raise RuntimeError(f"Unable to apply constraint '{constraint}' to schema of type '{schema_type}'")
  240. for annotation in other_metadata:
  241. if (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()):
  242. constraint = at_to_constraint_map[annotation_type]
  243. validator = NUMERIC_VALIDATOR_LOOKUP.get(constraint)
  244. if validator is None:
  245. raise ValueError(f'Unknown constraint {constraint}')
  246. schema = cs.no_info_after_validator_function(
  247. partial(validator, {constraint: getattr(annotation, constraint)}), schema
  248. )
  249. continue
  250. elif isinstance(annotation, (at.Predicate, at.Not)):
  251. predicate_name = f'{annotation.func.__qualname__}' if hasattr(annotation.func, '__qualname__') else ''
  252. def val_func(v: Any) -> Any:
  253. predicate_satisfied = annotation.func(v) # noqa: B023
  254. # annotation.func may also raise an exception, let it pass through
  255. if isinstance(annotation, at.Predicate): # noqa: B023
  256. if not predicate_satisfied:
  257. raise PydanticCustomError(
  258. 'predicate_failed',
  259. f'Predicate {predicate_name} failed', # type: ignore # noqa: B023
  260. )
  261. else:
  262. if predicate_satisfied:
  263. raise PydanticCustomError(
  264. 'not_operation_failed',
  265. f'Not of {predicate_name} failed', # type: ignore # noqa: B023
  266. )
  267. return v
  268. schema = cs.no_info_after_validator_function(val_func, schema)
  269. else:
  270. # ignore any other unknown metadata
  271. return None
  272. if chain_schema_steps:
  273. chain_schema_steps = [schema] + chain_schema_steps
  274. return cs.chain_schema(chain_schema_steps)
  275. return schema
  276. def collect_known_metadata(annotations: Iterable[Any]) -> tuple[dict[str, Any], list[Any]]:
  277. """Split `annotations` into known metadata and unknown annotations.
  278. Args:
  279. annotations: An iterable of annotations.
  280. Returns:
  281. A tuple contains a dict of known metadata and a list of unknown annotations.
  282. Example:
  283. ```python
  284. from annotated_types import Gt, Len
  285. from pydantic._internal._known_annotated_metadata import collect_known_metadata
  286. print(collect_known_metadata([Gt(1), Len(42), ...]))
  287. #> ({'gt': 1, 'min_length': 42}, [Ellipsis])
  288. ```
  289. """
  290. annotations = expand_grouped_metadata(annotations)
  291. res: dict[str, Any] = {}
  292. remaining: list[Any] = []
  293. for annotation in annotations:
  294. # isinstance(annotation, PydanticMetadata) also covers ._fields:_PydanticGeneralMetadata
  295. if isinstance(annotation, PydanticMetadata):
  296. res.update(annotation.__dict__)
  297. # we don't use dataclasses.asdict because that recursively calls asdict on the field values
  298. elif (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()):
  299. constraint = at_to_constraint_map[annotation_type]
  300. res[constraint] = getattr(annotation, constraint)
  301. elif isinstance(annotation, type) and issubclass(annotation, PydanticMetadata):
  302. # also support PydanticMetadata classes being used without initialisation,
  303. # e.g. `Annotated[int, Strict]` as well as `Annotated[int, Strict()]`
  304. res.update({k: v for k, v in vars(annotation).items() if not k.startswith('_')})
  305. else:
  306. remaining.append(annotation)
  307. # Nones can sneak in but pydantic-core will reject them
  308. # it'd be nice to clean things up so we don't put in None (we probably don't _need_ to, it was just easier)
  309. # but this is simple enough to kick that can down the road
  310. res = {k: v for k, v in res.items() if v is not None}
  311. return res, remaining
  312. def check_metadata(metadata: dict[str, Any], allowed: Iterable[str], source_type: Any) -> None:
  313. """A small utility function to validate that the given metadata can be applied to the target.
  314. More than saving lines of code, this gives us a consistent error message for all of our internal implementations.
  315. Args:
  316. metadata: A dict of metadata.
  317. allowed: An iterable of allowed metadata.
  318. source_type: The source type.
  319. Raises:
  320. TypeError: If there is metadatas that can't be applied on source type.
  321. """
  322. unknown = metadata.keys() - set(allowed)
  323. if unknown:
  324. raise TypeError(
  325. f'The following constraints cannot be applied to {source_type!r}: {", ".join([f"{k!r}" for k in unknown])}'
  326. )