expr.py (24476B)
1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Collection, 37 Dict, 38 Iterable, 39 List, 40 Optional, 41 Union, 42 cast, 43) 44 45from .common import c_name 46from .error import QAPISemError 47from .parser import QAPIDoc 48from .source import QAPISourceInfo 49 50 51# Deserialized JSON objects as returned by the parser. 52# The values of this mapping are not necessary to exhaustively type 53# here (and also not practical as long as mypy lacks recursive 54# types), because the purpose of this module is to interrogate that 55# type. 56_JSONObject = Dict[str, object] 57 58 59# See check_name_str(), below. 60valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 61 r'(x-)?' 62 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 63 64 65def check_name_is_str(name: object, 66 info: QAPISourceInfo, 67 source: str) -> None: 68 """ 69 Ensure that ``name`` is a ``str``. 70 71 :raise QAPISemError: When ``name`` fails validation. 72 """ 73 if not isinstance(name, str): 74 raise QAPISemError(info, "%s requires a string name" % source) 75 76 77def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 78 """ 79 Ensure that ``name`` is a valid QAPI name. 80 81 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 82 starting with a letter. It may be prefixed by a downstream prefix 83 of the form __RFQDN_, or the experimental prefix ``x-``. If both 84 prefixes are present, the __RFDQN_ prefix goes first. 85 86 A valid name cannot start with ``q_``, which is reserved. 87 88 :param name: Name to check. 89 :param info: QAPI schema source file information. 90 :param source: Error string describing what ``name`` belongs to. 91 92 :raise QAPISemError: When ``name`` fails validation. 93 :return: The stem of the valid name, with no prefixes. 94 """ 95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 96 # and 'q_obj_*' implicit type names. 97 match = valid_name.match(name) 98 if not match or c_name(name, False).startswith('q_'): 99 raise QAPISemError(info, "%s has an invalid name" % source) 100 return match.group(3) 101 102 103def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 104 """ 105 Ensure that ``name`` is a valid event name. 106 107 This means it must be a valid QAPI name as checked by 108 `check_name_str()`, but where the stem prohibits lowercase 109 characters and ``-``. 110 111 :param name: Name to check. 112 :param info: QAPI schema source file information. 113 :param source: Error string describing what ``name`` belongs to. 114 115 :raise QAPISemError: When ``name`` fails validation. 116 """ 117 stem = check_name_str(name, info, source) 118 if re.search(r'[a-z-]', stem): 119 raise QAPISemError( 120 info, "name of %s must not use lowercase or '-'" % source) 121 122 123def check_name_lower(name: str, info: QAPISourceInfo, source: str, 124 permit_upper: bool = False, 125 permit_underscore: bool = False) -> None: 126 """ 127 Ensure that ``name`` is a valid command or member name. 128 129 This means it must be a valid QAPI name as checked by 130 `check_name_str()`, but where the stem prohibits uppercase 131 characters and ``_``. 132 133 :param name: Name to check. 134 :param info: QAPI schema source file information. 135 :param source: Error string describing what ``name`` belongs to. 136 :param permit_upper: Additionally permit uppercase. 137 :param permit_underscore: Additionally permit ``_``. 138 139 :raise QAPISemError: When ``name`` fails validation. 140 """ 141 stem = check_name_str(name, info, source) 142 if ((not permit_upper and re.search(r'[A-Z]', stem)) 143 or (not permit_underscore and '_' in stem)): 144 raise QAPISemError( 145 info, "name of %s must not use uppercase or '_'" % source) 146 147 148def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 149 """ 150 Ensure that ``name`` is a valid user-defined type name. 151 152 This means it must be a valid QAPI name as checked by 153 `check_name_str()`, but where the stem must be in CamelCase. 154 155 :param name: Name to check. 156 :param info: QAPI schema source file information. 157 :param source: Error string describing what ``name`` belongs to. 158 159 :raise QAPISemError: When ``name`` fails validation. 160 """ 161 stem = check_name_str(name, info, source) 162 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 163 raise QAPISemError(info, "name of %s must use CamelCase" % source) 164 165 166def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 167 """ 168 Ensure that ``name`` is a valid definition name. 169 170 Based on the value of ``meta``, this means that: 171 - 'event' names adhere to `check_name_upper()`. 172 - 'command' names adhere to `check_name_lower()`. 173 - Else, meta is a type, and must pass `check_name_camel()`. 174 These names must not end with ``List``. 175 176 :param name: Name to check. 177 :param info: QAPI schema source file information. 178 :param meta: Meta-type name of the QAPI expression. 179 180 :raise QAPISemError: When ``name`` fails validation. 181 """ 182 if meta == 'event': 183 check_name_upper(name, info, meta) 184 elif meta == 'command': 185 check_name_lower( 186 name, info, meta, 187 permit_underscore=name in info.pragma.command_name_exceptions) 188 else: 189 check_name_camel(name, info, meta) 190 if name.endswith('List'): 191 raise QAPISemError( 192 info, "%s name should not end in 'List'" % meta) 193 194 195def check_keys(value: _JSONObject, 196 info: QAPISourceInfo, 197 source: str, 198 required: Collection[str], 199 optional: Collection[str]) -> None: 200 """ 201 Ensure that a dict has a specific set of keys. 202 203 :param value: The dict to check. 204 :param info: QAPI schema source file information. 205 :param source: Error string describing this ``value``. 206 :param required: Keys that *must* be present. 207 :param optional: Keys that *may* be present. 208 209 :raise QAPISemError: When unknown keys are present. 210 """ 211 212 def pprint(elems: Iterable[str]) -> str: 213 return ', '.join("'" + e + "'" for e in sorted(elems)) 214 215 missing = set(required) - set(value) 216 if missing: 217 raise QAPISemError( 218 info, 219 "%s misses key%s %s" 220 % (source, 's' if len(missing) > 1 else '', 221 pprint(missing))) 222 allowed = set(required) | set(optional) 223 unknown = set(value) - allowed 224 if unknown: 225 raise QAPISemError( 226 info, 227 "%s has unknown key%s %s\nValid keys are %s." 228 % (source, 's' if len(unknown) > 1 else '', 229 pprint(unknown), pprint(allowed))) 230 231 232def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 233 """ 234 Ensure flag members (if present) have valid values. 235 236 :param expr: The expression to validate. 237 :param info: QAPI schema source file information. 238 239 :raise QAPISemError: 240 When certain flags have an invalid value, or when 241 incompatible flags are present. 242 """ 243 for key in ('gen', 'success-response'): 244 if key in expr and expr[key] is not False: 245 raise QAPISemError( 246 info, "flag '%s' may only use false value" % key) 247 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 248 if key in expr and expr[key] is not True: 249 raise QAPISemError( 250 info, "flag '%s' may only use true value" % key) 251 if 'allow-oob' in expr and 'coroutine' in expr: 252 # This is not necessarily a fundamental incompatibility, but 253 # we don't have a use case and the desired semantics isn't 254 # obvious. The simplest solution is to forbid it until we get 255 # a use case for it. 256 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 257 "are incompatible") 258 259 260def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 261 """ 262 Validate the ``if`` member of an object. 263 264 The ``if`` member may be either a ``str`` or a dict. 265 266 :param expr: The expression containing the ``if`` member to validate. 267 :param info: QAPI schema source file information. 268 :param source: Error string describing ``expr``. 269 270 :raise QAPISemError: 271 When the "if" member fails validation, or when there are no 272 non-empty conditions. 273 :return: None 274 """ 275 276 def _check_if(cond: Union[str, object]) -> None: 277 if isinstance(cond, str): 278 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 279 raise QAPISemError( 280 info, 281 "'if' condition '%s' of %s is not a valid identifier" 282 % (cond, source)) 283 return 284 285 if not isinstance(cond, dict): 286 raise QAPISemError( 287 info, 288 "'if' condition of %s must be a string or an object" % source) 289 check_keys(cond, info, "'if' condition of %s" % source, [], 290 ["all", "any", "not"]) 291 if len(cond) != 1: 292 raise QAPISemError( 293 info, 294 "'if' condition of %s has conflicting keys" % source) 295 296 if 'not' in cond: 297 _check_if(cond['not']) 298 elif 'all' in cond: 299 _check_infix('all', cond['all']) 300 else: 301 _check_infix('any', cond['any']) 302 303 def _check_infix(operator: str, operands: object) -> None: 304 if not isinstance(operands, list): 305 raise QAPISemError( 306 info, 307 "'%s' condition of %s must be an array" 308 % (operator, source)) 309 if not operands: 310 raise QAPISemError( 311 info, "'if' condition [] of %s is useless" % source) 312 for operand in operands: 313 _check_if(operand) 314 315 ifcond = expr.get('if') 316 if ifcond is None: 317 return 318 319 _check_if(ifcond) 320 321 322def normalize_members(members: object) -> None: 323 """ 324 Normalize a "members" value. 325 326 If ``members`` is a dict, for every value in that dict, if that 327 value is not itself already a dict, normalize it to 328 ``{'type': value}``. 329 330 :forms: 331 :sugared: ``Dict[str, Union[str, TypeRef]]`` 332 :canonical: ``Dict[str, TypeRef]`` 333 334 :param members: The members value to normalize. 335 336 :return: None, ``members`` is normalized in-place as needed. 337 """ 338 if isinstance(members, dict): 339 for key, arg in members.items(): 340 if isinstance(arg, dict): 341 continue 342 members[key] = {'type': arg} 343 344 345def check_type(value: Optional[object], 346 info: QAPISourceInfo, 347 source: str, 348 allow_array: bool = False, 349 allow_dict: Union[bool, str] = False) -> None: 350 """ 351 Normalize and validate the QAPI type of ``value``. 352 353 Python types of ``str`` or ``None`` are always allowed. 354 355 :param value: The value to check. 356 :param info: QAPI schema source file information. 357 :param source: Error string describing this ``value``. 358 :param allow_array: 359 Allow a ``List[str]`` of length 1, which indicates an array of 360 the type named by the list element. 361 :param allow_dict: 362 Allow a dict. Its members can be struct type members or union 363 branches. When the value of ``allow_dict`` is in pragma 364 ``member-name-exceptions``, the dict's keys may violate the 365 member naming rules. The dict members are normalized in place. 366 367 :raise QAPISemError: When ``value`` fails validation. 368 :return: None, ``value`` is normalized in-place as needed. 369 """ 370 if value is None: 371 return 372 373 # Type name 374 if isinstance(value, str): 375 return 376 377 # Array type 378 if isinstance(value, list): 379 if not allow_array: 380 raise QAPISemError(info, "%s cannot be an array" % source) 381 if len(value) != 1 or not isinstance(value[0], str): 382 raise QAPISemError(info, 383 "%s: array type must contain single type name" % 384 source) 385 return 386 387 # Anonymous type 388 389 if not allow_dict: 390 raise QAPISemError(info, "%s should be a type name" % source) 391 392 if not isinstance(value, dict): 393 raise QAPISemError(info, 394 "%s should be an object or type name" % source) 395 396 permissive = False 397 if isinstance(allow_dict, str): 398 permissive = allow_dict in info.pragma.member_name_exceptions 399 400 # value is a dictionary, check that each member is okay 401 for (key, arg) in value.items(): 402 key_source = "%s member '%s'" % (source, key) 403 if key.startswith('*'): 404 key = key[1:] 405 check_name_lower(key, info, key_source, 406 permit_upper=permissive, 407 permit_underscore=permissive) 408 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 409 raise QAPISemError(info, "%s uses reserved name" % key_source) 410 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 411 check_if(arg, info, key_source) 412 check_features(arg.get('features'), info) 413 check_type(arg['type'], info, key_source, allow_array=True) 414 415 416def check_features(features: Optional[object], 417 info: QAPISourceInfo) -> None: 418 """ 419 Normalize and validate the ``features`` member. 420 421 ``features`` may be a ``list`` of either ``str`` or ``dict``. 422 Any ``str`` element will be normalized to ``{'name': element}``. 423 424 :forms: 425 :sugared: ``List[Union[str, Feature]]`` 426 :canonical: ``List[Feature]`` 427 428 :param features: The features member value to validate. 429 :param info: QAPI schema source file information. 430 431 :raise QAPISemError: When ``features`` fails validation. 432 :return: None, ``features`` is normalized in-place as needed. 433 """ 434 if features is None: 435 return 436 if not isinstance(features, list): 437 raise QAPISemError(info, "'features' must be an array") 438 features[:] = [f if isinstance(f, dict) else {'name': f} 439 for f in features] 440 for feat in features: 441 source = "'features' member" 442 assert isinstance(feat, dict) 443 check_keys(feat, info, source, ['name'], ['if']) 444 check_name_is_str(feat['name'], info, source) 445 source = "%s '%s'" % (source, feat['name']) 446 check_name_str(feat['name'], info, source) 447 check_if(feat, info, source) 448 449 450def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 451 """ 452 Normalize and validate this expression as an ``enum`` definition. 453 454 :param expr: The expression to validate. 455 :param info: QAPI schema source file information. 456 457 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 458 :return: None, ``expr`` is normalized in-place as needed. 459 """ 460 name = expr['enum'] 461 members = expr['data'] 462 prefix = expr.get('prefix') 463 464 if not isinstance(members, list): 465 raise QAPISemError(info, "'data' must be an array") 466 if prefix is not None and not isinstance(prefix, str): 467 raise QAPISemError(info, "'prefix' must be a string") 468 469 permissive = name in info.pragma.member_name_exceptions 470 471 members[:] = [m if isinstance(m, dict) else {'name': m} 472 for m in members] 473 for member in members: 474 source = "'data' member" 475 check_keys(member, info, source, ['name'], ['if']) 476 member_name = member['name'] 477 check_name_is_str(member_name, info, source) 478 source = "%s '%s'" % (source, member_name) 479 # Enum members may start with a digit 480 if member_name[0].isdigit(): 481 member_name = 'd' + member_name # Hack: hide the digit 482 check_name_lower(member_name, info, source, 483 permit_upper=permissive, 484 permit_underscore=permissive) 485 check_if(member, info, source) 486 487 488def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 489 """ 490 Normalize and validate this expression as a ``struct`` definition. 491 492 :param expr: The expression to validate. 493 :param info: QAPI schema source file information. 494 495 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 496 :return: None, ``expr`` is normalized in-place as needed. 497 """ 498 name = cast(str, expr['struct']) # Checked in check_exprs 499 members = expr['data'] 500 501 check_type(members, info, "'data'", allow_dict=name) 502 check_type(expr.get('base'), info, "'base'") 503 504 505def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 506 """ 507 Normalize and validate this expression as a ``union`` definition. 508 509 :param expr: The expression to validate. 510 :param info: QAPI schema source file information. 511 512 :raise QAPISemError: when ``expr`` is not a valid ``union``. 513 :return: None, ``expr`` is normalized in-place as needed. 514 """ 515 name = cast(str, expr['union']) # Checked in check_exprs 516 base = expr['base'] 517 discriminator = expr['discriminator'] 518 members = expr['data'] 519 520 check_type(base, info, "'base'", allow_dict=name) 521 check_name_is_str(discriminator, info, "'discriminator'") 522 523 if not isinstance(members, dict): 524 raise QAPISemError(info, "'data' must be an object") 525 526 for (key, value) in members.items(): 527 source = "'data' member '%s'" % key 528 check_keys(value, info, source, ['type'], ['if']) 529 check_if(value, info, source) 530 check_type(value['type'], info, source, allow_array=not base) 531 532 533def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 534 """ 535 Normalize and validate this expression as an ``alternate`` definition. 536 537 :param expr: The expression to validate. 538 :param info: QAPI schema source file information. 539 540 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 541 :return: None, ``expr`` is normalized in-place as needed. 542 """ 543 members = expr['data'] 544 545 if not members: 546 raise QAPISemError(info, "'data' must not be empty") 547 548 if not isinstance(members, dict): 549 raise QAPISemError(info, "'data' must be an object") 550 551 for (key, value) in members.items(): 552 source = "'data' member '%s'" % key 553 check_name_lower(key, info, source) 554 check_keys(value, info, source, ['type'], ['if']) 555 check_if(value, info, source) 556 check_type(value['type'], info, source) 557 558 559def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 560 """ 561 Normalize and validate this expression as a ``command`` definition. 562 563 :param expr: The expression to validate. 564 :param info: QAPI schema source file information. 565 566 :raise QAPISemError: When ``expr`` is not a valid ``command``. 567 :return: None, ``expr`` is normalized in-place as needed. 568 """ 569 args = expr.get('data') 570 rets = expr.get('returns') 571 boxed = expr.get('boxed', False) 572 573 if boxed and args is None: 574 raise QAPISemError(info, "'boxed': true requires 'data'") 575 check_type(args, info, "'data'", allow_dict=not boxed) 576 check_type(rets, info, "'returns'", allow_array=True) 577 578 579def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 580 """ 581 Normalize and validate this expression as an ``event`` definition. 582 583 :param expr: The expression to validate. 584 :param info: QAPI schema source file information. 585 586 :raise QAPISemError: When ``expr`` is not a valid ``event``. 587 :return: None, ``expr`` is normalized in-place as needed. 588 """ 589 args = expr.get('data') 590 boxed = expr.get('boxed', False) 591 592 if boxed and args is None: 593 raise QAPISemError(info, "'boxed': true requires 'data'") 594 check_type(args, info, "'data'", allow_dict=not boxed) 595 596 597def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 598 """ 599 Validate and normalize a list of parsed QAPI schema expressions. 600 601 This function accepts a list of expressions and metadata as returned 602 by the parser. It destructively normalizes the expressions in-place. 603 604 :param exprs: The list of expressions to normalize and validate. 605 606 :raise QAPISemError: When any expression fails validation. 607 :return: The same list of expressions (now modified). 608 """ 609 for expr_elem in exprs: 610 # Expression 611 assert isinstance(expr_elem['expr'], dict) 612 for key in expr_elem['expr'].keys(): 613 assert isinstance(key, str) 614 expr: _JSONObject = expr_elem['expr'] 615 616 # QAPISourceInfo 617 assert isinstance(expr_elem['info'], QAPISourceInfo) 618 info: QAPISourceInfo = expr_elem['info'] 619 620 # Optional[QAPIDoc] 621 tmp = expr_elem.get('doc') 622 assert tmp is None or isinstance(tmp, QAPIDoc) 623 doc: Optional[QAPIDoc] = tmp 624 625 if 'include' in expr: 626 continue 627 628 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 629 'command', 'event'} 630 if len(metas) != 1: 631 raise QAPISemError( 632 info, 633 "expression must have exactly one key" 634 " 'enum', 'struct', 'union', 'alternate'," 635 " 'command', 'event'") 636 meta = metas.pop() 637 638 check_name_is_str(expr[meta], info, "'%s'" % meta) 639 name = cast(str, expr[meta]) 640 info.set_defn(meta, name) 641 check_defn_name_str(name, info, meta) 642 643 if doc: 644 if doc.symbol != name: 645 raise QAPISemError( 646 info, "documentation comment is for '%s'" % doc.symbol) 647 doc.check_expr(expr) 648 elif info.pragma.doc_required: 649 raise QAPISemError(info, 650 "documentation comment required") 651 652 if meta == 'enum': 653 check_keys(expr, info, meta, 654 ['enum', 'data'], ['if', 'features', 'prefix']) 655 check_enum(expr, info) 656 elif meta == 'union': 657 check_keys(expr, info, meta, 658 ['union', 'base', 'discriminator', 'data'], 659 ['if', 'features']) 660 normalize_members(expr.get('base')) 661 normalize_members(expr['data']) 662 check_union(expr, info) 663 elif meta == 'alternate': 664 check_keys(expr, info, meta, 665 ['alternate', 'data'], ['if', 'features']) 666 normalize_members(expr['data']) 667 check_alternate(expr, info) 668 elif meta == 'struct': 669 check_keys(expr, info, meta, 670 ['struct', 'data'], ['base', 'if', 'features']) 671 normalize_members(expr['data']) 672 check_struct(expr, info) 673 elif meta == 'command': 674 check_keys(expr, info, meta, 675 ['command'], 676 ['data', 'returns', 'boxed', 'if', 'features', 677 'gen', 'success-response', 'allow-oob', 678 'allow-preconfig', 'coroutine']) 679 normalize_members(expr.get('data')) 680 check_command(expr, info) 681 elif meta == 'event': 682 check_keys(expr, info, meta, 683 ['event'], ['data', 'boxed', 'if', 'features']) 684 normalize_members(expr.get('data')) 685 check_event(expr, info) 686 else: 687 assert False, 'unexpected meta type' 688 689 check_if(expr, info, meta) 690 check_features(expr.get('features'), info) 691 check_flags(expr, info) 692 693 return exprs