1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Collection, 37 Dict, 38 Iterable, 39 List, 40 Optional, 41 Union, 42 cast, 43) 44 45from .common import c_name 46from .error import QAPISemError 47from .parser import QAPIDoc 48from .source import QAPISourceInfo 49 50 51# Deserialized JSON objects as returned by the parser. 52# The values of this mapping are not necessary to exhaustively type 53# here (and also not practical as long as mypy lacks recursive 54# types), because the purpose of this module is to interrogate that 55# type. 56_JSONObject = Dict[str, object] 57 58 59# See check_name_str(), below. 60valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 61 r'(x-)?' 62 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 63 64 65def check_name_is_str(name: object, 66 info: QAPISourceInfo, 67 source: str) -> None: 68 """ 69 Ensure that ``name`` is a ``str``. 70 71 :raise QAPISemError: When ``name`` fails validation. 72 """ 73 if not isinstance(name, str): 74 raise QAPISemError(info, "%s requires a string name" % source) 75 76 77def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 78 """ 79 Ensure that ``name`` is a valid QAPI name. 80 81 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 82 starting with a letter. It may be prefixed by a downstream prefix 83 of the form __RFQDN_, or the experimental prefix ``x-``. If both 84 prefixes are present, the __RFDQN_ prefix goes first. 85 86 A valid name cannot start with ``q_``, which is reserved. 87 88 :param name: Name to check. 89 :param info: QAPI schema source file information. 90 :param source: Error string describing what ``name`` belongs to. 91 92 :raise QAPISemError: When ``name`` fails validation. 93 :return: The stem of the valid name, with no prefixes. 94 """ 95 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 96 # and 'q_obj_*' implicit type names. 97 match = valid_name.match(name) 98 if not match or c_name(name, False).startswith('q_'): 99 raise QAPISemError(info, "%s has an invalid name" % source) 100 return match.group(3) 101 102 103def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 104 """ 105 Ensure that ``name`` is a valid event name. 106 107 This means it must be a valid QAPI name as checked by 108 `check_name_str()`, but where the stem prohibits lowercase 109 characters and ``-``. 110 111 :param name: Name to check. 112 :param info: QAPI schema source file information. 113 :param source: Error string describing what ``name`` belongs to. 114 115 :raise QAPISemError: When ``name`` fails validation. 116 """ 117 stem = check_name_str(name, info, source) 118 if re.search(r'[a-z-]', stem): 119 raise QAPISemError( 120 info, "name of %s must not use lowercase or '-'" % source) 121 122 123def check_name_lower(name: str, info: QAPISourceInfo, source: str, 124 permit_upper: bool = False, 125 permit_underscore: bool = False) -> None: 126 """ 127 Ensure that ``name`` is a valid command or member name. 128 129 This means it must be a valid QAPI name as checked by 130 `check_name_str()`, but where the stem prohibits uppercase 131 characters and ``_``. 132 133 :param name: Name to check. 134 :param info: QAPI schema source file information. 135 :param source: Error string describing what ``name`` belongs to. 136 :param permit_upper: Additionally permit uppercase. 137 :param permit_underscore: Additionally permit ``_``. 138 139 :raise QAPISemError: When ``name`` fails validation. 140 """ 141 stem = check_name_str(name, info, source) 142 if ((not permit_upper and re.search(r'[A-Z]', stem)) 143 or (not permit_underscore and '_' in stem)): 144 raise QAPISemError( 145 info, "name of %s must not use uppercase or '_'" % source) 146 147 148def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 149 """ 150 Ensure that ``name`` is a valid user-defined type name. 151 152 This means it must be a valid QAPI name as checked by 153 `check_name_str()`, but where the stem must be in CamelCase. 154 155 :param name: Name to check. 156 :param info: QAPI schema source file information. 157 :param source: Error string describing what ``name`` belongs to. 158 159 :raise QAPISemError: When ``name`` fails validation. 160 """ 161 stem = check_name_str(name, info, source) 162 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 163 raise QAPISemError(info, "name of %s must use CamelCase" % source) 164 165 166def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 167 """ 168 Ensure that ``name`` is a valid definition name. 169 170 Based on the value of ``meta``, this means that: 171 - 'event' names adhere to `check_name_upper()`. 172 - 'command' names adhere to `check_name_lower()`. 173 - Else, meta is a type, and must pass `check_name_camel()`. 174 These names must not end with ``Kind`` nor ``List``. 175 176 :param name: Name to check. 177 :param info: QAPI schema source file information. 178 :param meta: Meta-type name of the QAPI expression. 179 180 :raise QAPISemError: When ``name`` fails validation. 181 """ 182 if meta == 'event': 183 check_name_upper(name, info, meta) 184 elif meta == 'command': 185 check_name_lower( 186 name, info, meta, 187 permit_underscore=name in info.pragma.command_name_exceptions) 188 else: 189 check_name_camel(name, info, meta) 190 if name.endswith('Kind') or name.endswith('List'): 191 raise QAPISemError( 192 info, "%s name should not end in '%s'" % (meta, name[-4:])) 193 194 195def check_keys(value: _JSONObject, 196 info: QAPISourceInfo, 197 source: str, 198 required: Collection[str], 199 optional: Collection[str]) -> None: 200 """ 201 Ensure that a dict has a specific set of keys. 202 203 :param value: The dict to check. 204 :param info: QAPI schema source file information. 205 :param source: Error string describing this ``value``. 206 :param required: Keys that *must* be present. 207 :param optional: Keys that *may* be present. 208 209 :raise QAPISemError: When unknown keys are present. 210 """ 211 212 def pprint(elems: Iterable[str]) -> str: 213 return ', '.join("'" + e + "'" for e in sorted(elems)) 214 215 missing = set(required) - set(value) 216 if missing: 217 raise QAPISemError( 218 info, 219 "%s misses key%s %s" 220 % (source, 's' if len(missing) > 1 else '', 221 pprint(missing))) 222 allowed = set(required) | set(optional) 223 unknown = set(value) - allowed 224 if unknown: 225 raise QAPISemError( 226 info, 227 "%s has unknown key%s %s\nValid keys are %s." 228 % (source, 's' if len(unknown) > 1 else '', 229 pprint(unknown), pprint(allowed))) 230 231 232def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 233 """ 234 Ensure flag members (if present) have valid values. 235 236 :param expr: The expression to validate. 237 :param info: QAPI schema source file information. 238 239 :raise QAPISemError: 240 When certain flags have an invalid value, or when 241 incompatible flags are present. 242 """ 243 for key in ('gen', 'success-response'): 244 if key in expr and expr[key] is not False: 245 raise QAPISemError( 246 info, "flag '%s' may only use false value" % key) 247 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 248 if key in expr and expr[key] is not True: 249 raise QAPISemError( 250 info, "flag '%s' may only use true value" % key) 251 if 'allow-oob' in expr and 'coroutine' in expr: 252 # This is not necessarily a fundamental incompatibility, but 253 # we don't have a use case and the desired semantics isn't 254 # obvious. The simplest solution is to forbid it until we get 255 # a use case for it. 256 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 257 "are incompatible") 258 259 260def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 261 """ 262 Validate the ``if`` member of an object. 263 264 The ``if`` member may be either a ``str`` or a dict. 265 266 :param expr: The expression containing the ``if`` member to validate. 267 :param info: QAPI schema source file information. 268 :param source: Error string describing ``expr``. 269 270 :raise QAPISemError: 271 When the "if" member fails validation, or when there are no 272 non-empty conditions. 273 :return: None 274 """ 275 276 def _check_if(cond: Union[str, object]) -> None: 277 if isinstance(cond, str): 278 if not cond.strip(): 279 raise QAPISemError( 280 info, 281 "'if' condition '%s' of %s makes no sense" 282 % (cond, source)) 283 return 284 285 if not isinstance(cond, dict): 286 raise QAPISemError( 287 info, 288 "'if' condition of %s must be a string or an object" % source) 289 if len(cond) != 1: 290 raise QAPISemError( 291 info, 292 "'if' condition dict of %s must have one key: " 293 "'all'" % source) 294 check_keys(cond, info, "'if' condition", [], 295 ["all"]) 296 297 oper, operands = next(iter(cond.items())) 298 if not operands: 299 raise QAPISemError( 300 info, "'if' condition [] of %s is useless" % source) 301 302 if oper in ("all") and not isinstance(operands, list): 303 raise QAPISemError( 304 info, "'%s' condition of %s must be an array" % (oper, source)) 305 for operand in operands: 306 _check_if(operand) 307 308 ifcond = expr.get('if') 309 if ifcond is None: 310 return 311 312 _check_if(ifcond) 313 314 315def normalize_members(members: object) -> None: 316 """ 317 Normalize a "members" value. 318 319 If ``members`` is a dict, for every value in that dict, if that 320 value is not itself already a dict, normalize it to 321 ``{'type': value}``. 322 323 :forms: 324 :sugared: ``Dict[str, Union[str, TypeRef]]`` 325 :canonical: ``Dict[str, TypeRef]`` 326 327 :param members: The members value to normalize. 328 329 :return: None, ``members`` is normalized in-place as needed. 330 """ 331 if isinstance(members, dict): 332 for key, arg in members.items(): 333 if isinstance(arg, dict): 334 continue 335 members[key] = {'type': arg} 336 337 338def check_type(value: Optional[object], 339 info: QAPISourceInfo, 340 source: str, 341 allow_array: bool = False, 342 allow_dict: Union[bool, str] = False) -> None: 343 """ 344 Normalize and validate the QAPI type of ``value``. 345 346 Python types of ``str`` or ``None`` are always allowed. 347 348 :param value: The value to check. 349 :param info: QAPI schema source file information. 350 :param source: Error string describing this ``value``. 351 :param allow_array: 352 Allow a ``List[str]`` of length 1, which indicates an array of 353 the type named by the list element. 354 :param allow_dict: 355 Allow a dict. Its members can be struct type members or union 356 branches. When the value of ``allow_dict`` is in pragma 357 ``member-name-exceptions``, the dict's keys may violate the 358 member naming rules. The dict members are normalized in place. 359 360 :raise QAPISemError: When ``value`` fails validation. 361 :return: None, ``value`` is normalized in-place as needed. 362 """ 363 if value is None: 364 return 365 366 # Type name 367 if isinstance(value, str): 368 return 369 370 # Array type 371 if isinstance(value, list): 372 if not allow_array: 373 raise QAPISemError(info, "%s cannot be an array" % source) 374 if len(value) != 1 or not isinstance(value[0], str): 375 raise QAPISemError(info, 376 "%s: array type must contain single type name" % 377 source) 378 return 379 380 # Anonymous type 381 382 if not allow_dict: 383 raise QAPISemError(info, "%s should be a type name" % source) 384 385 if not isinstance(value, dict): 386 raise QAPISemError(info, 387 "%s should be an object or type name" % source) 388 389 permissive = False 390 if isinstance(allow_dict, str): 391 permissive = allow_dict in info.pragma.member_name_exceptions 392 393 # value is a dictionary, check that each member is okay 394 for (key, arg) in value.items(): 395 key_source = "%s member '%s'" % (source, key) 396 if key.startswith('*'): 397 key = key[1:] 398 check_name_lower(key, info, key_source, 399 permit_upper=permissive, 400 permit_underscore=permissive) 401 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 402 raise QAPISemError(info, "%s uses reserved name" % key_source) 403 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 404 check_if(arg, info, key_source) 405 check_features(arg.get('features'), info) 406 check_type(arg['type'], info, key_source, allow_array=True) 407 408 409def check_features(features: Optional[object], 410 info: QAPISourceInfo) -> None: 411 """ 412 Normalize and validate the ``features`` member. 413 414 ``features`` may be a ``list`` of either ``str`` or ``dict``. 415 Any ``str`` element will be normalized to ``{'name': element}``. 416 417 :forms: 418 :sugared: ``List[Union[str, Feature]]`` 419 :canonical: ``List[Feature]`` 420 421 :param features: The features member value to validate. 422 :param info: QAPI schema source file information. 423 424 :raise QAPISemError: When ``features`` fails validation. 425 :return: None, ``features`` is normalized in-place as needed. 426 """ 427 if features is None: 428 return 429 if not isinstance(features, list): 430 raise QAPISemError(info, "'features' must be an array") 431 features[:] = [f if isinstance(f, dict) else {'name': f} 432 for f in features] 433 for feat in features: 434 source = "'features' member" 435 assert isinstance(feat, dict) 436 check_keys(feat, info, source, ['name'], ['if']) 437 check_name_is_str(feat['name'], info, source) 438 source = "%s '%s'" % (source, feat['name']) 439 check_name_str(feat['name'], info, source) 440 check_if(feat, info, source) 441 442 443def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 444 """ 445 Normalize and validate this expression as an ``enum`` definition. 446 447 :param expr: The expression to validate. 448 :param info: QAPI schema source file information. 449 450 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 451 :return: None, ``expr`` is normalized in-place as needed. 452 """ 453 name = expr['enum'] 454 members = expr['data'] 455 prefix = expr.get('prefix') 456 457 if not isinstance(members, list): 458 raise QAPISemError(info, "'data' must be an array") 459 if prefix is not None and not isinstance(prefix, str): 460 raise QAPISemError(info, "'prefix' must be a string") 461 462 permissive = name in info.pragma.member_name_exceptions 463 464 members[:] = [m if isinstance(m, dict) else {'name': m} 465 for m in members] 466 for member in members: 467 source = "'data' member" 468 check_keys(member, info, source, ['name'], ['if']) 469 member_name = member['name'] 470 check_name_is_str(member_name, info, source) 471 source = "%s '%s'" % (source, member_name) 472 # Enum members may start with a digit 473 if member_name[0].isdigit(): 474 member_name = 'd' + member_name # Hack: hide the digit 475 check_name_lower(member_name, info, source, 476 permit_upper=permissive, 477 permit_underscore=permissive) 478 check_if(member, info, source) 479 480 481def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 482 """ 483 Normalize and validate this expression as a ``struct`` definition. 484 485 :param expr: The expression to validate. 486 :param info: QAPI schema source file information. 487 488 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 489 :return: None, ``expr`` is normalized in-place as needed. 490 """ 491 name = cast(str, expr['struct']) # Checked in check_exprs 492 members = expr['data'] 493 494 check_type(members, info, "'data'", allow_dict=name) 495 check_type(expr.get('base'), info, "'base'") 496 497 498def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 499 """ 500 Normalize and validate this expression as a ``union`` definition. 501 502 :param expr: The expression to validate. 503 :param info: QAPI schema source file information. 504 505 :raise QAPISemError: when ``expr`` is not a valid ``union``. 506 :return: None, ``expr`` is normalized in-place as needed. 507 """ 508 name = cast(str, expr['union']) # Checked in check_exprs 509 base = expr.get('base') 510 discriminator = expr.get('discriminator') 511 members = expr['data'] 512 513 if discriminator is None: # simple union 514 if base is not None: 515 raise QAPISemError(info, "'base' requires 'discriminator'") 516 else: # flat union 517 check_type(base, info, "'base'", allow_dict=name) 518 if not base: 519 raise QAPISemError(info, "'discriminator' requires 'base'") 520 check_name_is_str(discriminator, info, "'discriminator'") 521 522 if not isinstance(members, dict): 523 raise QAPISemError(info, "'data' must be an object") 524 525 for (key, value) in members.items(): 526 source = "'data' member '%s'" % key 527 if discriminator is None: 528 check_name_lower(key, info, source) 529 # else: name is in discriminator enum, which gets checked 530 check_keys(value, info, source, ['type'], ['if']) 531 check_if(value, info, source) 532 check_type(value['type'], info, source, allow_array=not base) 533 534 535def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 536 """ 537 Normalize and validate this expression as an ``alternate`` definition. 538 539 :param expr: The expression to validate. 540 :param info: QAPI schema source file information. 541 542 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 543 :return: None, ``expr`` is normalized in-place as needed. 544 """ 545 members = expr['data'] 546 547 if not members: 548 raise QAPISemError(info, "'data' must not be empty") 549 550 if not isinstance(members, dict): 551 raise QAPISemError(info, "'data' must be an object") 552 553 for (key, value) in members.items(): 554 source = "'data' member '%s'" % key 555 check_name_lower(key, info, source) 556 check_keys(value, info, source, ['type'], ['if']) 557 check_if(value, info, source) 558 check_type(value['type'], info, source) 559 560 561def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 562 """ 563 Normalize and validate this expression as a ``command`` definition. 564 565 :param expr: The expression to validate. 566 :param info: QAPI schema source file information. 567 568 :raise QAPISemError: When ``expr`` is not a valid ``command``. 569 :return: None, ``expr`` is normalized in-place as needed. 570 """ 571 args = expr.get('data') 572 rets = expr.get('returns') 573 boxed = expr.get('boxed', False) 574 575 if boxed and args is None: 576 raise QAPISemError(info, "'boxed': true requires 'data'") 577 check_type(args, info, "'data'", allow_dict=not boxed) 578 check_type(rets, info, "'returns'", allow_array=True) 579 580 581def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 582 """ 583 Normalize and validate this expression as an ``event`` definition. 584 585 :param expr: The expression to validate. 586 :param info: QAPI schema source file information. 587 588 :raise QAPISemError: When ``expr`` is not a valid ``event``. 589 :return: None, ``expr`` is normalized in-place as needed. 590 """ 591 args = expr.get('data') 592 boxed = expr.get('boxed', False) 593 594 if boxed and args is None: 595 raise QAPISemError(info, "'boxed': true requires 'data'") 596 check_type(args, info, "'data'", allow_dict=not boxed) 597 598 599def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 600 """ 601 Validate and normalize a list of parsed QAPI schema expressions. 602 603 This function accepts a list of expressions and metadata as returned 604 by the parser. It destructively normalizes the expressions in-place. 605 606 :param exprs: The list of expressions to normalize and validate. 607 608 :raise QAPISemError: When any expression fails validation. 609 :return: The same list of expressions (now modified). 610 """ 611 for expr_elem in exprs: 612 # Expression 613 assert isinstance(expr_elem['expr'], dict) 614 for key in expr_elem['expr'].keys(): 615 assert isinstance(key, str) 616 expr: _JSONObject = expr_elem['expr'] 617 618 # QAPISourceInfo 619 assert isinstance(expr_elem['info'], QAPISourceInfo) 620 info: QAPISourceInfo = expr_elem['info'] 621 622 # Optional[QAPIDoc] 623 tmp = expr_elem.get('doc') 624 assert tmp is None or isinstance(tmp, QAPIDoc) 625 doc: Optional[QAPIDoc] = tmp 626 627 if 'include' in expr: 628 continue 629 630 if 'enum' in expr: 631 meta = 'enum' 632 elif 'union' in expr: 633 meta = 'union' 634 elif 'alternate' in expr: 635 meta = 'alternate' 636 elif 'struct' in expr: 637 meta = 'struct' 638 elif 'command' in expr: 639 meta = 'command' 640 elif 'event' in expr: 641 meta = 'event' 642 else: 643 raise QAPISemError(info, "expression is missing metatype") 644 645 check_name_is_str(expr[meta], info, "'%s'" % meta) 646 name = cast(str, expr[meta]) 647 info.set_defn(meta, name) 648 check_defn_name_str(name, info, meta) 649 650 if doc: 651 if doc.symbol != name: 652 raise QAPISemError( 653 info, "documentation comment is for '%s'" % doc.symbol) 654 doc.check_expr(expr) 655 elif info.pragma.doc_required: 656 raise QAPISemError(info, 657 "documentation comment required") 658 659 if meta == 'enum': 660 check_keys(expr, info, meta, 661 ['enum', 'data'], ['if', 'features', 'prefix']) 662 check_enum(expr, info) 663 elif meta == 'union': 664 check_keys(expr, info, meta, 665 ['union', 'data'], 666 ['base', 'discriminator', 'if', 'features']) 667 normalize_members(expr.get('base')) 668 normalize_members(expr['data']) 669 check_union(expr, info) 670 elif meta == 'alternate': 671 check_keys(expr, info, meta, 672 ['alternate', 'data'], ['if', 'features']) 673 normalize_members(expr['data']) 674 check_alternate(expr, info) 675 elif meta == 'struct': 676 check_keys(expr, info, meta, 677 ['struct', 'data'], ['base', 'if', 'features']) 678 normalize_members(expr['data']) 679 check_struct(expr, info) 680 elif meta == 'command': 681 check_keys(expr, info, meta, 682 ['command'], 683 ['data', 'returns', 'boxed', 'if', 'features', 684 'gen', 'success-response', 'allow-oob', 685 'allow-preconfig', 'coroutine']) 686 normalize_members(expr.get('data')) 687 check_command(expr, info) 688 elif meta == 'event': 689 check_keys(expr, info, meta, 690 ['event'], ['data', 'boxed', 'if', 'features']) 691 normalize_members(expr.get('data')) 692 check_event(expr, info) 693 else: 694 assert False, 'unexpected meta type' 695 696 check_if(expr, info, meta) 697 check_features(expr.get('features'), info) 698 check_flags(expr, info) 699 700 return exprs 701