1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIDoc 47from .source import QAPISourceInfo 48 49 50# Deserialized JSON objects as returned by the parser. 51# The values of this mapping are not necessary to exhaustively type 52# here (and also not practical as long as mypy lacks recursive 53# types), because the purpose of this module is to interrogate that 54# type. 55_JSONObject = Dict[str, object] 56 57 58# See check_name_str(), below. 59valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 60 r'(x-)?' 61 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 62 63 64def check_name_is_str(name: object, 65 info: QAPISourceInfo, 66 source: str) -> None: 67 """ 68 Ensure that ``name`` is a ``str``. 69 70 :raise QAPISemError: When ``name`` fails validation. 71 """ 72 if not isinstance(name, str): 73 raise QAPISemError(info, "%s requires a string name" % source) 74 75 76def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 77 """ 78 Ensure that ``name`` is a valid QAPI name. 79 80 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 81 starting with a letter. It may be prefixed by a downstream prefix 82 of the form __RFQDN_, or the experimental prefix ``x-``. If both 83 prefixes are present, the __RFDQN_ prefix goes first. 84 85 A valid name cannot start with ``q_``, which is reserved. 86 87 :param name: Name to check. 88 :param info: QAPI schema source file information. 89 :param source: Error string describing what ``name`` belongs to. 90 91 :raise QAPISemError: When ``name`` fails validation. 92 :return: The stem of the valid name, with no prefixes. 93 """ 94 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 95 # and 'q_obj_*' implicit type names. 96 match = valid_name.match(name) 97 if not match or c_name(name, False).startswith('q_'): 98 raise QAPISemError(info, "%s has an invalid name" % source) 99 return match.group(3) 100 101 102def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 103 """ 104 Ensure that ``name`` is a valid event name. 105 106 This means it must be a valid QAPI name as checked by 107 `check_name_str()`, but where the stem prohibits lowercase 108 characters and ``-``. 109 110 :param name: Name to check. 111 :param info: QAPI schema source file information. 112 :param source: Error string describing what ``name`` belongs to. 113 114 :raise QAPISemError: When ``name`` fails validation. 115 """ 116 stem = check_name_str(name, info, source) 117 if re.search(r'[a-z-]', stem): 118 raise QAPISemError( 119 info, "name of %s must not use lowercase or '-'" % source) 120 121 122def check_name_lower(name: str, info: QAPISourceInfo, source: str, 123 permit_upper: bool = False, 124 permit_underscore: bool = False) -> None: 125 """ 126 Ensure that ``name`` is a valid command or member name. 127 128 This means it must be a valid QAPI name as checked by 129 `check_name_str()`, but where the stem prohibits uppercase 130 characters and ``_``. 131 132 :param name: Name to check. 133 :param info: QAPI schema source file information. 134 :param source: Error string describing what ``name`` belongs to. 135 :param permit_upper: Additionally permit uppercase. 136 :param permit_underscore: Additionally permit ``_``. 137 138 :raise QAPISemError: When ``name`` fails validation. 139 """ 140 stem = check_name_str(name, info, source) 141 if ((not permit_upper and re.search(r'[A-Z]', stem)) 142 or (not permit_underscore and '_' in stem)): 143 raise QAPISemError( 144 info, "name of %s must not use uppercase or '_'" % source) 145 146 147def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 148 """ 149 Ensure that ``name`` is a valid user-defined type name. 150 151 This means it must be a valid QAPI name as checked by 152 `check_name_str()`, but where the stem must be in CamelCase. 153 154 :param name: Name to check. 155 :param info: QAPI schema source file information. 156 :param source: Error string describing what ``name`` belongs to. 157 158 :raise QAPISemError: When ``name`` fails validation. 159 """ 160 stem = check_name_str(name, info, source) 161 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 162 raise QAPISemError(info, "name of %s must use CamelCase" % source) 163 164 165def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 166 """ 167 Ensure that ``name`` is a valid definition name. 168 169 Based on the value of ``meta``, this means that: 170 - 'event' names adhere to `check_name_upper()`. 171 - 'command' names adhere to `check_name_lower()`. 172 - Else, meta is a type, and must pass `check_name_camel()`. 173 These names must not end with ``List``. 174 175 :param name: Name to check. 176 :param info: QAPI schema source file information. 177 :param meta: Meta-type name of the QAPI expression. 178 179 :raise QAPISemError: When ``name`` fails validation. 180 """ 181 if meta == 'event': 182 check_name_upper(name, info, meta) 183 elif meta == 'command': 184 check_name_lower( 185 name, info, meta, 186 permit_underscore=name in info.pragma.command_name_exceptions) 187 else: 188 check_name_camel(name, info, meta) 189 if name.endswith('List'): 190 raise QAPISemError( 191 info, "%s name should not end in 'List'" % meta) 192 193 194def check_keys(value: _JSONObject, 195 info: QAPISourceInfo, 196 source: str, 197 required: List[str], 198 optional: List[str]) -> None: 199 """ 200 Ensure that a dict has a specific set of keys. 201 202 :param value: The dict to check. 203 :param info: QAPI schema source file information. 204 :param source: Error string describing this ``value``. 205 :param required: Keys that *must* be present. 206 :param optional: Keys that *may* be present. 207 208 :raise QAPISemError: When unknown keys are present. 209 """ 210 211 def pprint(elems: Iterable[str]) -> str: 212 return ', '.join("'" + e + "'" for e in sorted(elems)) 213 214 missing = set(required) - set(value) 215 if missing: 216 raise QAPISemError( 217 info, 218 "%s misses key%s %s" 219 % (source, 's' if len(missing) > 1 else '', 220 pprint(missing))) 221 allowed = set(required) | set(optional) 222 unknown = set(value) - allowed 223 if unknown: 224 raise QAPISemError( 225 info, 226 "%s has unknown key%s %s\nValid keys are %s." 227 % (source, 's' if len(unknown) > 1 else '', 228 pprint(unknown), pprint(allowed))) 229 230 231def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 232 """ 233 Ensure flag members (if present) have valid values. 234 235 :param expr: The expression to validate. 236 :param info: QAPI schema source file information. 237 238 :raise QAPISemError: 239 When certain flags have an invalid value, or when 240 incompatible flags are present. 241 """ 242 for key in ('gen', 'success-response'): 243 if key in expr and expr[key] is not False: 244 raise QAPISemError( 245 info, "flag '%s' may only use false value" % key) 246 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 247 if key in expr and expr[key] is not True: 248 raise QAPISemError( 249 info, "flag '%s' may only use true value" % key) 250 if 'allow-oob' in expr and 'coroutine' in expr: 251 # This is not necessarily a fundamental incompatibility, but 252 # we don't have a use case and the desired semantics isn't 253 # obvious. The simplest solution is to forbid it until we get 254 # a use case for it. 255 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 256 "are incompatible") 257 258 259def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 260 """ 261 Validate the ``if`` member of an object. 262 263 The ``if`` member may be either a ``str`` or a dict. 264 265 :param expr: The expression containing the ``if`` member to validate. 266 :param info: QAPI schema source file information. 267 :param source: Error string describing ``expr``. 268 269 :raise QAPISemError: 270 When the "if" member fails validation, or when there are no 271 non-empty conditions. 272 :return: None 273 """ 274 275 def _check_if(cond: Union[str, object]) -> None: 276 if isinstance(cond, str): 277 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 278 raise QAPISemError( 279 info, 280 "'if' condition '%s' of %s is not a valid identifier" 281 % (cond, source)) 282 return 283 284 if not isinstance(cond, dict): 285 raise QAPISemError( 286 info, 287 "'if' condition of %s must be a string or an object" % source) 288 check_keys(cond, info, "'if' condition of %s" % source, [], 289 ["all", "any", "not"]) 290 if len(cond) != 1: 291 raise QAPISemError( 292 info, 293 "'if' condition of %s has conflicting keys" % source) 294 295 if 'not' in cond: 296 _check_if(cond['not']) 297 elif 'all' in cond: 298 _check_infix('all', cond['all']) 299 else: 300 _check_infix('any', cond['any']) 301 302 def _check_infix(operator: str, operands: object) -> None: 303 if not isinstance(operands, list): 304 raise QAPISemError( 305 info, 306 "'%s' condition of %s must be an array" 307 % (operator, source)) 308 if not operands: 309 raise QAPISemError( 310 info, "'if' condition [] of %s is useless" % source) 311 for operand in operands: 312 _check_if(operand) 313 314 ifcond = expr.get('if') 315 if ifcond is None: 316 return 317 318 _check_if(ifcond) 319 320 321def normalize_members(members: object) -> None: 322 """ 323 Normalize a "members" value. 324 325 If ``members`` is a dict, for every value in that dict, if that 326 value is not itself already a dict, normalize it to 327 ``{'type': value}``. 328 329 :forms: 330 :sugared: ``Dict[str, Union[str, TypeRef]]`` 331 :canonical: ``Dict[str, TypeRef]`` 332 333 :param members: The members value to normalize. 334 335 :return: None, ``members`` is normalized in-place as needed. 336 """ 337 if isinstance(members, dict): 338 for key, arg in members.items(): 339 if isinstance(arg, dict): 340 continue 341 members[key] = {'type': arg} 342 343 344def check_type(value: Optional[object], 345 info: QAPISourceInfo, 346 source: str, 347 allow_array: bool = False, 348 allow_dict: Union[bool, str] = False) -> None: 349 """ 350 Normalize and validate the QAPI type of ``value``. 351 352 Python types of ``str`` or ``None`` are always allowed. 353 354 :param value: The value to check. 355 :param info: QAPI schema source file information. 356 :param source: Error string describing this ``value``. 357 :param allow_array: 358 Allow a ``List[str]`` of length 1, which indicates an array of 359 the type named by the list element. 360 :param allow_dict: 361 Allow a dict. Its members can be struct type members or union 362 branches. When the value of ``allow_dict`` is in pragma 363 ``member-name-exceptions``, the dict's keys may violate the 364 member naming rules. The dict members are normalized in place. 365 366 :raise QAPISemError: When ``value`` fails validation. 367 :return: None, ``value`` is normalized in-place as needed. 368 """ 369 if value is None: 370 return 371 372 # Type name 373 if isinstance(value, str): 374 return 375 376 # Array type 377 if isinstance(value, list): 378 if not allow_array: 379 raise QAPISemError(info, "%s cannot be an array" % source) 380 if len(value) != 1 or not isinstance(value[0], str): 381 raise QAPISemError(info, 382 "%s: array type must contain single type name" % 383 source) 384 return 385 386 # Anonymous type 387 388 if not allow_dict: 389 raise QAPISemError(info, "%s should be a type name" % source) 390 391 if not isinstance(value, dict): 392 raise QAPISemError(info, 393 "%s should be an object or type name" % source) 394 395 permissive = False 396 if isinstance(allow_dict, str): 397 permissive = allow_dict in info.pragma.member_name_exceptions 398 399 # value is a dictionary, check that each member is okay 400 for (key, arg) in value.items(): 401 key_source = "%s member '%s'" % (source, key) 402 if key.startswith('*'): 403 key = key[1:] 404 check_name_lower(key, info, key_source, 405 permit_upper=permissive, 406 permit_underscore=permissive) 407 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 408 raise QAPISemError(info, "%s uses reserved name" % key_source) 409 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 410 check_if(arg, info, key_source) 411 check_features(arg.get('features'), info) 412 check_type(arg['type'], info, key_source, allow_array=True) 413 414 415def check_features(features: Optional[object], 416 info: QAPISourceInfo) -> None: 417 """ 418 Normalize and validate the ``features`` member. 419 420 ``features`` may be a ``list`` of either ``str`` or ``dict``. 421 Any ``str`` element will be normalized to ``{'name': element}``. 422 423 :forms: 424 :sugared: ``List[Union[str, Feature]]`` 425 :canonical: ``List[Feature]`` 426 427 :param features: The features member value to validate. 428 :param info: QAPI schema source file information. 429 430 :raise QAPISemError: When ``features`` fails validation. 431 :return: None, ``features`` is normalized in-place as needed. 432 """ 433 if features is None: 434 return 435 if not isinstance(features, list): 436 raise QAPISemError(info, "'features' must be an array") 437 features[:] = [f if isinstance(f, dict) else {'name': f} 438 for f in features] 439 for feat in features: 440 source = "'features' member" 441 assert isinstance(feat, dict) 442 check_keys(feat, info, source, ['name'], ['if']) 443 check_name_is_str(feat['name'], info, source) 444 source = "%s '%s'" % (source, feat['name']) 445 check_name_lower(feat['name'], info, source) 446 check_if(feat, info, source) 447 448 449def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 450 """ 451 Normalize and validate this expression as an ``enum`` definition. 452 453 :param expr: The expression to validate. 454 :param info: QAPI schema source file information. 455 456 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 457 :return: None, ``expr`` is normalized in-place as needed. 458 """ 459 name = expr['enum'] 460 members = expr['data'] 461 prefix = expr.get('prefix') 462 463 if not isinstance(members, list): 464 raise QAPISemError(info, "'data' must be an array") 465 if prefix is not None and not isinstance(prefix, str): 466 raise QAPISemError(info, "'prefix' must be a string") 467 468 permissive = name in info.pragma.member_name_exceptions 469 470 members[:] = [m if isinstance(m, dict) else {'name': m} 471 for m in members] 472 for member in members: 473 source = "'data' member" 474 check_keys(member, info, source, ['name'], ['if', 'features']) 475 member_name = member['name'] 476 check_name_is_str(member_name, info, source) 477 source = "%s '%s'" % (source, member_name) 478 # Enum members may start with a digit 479 if member_name[0].isdigit(): 480 member_name = 'd' + member_name # Hack: hide the digit 481 check_name_lower(member_name, info, source, 482 permit_upper=permissive, 483 permit_underscore=permissive) 484 check_if(member, info, source) 485 check_features(member.get('features'), info) 486 487 488def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 489 """ 490 Normalize and validate this expression as a ``struct`` definition. 491 492 :param expr: The expression to validate. 493 :param info: QAPI schema source file information. 494 495 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 496 :return: None, ``expr`` is normalized in-place as needed. 497 """ 498 name = cast(str, expr['struct']) # Checked in check_exprs 499 members = expr['data'] 500 501 check_type(members, info, "'data'", allow_dict=name) 502 check_type(expr.get('base'), info, "'base'") 503 504 505def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 506 """ 507 Normalize and validate this expression as a ``union`` definition. 508 509 :param expr: The expression to validate. 510 :param info: QAPI schema source file information. 511 512 :raise QAPISemError: when ``expr`` is not a valid ``union``. 513 :return: None, ``expr`` is normalized in-place as needed. 514 """ 515 name = cast(str, expr['union']) # Checked in check_exprs 516 base = expr['base'] 517 discriminator = expr['discriminator'] 518 members = expr['data'] 519 520 check_type(base, info, "'base'", allow_dict=name) 521 check_name_is_str(discriminator, info, "'discriminator'") 522 523 if not isinstance(members, dict): 524 raise QAPISemError(info, "'data' must be an object") 525 526 for (key, value) in members.items(): 527 source = "'data' member '%s'" % key 528 check_keys(value, info, source, ['type'], ['if']) 529 check_if(value, info, source) 530 check_type(value['type'], info, source, allow_array=not base) 531 532 533def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 534 """ 535 Normalize and validate this expression as an ``alternate`` definition. 536 537 :param expr: The expression to validate. 538 :param info: QAPI schema source file information. 539 540 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 541 :return: None, ``expr`` is normalized in-place as needed. 542 """ 543 members = expr['data'] 544 545 if not members: 546 raise QAPISemError(info, "'data' must not be empty") 547 548 if not isinstance(members, dict): 549 raise QAPISemError(info, "'data' must be an object") 550 551 for (key, value) in members.items(): 552 source = "'data' member '%s'" % key 553 check_name_lower(key, info, source) 554 check_keys(value, info, source, ['type'], ['if']) 555 check_if(value, info, source) 556 check_type(value['type'], info, source, allow_array=True) 557 558 559def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 560 """ 561 Normalize and validate this expression as a ``command`` definition. 562 563 :param expr: The expression to validate. 564 :param info: QAPI schema source file information. 565 566 :raise QAPISemError: When ``expr`` is not a valid ``command``. 567 :return: None, ``expr`` is normalized in-place as needed. 568 """ 569 args = expr.get('data') 570 rets = expr.get('returns') 571 boxed = expr.get('boxed', False) 572 573 if boxed and args is None: 574 raise QAPISemError(info, "'boxed': true requires 'data'") 575 check_type(args, info, "'data'", allow_dict=not boxed) 576 check_type(rets, info, "'returns'", allow_array=True) 577 578 579def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 580 """ 581 Normalize and validate this expression as an ``event`` definition. 582 583 :param expr: The expression to validate. 584 :param info: QAPI schema source file information. 585 586 :raise QAPISemError: When ``expr`` is not a valid ``event``. 587 :return: None, ``expr`` is normalized in-place as needed. 588 """ 589 args = expr.get('data') 590 boxed = expr.get('boxed', False) 591 592 if boxed and args is None: 593 raise QAPISemError(info, "'boxed': true requires 'data'") 594 check_type(args, info, "'data'", allow_dict=not boxed) 595 596 597def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 598 """ 599 Validate and normalize a list of parsed QAPI schema expressions. 600 601 This function accepts a list of expressions and metadata as returned 602 by the parser. It destructively normalizes the expressions in-place. 603 604 :param exprs: The list of expressions to normalize and validate. 605 606 :raise QAPISemError: When any expression fails validation. 607 :return: The same list of expressions (now modified). 608 """ 609 for expr_elem in exprs: 610 # Expression 611 assert isinstance(expr_elem['expr'], dict) 612 for key in expr_elem['expr'].keys(): 613 assert isinstance(key, str) 614 expr: _JSONObject = expr_elem['expr'] 615 616 # QAPISourceInfo 617 assert isinstance(expr_elem['info'], QAPISourceInfo) 618 info: QAPISourceInfo = expr_elem['info'] 619 620 # Optional[QAPIDoc] 621 tmp = expr_elem.get('doc') 622 assert tmp is None or isinstance(tmp, QAPIDoc) 623 doc: Optional[QAPIDoc] = tmp 624 625 if 'include' in expr: 626 continue 627 628 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 629 'command', 'event'} 630 if len(metas) != 1: 631 raise QAPISemError( 632 info, 633 "expression must have exactly one key" 634 " 'enum', 'struct', 'union', 'alternate'," 635 " 'command', 'event'") 636 meta = metas.pop() 637 638 check_name_is_str(expr[meta], info, "'%s'" % meta) 639 name = cast(str, expr[meta]) 640 info.set_defn(meta, name) 641 check_defn_name_str(name, info, meta) 642 643 if doc: 644 if doc.symbol != name: 645 raise QAPISemError( 646 info, "documentation comment is for '%s'" % doc.symbol) 647 doc.check_expr(expr) 648 elif info.pragma.doc_required: 649 raise QAPISemError(info, 650 "documentation comment required") 651 652 if meta == 'enum': 653 check_keys(expr, info, meta, 654 ['enum', 'data'], ['if', 'features', 'prefix']) 655 check_enum(expr, info) 656 elif meta == 'union': 657 check_keys(expr, info, meta, 658 ['union', 'base', 'discriminator', 'data'], 659 ['if', 'features']) 660 normalize_members(expr.get('base')) 661 normalize_members(expr['data']) 662 check_union(expr, info) 663 elif meta == 'alternate': 664 check_keys(expr, info, meta, 665 ['alternate', 'data'], ['if', 'features']) 666 normalize_members(expr['data']) 667 check_alternate(expr, info) 668 elif meta == 'struct': 669 check_keys(expr, info, meta, 670 ['struct', 'data'], ['base', 'if', 'features']) 671 normalize_members(expr['data']) 672 check_struct(expr, info) 673 elif meta == 'command': 674 check_keys(expr, info, meta, 675 ['command'], 676 ['data', 'returns', 'boxed', 'if', 'features', 677 'gen', 'success-response', 'allow-oob', 678 'allow-preconfig', 'coroutine']) 679 normalize_members(expr.get('data')) 680 check_command(expr, info) 681 elif meta == 'event': 682 check_keys(expr, info, meta, 683 ['event'], ['data', 'boxed', 'if', 'features']) 684 normalize_members(expr.get('data')) 685 check_event(expr, info) 686 else: 687 assert False, 'unexpected meta type' 688 689 check_if(expr, info, meta) 690 check_features(expr.get('features'), info) 691 check_flags(expr, info) 692 693 return exprs 694