1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2019 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# 12# This work is licensed under the terms of the GNU GPL, version 2. 13# See the COPYING file in the top-level directory. 14 15""" 16Normalize and validate (context-free) QAPI schema expression structures. 17 18`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 19consisting of dict, list, str, bool, and int nodes. This module ensures 20that these nested structures have the correct type(s) and key(s) where 21appropriate for the QAPI context-free grammar. 22 23The QAPI schema expression language allows for certain syntactic sugar; 24this module also handles the normalization process of these nested 25structures. 26 27See `check_exprs` for the main entry point. 28 29See `schema.QAPISchema` for processing into native Python data 30structures and contextual semantic validation. 31""" 32 33import re 34from typing import ( 35 Collection, 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIDoc 47from .source import QAPISourceInfo 48 49 50# Deserialized JSON objects as returned by the parser. 51# The values of this mapping are not necessary to exhaustively type 52# here (and also not practical as long as mypy lacks recursive 53# types), because the purpose of this module is to interrogate that 54# type. 55_JSONObject = Dict[str, object] 56 57 58# See check_name_str(), below. 59valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 60 r'(x-)?' 61 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 62 63 64def check_name_is_str(name: object, 65 info: QAPISourceInfo, 66 source: str) -> None: 67 """ 68 Ensure that ``name`` is a ``str``. 69 70 :raise QAPISemError: When ``name`` fails validation. 71 """ 72 if not isinstance(name, str): 73 raise QAPISemError(info, "%s requires a string name" % source) 74 75 76def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 77 """ 78 Ensure that ``name`` is a valid QAPI name. 79 80 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 81 starting with a letter. It may be prefixed by a downstream prefix 82 of the form __RFQDN_, or the experimental prefix ``x-``. If both 83 prefixes are present, the __RFDQN_ prefix goes first. 84 85 A valid name cannot start with ``q_``, which is reserved. 86 87 :param name: Name to check. 88 :param info: QAPI schema source file information. 89 :param source: Error string describing what ``name`` belongs to. 90 91 :raise QAPISemError: When ``name`` fails validation. 92 :return: The stem of the valid name, with no prefixes. 93 """ 94 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 95 # and 'q_obj_*' implicit type names. 96 match = valid_name.match(name) 97 if not match or c_name(name, False).startswith('q_'): 98 raise QAPISemError(info, "%s has an invalid name" % source) 99 return match.group(3) 100 101 102def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 103 """ 104 Ensure that ``name`` is a valid event name. 105 106 This means it must be a valid QAPI name as checked by 107 `check_name_str()`, but where the stem prohibits lowercase 108 characters and ``-``. 109 110 :param name: Name to check. 111 :param info: QAPI schema source file information. 112 :param source: Error string describing what ``name`` belongs to. 113 114 :raise QAPISemError: When ``name`` fails validation. 115 """ 116 stem = check_name_str(name, info, source) 117 if re.search(r'[a-z-]', stem): 118 raise QAPISemError( 119 info, "name of %s must not use lowercase or '-'" % source) 120 121 122def check_name_lower(name: str, info: QAPISourceInfo, source: str, 123 permit_upper: bool = False, 124 permit_underscore: bool = False) -> None: 125 """ 126 Ensure that ``name`` is a valid command or member name. 127 128 This means it must be a valid QAPI name as checked by 129 `check_name_str()`, but where the stem prohibits uppercase 130 characters and ``_``. 131 132 :param name: Name to check. 133 :param info: QAPI schema source file information. 134 :param source: Error string describing what ``name`` belongs to. 135 :param permit_upper: Additionally permit uppercase. 136 :param permit_underscore: Additionally permit ``_``. 137 138 :raise QAPISemError: When ``name`` fails validation. 139 """ 140 stem = check_name_str(name, info, source) 141 if ((not permit_upper and re.search(r'[A-Z]', stem)) 142 or (not permit_underscore and '_' in stem)): 143 raise QAPISemError( 144 info, "name of %s must not use uppercase or '_'" % source) 145 146 147def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 148 """ 149 Ensure that ``name`` is a valid user-defined type name. 150 151 This means it must be a valid QAPI name as checked by 152 `check_name_str()`, but where the stem must be in CamelCase. 153 154 :param name: Name to check. 155 :param info: QAPI schema source file information. 156 :param source: Error string describing what ``name`` belongs to. 157 158 :raise QAPISemError: When ``name`` fails validation. 159 """ 160 stem = check_name_str(name, info, source) 161 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 162 raise QAPISemError(info, "name of %s must use CamelCase" % source) 163 164 165def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 166 """ 167 Ensure that ``name`` is a valid definition name. 168 169 Based on the value of ``meta``, this means that: 170 - 'event' names adhere to `check_name_upper()`. 171 - 'command' names adhere to `check_name_lower()`. 172 - Else, meta is a type, and must pass `check_name_camel()`. 173 These names must not end with ``Kind`` nor ``List``. 174 175 :param name: Name to check. 176 :param info: QAPI schema source file information. 177 :param meta: Meta-type name of the QAPI expression. 178 179 :raise QAPISemError: When ``name`` fails validation. 180 """ 181 if meta == 'event': 182 check_name_upper(name, info, meta) 183 elif meta == 'command': 184 check_name_lower( 185 name, info, meta, 186 permit_underscore=name in info.pragma.command_name_exceptions) 187 else: 188 check_name_camel(name, info, meta) 189 if name.endswith('Kind') or name.endswith('List'): 190 raise QAPISemError( 191 info, "%s name should not end in '%s'" % (meta, name[-4:])) 192 193 194def check_keys(value: _JSONObject, 195 info: QAPISourceInfo, 196 source: str, 197 required: Collection[str], 198 optional: Collection[str]) -> None: 199 """ 200 Ensure that a dict has a specific set of keys. 201 202 :param value: The dict to check. 203 :param info: QAPI schema source file information. 204 :param source: Error string describing this ``value``. 205 :param required: Keys that *must* be present. 206 :param optional: Keys that *may* be present. 207 208 :raise QAPISemError: When unknown keys are present. 209 """ 210 211 def pprint(elems: Iterable[str]) -> str: 212 return ', '.join("'" + e + "'" for e in sorted(elems)) 213 214 missing = set(required) - set(value) 215 if missing: 216 raise QAPISemError( 217 info, 218 "%s misses key%s %s" 219 % (source, 's' if len(missing) > 1 else '', 220 pprint(missing))) 221 allowed = set(required) | set(optional) 222 unknown = set(value) - allowed 223 if unknown: 224 raise QAPISemError( 225 info, 226 "%s has unknown key%s %s\nValid keys are %s." 227 % (source, 's' if len(unknown) > 1 else '', 228 pprint(unknown), pprint(allowed))) 229 230 231def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: 232 """ 233 Ensure flag members (if present) have valid values. 234 235 :param expr: The expression to validate. 236 :param info: QAPI schema source file information. 237 238 :raise QAPISemError: 239 When certain flags have an invalid value, or when 240 incompatible flags are present. 241 """ 242 for key in ['gen', 'success-response']: 243 if key in expr and expr[key] is not False: 244 raise QAPISemError( 245 info, "flag '%s' may only use false value" % key) 246 for key in ['boxed', 'allow-oob', 'allow-preconfig', 'coroutine']: 247 if key in expr and expr[key] is not True: 248 raise QAPISemError( 249 info, "flag '%s' may only use true value" % key) 250 if 'allow-oob' in expr and 'coroutine' in expr: 251 # This is not necessarily a fundamental incompatibility, but 252 # we don't have a use case and the desired semantics isn't 253 # obvious. The simplest solution is to forbid it until we get 254 # a use case for it. 255 raise QAPISemError(info, "flags 'allow-oob' and 'coroutine' " 256 "are incompatible") 257 258 259def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 260 """ 261 Normalize and validate the ``if`` member of an object. 262 263 The ``if`` member may be either a ``str`` or a ``List[str]``. 264 A ``str`` value will be normalized to ``List[str]``. 265 266 :forms: 267 :sugared: ``Union[str, List[str]]`` 268 :canonical: ``List[str]`` 269 270 :param expr: The expression containing the ``if`` member to validate. 271 :param info: QAPI schema source file information. 272 :param source: Error string describing ``expr``. 273 274 :raise QAPISemError: 275 When the "if" member fails validation, or when there are no 276 non-empty conditions. 277 :return: None, ``expr`` is normalized in-place as needed. 278 """ 279 ifcond = expr.get('if') 280 if ifcond is None: 281 return 282 283 if isinstance(ifcond, list): 284 if not ifcond: 285 raise QAPISemError( 286 info, "'if' condition [] of %s is useless" % source) 287 else: 288 # Normalize to a list 289 ifcond = expr['if'] = [ifcond] 290 291 for elt in ifcond: 292 if not isinstance(elt, str): 293 raise QAPISemError( 294 info, 295 "'if' condition of %s must be a string or a list of strings" 296 % source) 297 if not elt.strip(): 298 raise QAPISemError( 299 info, 300 "'if' condition '%s' of %s makes no sense" 301 % (elt, source)) 302 303 304def normalize_members(members: object) -> None: 305 """ 306 Normalize a "members" value. 307 308 If ``members`` is a dict, for every value in that dict, if that 309 value is not itself already a dict, normalize it to 310 ``{'type': value}``. 311 312 :forms: 313 :sugared: ``Dict[str, Union[str, TypeRef]]`` 314 :canonical: ``Dict[str, TypeRef]`` 315 316 :param members: The members value to normalize. 317 318 :return: None, ``members`` is normalized in-place as needed. 319 """ 320 if isinstance(members, dict): 321 for key, arg in members.items(): 322 if isinstance(arg, dict): 323 continue 324 members[key] = {'type': arg} 325 326 327def check_type(value: Optional[object], 328 info: QAPISourceInfo, 329 source: str, 330 allow_array: bool = False, 331 allow_dict: Union[bool, str] = False) -> None: 332 """ 333 Normalize and validate the QAPI type of ``value``. 334 335 Python types of ``str`` or ``None`` are always allowed. 336 337 :param value: The value to check. 338 :param info: QAPI schema source file information. 339 :param source: Error string describing this ``value``. 340 :param allow_array: 341 Allow a ``List[str]`` of length 1, which indicates an array of 342 the type named by the list element. 343 :param allow_dict: 344 Allow a dict. Its members can be struct type members or union 345 branches. When the value of ``allow_dict`` is in pragma 346 ``member-name-exceptions``, the dict's keys may violate the 347 member naming rules. The dict members are normalized in place. 348 349 :raise QAPISemError: When ``value`` fails validation. 350 :return: None, ``value`` is normalized in-place as needed. 351 """ 352 if value is None: 353 return 354 355 # Type name 356 if isinstance(value, str): 357 return 358 359 # Array type 360 if isinstance(value, list): 361 if not allow_array: 362 raise QAPISemError(info, "%s cannot be an array" % source) 363 if len(value) != 1 or not isinstance(value[0], str): 364 raise QAPISemError(info, 365 "%s: array type must contain single type name" % 366 source) 367 return 368 369 # Anonymous type 370 371 if not allow_dict: 372 raise QAPISemError(info, "%s should be a type name" % source) 373 374 if not isinstance(value, dict): 375 raise QAPISemError(info, 376 "%s should be an object or type name" % source) 377 378 permissive = False 379 if isinstance(allow_dict, str): 380 permissive = allow_dict in info.pragma.member_name_exceptions 381 382 # value is a dictionary, check that each member is okay 383 for (key, arg) in value.items(): 384 key_source = "%s member '%s'" % (source, key) 385 if key.startswith('*'): 386 key = key[1:] 387 check_name_lower(key, info, key_source, 388 permit_upper=permissive, 389 permit_underscore=permissive) 390 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 391 raise QAPISemError(info, "%s uses reserved name" % key_source) 392 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 393 check_if(arg, info, key_source) 394 check_features(arg.get('features'), info) 395 check_type(arg['type'], info, key_source, allow_array=True) 396 397 398def check_features(features: Optional[object], 399 info: QAPISourceInfo) -> None: 400 """ 401 Normalize and validate the ``features`` member. 402 403 ``features`` may be a ``list`` of either ``str`` or ``dict``. 404 Any ``str`` element will be normalized to ``{'name': element}``. 405 406 :forms: 407 :sugared: ``List[Union[str, Feature]]`` 408 :canonical: ``List[Feature]`` 409 410 :param features: The features member value to validate. 411 :param info: QAPI schema source file information. 412 413 :raise QAPISemError: When ``features`` fails validation. 414 :return: None, ``features`` is normalized in-place as needed. 415 """ 416 if features is None: 417 return 418 if not isinstance(features, list): 419 raise QAPISemError(info, "'features' must be an array") 420 features[:] = [f if isinstance(f, dict) else {'name': f} 421 for f in features] 422 for feat in features: 423 source = "'features' member" 424 assert isinstance(feat, dict) 425 check_keys(feat, info, source, ['name'], ['if']) 426 check_name_is_str(feat['name'], info, source) 427 source = "%s '%s'" % (source, feat['name']) 428 check_name_str(feat['name'], info, source) 429 check_if(feat, info, source) 430 431 432def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: 433 """ 434 Normalize and validate this expression as an ``enum`` definition. 435 436 :param expr: The expression to validate. 437 :param info: QAPI schema source file information. 438 439 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 440 :return: None, ``expr`` is normalized in-place as needed. 441 """ 442 name = expr['enum'] 443 members = expr['data'] 444 prefix = expr.get('prefix') 445 446 if not isinstance(members, list): 447 raise QAPISemError(info, "'data' must be an array") 448 if prefix is not None and not isinstance(prefix, str): 449 raise QAPISemError(info, "'prefix' must be a string") 450 451 permissive = name in info.pragma.member_name_exceptions 452 453 members[:] = [m if isinstance(m, dict) else {'name': m} 454 for m in members] 455 for member in members: 456 source = "'data' member" 457 member_name = member['name'] 458 check_keys(member, info, source, ['name'], ['if']) 459 check_name_is_str(member_name, info, source) 460 source = "%s '%s'" % (source, member_name) 461 # Enum members may start with a digit 462 if member_name[0].isdigit(): 463 member_name = 'd' + member_name # Hack: hide the digit 464 check_name_lower(member_name, info, source, 465 permit_upper=permissive, 466 permit_underscore=permissive) 467 check_if(member, info, source) 468 469 470def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: 471 """ 472 Normalize and validate this expression as a ``struct`` definition. 473 474 :param expr: The expression to validate. 475 :param info: QAPI schema source file information. 476 477 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 478 :return: None, ``expr`` is normalized in-place as needed. 479 """ 480 name = cast(str, expr['struct']) # Checked in check_exprs 481 members = expr['data'] 482 483 check_type(members, info, "'data'", allow_dict=name) 484 check_type(expr.get('base'), info, "'base'") 485 486 487def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: 488 """ 489 Normalize and validate this expression as a ``union`` definition. 490 491 :param expr: The expression to validate. 492 :param info: QAPI schema source file information. 493 494 :raise QAPISemError: when ``expr`` is not a valid ``union``. 495 :return: None, ``expr`` is normalized in-place as needed. 496 """ 497 name = cast(str, expr['union']) # Checked in check_exprs 498 base = expr.get('base') 499 discriminator = expr.get('discriminator') 500 members = expr['data'] 501 502 if discriminator is None: # simple union 503 if base is not None: 504 raise QAPISemError(info, "'base' requires 'discriminator'") 505 else: # flat union 506 check_type(base, info, "'base'", allow_dict=name) 507 if not base: 508 raise QAPISemError(info, "'discriminator' requires 'base'") 509 check_name_is_str(discriminator, info, "'discriminator'") 510 511 if not isinstance(members, dict): 512 raise QAPISemError(info, "'data' must be an object") 513 514 for (key, value) in members.items(): 515 source = "'data' member '%s'" % key 516 if discriminator is None: 517 check_name_lower(key, info, source) 518 # else: name is in discriminator enum, which gets checked 519 check_keys(value, info, source, ['type'], ['if']) 520 check_if(value, info, source) 521 check_type(value['type'], info, source, allow_array=not base) 522 523 524def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: 525 """ 526 Normalize and validate this expression as an ``alternate`` definition. 527 528 :param expr: The expression to validate. 529 :param info: QAPI schema source file information. 530 531 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 532 :return: None, ``expr`` is normalized in-place as needed. 533 """ 534 members = expr['data'] 535 536 if not members: 537 raise QAPISemError(info, "'data' must not be empty") 538 539 if not isinstance(members, dict): 540 raise QAPISemError(info, "'data' must be an object") 541 542 for (key, value) in members.items(): 543 source = "'data' member '%s'" % key 544 check_name_lower(key, info, source) 545 check_keys(value, info, source, ['type'], ['if']) 546 check_if(value, info, source) 547 check_type(value['type'], info, source) 548 549 550def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: 551 """ 552 Normalize and validate this expression as a ``command`` definition. 553 554 :param expr: The expression to validate. 555 :param info: QAPI schema source file information. 556 557 :raise QAPISemError: When ``expr`` is not a valid ``command``. 558 :return: None, ``expr`` is normalized in-place as needed. 559 """ 560 args = expr.get('data') 561 rets = expr.get('returns') 562 boxed = expr.get('boxed', False) 563 564 if boxed and args is None: 565 raise QAPISemError(info, "'boxed': true requires 'data'") 566 check_type(args, info, "'data'", allow_dict=not boxed) 567 check_type(rets, info, "'returns'", allow_array=True) 568 569 570def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: 571 """ 572 Normalize and validate this expression as an ``event`` definition. 573 574 :param expr: The expression to validate. 575 :param info: QAPI schema source file information. 576 577 :raise QAPISemError: When ``expr`` is not a valid ``event``. 578 :return: None, ``expr`` is normalized in-place as needed. 579 """ 580 args = expr.get('data') 581 boxed = expr.get('boxed', False) 582 583 if boxed and args is None: 584 raise QAPISemError(info, "'boxed': true requires 'data'") 585 check_type(args, info, "'data'", allow_dict=not boxed) 586 587 588def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: 589 """ 590 Validate and normalize a list of parsed QAPI schema expressions. 591 592 This function accepts a list of expressions and metadata as returned 593 by the parser. It destructively normalizes the expressions in-place. 594 595 :param exprs: The list of expressions to normalize and validate. 596 597 :raise QAPISemError: When any expression fails validation. 598 :return: The same list of expressions (now modified). 599 """ 600 for expr_elem in exprs: 601 # Expression 602 assert isinstance(expr_elem['expr'], dict) 603 for key in expr_elem['expr'].keys(): 604 assert isinstance(key, str) 605 expr: _JSONObject = expr_elem['expr'] 606 607 # QAPISourceInfo 608 assert isinstance(expr_elem['info'], QAPISourceInfo) 609 info: QAPISourceInfo = expr_elem['info'] 610 611 # Optional[QAPIDoc] 612 tmp = expr_elem.get('doc') 613 assert tmp is None or isinstance(tmp, QAPIDoc) 614 doc: Optional[QAPIDoc] = tmp 615 616 if 'include' in expr: 617 continue 618 619 if 'enum' in expr: 620 meta = 'enum' 621 elif 'union' in expr: 622 meta = 'union' 623 elif 'alternate' in expr: 624 meta = 'alternate' 625 elif 'struct' in expr: 626 meta = 'struct' 627 elif 'command' in expr: 628 meta = 'command' 629 elif 'event' in expr: 630 meta = 'event' 631 else: 632 raise QAPISemError(info, "expression is missing metatype") 633 634 check_name_is_str(expr[meta], info, "'%s'" % meta) 635 name = cast(str, expr[meta]) 636 info.set_defn(meta, name) 637 check_defn_name_str(name, info, meta) 638 639 if doc: 640 if doc.symbol != name: 641 raise QAPISemError( 642 info, "documentation comment is for '%s'" % doc.symbol) 643 doc.check_expr(expr) 644 elif info.pragma.doc_required: 645 raise QAPISemError(info, 646 "documentation comment required") 647 648 if meta == 'enum': 649 check_keys(expr, info, meta, 650 ['enum', 'data'], ['if', 'features', 'prefix']) 651 check_enum(expr, info) 652 elif meta == 'union': 653 check_keys(expr, info, meta, 654 ['union', 'data'], 655 ['base', 'discriminator', 'if', 'features']) 656 normalize_members(expr.get('base')) 657 normalize_members(expr['data']) 658 check_union(expr, info) 659 elif meta == 'alternate': 660 check_keys(expr, info, meta, 661 ['alternate', 'data'], ['if', 'features']) 662 normalize_members(expr['data']) 663 check_alternate(expr, info) 664 elif meta == 'struct': 665 check_keys(expr, info, meta, 666 ['struct', 'data'], ['base', 'if', 'features']) 667 normalize_members(expr['data']) 668 check_struct(expr, info) 669 elif meta == 'command': 670 check_keys(expr, info, meta, 671 ['command'], 672 ['data', 'returns', 'boxed', 'if', 'features', 673 'gen', 'success-response', 'allow-oob', 674 'allow-preconfig', 'coroutine']) 675 normalize_members(expr.get('data')) 676 check_command(expr, info) 677 elif meta == 'event': 678 check_keys(expr, info, meta, 679 ['event'], ['data', 'boxed', 'if', 'features']) 680 normalize_members(expr.get('data')) 681 check_event(expr, info) 682 else: 683 assert False, 'unexpected meta type' 684 685 check_if(expr, info, meta) 686 check_features(expr.get('features'), info) 687 check_flags(expr, info) 688 689 return exprs 690