1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# Deserialized JSON objects as returned by the parser. 51# The values of this mapping are not necessary to exhaustively type 52# here (and also not practical as long as mypy lacks recursive 53# types), because the purpose of this module is to interrogate that 54# type. 55_JSONObject = Dict[str, object] 56 57 58# See check_name_str(), below. 59valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 60 r'(x-)?' 61 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 62 63 64def check_name_is_str(name: object, 65 info: QAPISourceInfo, 66 source: str) -> None: 67 """ 68 Ensure that ``name`` is a ``str``. 69 70 :raise QAPISemError: When ``name`` fails validation. 71 """ 72 if not isinstance(name, str): 73 raise QAPISemError(info, "%s requires a string name" % source) 74 75 76def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 77 """ 78 Ensure that ``name`` is a valid QAPI name. 79 80 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 81 starting with a letter. It may be prefixed by a downstream prefix 82 of the form __RFQDN_, or the experimental prefix ``x-``. If both 83 prefixes are present, the __RFDQN_ prefix goes first. 84 85 A valid name cannot start with ``q_``, which is reserved. 86 87 :param name: Name to check. 88 :param info: QAPI schema source file information. 89 :param source: Error string describing what ``name`` belongs to. 90 91 :raise QAPISemError: When ``name`` fails validation. 92 :return: The stem of the valid name, with no prefixes. 93 """ 94 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 95 # and 'q_obj_*' implicit type names. 96 match = valid_name.match(name) 97 if not match or c_name(name, False).startswith('q_'): 98 raise QAPISemError(info, "%s has an invalid name" % source) 99 return match.group(3) 100 101 102def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 103 """ 104 Ensure that ``name`` is a valid event name. 105 106 This means it must be a valid QAPI name as checked by 107 `check_name_str()`, but where the stem prohibits lowercase 108 characters and ``-``. 109 110 :param name: Name to check. 111 :param info: QAPI schema source file information. 112 :param source: Error string describing what ``name`` belongs to. 113 114 :raise QAPISemError: When ``name`` fails validation. 115 """ 116 stem = check_name_str(name, info, source) 117 if re.search(r'[a-z-]', stem): 118 raise QAPISemError( 119 info, "name of %s must not use lowercase or '-'" % source) 120 121 122def check_name_lower(name: str, info: QAPISourceInfo, source: str, 123 permit_upper: bool = False, 124 permit_underscore: bool = False) -> None: 125 """ 126 Ensure that ``name`` is a valid command or member name. 127 128 This means it must be a valid QAPI name as checked by 129 `check_name_str()`, but where the stem prohibits uppercase 130 characters and ``_``. 131 132 :param name: Name to check. 133 :param info: QAPI schema source file information. 134 :param source: Error string describing what ``name`` belongs to. 135 :param permit_upper: Additionally permit uppercase. 136 :param permit_underscore: Additionally permit ``_``. 137 138 :raise QAPISemError: When ``name`` fails validation. 139 """ 140 stem = check_name_str(name, info, source) 141 if ((not permit_upper and re.search(r'[A-Z]', stem)) 142 or (not permit_underscore and '_' in stem)): 143 raise QAPISemError( 144 info, "name of %s must not use uppercase or '_'" % source) 145 146 147def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 148 """ 149 Ensure that ``name`` is a valid user-defined type name. 150 151 This means it must be a valid QAPI name as checked by 152 `check_name_str()`, but where the stem must be in CamelCase. 153 154 :param name: Name to check. 155 :param info: QAPI schema source file information. 156 :param source: Error string describing what ``name`` belongs to. 157 158 :raise QAPISemError: When ``name`` fails validation. 159 """ 160 stem = check_name_str(name, info, source) 161 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 162 raise QAPISemError(info, "name of %s must use CamelCase" % source) 163 164 165def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 166 """ 167 Ensure that ``name`` is a valid definition name. 168 169 Based on the value of ``meta``, this means that: 170 - 'event' names adhere to `check_name_upper()`. 171 - 'command' names adhere to `check_name_lower()`. 172 - Else, meta is a type, and must pass `check_name_camel()`. 173 These names must not end with ``List``. 174 175 :param name: Name to check. 176 :param info: QAPI schema source file information. 177 :param meta: Meta-type name of the QAPI expression. 178 179 :raise QAPISemError: When ``name`` fails validation. 180 """ 181 if meta == 'event': 182 check_name_upper(name, info, meta) 183 elif meta == 'command': 184 check_name_lower( 185 name, info, meta, 186 permit_underscore=name in info.pragma.command_name_exceptions) 187 else: 188 check_name_camel(name, info, meta) 189 if name.endswith('List'): 190 raise QAPISemError( 191 info, "%s name should not end in 'List'" % meta) 192 193 194def check_keys(value: _JSONObject, 195 info: QAPISourceInfo, 196 source: str, 197 required: List[str], 198 optional: List[str]) -> None: 199 """ 200 Ensure that a dict has a specific set of keys. 201 202 :param value: The dict to check. 203 :param info: QAPI schema source file information. 204 :param source: Error string describing this ``value``. 205 :param required: Keys that *must* be present. 206 :param optional: Keys that *may* be present. 207 208 :raise QAPISemError: When unknown keys are present. 209 """ 210 211 def pprint(elems: Iterable[str]) -> str: 212 return ', '.join("'" + e + "'" for e in sorted(elems)) 213 214 missing = set(required) - set(value) 215 if missing: 216 raise QAPISemError( 217 info, 218 "%s misses key%s %s" 219 % (source, 's' if len(missing) > 1 else '', 220 pprint(missing))) 221 allowed = set(required) | set(optional) 222 unknown = set(value) - allowed 223 if unknown: 224 raise QAPISemError( 225 info, 226 "%s has unknown key%s %s\nValid keys are %s." 227 % (source, 's' if len(unknown) > 1 else '', 228 pprint(unknown), pprint(allowed))) 229 230 231def check_flags(expr: QAPIExpression) -> None: 232 """ 233 Ensure flag members (if present) have valid values. 234 235 :param expr: The expression to validate. 236 237 :raise QAPISemError: 238 When certain flags have an invalid value, or when 239 incompatible flags are present. 240 """ 241 for key in ('gen', 'success-response'): 242 if key in expr and expr[key] is not False: 243 raise QAPISemError( 244 expr.info, "flag '%s' may only use false value" % key) 245 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 246 if key in expr and expr[key] is not True: 247 raise QAPISemError( 248 expr.info, "flag '%s' may only use true value" % key) 249 if 'allow-oob' in expr and 'coroutine' in expr: 250 # This is not necessarily a fundamental incompatibility, but 251 # we don't have a use case and the desired semantics isn't 252 # obvious. The simplest solution is to forbid it until we get 253 # a use case for it. 254 raise QAPISemError( 255 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 256 257 258def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: 259 """ 260 Validate the ``if`` member of an object. 261 262 The ``if`` member may be either a ``str`` or a dict. 263 264 :param expr: The expression containing the ``if`` member to validate. 265 :param info: QAPI schema source file information. 266 :param source: Error string describing ``expr``. 267 268 :raise QAPISemError: 269 When the "if" member fails validation, or when there are no 270 non-empty conditions. 271 :return: None 272 """ 273 274 def _check_if(cond: Union[str, object]) -> None: 275 if isinstance(cond, str): 276 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 277 raise QAPISemError( 278 info, 279 "'if' condition '%s' of %s is not a valid identifier" 280 % (cond, source)) 281 return 282 283 if not isinstance(cond, dict): 284 raise QAPISemError( 285 info, 286 "'if' condition of %s must be a string or an object" % source) 287 check_keys(cond, info, "'if' condition of %s" % source, [], 288 ["all", "any", "not"]) 289 if len(cond) != 1: 290 raise QAPISemError( 291 info, 292 "'if' condition of %s has conflicting keys" % source) 293 294 if 'not' in cond: 295 _check_if(cond['not']) 296 elif 'all' in cond: 297 _check_infix('all', cond['all']) 298 else: 299 _check_infix('any', cond['any']) 300 301 def _check_infix(operator: str, operands: object) -> None: 302 if not isinstance(operands, list): 303 raise QAPISemError( 304 info, 305 "'%s' condition of %s must be an array" 306 % (operator, source)) 307 if not operands: 308 raise QAPISemError( 309 info, "'if' condition [] of %s is useless" % source) 310 for operand in operands: 311 _check_if(operand) 312 313 ifcond = expr.get('if') 314 if ifcond is None: 315 return 316 317 _check_if(ifcond) 318 319 320def normalize_members(members: object) -> None: 321 """ 322 Normalize a "members" value. 323 324 If ``members`` is a dict, for every value in that dict, if that 325 value is not itself already a dict, normalize it to 326 ``{'type': value}``. 327 328 :forms: 329 :sugared: ``Dict[str, Union[str, TypeRef]]`` 330 :canonical: ``Dict[str, TypeRef]`` 331 332 :param members: The members value to normalize. 333 334 :return: None, ``members`` is normalized in-place as needed. 335 """ 336 if isinstance(members, dict): 337 for key, arg in members.items(): 338 if isinstance(arg, dict): 339 continue 340 members[key] = {'type': arg} 341 342 343def check_type(value: Optional[object], 344 info: QAPISourceInfo, 345 source: str, 346 allow_array: bool = False, 347 allow_dict: Union[bool, str] = False) -> None: 348 """ 349 Normalize and validate the QAPI type of ``value``. 350 351 Python types of ``str`` or ``None`` are always allowed. 352 353 :param value: The value to check. 354 :param info: QAPI schema source file information. 355 :param source: Error string describing this ``value``. 356 :param allow_array: 357 Allow a ``List[str]`` of length 1, which indicates an array of 358 the type named by the list element. 359 :param allow_dict: 360 Allow a dict. Its members can be struct type members or union 361 branches. When the value of ``allow_dict`` is in pragma 362 ``member-name-exceptions``, the dict's keys may violate the 363 member naming rules. The dict members are normalized in place. 364 365 :raise QAPISemError: When ``value`` fails validation. 366 :return: None, ``value`` is normalized in-place as needed. 367 """ 368 if value is None: 369 return 370 371 # Type name 372 if isinstance(value, str): 373 return 374 375 # Array type 376 if isinstance(value, list): 377 if not allow_array: 378 raise QAPISemError(info, "%s cannot be an array" % source) 379 if len(value) != 1 or not isinstance(value[0], str): 380 raise QAPISemError(info, 381 "%s: array type must contain single type name" % 382 source) 383 return 384 385 # Anonymous type 386 387 if not allow_dict: 388 raise QAPISemError(info, "%s should be a type name" % source) 389 390 if not isinstance(value, dict): 391 raise QAPISemError(info, 392 "%s should be an object or type name" % source) 393 394 permissive = False 395 if isinstance(allow_dict, str): 396 permissive = allow_dict in info.pragma.member_name_exceptions 397 398 # value is a dictionary, check that each member is okay 399 for (key, arg) in value.items(): 400 key_source = "%s member '%s'" % (source, key) 401 if key.startswith('*'): 402 key = key[1:] 403 check_name_lower(key, info, key_source, 404 permit_upper=permissive, 405 permit_underscore=permissive) 406 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 407 raise QAPISemError(info, "%s uses reserved name" % key_source) 408 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 409 check_if(arg, info, key_source) 410 check_features(arg.get('features'), info) 411 check_type(arg['type'], info, key_source, allow_array=True) 412 413 414def check_features(features: Optional[object], 415 info: QAPISourceInfo) -> None: 416 """ 417 Normalize and validate the ``features`` member. 418 419 ``features`` may be a ``list`` of either ``str`` or ``dict``. 420 Any ``str`` element will be normalized to ``{'name': element}``. 421 422 :forms: 423 :sugared: ``List[Union[str, Feature]]`` 424 :canonical: ``List[Feature]`` 425 426 :param features: The features member value to validate. 427 :param info: QAPI schema source file information. 428 429 :raise QAPISemError: When ``features`` fails validation. 430 :return: None, ``features`` is normalized in-place as needed. 431 """ 432 if features is None: 433 return 434 if not isinstance(features, list): 435 raise QAPISemError(info, "'features' must be an array") 436 features[:] = [f if isinstance(f, dict) else {'name': f} 437 for f in features] 438 for feat in features: 439 source = "'features' member" 440 assert isinstance(feat, dict) 441 check_keys(feat, info, source, ['name'], ['if']) 442 check_name_is_str(feat['name'], info, source) 443 source = "%s '%s'" % (source, feat['name']) 444 check_name_lower(feat['name'], info, source) 445 check_if(feat, info, source) 446 447 448def check_enum(expr: QAPIExpression) -> None: 449 """ 450 Normalize and validate this expression as an ``enum`` definition. 451 452 :param expr: The expression to validate. 453 454 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 455 :return: None, ``expr`` is normalized in-place as needed. 456 """ 457 name = expr['enum'] 458 members = expr['data'] 459 prefix = expr.get('prefix') 460 info = expr.info 461 462 if not isinstance(members, list): 463 raise QAPISemError(info, "'data' must be an array") 464 if prefix is not None and not isinstance(prefix, str): 465 raise QAPISemError(info, "'prefix' must be a string") 466 467 permissive = name in info.pragma.member_name_exceptions 468 469 members[:] = [m if isinstance(m, dict) else {'name': m} 470 for m in members] 471 for member in members: 472 source = "'data' member" 473 check_keys(member, info, source, ['name'], ['if', 'features']) 474 member_name = member['name'] 475 check_name_is_str(member_name, info, source) 476 source = "%s '%s'" % (source, member_name) 477 # Enum members may start with a digit 478 if member_name[0].isdigit(): 479 member_name = 'd' + member_name # Hack: hide the digit 480 check_name_lower(member_name, info, source, 481 permit_upper=permissive, 482 permit_underscore=permissive) 483 check_if(member, info, source) 484 check_features(member.get('features'), info) 485 486 487def check_struct(expr: QAPIExpression) -> None: 488 """ 489 Normalize and validate this expression as a ``struct`` definition. 490 491 :param expr: The expression to validate. 492 493 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 494 :return: None, ``expr`` is normalized in-place as needed. 495 """ 496 name = cast(str, expr['struct']) # Checked in check_exprs 497 members = expr['data'] 498 499 check_type(members, expr.info, "'data'", allow_dict=name) 500 check_type(expr.get('base'), expr.info, "'base'") 501 502 503def check_union(expr: QAPIExpression) -> None: 504 """ 505 Normalize and validate this expression as a ``union`` definition. 506 507 :param expr: The expression to validate. 508 509 :raise QAPISemError: when ``expr`` is not a valid ``union``. 510 :return: None, ``expr`` is normalized in-place as needed. 511 """ 512 name = cast(str, expr['union']) # Checked in check_exprs 513 base = expr['base'] 514 discriminator = expr['discriminator'] 515 members = expr['data'] 516 info = expr.info 517 518 check_type(base, info, "'base'", allow_dict=name) 519 check_name_is_str(discriminator, info, "'discriminator'") 520 521 if not isinstance(members, dict): 522 raise QAPISemError(info, "'data' must be an object") 523 524 for (key, value) in members.items(): 525 source = "'data' member '%s'" % key 526 check_keys(value, info, source, ['type'], ['if']) 527 check_if(value, info, source) 528 check_type(value['type'], info, source, allow_array=not base) 529 530 531def check_alternate(expr: QAPIExpression) -> None: 532 """ 533 Normalize and validate this expression as an ``alternate`` definition. 534 535 :param expr: The expression to validate. 536 537 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 538 :return: None, ``expr`` is normalized in-place as needed. 539 """ 540 members = expr['data'] 541 info = expr.info 542 543 if not members: 544 raise QAPISemError(info, "'data' must not be empty") 545 546 if not isinstance(members, dict): 547 raise QAPISemError(info, "'data' must be an object") 548 549 for (key, value) in members.items(): 550 source = "'data' member '%s'" % key 551 check_name_lower(key, info, source) 552 check_keys(value, info, source, ['type'], ['if']) 553 check_if(value, info, source) 554 check_type(value['type'], info, source, allow_array=True) 555 556 557def check_command(expr: QAPIExpression) -> None: 558 """ 559 Normalize and validate this expression as a ``command`` definition. 560 561 :param expr: The expression to validate. 562 563 :raise QAPISemError: When ``expr`` is not a valid ``command``. 564 :return: None, ``expr`` is normalized in-place as needed. 565 """ 566 args = expr.get('data') 567 rets = expr.get('returns') 568 boxed = expr.get('boxed', False) 569 570 if boxed and args is None: 571 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 572 check_type(args, expr.info, "'data'", allow_dict=not boxed) 573 check_type(rets, expr.info, "'returns'", allow_array=True) 574 575 576def check_event(expr: QAPIExpression) -> None: 577 """ 578 Normalize and validate this expression as an ``event`` definition. 579 580 :param expr: The expression to validate. 581 582 :raise QAPISemError: When ``expr`` is not a valid ``event``. 583 :return: None, ``expr`` is normalized in-place as needed. 584 """ 585 args = expr.get('data') 586 boxed = expr.get('boxed', False) 587 588 if boxed and args is None: 589 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 590 check_type(args, expr.info, "'data'", allow_dict=not boxed) 591 592 593def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 594 """ 595 Validate and normalize a list of parsed QAPI schema expressions. 596 597 This function accepts a list of expressions and metadata as returned 598 by the parser. It destructively normalizes the expressions in-place. 599 600 :param exprs: The list of expressions to normalize and validate. 601 602 :raise QAPISemError: When any expression fails validation. 603 :return: The same list of expressions (now modified). 604 """ 605 for expr in exprs: 606 info = expr.info 607 doc = expr.doc 608 609 if 'include' in expr: 610 continue 611 612 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 613 'command', 'event'} 614 if len(metas) != 1: 615 raise QAPISemError( 616 info, 617 "expression must have exactly one key" 618 " 'enum', 'struct', 'union', 'alternate'," 619 " 'command', 'event'") 620 meta = metas.pop() 621 622 check_name_is_str(expr[meta], info, "'%s'" % meta) 623 name = cast(str, expr[meta]) 624 info.set_defn(meta, name) 625 check_defn_name_str(name, info, meta) 626 627 if doc: 628 if doc.symbol != name: 629 raise QAPISemError( 630 info, "documentation comment is for '%s'" % doc.symbol) 631 doc.check_expr(expr) 632 elif info.pragma.doc_required: 633 raise QAPISemError(info, 634 "documentation comment required") 635 636 if meta == 'enum': 637 check_keys(expr, info, meta, 638 ['enum', 'data'], ['if', 'features', 'prefix']) 639 check_enum(expr) 640 elif meta == 'union': 641 check_keys(expr, info, meta, 642 ['union', 'base', 'discriminator', 'data'], 643 ['if', 'features']) 644 normalize_members(expr.get('base')) 645 normalize_members(expr['data']) 646 check_union(expr) 647 elif meta == 'alternate': 648 check_keys(expr, info, meta, 649 ['alternate', 'data'], ['if', 'features']) 650 normalize_members(expr['data']) 651 check_alternate(expr) 652 elif meta == 'struct': 653 check_keys(expr, info, meta, 654 ['struct', 'data'], ['base', 'if', 'features']) 655 normalize_members(expr['data']) 656 check_struct(expr) 657 elif meta == 'command': 658 check_keys(expr, info, meta, 659 ['command'], 660 ['data', 'returns', 'boxed', 'if', 'features', 661 'gen', 'success-response', 'allow-oob', 662 'allow-preconfig', 'coroutine']) 663 normalize_members(expr.get('data')) 664 check_command(expr) 665 elif meta == 'event': 666 check_keys(expr, info, meta, 667 ['event'], ['data', 'boxed', 'if', 'features']) 668 normalize_members(expr.get('data')) 669 check_event(expr) 670 else: 671 assert False, 'unexpected meta type' 672 673 check_if(expr, info, meta) 674 check_features(expr.get('features'), info) 675 check_flags(expr) 676 677 return exprs 678