1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# See check_name_str(), below. 51valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 52 r'(x-)?' 53 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 54 55 56def check_name_is_str(name: object, 57 info: QAPISourceInfo, 58 source: str) -> None: 59 """ 60 Ensure that ``name`` is a ``str``. 61 62 :raise QAPISemError: When ``name`` fails validation. 63 """ 64 if not isinstance(name, str): 65 raise QAPISemError(info, "%s requires a string name" % source) 66 67 68def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 69 """ 70 Ensure that ``name`` is a valid QAPI name. 71 72 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 73 starting with a letter. It may be prefixed by a downstream prefix 74 of the form __RFQDN_, or the experimental prefix ``x-``. If both 75 prefixes are present, the __RFDQN_ prefix goes first. 76 77 A valid name cannot start with ``q_``, which is reserved. 78 79 :param name: Name to check. 80 :param info: QAPI schema source file information. 81 :param source: Error string describing what ``name`` belongs to. 82 83 :raise QAPISemError: When ``name`` fails validation. 84 :return: The stem of the valid name, with no prefixes. 85 """ 86 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 87 # and 'q_obj_*' implicit type names. 88 match = valid_name.match(name) 89 if not match or c_name(name, False).startswith('q_'): 90 raise QAPISemError(info, "%s has an invalid name" % source) 91 return match.group(3) 92 93 94def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 95 """ 96 Ensure that ``name`` is a valid event name. 97 98 This means it must be a valid QAPI name as checked by 99 `check_name_str()`, but where the stem prohibits lowercase 100 characters and ``-``. 101 102 :param name: Name to check. 103 :param info: QAPI schema source file information. 104 :param source: Error string describing what ``name`` belongs to. 105 106 :raise QAPISemError: When ``name`` fails validation. 107 """ 108 stem = check_name_str(name, info, source) 109 if re.search(r'[a-z-]', stem): 110 raise QAPISemError( 111 info, "name of %s must not use lowercase or '-'" % source) 112 113 114def check_name_lower(name: str, info: QAPISourceInfo, source: str, 115 permit_upper: bool = False, 116 permit_underscore: bool = False) -> None: 117 """ 118 Ensure that ``name`` is a valid command or member name. 119 120 This means it must be a valid QAPI name as checked by 121 `check_name_str()`, but where the stem prohibits uppercase 122 characters and ``_``. 123 124 :param name: Name to check. 125 :param info: QAPI schema source file information. 126 :param source: Error string describing what ``name`` belongs to. 127 :param permit_upper: Additionally permit uppercase. 128 :param permit_underscore: Additionally permit ``_``. 129 130 :raise QAPISemError: When ``name`` fails validation. 131 """ 132 stem = check_name_str(name, info, source) 133 if ((not permit_upper and re.search(r'[A-Z]', stem)) 134 or (not permit_underscore and '_' in stem)): 135 raise QAPISemError( 136 info, "name of %s must not use uppercase or '_'" % source) 137 138 139def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 140 """ 141 Ensure that ``name`` is a valid user-defined type name. 142 143 This means it must be a valid QAPI name as checked by 144 `check_name_str()`, but where the stem must be in CamelCase. 145 146 :param name: Name to check. 147 :param info: QAPI schema source file information. 148 :param source: Error string describing what ``name`` belongs to. 149 150 :raise QAPISemError: When ``name`` fails validation. 151 """ 152 stem = check_name_str(name, info, source) 153 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 154 raise QAPISemError(info, "name of %s must use CamelCase" % source) 155 156 157def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 158 """ 159 Ensure that ``name`` is a valid definition name. 160 161 Based on the value of ``meta``, this means that: 162 - 'event' names adhere to `check_name_upper()`. 163 - 'command' names adhere to `check_name_lower()`. 164 - Else, meta is a type, and must pass `check_name_camel()`. 165 These names must not end with ``List``. 166 167 :param name: Name to check. 168 :param info: QAPI schema source file information. 169 :param meta: Meta-type name of the QAPI expression. 170 171 :raise QAPISemError: When ``name`` fails validation. 172 """ 173 if meta == 'event': 174 check_name_upper(name, info, meta) 175 elif meta == 'command': 176 check_name_lower( 177 name, info, meta, 178 permit_underscore=name in info.pragma.command_name_exceptions) 179 else: 180 check_name_camel(name, info, meta) 181 if name.endswith('List'): 182 raise QAPISemError( 183 info, "%s name should not end in 'List'" % meta) 184 185 186def check_keys(value: Dict[str, object], 187 info: QAPISourceInfo, 188 source: str, 189 required: List[str], 190 optional: List[str]) -> None: 191 """ 192 Ensure that a dict has a specific set of keys. 193 194 :param value: The dict to check. 195 :param info: QAPI schema source file information. 196 :param source: Error string describing this ``value``. 197 :param required: Keys that *must* be present. 198 :param optional: Keys that *may* be present. 199 200 :raise QAPISemError: When unknown keys are present. 201 """ 202 203 def pprint(elems: Iterable[str]) -> str: 204 return ', '.join("'" + e + "'" for e in sorted(elems)) 205 206 missing = set(required) - set(value) 207 if missing: 208 raise QAPISemError( 209 info, 210 "%s misses key%s %s" 211 % (source, 's' if len(missing) > 1 else '', 212 pprint(missing))) 213 allowed = set(required) | set(optional) 214 unknown = set(value) - allowed 215 if unknown: 216 raise QAPISemError( 217 info, 218 "%s has unknown key%s %s\nValid keys are %s." 219 % (source, 's' if len(unknown) > 1 else '', 220 pprint(unknown), pprint(allowed))) 221 222 223def check_flags(expr: QAPIExpression) -> None: 224 """ 225 Ensure flag members (if present) have valid values. 226 227 :param expr: The expression to validate. 228 229 :raise QAPISemError: 230 When certain flags have an invalid value, or when 231 incompatible flags are present. 232 """ 233 for key in ('gen', 'success-response'): 234 if key in expr and expr[key] is not False: 235 raise QAPISemError( 236 expr.info, "flag '%s' may only use false value" % key) 237 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 238 if key in expr and expr[key] is not True: 239 raise QAPISemError( 240 expr.info, "flag '%s' may only use true value" % key) 241 if 'allow-oob' in expr and 'coroutine' in expr: 242 # This is not necessarily a fundamental incompatibility, but 243 # we don't have a use case and the desired semantics isn't 244 # obvious. The simplest solution is to forbid it until we get 245 # a use case for it. 246 raise QAPISemError( 247 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 248 249 250def check_if(expr: Dict[str, object], 251 info: QAPISourceInfo, source: str) -> None: 252 """ 253 Validate the ``if`` member of an object. 254 255 The ``if`` member may be either a ``str`` or a dict. 256 257 :param expr: The expression containing the ``if`` member to validate. 258 :param info: QAPI schema source file information. 259 :param source: Error string describing ``expr``. 260 261 :raise QAPISemError: 262 When the "if" member fails validation, or when there are no 263 non-empty conditions. 264 :return: None 265 """ 266 267 def _check_if(cond: Union[str, object]) -> None: 268 if isinstance(cond, str): 269 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 270 raise QAPISemError( 271 info, 272 "'if' condition '%s' of %s is not a valid identifier" 273 % (cond, source)) 274 return 275 276 if not isinstance(cond, dict): 277 raise QAPISemError( 278 info, 279 "'if' condition of %s must be a string or an object" % source) 280 check_keys(cond, info, "'if' condition of %s" % source, [], 281 ["all", "any", "not"]) 282 if len(cond) != 1: 283 raise QAPISemError( 284 info, 285 "'if' condition of %s has conflicting keys" % source) 286 287 if 'not' in cond: 288 _check_if(cond['not']) 289 elif 'all' in cond: 290 _check_infix('all', cond['all']) 291 else: 292 _check_infix('any', cond['any']) 293 294 def _check_infix(operator: str, operands: object) -> None: 295 if not isinstance(operands, list): 296 raise QAPISemError( 297 info, 298 "'%s' condition of %s must be an array" 299 % (operator, source)) 300 if not operands: 301 raise QAPISemError( 302 info, "'if' condition [] of %s is useless" % source) 303 for operand in operands: 304 _check_if(operand) 305 306 ifcond = expr.get('if') 307 if ifcond is None: 308 return 309 310 _check_if(ifcond) 311 312 313def normalize_members(members: object) -> None: 314 """ 315 Normalize a "members" value. 316 317 If ``members`` is a dict, for every value in that dict, if that 318 value is not itself already a dict, normalize it to 319 ``{'type': value}``. 320 321 :forms: 322 :sugared: ``Dict[str, Union[str, TypeRef]]`` 323 :canonical: ``Dict[str, TypeRef]`` 324 325 :param members: The members value to normalize. 326 327 :return: None, ``members`` is normalized in-place as needed. 328 """ 329 if isinstance(members, dict): 330 for key, arg in members.items(): 331 if isinstance(arg, dict): 332 continue 333 members[key] = {'type': arg} 334 335 336def check_type_name(value: Optional[object], 337 info: QAPISourceInfo, source: str) -> None: 338 if value is None: 339 return 340 341 if isinstance(value, str): 342 return 343 344 if isinstance(value, list): 345 raise QAPISemError(info, "%s cannot be an array" % source) 346 347 raise QAPISemError(info, "%s should be a type name" % source) 348 349 350def check_type_name_or_array(value: Optional[object], 351 info: QAPISourceInfo, source: str) -> None: 352 if value is None: 353 return 354 355 if isinstance(value, str): 356 return 357 358 if isinstance(value, list): 359 if len(value) != 1 or not isinstance(value[0], str): 360 raise QAPISemError(info, 361 "%s: array type must contain single type name" % 362 source) 363 return 364 365 raise QAPISemError(info, 366 "%s should be a type name" % source) 367 368 369def check_type_name_or_implicit(value: Optional[object], 370 info: QAPISourceInfo, source: str, 371 parent_name: Optional[str]) -> None: 372 """ 373 Normalize and validate an optional implicit struct type. 374 375 Accept ``None``, ``str``, or a ``dict`` defining an implicit 376 struct type. The latter is normalized in place. 377 378 :param value: The value to check. 379 :param info: QAPI schema source file information. 380 :param source: Error string describing this ``value``. 381 :param parent_name: 382 When the value of ``parent_name`` is in pragma 383 ``member-name-exceptions``, an implicit struct type may 384 violate the member naming rules. 385 386 :raise QAPISemError: When ``value`` fails validation. 387 :return: None 388 """ 389 if value is None: 390 return 391 392 if isinstance(value, str): 393 return 394 395 if isinstance(value, list): 396 raise QAPISemError(info, "%s cannot be an array" % source) 397 398 if not isinstance(value, dict): 399 raise QAPISemError(info, 400 "%s should be an object or type name" % source) 401 402 permissive = parent_name in info.pragma.member_name_exceptions 403 404 for (key, arg) in value.items(): 405 key_source = "%s member '%s'" % (source, key) 406 if key.startswith('*'): 407 key = key[1:] 408 check_name_lower(key, info, key_source, 409 permit_upper=permissive, 410 permit_underscore=permissive) 411 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 412 raise QAPISemError(info, "%s uses reserved name" % key_source) 413 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 414 check_if(arg, info, key_source) 415 check_features(arg.get('features'), info) 416 check_type_name_or_array(arg['type'], info, key_source) 417 418 419def check_features(features: Optional[object], 420 info: QAPISourceInfo) -> None: 421 """ 422 Normalize and validate the ``features`` member. 423 424 ``features`` may be a ``list`` of either ``str`` or ``dict``. 425 Any ``str`` element will be normalized to ``{'name': element}``. 426 427 :forms: 428 :sugared: ``List[Union[str, Feature]]`` 429 :canonical: ``List[Feature]`` 430 431 :param features: The features member value to validate. 432 :param info: QAPI schema source file information. 433 434 :raise QAPISemError: When ``features`` fails validation. 435 :return: None, ``features`` is normalized in-place as needed. 436 """ 437 if features is None: 438 return 439 if not isinstance(features, list): 440 raise QAPISemError(info, "'features' must be an array") 441 features[:] = [f if isinstance(f, dict) else {'name': f} 442 for f in features] 443 for feat in features: 444 source = "'features' member" 445 assert isinstance(feat, dict) 446 check_keys(feat, info, source, ['name'], ['if']) 447 check_name_is_str(feat['name'], info, source) 448 source = "%s '%s'" % (source, feat['name']) 449 check_name_lower(feat['name'], info, source) 450 check_if(feat, info, source) 451 452 453def check_enum(expr: QAPIExpression) -> None: 454 """ 455 Normalize and validate this expression as an ``enum`` definition. 456 457 :param expr: The expression to validate. 458 459 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 460 :return: None, ``expr`` is normalized in-place as needed. 461 """ 462 name = expr['enum'] 463 members = expr['data'] 464 prefix = expr.get('prefix') 465 info = expr.info 466 467 if not isinstance(members, list): 468 raise QAPISemError(info, "'data' must be an array") 469 if prefix is not None and not isinstance(prefix, str): 470 raise QAPISemError(info, "'prefix' must be a string") 471 472 permissive = name in info.pragma.member_name_exceptions 473 474 members[:] = [m if isinstance(m, dict) else {'name': m} 475 for m in members] 476 for member in members: 477 source = "'data' member" 478 check_keys(member, info, source, ['name'], ['if', 'features']) 479 member_name = member['name'] 480 check_name_is_str(member_name, info, source) 481 source = "%s '%s'" % (source, member_name) 482 # Enum members may start with a digit 483 if member_name[0].isdigit(): 484 member_name = 'd' + member_name # Hack: hide the digit 485 check_name_lower(member_name, info, source, 486 permit_upper=permissive, 487 permit_underscore=permissive) 488 check_if(member, info, source) 489 check_features(member.get('features'), info) 490 491 492def check_struct(expr: QAPIExpression) -> None: 493 """ 494 Normalize and validate this expression as a ``struct`` definition. 495 496 :param expr: The expression to validate. 497 498 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 499 :return: None, ``expr`` is normalized in-place as needed. 500 """ 501 name = cast(str, expr['struct']) # Checked in check_exprs 502 members = expr['data'] 503 504 check_type_name_or_implicit(members, expr.info, "'data'", name) 505 check_type_name(expr.get('base'), expr.info, "'base'") 506 507 508def check_union(expr: QAPIExpression) -> None: 509 """ 510 Normalize and validate this expression as a ``union`` definition. 511 512 :param expr: The expression to validate. 513 514 :raise QAPISemError: when ``expr`` is not a valid ``union``. 515 :return: None, ``expr`` is normalized in-place as needed. 516 """ 517 name = cast(str, expr['union']) # Checked in check_exprs 518 base = expr['base'] 519 discriminator = expr['discriminator'] 520 members = expr['data'] 521 info = expr.info 522 523 check_type_name_or_implicit(base, info, "'base'", name) 524 check_name_is_str(discriminator, info, "'discriminator'") 525 526 if not isinstance(members, dict): 527 raise QAPISemError(info, "'data' must be an object") 528 529 for (key, value) in members.items(): 530 source = "'data' member '%s'" % key 531 check_keys(value, info, source, ['type'], ['if']) 532 check_if(value, info, source) 533 check_type_name(value['type'], info, source) 534 535 536def check_alternate(expr: QAPIExpression) -> None: 537 """ 538 Normalize and validate this expression as an ``alternate`` definition. 539 540 :param expr: The expression to validate. 541 542 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 543 :return: None, ``expr`` is normalized in-place as needed. 544 """ 545 members = expr['data'] 546 info = expr.info 547 548 if not members: 549 raise QAPISemError(info, "'data' must not be empty") 550 551 if not isinstance(members, dict): 552 raise QAPISemError(info, "'data' must be an object") 553 554 for (key, value) in members.items(): 555 source = "'data' member '%s'" % key 556 check_name_lower(key, info, source) 557 check_keys(value, info, source, ['type'], ['if']) 558 check_if(value, info, source) 559 check_type_name_or_array(value['type'], info, source) 560 561 562def check_command(expr: QAPIExpression) -> None: 563 """ 564 Normalize and validate this expression as a ``command`` definition. 565 566 :param expr: The expression to validate. 567 568 :raise QAPISemError: When ``expr`` is not a valid ``command``. 569 :return: None, ``expr`` is normalized in-place as needed. 570 """ 571 args = expr.get('data') 572 rets = expr.get('returns') 573 boxed = expr.get('boxed', False) 574 575 if boxed: 576 if args is None: 577 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 578 check_type_name(args, expr.info, "'data'") 579 else: 580 check_type_name_or_implicit(args, expr.info, "'data'", None) 581 check_type_name_or_array(rets, expr.info, "'returns'") 582 583 584def check_event(expr: QAPIExpression) -> None: 585 """ 586 Normalize and validate this expression as an ``event`` definition. 587 588 :param expr: The expression to validate. 589 590 :raise QAPISemError: When ``expr`` is not a valid ``event``. 591 :return: None, ``expr`` is normalized in-place as needed. 592 """ 593 args = expr.get('data') 594 boxed = expr.get('boxed', False) 595 596 if boxed: 597 if args is None: 598 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 599 check_type_name(args, expr.info, "'data'") 600 else: 601 check_type_name_or_implicit(args, expr.info, "'data'", None) 602 603 604def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 605 """ 606 Validate and normalize a list of parsed QAPI schema expressions. 607 608 This function accepts a list of expressions and metadata as returned 609 by the parser. It destructively normalizes the expressions in-place. 610 611 :param exprs: The list of expressions to normalize and validate. 612 613 :raise QAPISemError: When any expression fails validation. 614 :return: The same list of expressions (now modified). 615 """ 616 for expr in exprs: 617 info = expr.info 618 doc = expr.doc 619 620 if 'include' in expr: 621 continue 622 623 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 624 'command', 'event'} 625 if len(metas) != 1: 626 raise QAPISemError( 627 info, 628 "expression must have exactly one key" 629 " 'enum', 'struct', 'union', 'alternate'," 630 " 'command', 'event'") 631 meta = metas.pop() 632 633 check_name_is_str(expr[meta], info, "'%s'" % meta) 634 name = cast(str, expr[meta]) 635 info.set_defn(meta, name) 636 check_defn_name_str(name, info, meta) 637 638 if doc: 639 if doc.symbol != name: 640 raise QAPISemError( 641 info, "documentation comment is for '%s'" % doc.symbol) 642 doc.check_expr(expr) 643 elif info.pragma.doc_required: 644 raise QAPISemError(info, 645 "documentation comment required") 646 647 if meta == 'enum': 648 check_keys(expr, info, meta, 649 ['enum', 'data'], ['if', 'features', 'prefix']) 650 check_enum(expr) 651 elif meta == 'union': 652 check_keys(expr, info, meta, 653 ['union', 'base', 'discriminator', 'data'], 654 ['if', 'features']) 655 normalize_members(expr.get('base')) 656 normalize_members(expr['data']) 657 check_union(expr) 658 elif meta == 'alternate': 659 check_keys(expr, info, meta, 660 ['alternate', 'data'], ['if', 'features']) 661 normalize_members(expr['data']) 662 check_alternate(expr) 663 elif meta == 'struct': 664 check_keys(expr, info, meta, 665 ['struct', 'data'], ['base', 'if', 'features']) 666 normalize_members(expr['data']) 667 check_struct(expr) 668 elif meta == 'command': 669 check_keys(expr, info, meta, 670 ['command'], 671 ['data', 'returns', 'boxed', 'if', 'features', 672 'gen', 'success-response', 'allow-oob', 673 'allow-preconfig', 'coroutine']) 674 normalize_members(expr.get('data')) 675 check_command(expr) 676 elif meta == 'event': 677 check_keys(expr, info, meta, 678 ['event'], ['data', 'boxed', 'if', 'features']) 679 normalize_members(expr.get('data')) 680 check_event(expr) 681 else: 682 assert False, 'unexpected meta type' 683 684 check_if(expr, info, meta) 685 check_features(expr.get('features'), info) 686 check_flags(expr) 687 688 return exprs 689