1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# See check_name_str(), below. 51valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 52 r'(x-)?' 53 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 54 55 56def check_name_is_str(name: object, 57 info: QAPISourceInfo, 58 source: str) -> None: 59 """ 60 Ensure that ``name`` is a ``str``. 61 62 :raise QAPISemError: When ``name`` fails validation. 63 """ 64 if not isinstance(name, str): 65 raise QAPISemError(info, "%s requires a string name" % source) 66 67 68def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 69 """ 70 Ensure that ``name`` is a valid QAPI name. 71 72 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 73 starting with a letter. It may be prefixed by a downstream prefix 74 of the form __RFQDN_, or the experimental prefix ``x-``. If both 75 prefixes are present, the __RFDQN_ prefix goes first. 76 77 A valid name cannot start with ``q_``, which is reserved. 78 79 :param name: Name to check. 80 :param info: QAPI schema source file information. 81 :param source: Error string describing what ``name`` belongs to. 82 83 :raise QAPISemError: When ``name`` fails validation. 84 :return: The stem of the valid name, with no prefixes. 85 """ 86 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 87 # and 'q_obj_*' implicit type names. 88 match = valid_name.match(name) 89 if not match or c_name(name, False).startswith('q_'): 90 raise QAPISemError(info, "%s has an invalid name" % source) 91 return match.group(3) 92 93 94def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 95 """ 96 Ensure that ``name`` is a valid event name. 97 98 This means it must be a valid QAPI name as checked by 99 `check_name_str()`, but where the stem prohibits lowercase 100 characters and ``-``. 101 102 :param name: Name to check. 103 :param info: QAPI schema source file information. 104 :param source: Error string describing what ``name`` belongs to. 105 106 :raise QAPISemError: When ``name`` fails validation. 107 """ 108 stem = check_name_str(name, info, source) 109 if re.search(r'[a-z-]', stem): 110 raise QAPISemError( 111 info, "name of %s must not use lowercase or '-'" % source) 112 113 114def check_name_lower(name: str, info: QAPISourceInfo, source: str, 115 permit_upper: bool = False, 116 permit_underscore: bool = False) -> None: 117 """ 118 Ensure that ``name`` is a valid command or member name. 119 120 This means it must be a valid QAPI name as checked by 121 `check_name_str()`, but where the stem prohibits uppercase 122 characters and ``_``. 123 124 :param name: Name to check. 125 :param info: QAPI schema source file information. 126 :param source: Error string describing what ``name`` belongs to. 127 :param permit_upper: Additionally permit uppercase. 128 :param permit_underscore: Additionally permit ``_``. 129 130 :raise QAPISemError: When ``name`` fails validation. 131 """ 132 stem = check_name_str(name, info, source) 133 if ((not permit_upper and re.search(r'[A-Z]', stem)) 134 or (not permit_underscore and '_' in stem)): 135 raise QAPISemError( 136 info, "name of %s must not use uppercase or '_'" % source) 137 138 139def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 140 """ 141 Ensure that ``name`` is a valid user-defined type name. 142 143 This means it must be a valid QAPI name as checked by 144 `check_name_str()`, but where the stem must be in CamelCase. 145 146 :param name: Name to check. 147 :param info: QAPI schema source file information. 148 :param source: Error string describing what ``name`` belongs to. 149 150 :raise QAPISemError: When ``name`` fails validation. 151 """ 152 stem = check_name_str(name, info, source) 153 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 154 raise QAPISemError(info, "name of %s must use CamelCase" % source) 155 156 157def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 158 """ 159 Ensure that ``name`` is a valid definition name. 160 161 Based on the value of ``meta``, this means that: 162 - 'event' names adhere to `check_name_upper()`. 163 - 'command' names adhere to `check_name_lower()`. 164 - Else, meta is a type, and must pass `check_name_camel()`. 165 These names must not end with ``List``. 166 167 :param name: Name to check. 168 :param info: QAPI schema source file information. 169 :param meta: Meta-type name of the QAPI expression. 170 171 :raise QAPISemError: When ``name`` fails validation. 172 """ 173 if meta == 'event': 174 check_name_upper(name, info, meta) 175 elif meta == 'command': 176 check_name_lower( 177 name, info, meta, 178 permit_underscore=name in info.pragma.command_name_exceptions) 179 else: 180 check_name_camel(name, info, meta) 181 if name.endswith('List'): 182 raise QAPISemError( 183 info, "%s name should not end in 'List'" % meta) 184 185 186def check_keys(value: Dict[str, object], 187 info: QAPISourceInfo, 188 source: str, 189 required: List[str], 190 optional: List[str]) -> None: 191 """ 192 Ensure that a dict has a specific set of keys. 193 194 :param value: The dict to check. 195 :param info: QAPI schema source file information. 196 :param source: Error string describing this ``value``. 197 :param required: Keys that *must* be present. 198 :param optional: Keys that *may* be present. 199 200 :raise QAPISemError: When unknown keys are present. 201 """ 202 203 def pprint(elems: Iterable[str]) -> str: 204 return ', '.join("'" + e + "'" for e in sorted(elems)) 205 206 missing = set(required) - set(value) 207 if missing: 208 raise QAPISemError( 209 info, 210 "%s misses key%s %s" 211 % (source, 's' if len(missing) > 1 else '', 212 pprint(missing))) 213 allowed = set(required) | set(optional) 214 unknown = set(value) - allowed 215 if unknown: 216 raise QAPISemError( 217 info, 218 "%s has unknown key%s %s\nValid keys are %s." 219 % (source, 's' if len(unknown) > 1 else '', 220 pprint(unknown), pprint(allowed))) 221 222 223def check_flags(expr: QAPIExpression) -> None: 224 """ 225 Ensure flag members (if present) have valid values. 226 227 :param expr: The expression to validate. 228 229 :raise QAPISemError: 230 When certain flags have an invalid value, or when 231 incompatible flags are present. 232 """ 233 for key in ('gen', 'success-response'): 234 if key in expr and expr[key] is not False: 235 raise QAPISemError( 236 expr.info, "flag '%s' may only use false value" % key) 237 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 238 if key in expr and expr[key] is not True: 239 raise QAPISemError( 240 expr.info, "flag '%s' may only use true value" % key) 241 if 'allow-oob' in expr and 'coroutine' in expr: 242 # This is not necessarily a fundamental incompatibility, but 243 # we don't have a use case and the desired semantics isn't 244 # obvious. The simplest solution is to forbid it until we get 245 # a use case for it. 246 raise QAPISemError( 247 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 248 249 250def check_if(expr: Dict[str, object], 251 info: QAPISourceInfo, source: str) -> None: 252 """ 253 Validate the ``if`` member of an object. 254 255 The ``if`` member may be either a ``str`` or a dict. 256 257 :param expr: The expression containing the ``if`` member to validate. 258 :param info: QAPI schema source file information. 259 :param source: Error string describing ``expr``. 260 261 :raise QAPISemError: 262 When the "if" member fails validation, or when there are no 263 non-empty conditions. 264 :return: None 265 """ 266 267 def _check_if(cond: Union[str, object]) -> None: 268 if isinstance(cond, str): 269 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 270 raise QAPISemError( 271 info, 272 "'if' condition '%s' of %s is not a valid identifier" 273 % (cond, source)) 274 return 275 276 if not isinstance(cond, dict): 277 raise QAPISemError( 278 info, 279 "'if' condition of %s must be a string or an object" % source) 280 check_keys(cond, info, "'if' condition of %s" % source, [], 281 ["all", "any", "not"]) 282 if len(cond) != 1: 283 raise QAPISemError( 284 info, 285 "'if' condition of %s has conflicting keys" % source) 286 287 if 'not' in cond: 288 _check_if(cond['not']) 289 elif 'all' in cond: 290 _check_infix('all', cond['all']) 291 else: 292 _check_infix('any', cond['any']) 293 294 def _check_infix(operator: str, operands: object) -> None: 295 if not isinstance(operands, list): 296 raise QAPISemError( 297 info, 298 "'%s' condition of %s must be an array" 299 % (operator, source)) 300 if not operands: 301 raise QAPISemError( 302 info, "'if' condition [] of %s is useless" % source) 303 for operand in operands: 304 _check_if(operand) 305 306 ifcond = expr.get('if') 307 if ifcond is None: 308 return 309 310 _check_if(ifcond) 311 312 313def normalize_members(members: object) -> None: 314 """ 315 Normalize a "members" value. 316 317 If ``members`` is a dict, for every value in that dict, if that 318 value is not itself already a dict, normalize it to 319 ``{'type': value}``. 320 321 :forms: 322 :sugared: ``Dict[str, Union[str, TypeRef]]`` 323 :canonical: ``Dict[str, TypeRef]`` 324 325 :param members: The members value to normalize. 326 327 :return: None, ``members`` is normalized in-place as needed. 328 """ 329 if isinstance(members, dict): 330 for key, arg in members.items(): 331 if isinstance(arg, dict): 332 continue 333 members[key] = {'type': arg} 334 335 336def check_type_name(value: Optional[object], 337 info: QAPISourceInfo, source: str) -> None: 338 if value is None: 339 return 340 341 if isinstance(value, str): 342 return 343 344 raise QAPISemError(info, "%s should be a type name" % source) 345 346 347def check_type_name_or_array(value: Optional[object], 348 info: QAPISourceInfo, source: str) -> None: 349 if value is None: 350 return 351 352 if isinstance(value, str): 353 return 354 355 if isinstance(value, list): 356 if len(value) != 1 or not isinstance(value[0], str): 357 raise QAPISemError(info, 358 "%s: array type must contain single type name" % 359 source) 360 return 361 362 raise QAPISemError(info, 363 "%s should be a type name" % source) 364 365 366def check_type_name_or_implicit(value: Optional[object], 367 info: QAPISourceInfo, source: str, 368 parent_name: Optional[str]) -> None: 369 """ 370 Normalize and validate an optional implicit struct type. 371 372 Accept ``None``, ``str``, or a ``dict`` defining an implicit 373 struct type. The latter is normalized in place. 374 375 :param value: The value to check. 376 :param info: QAPI schema source file information. 377 :param source: Error string describing this ``value``. 378 :param parent_name: 379 When the value of ``parent_name`` is in pragma 380 ``member-name-exceptions``, an implicit struct type may 381 violate the member naming rules. 382 383 :raise QAPISemError: When ``value`` fails validation. 384 :return: None 385 """ 386 if value is None: 387 return 388 389 if isinstance(value, str): 390 return 391 392 if not isinstance(value, dict): 393 raise QAPISemError(info, 394 "%s should be an object or type name" % source) 395 396 permissive = parent_name in info.pragma.member_name_exceptions 397 398 for (key, arg) in value.items(): 399 key_source = "%s member '%s'" % (source, key) 400 if key.startswith('*'): 401 key = key[1:] 402 check_name_lower(key, info, key_source, 403 permit_upper=permissive, 404 permit_underscore=permissive) 405 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 406 raise QAPISemError(info, "%s uses reserved name" % key_source) 407 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 408 check_if(arg, info, key_source) 409 check_features(arg.get('features'), info) 410 check_type_name_or_array(arg['type'], info, key_source) 411 412 413def check_features(features: Optional[object], 414 info: QAPISourceInfo) -> None: 415 """ 416 Normalize and validate the ``features`` member. 417 418 ``features`` may be a ``list`` of either ``str`` or ``dict``. 419 Any ``str`` element will be normalized to ``{'name': element}``. 420 421 :forms: 422 :sugared: ``List[Union[str, Feature]]`` 423 :canonical: ``List[Feature]`` 424 425 :param features: The features member value to validate. 426 :param info: QAPI schema source file information. 427 428 :raise QAPISemError: When ``features`` fails validation. 429 :return: None, ``features`` is normalized in-place as needed. 430 """ 431 if features is None: 432 return 433 if not isinstance(features, list): 434 raise QAPISemError(info, "'features' must be an array") 435 features[:] = [f if isinstance(f, dict) else {'name': f} 436 for f in features] 437 for feat in features: 438 source = "'features' member" 439 assert isinstance(feat, dict) 440 check_keys(feat, info, source, ['name'], ['if']) 441 check_name_is_str(feat['name'], info, source) 442 source = "%s '%s'" % (source, feat['name']) 443 check_name_lower(feat['name'], info, source) 444 check_if(feat, info, source) 445 446 447def check_enum(expr: QAPIExpression) -> None: 448 """ 449 Normalize and validate this expression as an ``enum`` definition. 450 451 :param expr: The expression to validate. 452 453 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 454 :return: None, ``expr`` is normalized in-place as needed. 455 """ 456 name = expr['enum'] 457 members = expr['data'] 458 prefix = expr.get('prefix') 459 info = expr.info 460 461 if not isinstance(members, list): 462 raise QAPISemError(info, "'data' must be an array") 463 if prefix is not None and not isinstance(prefix, str): 464 raise QAPISemError(info, "'prefix' must be a string") 465 466 permissive = name in info.pragma.member_name_exceptions 467 468 members[:] = [m if isinstance(m, dict) else {'name': m} 469 for m in members] 470 for member in members: 471 source = "'data' member" 472 check_keys(member, info, source, ['name'], ['if', 'features']) 473 member_name = member['name'] 474 check_name_is_str(member_name, info, source) 475 source = "%s '%s'" % (source, member_name) 476 # Enum members may start with a digit 477 if member_name[0].isdigit(): 478 member_name = 'd' + member_name # Hack: hide the digit 479 check_name_lower(member_name, info, source, 480 permit_upper=permissive, 481 permit_underscore=permissive) 482 check_if(member, info, source) 483 check_features(member.get('features'), info) 484 485 486def check_struct(expr: QAPIExpression) -> None: 487 """ 488 Normalize and validate this expression as a ``struct`` definition. 489 490 :param expr: The expression to validate. 491 492 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 493 :return: None, ``expr`` is normalized in-place as needed. 494 """ 495 name = cast(str, expr['struct']) # Checked in check_exprs 496 members = expr['data'] 497 498 check_type_name_or_implicit(members, expr.info, "'data'", name) 499 check_type_name(expr.get('base'), expr.info, "'base'") 500 501 502def check_union(expr: QAPIExpression) -> None: 503 """ 504 Normalize and validate this expression as a ``union`` definition. 505 506 :param expr: The expression to validate. 507 508 :raise QAPISemError: when ``expr`` is not a valid ``union``. 509 :return: None, ``expr`` is normalized in-place as needed. 510 """ 511 name = cast(str, expr['union']) # Checked in check_exprs 512 base = expr['base'] 513 discriminator = expr['discriminator'] 514 members = expr['data'] 515 info = expr.info 516 517 check_type_name_or_implicit(base, info, "'base'", name) 518 check_name_is_str(discriminator, info, "'discriminator'") 519 520 if not isinstance(members, dict): 521 raise QAPISemError(info, "'data' must be an object") 522 523 for (key, value) in members.items(): 524 source = "'data' member '%s'" % key 525 check_keys(value, info, source, ['type'], ['if']) 526 check_if(value, info, source) 527 check_type_name(value['type'], info, source) 528 529 530def check_alternate(expr: QAPIExpression) -> None: 531 """ 532 Normalize and validate this expression as an ``alternate`` definition. 533 534 :param expr: The expression to validate. 535 536 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 537 :return: None, ``expr`` is normalized in-place as needed. 538 """ 539 members = expr['data'] 540 info = expr.info 541 542 if not members: 543 raise QAPISemError(info, "'data' must not be empty") 544 545 if not isinstance(members, dict): 546 raise QAPISemError(info, "'data' must be an object") 547 548 for (key, value) in members.items(): 549 source = "'data' member '%s'" % key 550 check_name_lower(key, info, source) 551 check_keys(value, info, source, ['type'], ['if']) 552 check_if(value, info, source) 553 check_type_name_or_array(value['type'], info, source) 554 555 556def check_command(expr: QAPIExpression) -> None: 557 """ 558 Normalize and validate this expression as a ``command`` definition. 559 560 :param expr: The expression to validate. 561 562 :raise QAPISemError: When ``expr`` is not a valid ``command``. 563 :return: None, ``expr`` is normalized in-place as needed. 564 """ 565 args = expr.get('data') 566 rets = expr.get('returns') 567 boxed = expr.get('boxed', False) 568 569 if boxed: 570 if args is None: 571 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 572 check_type_name(args, expr.info, "'data'") 573 else: 574 check_type_name_or_implicit(args, expr.info, "'data'", None) 575 check_type_name_or_array(rets, expr.info, "'returns'") 576 577 578def check_event(expr: QAPIExpression) -> None: 579 """ 580 Normalize and validate this expression as an ``event`` definition. 581 582 :param expr: The expression to validate. 583 584 :raise QAPISemError: When ``expr`` is not a valid ``event``. 585 :return: None, ``expr`` is normalized in-place as needed. 586 """ 587 args = expr.get('data') 588 boxed = expr.get('boxed', False) 589 590 if boxed: 591 if args is None: 592 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 593 check_type_name(args, expr.info, "'data'") 594 else: 595 check_type_name_or_implicit(args, expr.info, "'data'", None) 596 597 598def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 599 """ 600 Validate and normalize a list of parsed QAPI schema expressions. 601 602 This function accepts a list of expressions and metadata as returned 603 by the parser. It destructively normalizes the expressions in-place. 604 605 :param exprs: The list of expressions to normalize and validate. 606 607 :raise QAPISemError: When any expression fails validation. 608 :return: The same list of expressions (now modified). 609 """ 610 for expr in exprs: 611 info = expr.info 612 doc = expr.doc 613 614 if 'include' in expr: 615 continue 616 617 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 618 'command', 'event'} 619 if len(metas) != 1: 620 raise QAPISemError( 621 info, 622 "expression must have exactly one key" 623 " 'enum', 'struct', 'union', 'alternate'," 624 " 'command', 'event'") 625 meta = metas.pop() 626 627 check_name_is_str(expr[meta], info, "'%s'" % meta) 628 name = cast(str, expr[meta]) 629 info.set_defn(meta, name) 630 check_defn_name_str(name, info, meta) 631 632 if doc: 633 if doc.symbol != name: 634 raise QAPISemError( 635 info, "documentation comment is for '%s'" % doc.symbol) 636 doc.check_expr(expr) 637 elif info.pragma.doc_required: 638 raise QAPISemError(info, 639 "documentation comment required") 640 641 if meta == 'enum': 642 check_keys(expr, info, meta, 643 ['enum', 'data'], ['if', 'features', 'prefix']) 644 check_enum(expr) 645 elif meta == 'union': 646 check_keys(expr, info, meta, 647 ['union', 'base', 'discriminator', 'data'], 648 ['if', 'features']) 649 normalize_members(expr.get('base')) 650 normalize_members(expr['data']) 651 check_union(expr) 652 elif meta == 'alternate': 653 check_keys(expr, info, meta, 654 ['alternate', 'data'], ['if', 'features']) 655 normalize_members(expr['data']) 656 check_alternate(expr) 657 elif meta == 'struct': 658 check_keys(expr, info, meta, 659 ['struct', 'data'], ['base', 'if', 'features']) 660 normalize_members(expr['data']) 661 check_struct(expr) 662 elif meta == 'command': 663 check_keys(expr, info, meta, 664 ['command'], 665 ['data', 'returns', 'boxed', 'if', 'features', 666 'gen', 'success-response', 'allow-oob', 667 'allow-preconfig', 'coroutine']) 668 normalize_members(expr.get('data')) 669 check_command(expr) 670 elif meta == 'event': 671 check_keys(expr, info, meta, 672 ['event'], ['data', 'boxed', 'if', 'features']) 673 normalize_members(expr.get('data')) 674 check_event(expr) 675 else: 676 assert False, 'unexpected meta type' 677 678 check_if(expr, info, meta) 679 check_features(expr.get('features'), info) 680 check_flags(expr) 681 682 return exprs 683