1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# See check_name_str(), below. 51valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 52 r'(x-)?' 53 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 54 55 56def check_name_is_str(name: object, 57 info: QAPISourceInfo, 58 source: str) -> None: 59 """ 60 Ensure that ``name`` is a ``str``. 61 62 :raise QAPISemError: When ``name`` fails validation. 63 """ 64 if not isinstance(name, str): 65 raise QAPISemError(info, "%s requires a string name" % source) 66 67 68def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 69 """ 70 Ensure that ``name`` is a valid QAPI name. 71 72 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 73 starting with a letter. It may be prefixed by a downstream prefix 74 of the form __RFQDN_, or the experimental prefix ``x-``. If both 75 prefixes are present, the __RFDQN_ prefix goes first. 76 77 A valid name cannot start with ``q_``, which is reserved. 78 79 :param name: Name to check. 80 :param info: QAPI schema source file information. 81 :param source: Error string describing what ``name`` belongs to. 82 83 :raise QAPISemError: When ``name`` fails validation. 84 :return: The stem of the valid name, with no prefixes. 85 """ 86 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 87 # and 'q_obj_*' implicit type names. 88 match = valid_name.match(name) 89 if not match or c_name(name, False).startswith('q_'): 90 raise QAPISemError(info, "%s has an invalid name" % source) 91 return match.group(3) 92 93 94def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 95 """ 96 Ensure that ``name`` is a valid event name. 97 98 This means it must be a valid QAPI name as checked by 99 `check_name_str()`, but where the stem prohibits lowercase 100 characters and ``-``. 101 102 :param name: Name to check. 103 :param info: QAPI schema source file information. 104 :param source: Error string describing what ``name`` belongs to. 105 106 :raise QAPISemError: When ``name`` fails validation. 107 """ 108 stem = check_name_str(name, info, source) 109 if re.search(r'[a-z-]', stem): 110 raise QAPISemError( 111 info, "name of %s must not use lowercase or '-'" % source) 112 113 114def check_name_lower(name: str, info: QAPISourceInfo, source: str, 115 permit_upper: bool = False, 116 permit_underscore: bool = False) -> None: 117 """ 118 Ensure that ``name`` is a valid command or member name. 119 120 This means it must be a valid QAPI name as checked by 121 `check_name_str()`, but where the stem prohibits uppercase 122 characters and ``_``. 123 124 :param name: Name to check. 125 :param info: QAPI schema source file information. 126 :param source: Error string describing what ``name`` belongs to. 127 :param permit_upper: Additionally permit uppercase. 128 :param permit_underscore: Additionally permit ``_``. 129 130 :raise QAPISemError: When ``name`` fails validation. 131 """ 132 stem = check_name_str(name, info, source) 133 if ((not permit_upper and re.search(r'[A-Z]', stem)) 134 or (not permit_underscore and '_' in stem)): 135 raise QAPISemError( 136 info, "name of %s must not use uppercase or '_'" % source) 137 138 139def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 140 """ 141 Ensure that ``name`` is a valid user-defined type name. 142 143 This means it must be a valid QAPI name as checked by 144 `check_name_str()`, but where the stem must be in CamelCase. 145 146 :param name: Name to check. 147 :param info: QAPI schema source file information. 148 :param source: Error string describing what ``name`` belongs to. 149 150 :raise QAPISemError: When ``name`` fails validation. 151 """ 152 stem = check_name_str(name, info, source) 153 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 154 raise QAPISemError(info, "name of %s must use CamelCase" % source) 155 156 157def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 158 """ 159 Ensure that ``name`` is a valid definition name. 160 161 Based on the value of ``meta``, this means that: 162 - 'event' names adhere to `check_name_upper()`. 163 - 'command' names adhere to `check_name_lower()`. 164 - Else, meta is a type, and must pass `check_name_camel()`. 165 These names must not end with ``List``. 166 167 :param name: Name to check. 168 :param info: QAPI schema source file information. 169 :param meta: Meta-type name of the QAPI expression. 170 171 :raise QAPISemError: When ``name`` fails validation. 172 """ 173 if meta == 'event': 174 check_name_upper(name, info, meta) 175 elif meta == 'command': 176 check_name_lower( 177 name, info, meta, 178 permit_underscore=name in info.pragma.command_name_exceptions) 179 else: 180 check_name_camel(name, info, meta) 181 if name.endswith('List'): 182 raise QAPISemError( 183 info, "%s name should not end in 'List'" % meta) 184 185 186def check_keys(value: Dict[str, object], 187 info: QAPISourceInfo, 188 source: str, 189 required: List[str], 190 optional: List[str]) -> None: 191 """ 192 Ensure that a dict has a specific set of keys. 193 194 :param value: The dict to check. 195 :param info: QAPI schema source file information. 196 :param source: Error string describing this ``value``. 197 :param required: Keys that *must* be present. 198 :param optional: Keys that *may* be present. 199 200 :raise QAPISemError: When unknown keys are present. 201 """ 202 203 def pprint(elems: Iterable[str]) -> str: 204 return ', '.join("'" + e + "'" for e in sorted(elems)) 205 206 missing = set(required) - set(value) 207 if missing: 208 raise QAPISemError( 209 info, 210 "%s misses key%s %s" 211 % (source, 's' if len(missing) > 1 else '', 212 pprint(missing))) 213 allowed = set(required) | set(optional) 214 unknown = set(value) - allowed 215 if unknown: 216 raise QAPISemError( 217 info, 218 "%s has unknown key%s %s\nValid keys are %s." 219 % (source, 's' if len(unknown) > 1 else '', 220 pprint(unknown), pprint(allowed))) 221 222 223def check_flags(expr: QAPIExpression) -> None: 224 """ 225 Ensure flag members (if present) have valid values. 226 227 :param expr: The expression to validate. 228 229 :raise QAPISemError: 230 When certain flags have an invalid value, or when 231 incompatible flags are present. 232 """ 233 for key in ('gen', 'success-response'): 234 if key in expr and expr[key] is not False: 235 raise QAPISemError( 236 expr.info, "flag '%s' may only use false value" % key) 237 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 238 if key in expr and expr[key] is not True: 239 raise QAPISemError( 240 expr.info, "flag '%s' may only use true value" % key) 241 if 'allow-oob' in expr and 'coroutine' in expr: 242 # This is not necessarily a fundamental incompatibility, but 243 # we don't have a use case and the desired semantics isn't 244 # obvious. The simplest solution is to forbid it until we get 245 # a use case for it. 246 raise QAPISemError( 247 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 248 249 250def check_if(expr: Dict[str, object], 251 info: QAPISourceInfo, source: str) -> None: 252 """ 253 Validate the ``if`` member of an object. 254 255 The ``if`` member may be either a ``str`` or a dict. 256 257 :param expr: The expression containing the ``if`` member to validate. 258 :param info: QAPI schema source file information. 259 :param source: Error string describing ``expr``. 260 261 :raise QAPISemError: 262 When the "if" member fails validation, or when there are no 263 non-empty conditions. 264 :return: None 265 """ 266 267 def _check_if(cond: Union[str, object]) -> None: 268 if isinstance(cond, str): 269 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 270 raise QAPISemError( 271 info, 272 "'if' condition '%s' of %s is not a valid identifier" 273 % (cond, source)) 274 return 275 276 if not isinstance(cond, dict): 277 raise QAPISemError( 278 info, 279 "'if' condition of %s must be a string or an object" % source) 280 check_keys(cond, info, "'if' condition of %s" % source, [], 281 ["all", "any", "not"]) 282 if len(cond) != 1: 283 raise QAPISemError( 284 info, 285 "'if' condition of %s has conflicting keys" % source) 286 287 if 'not' in cond: 288 _check_if(cond['not']) 289 elif 'all' in cond: 290 _check_infix('all', cond['all']) 291 else: 292 _check_infix('any', cond['any']) 293 294 def _check_infix(operator: str, operands: object) -> None: 295 if not isinstance(operands, list): 296 raise QAPISemError( 297 info, 298 "'%s' condition of %s must be an array" 299 % (operator, source)) 300 if not operands: 301 raise QAPISemError( 302 info, "'if' condition [] of %s is useless" % source) 303 for operand in operands: 304 _check_if(operand) 305 306 ifcond = expr.get('if') 307 if ifcond is None: 308 return 309 310 _check_if(ifcond) 311 312 313def normalize_members(members: object) -> None: 314 """ 315 Normalize a "members" value. 316 317 If ``members`` is a dict, for every value in that dict, if that 318 value is not itself already a dict, normalize it to 319 ``{'type': value}``. 320 321 :forms: 322 :sugared: ``Dict[str, Union[str, TypeRef]]`` 323 :canonical: ``Dict[str, TypeRef]`` 324 325 :param members: The members value to normalize. 326 327 :return: None, ``members`` is normalized in-place as needed. 328 """ 329 if isinstance(members, dict): 330 for key, arg in members.items(): 331 if isinstance(arg, dict): 332 continue 333 members[key] = {'type': arg} 334 335 336def check_type_name(value: Optional[object], 337 info: QAPISourceInfo, source: str) -> None: 338 if value is not None and not isinstance(value, str): 339 raise QAPISemError(info, "%s should be a type name" % source) 340 341 342def check_type_name_or_array(value: Optional[object], 343 info: QAPISourceInfo, source: str) -> None: 344 if value is None or isinstance(value, str): 345 return 346 347 if isinstance(value, list): 348 if len(value) != 1 or not isinstance(value[0], str): 349 raise QAPISemError(info, 350 "%s: array type must contain single type name" % 351 source) 352 return 353 354 raise QAPISemError(info, 355 "%s should be a type name" % source) 356 357 358def check_type_name_or_implicit(value: Optional[object], 359 info: QAPISourceInfo, source: str, 360 parent_name: Optional[str]) -> None: 361 """ 362 Normalize and validate an optional implicit struct type. 363 364 Accept ``None``, ``str``, or a ``dict`` defining an implicit 365 struct type. The latter is normalized in place. 366 367 :param value: The value to check. 368 :param info: QAPI schema source file information. 369 :param source: Error string describing this ``value``. 370 :param parent_name: 371 When the value of ``parent_name`` is in pragma 372 ``member-name-exceptions``, an implicit struct type may 373 violate the member naming rules. 374 375 :raise QAPISemError: When ``value`` fails validation. 376 :return: None 377 """ 378 if value is None: 379 return 380 381 if isinstance(value, str): 382 return 383 384 if not isinstance(value, dict): 385 raise QAPISemError(info, 386 "%s should be an object or type name" % source) 387 388 permissive = parent_name in info.pragma.member_name_exceptions 389 390 for (key, arg) in value.items(): 391 key_source = "%s member '%s'" % (source, key) 392 if key.startswith('*'): 393 key = key[1:] 394 check_name_lower(key, info, key_source, 395 permit_upper=permissive, 396 permit_underscore=permissive) 397 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 398 raise QAPISemError(info, "%s uses reserved name" % key_source) 399 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 400 check_if(arg, info, key_source) 401 check_features(arg.get('features'), info) 402 check_type_name_or_array(arg['type'], info, key_source) 403 404 405def check_features(features: Optional[object], 406 info: QAPISourceInfo) -> None: 407 """ 408 Normalize and validate the ``features`` member. 409 410 ``features`` may be a ``list`` of either ``str`` or ``dict``. 411 Any ``str`` element will be normalized to ``{'name': element}``. 412 413 :forms: 414 :sugared: ``List[Union[str, Feature]]`` 415 :canonical: ``List[Feature]`` 416 417 :param features: The features member value to validate. 418 :param info: QAPI schema source file information. 419 420 :raise QAPISemError: When ``features`` fails validation. 421 :return: None, ``features`` is normalized in-place as needed. 422 """ 423 if features is None: 424 return 425 if not isinstance(features, list): 426 raise QAPISemError(info, "'features' must be an array") 427 features[:] = [f if isinstance(f, dict) else {'name': f} 428 for f in features] 429 for feat in features: 430 source = "'features' member" 431 assert isinstance(feat, dict) 432 check_keys(feat, info, source, ['name'], ['if']) 433 check_name_is_str(feat['name'], info, source) 434 source = "%s '%s'" % (source, feat['name']) 435 check_name_lower(feat['name'], info, source) 436 check_if(feat, info, source) 437 438 439def check_enum(expr: QAPIExpression) -> None: 440 """ 441 Normalize and validate this expression as an ``enum`` definition. 442 443 :param expr: The expression to validate. 444 445 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 446 :return: None, ``expr`` is normalized in-place as needed. 447 """ 448 name = expr['enum'] 449 members = expr['data'] 450 prefix = expr.get('prefix') 451 info = expr.info 452 453 if not isinstance(members, list): 454 raise QAPISemError(info, "'data' must be an array") 455 if prefix is not None and not isinstance(prefix, str): 456 raise QAPISemError(info, "'prefix' must be a string") 457 458 permissive = name in info.pragma.member_name_exceptions 459 460 members[:] = [m if isinstance(m, dict) else {'name': m} 461 for m in members] 462 for member in members: 463 source = "'data' member" 464 check_keys(member, info, source, ['name'], ['if', 'features']) 465 member_name = member['name'] 466 check_name_is_str(member_name, info, source) 467 source = "%s '%s'" % (source, member_name) 468 # Enum members may start with a digit 469 if member_name[0].isdigit(): 470 member_name = 'd' + member_name # Hack: hide the digit 471 check_name_lower(member_name, info, source, 472 permit_upper=permissive, 473 permit_underscore=permissive) 474 check_if(member, info, source) 475 check_features(member.get('features'), info) 476 477 478def check_struct(expr: QAPIExpression) -> None: 479 """ 480 Normalize and validate this expression as a ``struct`` definition. 481 482 :param expr: The expression to validate. 483 484 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 485 :return: None, ``expr`` is normalized in-place as needed. 486 """ 487 name = cast(str, expr['struct']) # Checked in check_exprs 488 members = expr['data'] 489 490 check_type_name_or_implicit(members, expr.info, "'data'", name) 491 check_type_name(expr.get('base'), expr.info, "'base'") 492 493 494def check_union(expr: QAPIExpression) -> None: 495 """ 496 Normalize and validate this expression as a ``union`` definition. 497 498 :param expr: The expression to validate. 499 500 :raise QAPISemError: when ``expr`` is not a valid ``union``. 501 :return: None, ``expr`` is normalized in-place as needed. 502 """ 503 name = cast(str, expr['union']) # Checked in check_exprs 504 base = expr['base'] 505 discriminator = expr['discriminator'] 506 members = expr['data'] 507 info = expr.info 508 509 check_type_name_or_implicit(base, info, "'base'", name) 510 check_name_is_str(discriminator, info, "'discriminator'") 511 512 if not isinstance(members, dict): 513 raise QAPISemError(info, "'data' must be an object") 514 515 for (key, value) in members.items(): 516 source = "'data' member '%s'" % key 517 check_keys(value, info, source, ['type'], ['if']) 518 check_if(value, info, source) 519 check_type_name(value['type'], info, source) 520 521 522def check_alternate(expr: QAPIExpression) -> None: 523 """ 524 Normalize and validate this expression as an ``alternate`` definition. 525 526 :param expr: The expression to validate. 527 528 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 529 :return: None, ``expr`` is normalized in-place as needed. 530 """ 531 members = expr['data'] 532 info = expr.info 533 534 if not members: 535 raise QAPISemError(info, "'data' must not be empty") 536 537 if not isinstance(members, dict): 538 raise QAPISemError(info, "'data' must be an object") 539 540 for (key, value) in members.items(): 541 source = "'data' member '%s'" % key 542 check_name_lower(key, info, source) 543 check_keys(value, info, source, ['type'], ['if']) 544 check_if(value, info, source) 545 check_type_name_or_array(value['type'], info, source) 546 547 548def check_command(expr: QAPIExpression) -> None: 549 """ 550 Normalize and validate this expression as a ``command`` definition. 551 552 :param expr: The expression to validate. 553 554 :raise QAPISemError: When ``expr`` is not a valid ``command``. 555 :return: None, ``expr`` is normalized in-place as needed. 556 """ 557 args = expr.get('data') 558 rets = expr.get('returns') 559 boxed = expr.get('boxed', False) 560 561 if boxed: 562 if args is None: 563 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 564 check_type_name(args, expr.info, "'data'") 565 else: 566 check_type_name_or_implicit(args, expr.info, "'data'", None) 567 check_type_name_or_array(rets, expr.info, "'returns'") 568 569 570def check_event(expr: QAPIExpression) -> None: 571 """ 572 Normalize and validate this expression as an ``event`` definition. 573 574 :param expr: The expression to validate. 575 576 :raise QAPISemError: When ``expr`` is not a valid ``event``. 577 :return: None, ``expr`` is normalized in-place as needed. 578 """ 579 args = expr.get('data') 580 boxed = expr.get('boxed', False) 581 582 if boxed: 583 if args is None: 584 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 585 check_type_name(args, expr.info, "'data'") 586 else: 587 check_type_name_or_implicit(args, expr.info, "'data'", None) 588 589 590def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 591 """ 592 Validate and normalize a list of parsed QAPI schema expressions. 593 594 This function accepts a list of expressions and metadata as returned 595 by the parser. It destructively normalizes the expressions in-place. 596 597 :param exprs: The list of expressions to normalize and validate. 598 599 :raise QAPISemError: When any expression fails validation. 600 :return: The same list of expressions (now modified). 601 """ 602 for expr in exprs: 603 info = expr.info 604 doc = expr.doc 605 606 if 'include' in expr: 607 continue 608 609 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 610 'command', 'event'} 611 if len(metas) != 1: 612 raise QAPISemError( 613 info, 614 "expression must have exactly one key" 615 " 'enum', 'struct', 'union', 'alternate'," 616 " 'command', 'event'") 617 meta = metas.pop() 618 619 check_name_is_str(expr[meta], info, "'%s'" % meta) 620 name = cast(str, expr[meta]) 621 info.set_defn(meta, name) 622 check_defn_name_str(name, info, meta) 623 624 if doc: 625 if doc.symbol != name: 626 raise QAPISemError( 627 info, "documentation comment is for '%s'" % doc.symbol) 628 doc.check_expr(expr) 629 elif info.pragma.doc_required: 630 raise QAPISemError(info, 631 "documentation comment required") 632 633 if meta == 'enum': 634 check_keys(expr, info, meta, 635 ['enum', 'data'], ['if', 'features', 'prefix']) 636 check_enum(expr) 637 elif meta == 'union': 638 check_keys(expr, info, meta, 639 ['union', 'base', 'discriminator', 'data'], 640 ['if', 'features']) 641 normalize_members(expr.get('base')) 642 normalize_members(expr['data']) 643 check_union(expr) 644 elif meta == 'alternate': 645 check_keys(expr, info, meta, 646 ['alternate', 'data'], ['if', 'features']) 647 normalize_members(expr['data']) 648 check_alternate(expr) 649 elif meta == 'struct': 650 check_keys(expr, info, meta, 651 ['struct', 'data'], ['base', 'if', 'features']) 652 normalize_members(expr['data']) 653 check_struct(expr) 654 elif meta == 'command': 655 check_keys(expr, info, meta, 656 ['command'], 657 ['data', 'returns', 'boxed', 'if', 'features', 658 'gen', 'success-response', 'allow-oob', 659 'allow-preconfig', 'coroutine']) 660 normalize_members(expr.get('data')) 661 check_command(expr) 662 elif meta == 'event': 663 check_keys(expr, info, meta, 664 ['event'], ['data', 'boxed', 'if', 'features']) 665 normalize_members(expr.get('data')) 666 check_event(expr) 667 else: 668 assert False, 'unexpected meta type' 669 670 check_if(expr, info, meta) 671 check_features(expr.get('features'), info) 672 check_flags(expr) 673 674 return exprs 675