1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# See check_name_str(), below. 51valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 52 r'(x-)?' 53 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 54 55 56def check_name_is_str(name: object, 57 info: QAPISourceInfo, 58 source: str) -> None: 59 """ 60 Ensure that ``name`` is a ``str``. 61 62 :raise QAPISemError: When ``name`` fails validation. 63 """ 64 if not isinstance(name, str): 65 raise QAPISemError(info, "%s requires a string name" % source) 66 67 68def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 69 """ 70 Ensure that ``name`` is a valid QAPI name. 71 72 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 73 starting with a letter. It may be prefixed by a downstream prefix 74 of the form __RFQDN_, or the experimental prefix ``x-``. If both 75 prefixes are present, the __RFDQN_ prefix goes first. 76 77 A valid name cannot start with ``q_``, which is reserved. 78 79 :param name: Name to check. 80 :param info: QAPI schema source file information. 81 :param source: Error string describing what ``name`` belongs to. 82 83 :raise QAPISemError: When ``name`` fails validation. 84 :return: The stem of the valid name, with no prefixes. 85 """ 86 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 87 # and 'q_obj_*' implicit type names. 88 match = valid_name.match(name) 89 if not match or c_name(name, False).startswith('q_'): 90 raise QAPISemError(info, "%s has an invalid name" % source) 91 return match.group(3) 92 93 94def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 95 """ 96 Ensure that ``name`` is a valid event name. 97 98 This means it must be a valid QAPI name as checked by 99 `check_name_str()`, but where the stem prohibits lowercase 100 characters and ``-``. 101 102 :param name: Name to check. 103 :param info: QAPI schema source file information. 104 :param source: Error string describing what ``name`` belongs to. 105 106 :raise QAPISemError: When ``name`` fails validation. 107 """ 108 stem = check_name_str(name, info, source) 109 if re.search(r'[a-z-]', stem): 110 raise QAPISemError( 111 info, "name of %s must not use lowercase or '-'" % source) 112 113 114def check_name_lower(name: str, info: QAPISourceInfo, source: str, 115 permit_upper: bool = False, 116 permit_underscore: bool = False) -> None: 117 """ 118 Ensure that ``name`` is a valid command or member name. 119 120 This means it must be a valid QAPI name as checked by 121 `check_name_str()`, but where the stem prohibits uppercase 122 characters and ``_``. 123 124 :param name: Name to check. 125 :param info: QAPI schema source file information. 126 :param source: Error string describing what ``name`` belongs to. 127 :param permit_upper: Additionally permit uppercase. 128 :param permit_underscore: Additionally permit ``_``. 129 130 :raise QAPISemError: When ``name`` fails validation. 131 """ 132 stem = check_name_str(name, info, source) 133 if ((not permit_upper and re.search(r'[A-Z]', stem)) 134 or (not permit_underscore and '_' in stem)): 135 raise QAPISemError( 136 info, "name of %s must not use uppercase or '_'" % source) 137 138 139def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 140 """ 141 Ensure that ``name`` is a valid user-defined type name. 142 143 This means it must be a valid QAPI name as checked by 144 `check_name_str()`, but where the stem must be in CamelCase. 145 146 :param name: Name to check. 147 :param info: QAPI schema source file information. 148 :param source: Error string describing what ``name`` belongs to. 149 150 :raise QAPISemError: When ``name`` fails validation. 151 """ 152 stem = check_name_str(name, info, source) 153 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 154 raise QAPISemError(info, "name of %s must use CamelCase" % source) 155 156 157def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 158 """ 159 Ensure that ``name`` is a valid definition name. 160 161 Based on the value of ``meta``, this means that: 162 - 'event' names adhere to `check_name_upper()`. 163 - 'command' names adhere to `check_name_lower()`. 164 - Else, meta is a type, and must pass `check_name_camel()`. 165 These names must not end with ``List``. 166 167 :param name: Name to check. 168 :param info: QAPI schema source file information. 169 :param meta: Meta-type name of the QAPI expression. 170 171 :raise QAPISemError: When ``name`` fails validation. 172 """ 173 if meta == 'event': 174 check_name_upper(name, info, meta) 175 elif meta == 'command': 176 check_name_lower( 177 name, info, meta, 178 permit_underscore=name in info.pragma.command_name_exceptions) 179 else: 180 check_name_camel(name, info, meta) 181 if name.endswith('List'): 182 raise QAPISemError( 183 info, "%s name should not end in 'List'" % meta) 184 185 186def check_keys(value: Dict[str, object], 187 info: QAPISourceInfo, 188 source: str, 189 required: List[str], 190 optional: List[str]) -> None: 191 """ 192 Ensure that a dict has a specific set of keys. 193 194 :param value: The dict to check. 195 :param info: QAPI schema source file information. 196 :param source: Error string describing this ``value``. 197 :param required: Keys that *must* be present. 198 :param optional: Keys that *may* be present. 199 200 :raise QAPISemError: When unknown keys are present. 201 """ 202 203 def pprint(elems: Iterable[str]) -> str: 204 return ', '.join("'" + e + "'" for e in sorted(elems)) 205 206 missing = set(required) - set(value) 207 if missing: 208 raise QAPISemError( 209 info, 210 "%s misses key%s %s" 211 % (source, 's' if len(missing) > 1 else '', 212 pprint(missing))) 213 allowed = set(required) | set(optional) 214 unknown = set(value) - allowed 215 if unknown: 216 raise QAPISemError( 217 info, 218 "%s has unknown key%s %s\nValid keys are %s." 219 % (source, 's' if len(unknown) > 1 else '', 220 pprint(unknown), pprint(allowed))) 221 222 223def check_flags(expr: QAPIExpression) -> None: 224 """ 225 Ensure flag members (if present) have valid values. 226 227 :param expr: The expression to validate. 228 229 :raise QAPISemError: 230 When certain flags have an invalid value, or when 231 incompatible flags are present. 232 """ 233 for key in ('gen', 'success-response'): 234 if key in expr and expr[key] is not False: 235 raise QAPISemError( 236 expr.info, "flag '%s' may only use false value" % key) 237 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 238 if key in expr and expr[key] is not True: 239 raise QAPISemError( 240 expr.info, "flag '%s' may only use true value" % key) 241 if 'allow-oob' in expr and 'coroutine' in expr: 242 # This is not necessarily a fundamental incompatibility, but 243 # we don't have a use case and the desired semantics isn't 244 # obvious. The simplest solution is to forbid it until we get 245 # a use case for it. 246 raise QAPISemError( 247 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 248 249 250def check_if(expr: Dict[str, object], 251 info: QAPISourceInfo, source: str) -> None: 252 """ 253 Validate the ``if`` member of an object. 254 255 The ``if`` member may be either a ``str`` or a dict. 256 257 :param expr: The expression containing the ``if`` member to validate. 258 :param info: QAPI schema source file information. 259 :param source: Error string describing ``expr``. 260 261 :raise QAPISemError: 262 When the "if" member fails validation, or when there are no 263 non-empty conditions. 264 :return: None 265 """ 266 267 def _check_if(cond: Union[str, object]) -> None: 268 if isinstance(cond, str): 269 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 270 raise QAPISemError( 271 info, 272 "'if' condition '%s' of %s is not a valid identifier" 273 % (cond, source)) 274 return 275 276 if not isinstance(cond, dict): 277 raise QAPISemError( 278 info, 279 "'if' condition of %s must be a string or an object" % source) 280 check_keys(cond, info, "'if' condition of %s" % source, [], 281 ["all", "any", "not"]) 282 if len(cond) != 1: 283 raise QAPISemError( 284 info, 285 "'if' condition of %s has conflicting keys" % source) 286 287 if 'not' in cond: 288 _check_if(cond['not']) 289 elif 'all' in cond: 290 _check_infix('all', cond['all']) 291 else: 292 _check_infix('any', cond['any']) 293 294 def _check_infix(operator: str, operands: object) -> None: 295 if not isinstance(operands, list): 296 raise QAPISemError( 297 info, 298 "'%s' condition of %s must be an array" 299 % (operator, source)) 300 if not operands: 301 raise QAPISemError( 302 info, "'if' condition [] of %s is useless" % source) 303 for operand in operands: 304 _check_if(operand) 305 306 ifcond = expr.get('if') 307 if ifcond is None: 308 return 309 310 _check_if(ifcond) 311 312 313def normalize_members(members: object) -> None: 314 """ 315 Normalize a "members" value. 316 317 If ``members`` is a dict, for every value in that dict, if that 318 value is not itself already a dict, normalize it to 319 ``{'type': value}``. 320 321 :forms: 322 :sugared: ``Dict[str, Union[str, TypeRef]]`` 323 :canonical: ``Dict[str, TypeRef]`` 324 325 :param members: The members value to normalize. 326 327 :return: None, ``members`` is normalized in-place as needed. 328 """ 329 if isinstance(members, dict): 330 for key, arg in members.items(): 331 if isinstance(arg, dict): 332 continue 333 members[key] = {'type': arg} 334 335 336def check_type(value: Optional[object], 337 info: QAPISourceInfo, 338 source: str, 339 allow_array: bool = False, 340 allow_dict: Union[bool, str] = False) -> None: 341 """ 342 Normalize and validate the QAPI type of ``value``. 343 344 Python types of ``str`` or ``None`` are always allowed. 345 346 :param value: The value to check. 347 :param info: QAPI schema source file information. 348 :param source: Error string describing this ``value``. 349 :param allow_array: 350 Allow a ``List[str]`` of length 1, which indicates an array of 351 the type named by the list element. 352 :param allow_dict: 353 Allow a dict. Its members can be struct type members or union 354 branches. When the value of ``allow_dict`` is in pragma 355 ``member-name-exceptions``, the dict's keys may violate the 356 member naming rules. The dict members are normalized in place. 357 358 :raise QAPISemError: When ``value`` fails validation. 359 :return: None, ``value`` is normalized in-place as needed. 360 """ 361 if value is None: 362 return 363 364 # Type name 365 if isinstance(value, str): 366 return 367 368 # Array type 369 if isinstance(value, list): 370 if not allow_array: 371 raise QAPISemError(info, "%s cannot be an array" % source) 372 if len(value) != 1 or not isinstance(value[0], str): 373 raise QAPISemError(info, 374 "%s: array type must contain single type name" % 375 source) 376 return 377 378 # Anonymous type 379 380 if not allow_dict: 381 raise QAPISemError(info, "%s should be a type name" % source) 382 383 if not isinstance(value, dict): 384 raise QAPISemError(info, 385 "%s should be an object or type name" % source) 386 387 permissive = False 388 if isinstance(allow_dict, str): 389 permissive = allow_dict in info.pragma.member_name_exceptions 390 391 # value is a dictionary, check that each member is okay 392 for (key, arg) in value.items(): 393 key_source = "%s member '%s'" % (source, key) 394 if key.startswith('*'): 395 key = key[1:] 396 check_name_lower(key, info, key_source, 397 permit_upper=permissive, 398 permit_underscore=permissive) 399 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 400 raise QAPISemError(info, "%s uses reserved name" % key_source) 401 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 402 check_if(arg, info, key_source) 403 check_features(arg.get('features'), info) 404 check_type(arg['type'], info, key_source, allow_array=True) 405 406 407def check_features(features: Optional[object], 408 info: QAPISourceInfo) -> None: 409 """ 410 Normalize and validate the ``features`` member. 411 412 ``features`` may be a ``list`` of either ``str`` or ``dict``. 413 Any ``str`` element will be normalized to ``{'name': element}``. 414 415 :forms: 416 :sugared: ``List[Union[str, Feature]]`` 417 :canonical: ``List[Feature]`` 418 419 :param features: The features member value to validate. 420 :param info: QAPI schema source file information. 421 422 :raise QAPISemError: When ``features`` fails validation. 423 :return: None, ``features`` is normalized in-place as needed. 424 """ 425 if features is None: 426 return 427 if not isinstance(features, list): 428 raise QAPISemError(info, "'features' must be an array") 429 features[:] = [f if isinstance(f, dict) else {'name': f} 430 for f in features] 431 for feat in features: 432 source = "'features' member" 433 assert isinstance(feat, dict) 434 check_keys(feat, info, source, ['name'], ['if']) 435 check_name_is_str(feat['name'], info, source) 436 source = "%s '%s'" % (source, feat['name']) 437 check_name_lower(feat['name'], info, source) 438 check_if(feat, info, source) 439 440 441def check_enum(expr: QAPIExpression) -> None: 442 """ 443 Normalize and validate this expression as an ``enum`` definition. 444 445 :param expr: The expression to validate. 446 447 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 448 :return: None, ``expr`` is normalized in-place as needed. 449 """ 450 name = expr['enum'] 451 members = expr['data'] 452 prefix = expr.get('prefix') 453 info = expr.info 454 455 if not isinstance(members, list): 456 raise QAPISemError(info, "'data' must be an array") 457 if prefix is not None and not isinstance(prefix, str): 458 raise QAPISemError(info, "'prefix' must be a string") 459 460 permissive = name in info.pragma.member_name_exceptions 461 462 members[:] = [m if isinstance(m, dict) else {'name': m} 463 for m in members] 464 for member in members: 465 source = "'data' member" 466 check_keys(member, info, source, ['name'], ['if', 'features']) 467 member_name = member['name'] 468 check_name_is_str(member_name, info, source) 469 source = "%s '%s'" % (source, member_name) 470 # Enum members may start with a digit 471 if member_name[0].isdigit(): 472 member_name = 'd' + member_name # Hack: hide the digit 473 check_name_lower(member_name, info, source, 474 permit_upper=permissive, 475 permit_underscore=permissive) 476 check_if(member, info, source) 477 check_features(member.get('features'), info) 478 479 480def check_struct(expr: QAPIExpression) -> None: 481 """ 482 Normalize and validate this expression as a ``struct`` definition. 483 484 :param expr: The expression to validate. 485 486 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 487 :return: None, ``expr`` is normalized in-place as needed. 488 """ 489 name = cast(str, expr['struct']) # Checked in check_exprs 490 members = expr['data'] 491 492 check_type(members, expr.info, "'data'", allow_dict=name) 493 check_type(expr.get('base'), expr.info, "'base'") 494 495 496def check_union(expr: QAPIExpression) -> None: 497 """ 498 Normalize and validate this expression as a ``union`` definition. 499 500 :param expr: The expression to validate. 501 502 :raise QAPISemError: when ``expr`` is not a valid ``union``. 503 :return: None, ``expr`` is normalized in-place as needed. 504 """ 505 name = cast(str, expr['union']) # Checked in check_exprs 506 base = expr['base'] 507 discriminator = expr['discriminator'] 508 members = expr['data'] 509 info = expr.info 510 511 check_type(base, info, "'base'", allow_dict=name) 512 check_name_is_str(discriminator, info, "'discriminator'") 513 514 if not isinstance(members, dict): 515 raise QAPISemError(info, "'data' must be an object") 516 517 for (key, value) in members.items(): 518 source = "'data' member '%s'" % key 519 check_keys(value, info, source, ['type'], ['if']) 520 check_if(value, info, source) 521 check_type(value['type'], info, source, allow_array=not base) 522 523 524def check_alternate(expr: QAPIExpression) -> None: 525 """ 526 Normalize and validate this expression as an ``alternate`` definition. 527 528 :param expr: The expression to validate. 529 530 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 531 :return: None, ``expr`` is normalized in-place as needed. 532 """ 533 members = expr['data'] 534 info = expr.info 535 536 if not members: 537 raise QAPISemError(info, "'data' must not be empty") 538 539 if not isinstance(members, dict): 540 raise QAPISemError(info, "'data' must be an object") 541 542 for (key, value) in members.items(): 543 source = "'data' member '%s'" % key 544 check_name_lower(key, info, source) 545 check_keys(value, info, source, ['type'], ['if']) 546 check_if(value, info, source) 547 check_type(value['type'], info, source, allow_array=True) 548 549 550def check_command(expr: QAPIExpression) -> None: 551 """ 552 Normalize and validate this expression as a ``command`` definition. 553 554 :param expr: The expression to validate. 555 556 :raise QAPISemError: When ``expr`` is not a valid ``command``. 557 :return: None, ``expr`` is normalized in-place as needed. 558 """ 559 args = expr.get('data') 560 rets = expr.get('returns') 561 boxed = expr.get('boxed', False) 562 563 if boxed and args is None: 564 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 565 check_type(args, expr.info, "'data'", allow_dict=not boxed) 566 check_type(rets, expr.info, "'returns'", allow_array=True) 567 568 569def check_event(expr: QAPIExpression) -> None: 570 """ 571 Normalize and validate this expression as an ``event`` definition. 572 573 :param expr: The expression to validate. 574 575 :raise QAPISemError: When ``expr`` is not a valid ``event``. 576 :return: None, ``expr`` is normalized in-place as needed. 577 """ 578 args = expr.get('data') 579 boxed = expr.get('boxed', False) 580 581 if boxed and args is None: 582 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 583 check_type(args, expr.info, "'data'", allow_dict=not boxed) 584 585 586def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 587 """ 588 Validate and normalize a list of parsed QAPI schema expressions. 589 590 This function accepts a list of expressions and metadata as returned 591 by the parser. It destructively normalizes the expressions in-place. 592 593 :param exprs: The list of expressions to normalize and validate. 594 595 :raise QAPISemError: When any expression fails validation. 596 :return: The same list of expressions (now modified). 597 """ 598 for expr in exprs: 599 info = expr.info 600 doc = expr.doc 601 602 if 'include' in expr: 603 continue 604 605 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 606 'command', 'event'} 607 if len(metas) != 1: 608 raise QAPISemError( 609 info, 610 "expression must have exactly one key" 611 " 'enum', 'struct', 'union', 'alternate'," 612 " 'command', 'event'") 613 meta = metas.pop() 614 615 check_name_is_str(expr[meta], info, "'%s'" % meta) 616 name = cast(str, expr[meta]) 617 info.set_defn(meta, name) 618 check_defn_name_str(name, info, meta) 619 620 if doc: 621 if doc.symbol != name: 622 raise QAPISemError( 623 info, "documentation comment is for '%s'" % doc.symbol) 624 doc.check_expr(expr) 625 elif info.pragma.doc_required: 626 raise QAPISemError(info, 627 "documentation comment required") 628 629 if meta == 'enum': 630 check_keys(expr, info, meta, 631 ['enum', 'data'], ['if', 'features', 'prefix']) 632 check_enum(expr) 633 elif meta == 'union': 634 check_keys(expr, info, meta, 635 ['union', 'base', 'discriminator', 'data'], 636 ['if', 'features']) 637 normalize_members(expr.get('base')) 638 normalize_members(expr['data']) 639 check_union(expr) 640 elif meta == 'alternate': 641 check_keys(expr, info, meta, 642 ['alternate', 'data'], ['if', 'features']) 643 normalize_members(expr['data']) 644 check_alternate(expr) 645 elif meta == 'struct': 646 check_keys(expr, info, meta, 647 ['struct', 'data'], ['base', 'if', 'features']) 648 normalize_members(expr['data']) 649 check_struct(expr) 650 elif meta == 'command': 651 check_keys(expr, info, meta, 652 ['command'], 653 ['data', 'returns', 'boxed', 'if', 'features', 654 'gen', 'success-response', 'allow-oob', 655 'allow-preconfig', 'coroutine']) 656 normalize_members(expr.get('data')) 657 check_command(expr) 658 elif meta == 'event': 659 check_keys(expr, info, meta, 660 ['event'], ['data', 'boxed', 'if', 'features']) 661 normalize_members(expr.get('data')) 662 check_event(expr) 663 else: 664 assert False, 'unexpected meta type' 665 666 check_if(expr, info, meta) 667 check_features(expr.get('features'), info) 668 check_flags(expr) 669 670 return exprs 671