1# -*- coding: utf-8 -*- 2# 3# Copyright IBM, Corp. 2011 4# Copyright (c) 2013-2021 Red Hat Inc. 5# 6# Authors: 7# Anthony Liguori <aliguori@us.ibm.com> 8# Markus Armbruster <armbru@redhat.com> 9# Eric Blake <eblake@redhat.com> 10# Marc-André Lureau <marcandre.lureau@redhat.com> 11# John Snow <jsnow@redhat.com> 12# 13# This work is licensed under the terms of the GNU GPL, version 2. 14# See the COPYING file in the top-level directory. 15 16""" 17Normalize and validate (context-free) QAPI schema expression structures. 18 19`QAPISchemaParser` parses a QAPI schema into abstract syntax trees 20consisting of dict, list, str, bool, and int nodes. This module ensures 21that these nested structures have the correct type(s) and key(s) where 22appropriate for the QAPI context-free grammar. 23 24The QAPI schema expression language allows for certain syntactic sugar; 25this module also handles the normalization process of these nested 26structures. 27 28See `check_exprs` for the main entry point. 29 30See `schema.QAPISchema` for processing into native Python data 31structures and contextual semantic validation. 32""" 33 34import re 35from typing import ( 36 Dict, 37 Iterable, 38 List, 39 Optional, 40 Union, 41 cast, 42) 43 44from .common import c_name 45from .error import QAPISemError 46from .parser import QAPIExpression 47from .source import QAPISourceInfo 48 49 50# See check_name_str(), below. 51valid_name = re.compile(r'(__[a-z0-9.-]+_)?' 52 r'(x-)?' 53 r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) 54 55 56def check_name_is_str(name: object, 57 info: QAPISourceInfo, 58 source: str) -> None: 59 """ 60 Ensure that ``name`` is a ``str``. 61 62 :raise QAPISemError: When ``name`` fails validation. 63 """ 64 if not isinstance(name, str): 65 raise QAPISemError(info, "%s requires a string name" % source) 66 67 68def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: 69 """ 70 Ensure that ``name`` is a valid QAPI name. 71 72 A valid name consists of ASCII letters, digits, ``-``, and ``_``, 73 starting with a letter. It may be prefixed by a downstream prefix 74 of the form __RFQDN_, or the experimental prefix ``x-``. If both 75 prefixes are present, the __RFDQN_ prefix goes first. 76 77 A valid name cannot start with ``q_``, which is reserved. 78 79 :param name: Name to check. 80 :param info: QAPI schema source file information. 81 :param source: Error string describing what ``name`` belongs to. 82 83 :raise QAPISemError: When ``name`` fails validation. 84 :return: The stem of the valid name, with no prefixes. 85 """ 86 # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' 87 # and 'q_obj_*' implicit type names. 88 match = valid_name.match(name) 89 if not match or c_name(name, False).startswith('q_'): 90 raise QAPISemError(info, "%s has an invalid name" % source) 91 return match.group(3) 92 93 94def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: 95 """ 96 Ensure that ``name`` is a valid event name. 97 98 This means it must be a valid QAPI name as checked by 99 `check_name_str()`, but where the stem prohibits lowercase 100 characters and ``-``. 101 102 :param name: Name to check. 103 :param info: QAPI schema source file information. 104 :param source: Error string describing what ``name`` belongs to. 105 106 :raise QAPISemError: When ``name`` fails validation. 107 """ 108 stem = check_name_str(name, info, source) 109 if re.search(r'[a-z-]', stem): 110 raise QAPISemError( 111 info, "name of %s must not use lowercase or '-'" % source) 112 113 114def check_name_lower(name: str, info: QAPISourceInfo, source: str, 115 permit_upper: bool = False, 116 permit_underscore: bool = False) -> None: 117 """ 118 Ensure that ``name`` is a valid command or member name. 119 120 This means it must be a valid QAPI name as checked by 121 `check_name_str()`, but where the stem prohibits uppercase 122 characters and ``_``. 123 124 :param name: Name to check. 125 :param info: QAPI schema source file information. 126 :param source: Error string describing what ``name`` belongs to. 127 :param permit_upper: Additionally permit uppercase. 128 :param permit_underscore: Additionally permit ``_``. 129 130 :raise QAPISemError: When ``name`` fails validation. 131 """ 132 stem = check_name_str(name, info, source) 133 if ((not permit_upper and re.search(r'[A-Z]', stem)) 134 or (not permit_underscore and '_' in stem)): 135 raise QAPISemError( 136 info, "name of %s must not use uppercase or '_'" % source) 137 138 139def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: 140 """ 141 Ensure that ``name`` is a valid user-defined type name. 142 143 This means it must be a valid QAPI name as checked by 144 `check_name_str()`, but where the stem must be in CamelCase. 145 146 :param name: Name to check. 147 :param info: QAPI schema source file information. 148 :param source: Error string describing what ``name`` belongs to. 149 150 :raise QAPISemError: When ``name`` fails validation. 151 """ 152 stem = check_name_str(name, info, source) 153 if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): 154 raise QAPISemError(info, "name of %s must use CamelCase" % source) 155 156 157def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: 158 """ 159 Ensure that ``name`` is a valid definition name. 160 161 Based on the value of ``meta``, this means that: 162 - 'event' names adhere to `check_name_upper()`. 163 - 'command' names adhere to `check_name_lower()`. 164 - Else, meta is a type, and must pass `check_name_camel()`. 165 These names must not end with ``List``. 166 167 :param name: Name to check. 168 :param info: QAPI schema source file information. 169 :param meta: Meta-type name of the QAPI expression. 170 171 :raise QAPISemError: When ``name`` fails validation. 172 """ 173 if meta == 'event': 174 check_name_upper(name, info, meta) 175 elif meta == 'command': 176 check_name_lower( 177 name, info, meta, 178 permit_underscore=name in info.pragma.command_name_exceptions) 179 else: 180 check_name_camel(name, info, meta) 181 if name.endswith('List'): 182 raise QAPISemError( 183 info, "%s name should not end in 'List'" % meta) 184 185 186def check_keys(value: Dict[str, object], 187 info: QAPISourceInfo, 188 source: str, 189 required: List[str], 190 optional: List[str]) -> None: 191 """ 192 Ensure that a dict has a specific set of keys. 193 194 :param value: The dict to check. 195 :param info: QAPI schema source file information. 196 :param source: Error string describing this ``value``. 197 :param required: Keys that *must* be present. 198 :param optional: Keys that *may* be present. 199 200 :raise QAPISemError: When unknown keys are present. 201 """ 202 203 def pprint(elems: Iterable[str]) -> str: 204 return ', '.join("'" + e + "'" for e in sorted(elems)) 205 206 missing = set(required) - set(value) 207 if missing: 208 raise QAPISemError( 209 info, 210 "%s misses key%s %s" 211 % (source, 's' if len(missing) > 1 else '', 212 pprint(missing))) 213 allowed = set(required) | set(optional) 214 unknown = set(value) - allowed 215 if unknown: 216 raise QAPISemError( 217 info, 218 "%s has unknown key%s %s\nValid keys are %s." 219 % (source, 's' if len(unknown) > 1 else '', 220 pprint(unknown), pprint(allowed))) 221 222 223def check_flags(expr: QAPIExpression) -> None: 224 """ 225 Ensure flag members (if present) have valid values. 226 227 :param expr: The expression to validate. 228 229 :raise QAPISemError: 230 When certain flags have an invalid value, or when 231 incompatible flags are present. 232 """ 233 for key in ('gen', 'success-response'): 234 if key in expr and expr[key] is not False: 235 raise QAPISemError( 236 expr.info, "flag '%s' may only use false value" % key) 237 for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): 238 if key in expr and expr[key] is not True: 239 raise QAPISemError( 240 expr.info, "flag '%s' may only use true value" % key) 241 if 'allow-oob' in expr and 'coroutine' in expr: 242 # This is not necessarily a fundamental incompatibility, but 243 # we don't have a use case and the desired semantics isn't 244 # obvious. The simplest solution is to forbid it until we get 245 # a use case for it. 246 raise QAPISemError( 247 expr.info, "flags 'allow-oob' and 'coroutine' are incompatible") 248 249 250def check_if(expr: Dict[str, object], 251 info: QAPISourceInfo, source: str) -> None: 252 """ 253 Validate the ``if`` member of an object. 254 255 The ``if`` member may be either a ``str`` or a dict. 256 257 :param expr: The expression containing the ``if`` member to validate. 258 :param info: QAPI schema source file information. 259 :param source: Error string describing ``expr``. 260 261 :raise QAPISemError: 262 When the "if" member fails validation, or when there are no 263 non-empty conditions. 264 :return: None 265 """ 266 267 def _check_if(cond: Union[str, object]) -> None: 268 if isinstance(cond, str): 269 if not re.fullmatch(r'[A-Z][A-Z0-9_]*', cond): 270 raise QAPISemError( 271 info, 272 "'if' condition '%s' of %s is not a valid identifier" 273 % (cond, source)) 274 return 275 276 if not isinstance(cond, dict): 277 raise QAPISemError( 278 info, 279 "'if' condition of %s must be a string or an object" % source) 280 check_keys(cond, info, "'if' condition of %s" % source, [], 281 ["all", "any", "not"]) 282 if len(cond) != 1: 283 raise QAPISemError( 284 info, 285 "'if' condition of %s has conflicting keys" % source) 286 287 if 'not' in cond: 288 _check_if(cond['not']) 289 elif 'all' in cond: 290 _check_infix('all', cond['all']) 291 else: 292 _check_infix('any', cond['any']) 293 294 def _check_infix(operator: str, operands: object) -> None: 295 if not isinstance(operands, list): 296 raise QAPISemError( 297 info, 298 "'%s' condition of %s must be an array" 299 % (operator, source)) 300 if not operands: 301 raise QAPISemError( 302 info, "'if' condition [] of %s is useless" % source) 303 for operand in operands: 304 _check_if(operand) 305 306 ifcond = expr.get('if') 307 if ifcond is None: 308 return 309 310 _check_if(ifcond) 311 312 313def normalize_members(members: object) -> None: 314 """ 315 Normalize a "members" value. 316 317 If ``members`` is a dict, for every value in that dict, if that 318 value is not itself already a dict, normalize it to 319 ``{'type': value}``. 320 321 :forms: 322 :sugared: ``Dict[str, Union[str, TypeRef]]`` 323 :canonical: ``Dict[str, TypeRef]`` 324 325 :param members: The members value to normalize. 326 327 :return: None, ``members`` is normalized in-place as needed. 328 """ 329 if isinstance(members, dict): 330 for key, arg in members.items(): 331 if isinstance(arg, dict): 332 continue 333 members[key] = {'type': arg} 334 335 336def check_type_name(value: Optional[object], 337 info: QAPISourceInfo, source: str) -> None: 338 if value is not None and not isinstance(value, str): 339 raise QAPISemError(info, "%s should be a type name" % source) 340 341 342def check_type_name_or_array(value: Optional[object], 343 info: QAPISourceInfo, source: str) -> None: 344 if value is None or isinstance(value, str): 345 return 346 347 if not isinstance(value, list): 348 raise QAPISemError(info, 349 "%s should be a type name or array" % source) 350 351 if len(value) != 1 or not isinstance(value[0], str): 352 raise QAPISemError(info, 353 "%s: array type must contain single type name" % 354 source) 355 356 357def check_type_name_or_implicit(value: Optional[object], 358 info: QAPISourceInfo, source: str, 359 parent_name: Optional[str]) -> None: 360 """ 361 Normalize and validate an optional implicit struct type. 362 363 Accept ``None``, ``str``, or a ``dict`` defining an implicit 364 struct type. The latter is normalized in place. 365 366 :param value: The value to check. 367 :param info: QAPI schema source file information. 368 :param source: Error string describing this ``value``. 369 :param parent_name: 370 When the value of ``parent_name`` is in pragma 371 ``member-name-exceptions``, an implicit struct type may 372 violate the member naming rules. 373 374 :raise QAPISemError: When ``value`` fails validation. 375 :return: None 376 """ 377 if value is None: 378 return 379 380 if isinstance(value, str): 381 return 382 383 if not isinstance(value, dict): 384 raise QAPISemError(info, 385 "%s should be an object or type name" % source) 386 387 permissive = parent_name in info.pragma.member_name_exceptions 388 389 for (key, arg) in value.items(): 390 key_source = "%s member '%s'" % (source, key) 391 if key.startswith('*'): 392 key = key[1:] 393 check_name_lower(key, info, key_source, 394 permit_upper=permissive, 395 permit_underscore=permissive) 396 if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): 397 raise QAPISemError(info, "%s uses reserved name" % key_source) 398 check_keys(arg, info, key_source, ['type'], ['if', 'features']) 399 check_if(arg, info, key_source) 400 check_features(arg.get('features'), info) 401 check_type_name_or_array(arg['type'], info, key_source) 402 403 404def check_features(features: Optional[object], 405 info: QAPISourceInfo) -> None: 406 """ 407 Normalize and validate the ``features`` member. 408 409 ``features`` may be a ``list`` of either ``str`` or ``dict``. 410 Any ``str`` element will be normalized to ``{'name': element}``. 411 412 :forms: 413 :sugared: ``List[Union[str, Feature]]`` 414 :canonical: ``List[Feature]`` 415 416 :param features: The features member value to validate. 417 :param info: QAPI schema source file information. 418 419 :raise QAPISemError: When ``features`` fails validation. 420 :return: None, ``features`` is normalized in-place as needed. 421 """ 422 if features is None: 423 return 424 if not isinstance(features, list): 425 raise QAPISemError(info, "'features' must be an array") 426 features[:] = [f if isinstance(f, dict) else {'name': f} 427 for f in features] 428 for feat in features: 429 source = "'features' member" 430 assert isinstance(feat, dict) 431 check_keys(feat, info, source, ['name'], ['if']) 432 check_name_is_str(feat['name'], info, source) 433 source = "%s '%s'" % (source, feat['name']) 434 check_name_lower(feat['name'], info, source) 435 check_if(feat, info, source) 436 437 438def check_enum(expr: QAPIExpression) -> None: 439 """ 440 Normalize and validate this expression as an ``enum`` definition. 441 442 :param expr: The expression to validate. 443 444 :raise QAPISemError: When ``expr`` is not a valid ``enum``. 445 :return: None, ``expr`` is normalized in-place as needed. 446 """ 447 name = expr['enum'] 448 members = expr['data'] 449 prefix = expr.get('prefix') 450 info = expr.info 451 452 if not isinstance(members, list): 453 raise QAPISemError(info, "'data' must be an array") 454 if prefix is not None and not isinstance(prefix, str): 455 raise QAPISemError(info, "'prefix' must be a string") 456 457 permissive = name in info.pragma.member_name_exceptions 458 459 members[:] = [m if isinstance(m, dict) else {'name': m} 460 for m in members] 461 for member in members: 462 source = "'data' member" 463 check_keys(member, info, source, ['name'], ['if', 'features']) 464 member_name = member['name'] 465 check_name_is_str(member_name, info, source) 466 source = "%s '%s'" % (source, member_name) 467 # Enum members may start with a digit 468 if member_name[0].isdigit(): 469 member_name = 'd' + member_name # Hack: hide the digit 470 check_name_lower(member_name, info, source, 471 permit_upper=permissive, 472 permit_underscore=permissive) 473 check_if(member, info, source) 474 check_features(member.get('features'), info) 475 476 477def check_struct(expr: QAPIExpression) -> None: 478 """ 479 Normalize and validate this expression as a ``struct`` definition. 480 481 :param expr: The expression to validate. 482 483 :raise QAPISemError: When ``expr`` is not a valid ``struct``. 484 :return: None, ``expr`` is normalized in-place as needed. 485 """ 486 name = cast(str, expr['struct']) # Checked in check_exprs 487 members = expr['data'] 488 489 check_type_name_or_implicit(members, expr.info, "'data'", name) 490 check_type_name(expr.get('base'), expr.info, "'base'") 491 492 493def check_union(expr: QAPIExpression) -> None: 494 """ 495 Normalize and validate this expression as a ``union`` definition. 496 497 :param expr: The expression to validate. 498 499 :raise QAPISemError: when ``expr`` is not a valid ``union``. 500 :return: None, ``expr`` is normalized in-place as needed. 501 """ 502 name = cast(str, expr['union']) # Checked in check_exprs 503 base = expr['base'] 504 discriminator = expr['discriminator'] 505 members = expr['data'] 506 info = expr.info 507 508 check_type_name_or_implicit(base, info, "'base'", name) 509 check_name_is_str(discriminator, info, "'discriminator'") 510 511 if not isinstance(members, dict): 512 raise QAPISemError(info, "'data' must be an object") 513 514 for (key, value) in members.items(): 515 source = "'data' member '%s'" % key 516 check_keys(value, info, source, ['type'], ['if']) 517 check_if(value, info, source) 518 check_type_name(value['type'], info, source) 519 520 521def check_alternate(expr: QAPIExpression) -> None: 522 """ 523 Normalize and validate this expression as an ``alternate`` definition. 524 525 :param expr: The expression to validate. 526 527 :raise QAPISemError: When ``expr`` is not a valid ``alternate``. 528 :return: None, ``expr`` is normalized in-place as needed. 529 """ 530 members = expr['data'] 531 info = expr.info 532 533 if not members: 534 raise QAPISemError(info, "'data' must not be empty") 535 536 if not isinstance(members, dict): 537 raise QAPISemError(info, "'data' must be an object") 538 539 for (key, value) in members.items(): 540 source = "'data' member '%s'" % key 541 check_name_lower(key, info, source) 542 check_keys(value, info, source, ['type'], ['if']) 543 check_if(value, info, source) 544 check_type_name_or_array(value['type'], info, source) 545 546 547def check_command(expr: QAPIExpression) -> None: 548 """ 549 Normalize and validate this expression as a ``command`` definition. 550 551 :param expr: The expression to validate. 552 553 :raise QAPISemError: When ``expr`` is not a valid ``command``. 554 :return: None, ``expr`` is normalized in-place as needed. 555 """ 556 args = expr.get('data') 557 rets = expr.get('returns') 558 boxed = expr.get('boxed', False) 559 560 if boxed: 561 if args is None: 562 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 563 check_type_name(args, expr.info, "'data'") 564 else: 565 check_type_name_or_implicit(args, expr.info, "'data'", None) 566 check_type_name_or_array(rets, expr.info, "'returns'") 567 568 569def check_event(expr: QAPIExpression) -> None: 570 """ 571 Normalize and validate this expression as an ``event`` definition. 572 573 :param expr: The expression to validate. 574 575 :raise QAPISemError: When ``expr`` is not a valid ``event``. 576 :return: None, ``expr`` is normalized in-place as needed. 577 """ 578 args = expr.get('data') 579 boxed = expr.get('boxed', False) 580 581 if boxed: 582 if args is None: 583 raise QAPISemError(expr.info, "'boxed': true requires 'data'") 584 check_type_name(args, expr.info, "'data'") 585 else: 586 check_type_name_or_implicit(args, expr.info, "'data'", None) 587 588 589def check_exprs(exprs: List[QAPIExpression]) -> List[QAPIExpression]: 590 """ 591 Validate and normalize a list of parsed QAPI schema expressions. 592 593 This function accepts a list of expressions and metadata as returned 594 by the parser. It destructively normalizes the expressions in-place. 595 596 :param exprs: The list of expressions to normalize and validate. 597 598 :raise QAPISemError: When any expression fails validation. 599 :return: The same list of expressions (now modified). 600 """ 601 for expr in exprs: 602 info = expr.info 603 doc = expr.doc 604 605 if 'include' in expr: 606 continue 607 608 metas = expr.keys() & {'enum', 'struct', 'union', 'alternate', 609 'command', 'event'} 610 if len(metas) != 1: 611 raise QAPISemError( 612 info, 613 "expression must have exactly one key" 614 " 'enum', 'struct', 'union', 'alternate'," 615 " 'command', 'event'") 616 meta = metas.pop() 617 618 check_name_is_str(expr[meta], info, "'%s'" % meta) 619 name = cast(str, expr[meta]) 620 info.set_defn(meta, name) 621 check_defn_name_str(name, info, meta) 622 623 if doc: 624 if doc.symbol != name: 625 raise QAPISemError( 626 info, "documentation comment is for '%s'" % doc.symbol) 627 doc.check_expr(expr) 628 elif info.pragma.doc_required: 629 raise QAPISemError(info, 630 "documentation comment required") 631 632 if meta == 'enum': 633 check_keys(expr, info, meta, 634 ['enum', 'data'], ['if', 'features', 'prefix']) 635 check_enum(expr) 636 elif meta == 'union': 637 check_keys(expr, info, meta, 638 ['union', 'base', 'discriminator', 'data'], 639 ['if', 'features']) 640 normalize_members(expr.get('base')) 641 normalize_members(expr['data']) 642 check_union(expr) 643 elif meta == 'alternate': 644 check_keys(expr, info, meta, 645 ['alternate', 'data'], ['if', 'features']) 646 normalize_members(expr['data']) 647 check_alternate(expr) 648 elif meta == 'struct': 649 check_keys(expr, info, meta, 650 ['struct', 'data'], ['base', 'if', 'features']) 651 normalize_members(expr['data']) 652 check_struct(expr) 653 elif meta == 'command': 654 check_keys(expr, info, meta, 655 ['command'], 656 ['data', 'returns', 'boxed', 'if', 'features', 657 'gen', 'success-response', 'allow-oob', 658 'allow-preconfig', 'coroutine']) 659 normalize_members(expr.get('data')) 660 check_command(expr) 661 elif meta == 'event': 662 check_keys(expr, info, meta, 663 ['event'], ['data', 'boxed', 'if', 'features']) 664 normalize_members(expr.get('data')) 665 check_event(expr) 666 else: 667 assert False, 'unexpected meta type' 668 669 check_if(expr, info, meta) 670 check_features(expr.get('features'), info) 671 check_flags(expr) 672 673 return exprs 674