1# -*- coding: utf-8 -*- 2# 3# QAPI schema parser 4# 5# Copyright IBM, Corp. 2011 6# Copyright (c) 2013-2019 Red Hat Inc. 7# 8# Authors: 9# Anthony Liguori <aliguori@us.ibm.com> 10# Markus Armbruster <armbru@redhat.com> 11# Marc-André Lureau <marcandre.lureau@redhat.com> 12# Kevin Wolf <kwolf@redhat.com> 13# 14# This work is licensed under the terms of the GNU GPL, version 2. 15# See the COPYING file in the top-level directory. 16 17import os 18import re 19from collections import OrderedDict 20 21from qapi.error import QAPIParseError, QAPISemError 22from qapi.source import QAPISourceInfo 23 24 25class QAPISchemaParser: 26 27 def __init__(self, fname, previously_included=None, incl_info=None): 28 previously_included = previously_included or set() 29 previously_included.add(os.path.abspath(fname)) 30 31 try: 32 fp = open(fname, 'r', encoding='utf-8') 33 self.src = fp.read() 34 except IOError as e: 35 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), 36 "can't read %s file '%s': %s" 37 % ("include" if incl_info else "schema", 38 fname, 39 e.strerror)) 40 41 if self.src == '' or self.src[-1] != '\n': 42 self.src += '\n' 43 self.cursor = 0 44 self.info = QAPISourceInfo(fname, 1, incl_info) 45 self.line_pos = 0 46 self.exprs = [] 47 self.docs = [] 48 self.accept() 49 cur_doc = None 50 51 while self.tok is not None: 52 info = self.info 53 if self.tok == '#': 54 self.reject_expr_doc(cur_doc) 55 for cur_doc in self.get_doc(info): 56 self.docs.append(cur_doc) 57 continue 58 59 expr = self.get_expr(False) 60 if 'include' in expr: 61 self.reject_expr_doc(cur_doc) 62 if len(expr) != 1: 63 raise QAPISemError(info, "invalid 'include' directive") 64 include = expr['include'] 65 if not isinstance(include, str): 66 raise QAPISemError(info, 67 "value of 'include' must be a string") 68 incl_fname = os.path.join(os.path.dirname(fname), 69 include) 70 self.exprs.append({'expr': {'include': incl_fname}, 71 'info': info}) 72 exprs_include = self._include(include, info, incl_fname, 73 previously_included) 74 if exprs_include: 75 self.exprs.extend(exprs_include.exprs) 76 self.docs.extend(exprs_include.docs) 77 elif "pragma" in expr: 78 self.reject_expr_doc(cur_doc) 79 if len(expr) != 1: 80 raise QAPISemError(info, "invalid 'pragma' directive") 81 pragma = expr['pragma'] 82 if not isinstance(pragma, dict): 83 raise QAPISemError( 84 info, "value of 'pragma' must be an object") 85 for name, value in pragma.items(): 86 self._pragma(name, value, info) 87 else: 88 expr_elem = {'expr': expr, 89 'info': info} 90 if cur_doc: 91 if not cur_doc.symbol: 92 raise QAPISemError( 93 cur_doc.info, "definition documentation required") 94 expr_elem['doc'] = cur_doc 95 self.exprs.append(expr_elem) 96 cur_doc = None 97 self.reject_expr_doc(cur_doc) 98 99 @staticmethod 100 def reject_expr_doc(doc): 101 if doc and doc.symbol: 102 raise QAPISemError( 103 doc.info, 104 "documentation for '%s' is not followed by the definition" 105 % doc.symbol) 106 107 def _include(self, include, info, incl_fname, previously_included): 108 incl_abs_fname = os.path.abspath(incl_fname) 109 # catch inclusion cycle 110 inf = info 111 while inf: 112 if incl_abs_fname == os.path.abspath(inf.fname): 113 raise QAPISemError(info, "inclusion loop for %s" % include) 114 inf = inf.parent 115 116 # skip multiple include of the same file 117 if incl_abs_fname in previously_included: 118 return None 119 120 return QAPISchemaParser(incl_fname, previously_included, info) 121 122 def _pragma(self, name, value, info): 123 if name == 'doc-required': 124 if not isinstance(value, bool): 125 raise QAPISemError(info, 126 "pragma 'doc-required' must be boolean") 127 info.pragma.doc_required = value 128 elif name == 'returns-whitelist': 129 if (not isinstance(value, list) 130 or any([not isinstance(elt, str) for elt in value])): 131 raise QAPISemError( 132 info, 133 "pragma returns-whitelist must be a list of strings") 134 info.pragma.returns_whitelist = value 135 elif name == 'name-case-whitelist': 136 if (not isinstance(value, list) 137 or any([not isinstance(elt, str) for elt in value])): 138 raise QAPISemError( 139 info, 140 "pragma name-case-whitelist must be a list of strings") 141 info.pragma.name_case_whitelist = value 142 else: 143 raise QAPISemError(info, "unknown pragma '%s'" % name) 144 145 def accept(self, skip_comment=True): 146 while True: 147 self.tok = self.src[self.cursor] 148 self.pos = self.cursor 149 self.cursor += 1 150 self.val = None 151 152 if self.tok == '#': 153 if self.src[self.cursor] == '#': 154 # Start of doc comment 155 skip_comment = False 156 self.cursor = self.src.find('\n', self.cursor) 157 if not skip_comment: 158 self.val = self.src[self.pos:self.cursor] 159 return 160 elif self.tok in '{}:,[]': 161 return 162 elif self.tok == "'": 163 # Note: we accept only printable ASCII 164 string = '' 165 esc = False 166 while True: 167 ch = self.src[self.cursor] 168 self.cursor += 1 169 if ch == '\n': 170 raise QAPIParseError(self, "missing terminating \"'\"") 171 if esc: 172 # Note: we recognize only \\ because we have 173 # no use for funny characters in strings 174 if ch != '\\': 175 raise QAPIParseError(self, 176 "unknown escape \\%s" % ch) 177 esc = False 178 elif ch == '\\': 179 esc = True 180 continue 181 elif ch == "'": 182 self.val = string 183 return 184 if ord(ch) < 32 or ord(ch) >= 127: 185 raise QAPIParseError( 186 self, "funny character in string") 187 string += ch 188 elif self.src.startswith('true', self.pos): 189 self.val = True 190 self.cursor += 3 191 return 192 elif self.src.startswith('false', self.pos): 193 self.val = False 194 self.cursor += 4 195 return 196 elif self.tok == '\n': 197 if self.cursor == len(self.src): 198 self.tok = None 199 return 200 self.info = self.info.next_line() 201 self.line_pos = self.cursor 202 elif not self.tok.isspace(): 203 # Show up to next structural, whitespace or quote 204 # character 205 match = re.match('[^[\\]{}:,\\s\'"]+', 206 self.src[self.cursor-1:]) 207 raise QAPIParseError(self, "stray '%s'" % match.group(0)) 208 209 def get_members(self): 210 expr = OrderedDict() 211 if self.tok == '}': 212 self.accept() 213 return expr 214 if self.tok != "'": 215 raise QAPIParseError(self, "expected string or '}'") 216 while True: 217 key = self.val 218 self.accept() 219 if self.tok != ':': 220 raise QAPIParseError(self, "expected ':'") 221 self.accept() 222 if key in expr: 223 raise QAPIParseError(self, "duplicate key '%s'" % key) 224 expr[key] = self.get_expr(True) 225 if self.tok == '}': 226 self.accept() 227 return expr 228 if self.tok != ',': 229 raise QAPIParseError(self, "expected ',' or '}'") 230 self.accept() 231 if self.tok != "'": 232 raise QAPIParseError(self, "expected string") 233 234 def get_values(self): 235 expr = [] 236 if self.tok == ']': 237 self.accept() 238 return expr 239 if self.tok not in "{['tfn": 240 raise QAPIParseError( 241 self, "expected '{', '[', ']', string, boolean or 'null'") 242 while True: 243 expr.append(self.get_expr(True)) 244 if self.tok == ']': 245 self.accept() 246 return expr 247 if self.tok != ',': 248 raise QAPIParseError(self, "expected ',' or ']'") 249 self.accept() 250 251 def get_expr(self, nested): 252 if self.tok != '{' and not nested: 253 raise QAPIParseError(self, "expected '{'") 254 if self.tok == '{': 255 self.accept() 256 expr = self.get_members() 257 elif self.tok == '[': 258 self.accept() 259 expr = self.get_values() 260 elif self.tok in "'tfn": 261 expr = self.val 262 self.accept() 263 else: 264 raise QAPIParseError( 265 self, "expected '{', '[', string, boolean or 'null'") 266 return expr 267 268 def get_doc(self, info): 269 if self.val != '##': 270 raise QAPIParseError( 271 self, "junk after '##' at start of documentation comment") 272 273 docs = [] 274 cur_doc = QAPIDoc(self, info) 275 self.accept(False) 276 while self.tok == '#': 277 if self.val.startswith('##'): 278 # End of doc comment 279 if self.val != '##': 280 raise QAPIParseError( 281 self, 282 "junk after '##' at end of documentation comment") 283 cur_doc.end_comment() 284 docs.append(cur_doc) 285 self.accept() 286 return docs 287 if self.val.startswith('# ='): 288 if cur_doc.symbol: 289 raise QAPIParseError( 290 self, 291 "unexpected '=' markup in definition documentation") 292 if cur_doc.body.text: 293 cur_doc.end_comment() 294 docs.append(cur_doc) 295 cur_doc = QAPIDoc(self, info) 296 cur_doc.append(self.val) 297 self.accept(False) 298 299 raise QAPIParseError(self, "documentation comment must end with '##'") 300 301 302class QAPIDoc: 303 """ 304 A documentation comment block, either definition or free-form 305 306 Definition documentation blocks consist of 307 308 * a body section: one line naming the definition, followed by an 309 overview (any number of lines) 310 311 * argument sections: a description of each argument (for commands 312 and events) or member (for structs, unions and alternates) 313 314 * features sections: a description of each feature flag 315 316 * additional (non-argument) sections, possibly tagged 317 318 Free-form documentation blocks consist only of a body section. 319 """ 320 321 class Section: 322 def __init__(self, name=None): 323 # optional section name (argument/member or section name) 324 self.name = name 325 self.text = '' 326 327 def append(self, line): 328 self.text += line.rstrip() + '\n' 329 330 class ArgSection(Section): 331 def __init__(self, name): 332 super().__init__(name) 333 self.member = None 334 335 def connect(self, member): 336 self.member = member 337 338 def __init__(self, parser, info): 339 # self._parser is used to report errors with QAPIParseError. The 340 # resulting error position depends on the state of the parser. 341 # It happens to be the beginning of the comment. More or less 342 # servicable, but action at a distance. 343 self._parser = parser 344 self.info = info 345 self.symbol = None 346 self.body = QAPIDoc.Section() 347 # dict mapping parameter name to ArgSection 348 self.args = OrderedDict() 349 self.features = OrderedDict() 350 # a list of Section 351 self.sections = [] 352 # the current section 353 self._section = self.body 354 self._append_line = self._append_body_line 355 356 def has_section(self, name): 357 """Return True if we have a section with this name.""" 358 for i in self.sections: 359 if i.name == name: 360 return True 361 return False 362 363 def append(self, line): 364 """ 365 Parse a comment line and add it to the documentation. 366 367 The way that the line is dealt with depends on which part of 368 the documentation we're parsing right now: 369 * The body section: ._append_line is ._append_body_line 370 * An argument section: ._append_line is ._append_args_line 371 * A features section: ._append_line is ._append_features_line 372 * An additional section: ._append_line is ._append_various_line 373 """ 374 line = line[1:] 375 if not line: 376 self._append_freeform(line) 377 return 378 379 if line[0] != ' ': 380 raise QAPIParseError(self._parser, "missing space after #") 381 line = line[1:] 382 self._append_line(line) 383 384 def end_comment(self): 385 self._end_section() 386 387 @staticmethod 388 def _is_section_tag(name): 389 return name in ('Returns:', 'Since:', 390 # those are often singular or plural 391 'Note:', 'Notes:', 392 'Example:', 'Examples:', 393 'TODO:') 394 395 def _append_body_line(self, line): 396 """ 397 Process a line of documentation text in the body section. 398 399 If this a symbol line and it is the section's first line, this 400 is a definition documentation block for that symbol. 401 402 If it's a definition documentation block, another symbol line 403 begins the argument section for the argument named by it, and 404 a section tag begins an additional section. Start that 405 section and append the line to it. 406 407 Else, append the line to the current section. 408 """ 409 name = line.split(' ', 1)[0] 410 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't 411 # recognized, and get silently treated as ordinary text 412 if not self.symbol and not self.body.text and line.startswith('@'): 413 if not line.endswith(':'): 414 raise QAPIParseError(self._parser, "line should end with ':'") 415 self.symbol = line[1:-1] 416 # FIXME invalid names other than the empty string aren't flagged 417 if not self.symbol: 418 raise QAPIParseError(self._parser, "invalid name") 419 elif self.symbol: 420 # This is a definition documentation block 421 if name.startswith('@') and name.endswith(':'): 422 self._append_line = self._append_args_line 423 self._append_args_line(line) 424 elif line == 'Features:': 425 self._append_line = self._append_features_line 426 elif self._is_section_tag(name): 427 self._append_line = self._append_various_line 428 self._append_various_line(line) 429 else: 430 self._append_freeform(line.strip()) 431 else: 432 # This is a free-form documentation block 433 self._append_freeform(line.strip()) 434 435 def _append_args_line(self, line): 436 """ 437 Process a line of documentation text in an argument section. 438 439 A symbol line begins the next argument section, a section tag 440 section or a non-indented line after a blank line begins an 441 additional section. Start that section and append the line to 442 it. 443 444 Else, append the line to the current section. 445 446 """ 447 name = line.split(' ', 1)[0] 448 449 if name.startswith('@') and name.endswith(':'): 450 line = line[len(name)+1:] 451 self._start_args_section(name[1:-1]) 452 elif self._is_section_tag(name): 453 self._append_line = self._append_various_line 454 self._append_various_line(line) 455 return 456 elif (self._section.text.endswith('\n\n') 457 and line and not line[0].isspace()): 458 if line == 'Features:': 459 self._append_line = self._append_features_line 460 else: 461 self._start_section() 462 self._append_line = self._append_various_line 463 self._append_various_line(line) 464 return 465 466 self._append_freeform(line.strip()) 467 468 def _append_features_line(self, line): 469 name = line.split(' ', 1)[0] 470 471 if name.startswith('@') and name.endswith(':'): 472 line = line[len(name)+1:] 473 self._start_features_section(name[1:-1]) 474 elif self._is_section_tag(name): 475 self._append_line = self._append_various_line 476 self._append_various_line(line) 477 return 478 elif (self._section.text.endswith('\n\n') 479 and line and not line[0].isspace()): 480 self._start_section() 481 self._append_line = self._append_various_line 482 self._append_various_line(line) 483 return 484 485 self._append_freeform(line.strip()) 486 487 def _append_various_line(self, line): 488 """ 489 Process a line of documentation text in an additional section. 490 491 A symbol line is an error. 492 493 A section tag begins an additional section. Start that 494 section and append the line to it. 495 496 Else, append the line to the current section. 497 """ 498 name = line.split(' ', 1)[0] 499 500 if name.startswith('@') and name.endswith(':'): 501 raise QAPIParseError(self._parser, 502 "'%s' can't follow '%s' section" 503 % (name, self.sections[0].name)) 504 if self._is_section_tag(name): 505 line = line[len(name)+1:] 506 self._start_section(name[:-1]) 507 508 if (not self._section.name or 509 not self._section.name.startswith('Example')): 510 line = line.strip() 511 512 self._append_freeform(line) 513 514 def _start_symbol_section(self, symbols_dict, name): 515 # FIXME invalid names other than the empty string aren't flagged 516 if not name: 517 raise QAPIParseError(self._parser, "invalid parameter name") 518 if name in symbols_dict: 519 raise QAPIParseError(self._parser, 520 "'%s' parameter name duplicated" % name) 521 assert not self.sections 522 self._end_section() 523 self._section = QAPIDoc.ArgSection(name) 524 symbols_dict[name] = self._section 525 526 def _start_args_section(self, name): 527 self._start_symbol_section(self.args, name) 528 529 def _start_features_section(self, name): 530 self._start_symbol_section(self.features, name) 531 532 def _start_section(self, name=None): 533 if name in ('Returns', 'Since') and self.has_section(name): 534 raise QAPIParseError(self._parser, 535 "duplicated '%s' section" % name) 536 self._end_section() 537 self._section = QAPIDoc.Section(name) 538 self.sections.append(self._section) 539 540 def _end_section(self): 541 if self._section: 542 text = self._section.text = self._section.text.strip() 543 if self._section.name and (not text or text.isspace()): 544 raise QAPIParseError( 545 self._parser, 546 "empty doc section '%s'" % self._section.name) 547 self._section = None 548 549 def _append_freeform(self, line): 550 match = re.match(r'(@\S+:)', line) 551 if match: 552 raise QAPIParseError(self._parser, 553 "'%s' not allowed in free-form documentation" 554 % match.group(1)) 555 self._section.append(line) 556 557 def connect_member(self, member): 558 if member.name not in self.args: 559 # Undocumented TODO outlaw 560 self.args[member.name] = QAPIDoc.ArgSection(member.name) 561 self.args[member.name].connect(member) 562 563 def connect_feature(self, feature): 564 if feature.name not in self.features: 565 raise QAPISemError(feature.info, 566 "feature '%s' lacks documentation" 567 % feature.name) 568 self.features[feature.name].connect(feature) 569 570 def check_expr(self, expr): 571 if self.has_section('Returns') and 'command' not in expr: 572 raise QAPISemError(self.info, 573 "'Returns:' is only valid for commands") 574 575 def check(self): 576 577 def check_args_section(args, info, what): 578 bogus = [name for name, section in args.items() 579 if not section.member] 580 if bogus: 581 raise QAPISemError( 582 self.info, 583 "documented member%s '%s' %s not exist" 584 % ("s" if len(bogus) > 1 else "", 585 "', '".join(bogus), 586 "do" if len(bogus) > 1 else "does")) 587 588 check_args_section(self.args, self.info, 'members') 589 check_args_section(self.features, self.info, 'features') 590