1# -*- coding: utf-8 -*- 2# 3# QAPI schema parser 4# 5# Copyright IBM, Corp. 2011 6# Copyright (c) 2013-2019 Red Hat Inc. 7# 8# Authors: 9# Anthony Liguori <aliguori@us.ibm.com> 10# Markus Armbruster <armbru@redhat.com> 11# Marc-André Lureau <marcandre.lureau@redhat.com> 12# Kevin Wolf <kwolf@redhat.com> 13# 14# This work is licensed under the terms of the GNU GPL, version 2. 15# See the COPYING file in the top-level directory. 16 17import os 18import re 19from collections import OrderedDict 20 21from qapi.error import QAPIParseError, QAPISemError 22from qapi.source import QAPISourceInfo 23 24 25class QAPISchemaParser: 26 27 def __init__(self, fname, previously_included=None, incl_info=None): 28 previously_included = previously_included or set() 29 previously_included.add(os.path.abspath(fname)) 30 31 try: 32 fp = open(fname, 'r', encoding='utf-8') 33 self.src = fp.read() 34 except IOError as e: 35 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), 36 "can't read %s file '%s': %s" 37 % ("include" if incl_info else "schema", 38 fname, 39 e.strerror)) 40 41 if self.src == '' or self.src[-1] != '\n': 42 self.src += '\n' 43 self.cursor = 0 44 self.info = QAPISourceInfo(fname, 1, incl_info) 45 self.line_pos = 0 46 self.exprs = [] 47 self.docs = [] 48 self.accept() 49 cur_doc = None 50 51 while self.tok is not None: 52 info = self.info 53 if self.tok == '#': 54 self.reject_expr_doc(cur_doc) 55 cur_doc = self.get_doc(info) 56 self.docs.append(cur_doc) 57 continue 58 59 expr = self.get_expr(False) 60 if 'include' in expr: 61 self.reject_expr_doc(cur_doc) 62 if len(expr) != 1: 63 raise QAPISemError(info, "invalid 'include' directive") 64 include = expr['include'] 65 if not isinstance(include, str): 66 raise QAPISemError(info, 67 "value of 'include' must be a string") 68 incl_fname = os.path.join(os.path.dirname(fname), 69 include) 70 self.exprs.append({'expr': {'include': incl_fname}, 71 'info': info}) 72 exprs_include = self._include(include, info, incl_fname, 73 previously_included) 74 if exprs_include: 75 self.exprs.extend(exprs_include.exprs) 76 self.docs.extend(exprs_include.docs) 77 elif "pragma" in expr: 78 self.reject_expr_doc(cur_doc) 79 if len(expr) != 1: 80 raise QAPISemError(info, "invalid 'pragma' directive") 81 pragma = expr['pragma'] 82 if not isinstance(pragma, dict): 83 raise QAPISemError( 84 info, "value of 'pragma' must be an object") 85 for name, value in pragma.items(): 86 self._pragma(name, value, info) 87 else: 88 expr_elem = {'expr': expr, 89 'info': info} 90 if cur_doc: 91 if not cur_doc.symbol: 92 raise QAPISemError( 93 cur_doc.info, "definition documentation required") 94 expr_elem['doc'] = cur_doc 95 self.exprs.append(expr_elem) 96 cur_doc = None 97 self.reject_expr_doc(cur_doc) 98 99 @staticmethod 100 def reject_expr_doc(doc): 101 if doc and doc.symbol: 102 raise QAPISemError( 103 doc.info, 104 "documentation for '%s' is not followed by the definition" 105 % doc.symbol) 106 107 def _include(self, include, info, incl_fname, previously_included): 108 incl_abs_fname = os.path.abspath(incl_fname) 109 # catch inclusion cycle 110 inf = info 111 while inf: 112 if incl_abs_fname == os.path.abspath(inf.fname): 113 raise QAPISemError(info, "inclusion loop for %s" % include) 114 inf = inf.parent 115 116 # skip multiple include of the same file 117 if incl_abs_fname in previously_included: 118 return None 119 120 return QAPISchemaParser(incl_fname, previously_included, info) 121 122 def _pragma(self, name, value, info): 123 if name == 'doc-required': 124 if not isinstance(value, bool): 125 raise QAPISemError(info, 126 "pragma 'doc-required' must be boolean") 127 info.pragma.doc_required = value 128 elif name == 'returns-whitelist': 129 if (not isinstance(value, list) 130 or any([not isinstance(elt, str) for elt in value])): 131 raise QAPISemError( 132 info, 133 "pragma returns-whitelist must be a list of strings") 134 info.pragma.returns_whitelist = value 135 elif name == 'name-case-whitelist': 136 if (not isinstance(value, list) 137 or any([not isinstance(elt, str) for elt in value])): 138 raise QAPISemError( 139 info, 140 "pragma name-case-whitelist must be a list of strings") 141 info.pragma.name_case_whitelist = value 142 else: 143 raise QAPISemError(info, "unknown pragma '%s'" % name) 144 145 def accept(self, skip_comment=True): 146 while True: 147 self.tok = self.src[self.cursor] 148 self.pos = self.cursor 149 self.cursor += 1 150 self.val = None 151 152 if self.tok == '#': 153 if self.src[self.cursor] == '#': 154 # Start of doc comment 155 skip_comment = False 156 self.cursor = self.src.find('\n', self.cursor) 157 if not skip_comment: 158 self.val = self.src[self.pos:self.cursor] 159 return 160 elif self.tok in '{}:,[]': 161 return 162 elif self.tok == "'": 163 # Note: we accept only printable ASCII 164 string = '' 165 esc = False 166 while True: 167 ch = self.src[self.cursor] 168 self.cursor += 1 169 if ch == '\n': 170 raise QAPIParseError(self, "missing terminating \"'\"") 171 if esc: 172 # Note: we recognize only \\ because we have 173 # no use for funny characters in strings 174 if ch != '\\': 175 raise QAPIParseError(self, 176 "unknown escape \\%s" % ch) 177 esc = False 178 elif ch == '\\': 179 esc = True 180 continue 181 elif ch == "'": 182 self.val = string 183 return 184 if ord(ch) < 32 or ord(ch) >= 127: 185 raise QAPIParseError( 186 self, "funny character in string") 187 string += ch 188 elif self.src.startswith('true', self.pos): 189 self.val = True 190 self.cursor += 3 191 return 192 elif self.src.startswith('false', self.pos): 193 self.val = False 194 self.cursor += 4 195 return 196 elif self.tok == '\n': 197 if self.cursor == len(self.src): 198 self.tok = None 199 return 200 self.info = self.info.next_line() 201 self.line_pos = self.cursor 202 elif not self.tok.isspace(): 203 # Show up to next structural, whitespace or quote 204 # character 205 match = re.match('[^[\\]{}:,\\s\'"]+', 206 self.src[self.cursor-1:]) 207 raise QAPIParseError(self, "stray '%s'" % match.group(0)) 208 209 def get_members(self): 210 expr = OrderedDict() 211 if self.tok == '}': 212 self.accept() 213 return expr 214 if self.tok != "'": 215 raise QAPIParseError(self, "expected string or '}'") 216 while True: 217 key = self.val 218 self.accept() 219 if self.tok != ':': 220 raise QAPIParseError(self, "expected ':'") 221 self.accept() 222 if key in expr: 223 raise QAPIParseError(self, "duplicate key '%s'" % key) 224 expr[key] = self.get_expr(True) 225 if self.tok == '}': 226 self.accept() 227 return expr 228 if self.tok != ',': 229 raise QAPIParseError(self, "expected ',' or '}'") 230 self.accept() 231 if self.tok != "'": 232 raise QAPIParseError(self, "expected string") 233 234 def get_values(self): 235 expr = [] 236 if self.tok == ']': 237 self.accept() 238 return expr 239 if self.tok not in "{['tfn": 240 raise QAPIParseError( 241 self, "expected '{', '[', ']', string, boolean or 'null'") 242 while True: 243 expr.append(self.get_expr(True)) 244 if self.tok == ']': 245 self.accept() 246 return expr 247 if self.tok != ',': 248 raise QAPIParseError(self, "expected ',' or ']'") 249 self.accept() 250 251 def get_expr(self, nested): 252 if self.tok != '{' and not nested: 253 raise QAPIParseError(self, "expected '{'") 254 if self.tok == '{': 255 self.accept() 256 expr = self.get_members() 257 elif self.tok == '[': 258 self.accept() 259 expr = self.get_values() 260 elif self.tok in "'tfn": 261 expr = self.val 262 self.accept() 263 else: 264 raise QAPIParseError( 265 self, "expected '{', '[', string, boolean or 'null'") 266 return expr 267 268 def get_doc(self, info): 269 if self.val != '##': 270 raise QAPIParseError( 271 self, "junk after '##' at start of documentation comment") 272 273 doc = QAPIDoc(self, info) 274 self.accept(False) 275 while self.tok == '#': 276 if self.val.startswith('##'): 277 # End of doc comment 278 if self.val != '##': 279 raise QAPIParseError( 280 self, 281 "junk after '##' at end of documentation comment") 282 doc.end_comment() 283 self.accept() 284 return doc 285 if self.val.startswith('# ='): 286 if doc.symbol: 287 raise QAPIParseError( 288 self, 289 "unexpected '=' markup in definition documentation") 290 doc.append(self.val) 291 self.accept(False) 292 293 raise QAPIParseError(self, "documentation comment must end with '##'") 294 295 296class QAPIDoc: 297 """ 298 A documentation comment block, either definition or free-form 299 300 Definition documentation blocks consist of 301 302 * a body section: one line naming the definition, followed by an 303 overview (any number of lines) 304 305 * argument sections: a description of each argument (for commands 306 and events) or member (for structs, unions and alternates) 307 308 * features sections: a description of each feature flag 309 310 * additional (non-argument) sections, possibly tagged 311 312 Free-form documentation blocks consist only of a body section. 313 """ 314 315 class Section: 316 def __init__(self, name=None): 317 # optional section name (argument/member or section name) 318 self.name = name 319 # the list of lines for this section 320 self.text = '' 321 322 def append(self, line): 323 self.text += line.rstrip() + '\n' 324 325 class ArgSection(Section): 326 def __init__(self, name): 327 super().__init__(name) 328 self.member = None 329 330 def connect(self, member): 331 self.member = member 332 333 def __init__(self, parser, info): 334 # self._parser is used to report errors with QAPIParseError. The 335 # resulting error position depends on the state of the parser. 336 # It happens to be the beginning of the comment. More or less 337 # servicable, but action at a distance. 338 self._parser = parser 339 self.info = info 340 self.symbol = None 341 self.body = QAPIDoc.Section() 342 # dict mapping parameter name to ArgSection 343 self.args = OrderedDict() 344 self.features = OrderedDict() 345 # a list of Section 346 self.sections = [] 347 # the current section 348 self._section = self.body 349 self._append_line = self._append_body_line 350 351 def has_section(self, name): 352 """Return True if we have a section with this name.""" 353 for i in self.sections: 354 if i.name == name: 355 return True 356 return False 357 358 def append(self, line): 359 """ 360 Parse a comment line and add it to the documentation. 361 362 The way that the line is dealt with depends on which part of 363 the documentation we're parsing right now: 364 * The body section: ._append_line is ._append_body_line 365 * An argument section: ._append_line is ._append_args_line 366 * A features section: ._append_line is ._append_features_line 367 * An additional section: ._append_line is ._append_various_line 368 """ 369 line = line[1:] 370 if not line: 371 self._append_freeform(line) 372 return 373 374 if line[0] != ' ': 375 raise QAPIParseError(self._parser, "missing space after #") 376 line = line[1:] 377 self._append_line(line) 378 379 def end_comment(self): 380 self._end_section() 381 382 @staticmethod 383 def _is_section_tag(name): 384 return name in ('Returns:', 'Since:', 385 # those are often singular or plural 386 'Note:', 'Notes:', 387 'Example:', 'Examples:', 388 'TODO:') 389 390 def _append_body_line(self, line): 391 """ 392 Process a line of documentation text in the body section. 393 394 If this a symbol line and it is the section's first line, this 395 is a definition documentation block for that symbol. 396 397 If it's a definition documentation block, another symbol line 398 begins the argument section for the argument named by it, and 399 a section tag begins an additional section. Start that 400 section and append the line to it. 401 402 Else, append the line to the current section. 403 """ 404 name = line.split(' ', 1)[0] 405 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't 406 # recognized, and get silently treated as ordinary text 407 if not self.symbol and not self.body.text and line.startswith('@'): 408 if not line.endswith(':'): 409 raise QAPIParseError(self._parser, "line should end with ':'") 410 self.symbol = line[1:-1] 411 # FIXME invalid names other than the empty string aren't flagged 412 if not self.symbol: 413 raise QAPIParseError(self._parser, "invalid name") 414 elif self.symbol: 415 # This is a definition documentation block 416 if name.startswith('@') and name.endswith(':'): 417 self._append_line = self._append_args_line 418 self._append_args_line(line) 419 elif line == 'Features:': 420 self._append_line = self._append_features_line 421 elif self._is_section_tag(name): 422 self._append_line = self._append_various_line 423 self._append_various_line(line) 424 else: 425 self._append_freeform(line.strip()) 426 else: 427 # This is a free-form documentation block 428 self._append_freeform(line.strip()) 429 430 def _append_args_line(self, line): 431 """ 432 Process a line of documentation text in an argument section. 433 434 A symbol line begins the next argument section, a section tag 435 section or a non-indented line after a blank line begins an 436 additional section. Start that section and append the line to 437 it. 438 439 Else, append the line to the current section. 440 441 """ 442 name = line.split(' ', 1)[0] 443 444 if name.startswith('@') and name.endswith(':'): 445 line = line[len(name)+1:] 446 self._start_args_section(name[1:-1]) 447 elif self._is_section_tag(name): 448 self._append_line = self._append_various_line 449 self._append_various_line(line) 450 return 451 elif (self._section.text.endswith('\n\n') 452 and line and not line[0].isspace()): 453 if line == 'Features:': 454 self._append_line = self._append_features_line 455 else: 456 self._start_section() 457 self._append_line = self._append_various_line 458 self._append_various_line(line) 459 return 460 461 self._append_freeform(line.strip()) 462 463 def _append_features_line(self, line): 464 name = line.split(' ', 1)[0] 465 466 if name.startswith('@') and name.endswith(':'): 467 line = line[len(name)+1:] 468 self._start_features_section(name[1:-1]) 469 elif self._is_section_tag(name): 470 self._append_line = self._append_various_line 471 self._append_various_line(line) 472 return 473 elif (self._section.text.endswith('\n\n') 474 and line and not line[0].isspace()): 475 self._start_section() 476 self._append_line = self._append_various_line 477 self._append_various_line(line) 478 return 479 480 self._append_freeform(line.strip()) 481 482 def _append_various_line(self, line): 483 """ 484 Process a line of documentation text in an additional section. 485 486 A symbol line is an error. 487 488 A section tag begins an additional section. Start that 489 section and append the line to it. 490 491 Else, append the line to the current section. 492 """ 493 name = line.split(' ', 1)[0] 494 495 if name.startswith('@') and name.endswith(':'): 496 raise QAPIParseError(self._parser, 497 "'%s' can't follow '%s' section" 498 % (name, self.sections[0].name)) 499 if self._is_section_tag(name): 500 line = line[len(name)+1:] 501 self._start_section(name[:-1]) 502 503 if (not self._section.name or 504 not self._section.name.startswith('Example')): 505 line = line.strip() 506 507 self._append_freeform(line) 508 509 def _start_symbol_section(self, symbols_dict, name): 510 # FIXME invalid names other than the empty string aren't flagged 511 if not name: 512 raise QAPIParseError(self._parser, "invalid parameter name") 513 if name in symbols_dict: 514 raise QAPIParseError(self._parser, 515 "'%s' parameter name duplicated" % name) 516 assert not self.sections 517 self._end_section() 518 self._section = QAPIDoc.ArgSection(name) 519 symbols_dict[name] = self._section 520 521 def _start_args_section(self, name): 522 self._start_symbol_section(self.args, name) 523 524 def _start_features_section(self, name): 525 self._start_symbol_section(self.features, name) 526 527 def _start_section(self, name=None): 528 if name in ('Returns', 'Since') and self.has_section(name): 529 raise QAPIParseError(self._parser, 530 "duplicated '%s' section" % name) 531 self._end_section() 532 self._section = QAPIDoc.Section(name) 533 self.sections.append(self._section) 534 535 def _end_section(self): 536 if self._section: 537 text = self._section.text = self._section.text.strip() 538 if self._section.name and (not text or text.isspace()): 539 raise QAPIParseError( 540 self._parser, 541 "empty doc section '%s'" % self._section.name) 542 self._section = None 543 544 def _append_freeform(self, line): 545 match = re.match(r'(@\S+:)', line) 546 if match: 547 raise QAPIParseError(self._parser, 548 "'%s' not allowed in free-form documentation" 549 % match.group(1)) 550 self._section.append(line) 551 552 def connect_member(self, member): 553 if member.name not in self.args: 554 # Undocumented TODO outlaw 555 self.args[member.name] = QAPIDoc.ArgSection(member.name) 556 self.args[member.name].connect(member) 557 558 def connect_feature(self, feature): 559 if feature.name not in self.features: 560 raise QAPISemError(feature.info, 561 "feature '%s' lacks documentation" 562 % feature.name) 563 self.features[feature.name].connect(feature) 564 565 def check_expr(self, expr): 566 if self.has_section('Returns') and 'command' not in expr: 567 raise QAPISemError(self.info, 568 "'Returns:' is only valid for commands") 569 570 def check(self): 571 572 def check_args_section(args, info, what): 573 bogus = [name for name, section in args.items() 574 if not section.member] 575 if bogus: 576 raise QAPISemError( 577 self.info, 578 "documented member%s '%s' %s not exist" 579 % ("s" if len(bogus) > 1 else "", 580 "', '".join(bogus), 581 "do" if len(bogus) > 1 else "does")) 582 583 check_args_section(self.args, self.info, 'members') 584 check_args_section(self.features, self.info, 'features') 585