1# -*- coding: utf-8 -*- 2# 3# QAPI schema parser 4# 5# Copyright IBM, Corp. 2011 6# Copyright (c) 2013-2019 Red Hat Inc. 7# 8# Authors: 9# Anthony Liguori <aliguori@us.ibm.com> 10# Markus Armbruster <armbru@redhat.com> 11# Marc-André Lureau <marcandre.lureau@redhat.com> 12# Kevin Wolf <kwolf@redhat.com> 13# 14# This work is licensed under the terms of the GNU GPL, version 2. 15# See the COPYING file in the top-level directory. 16 17import os 18import re 19from collections import OrderedDict 20 21from qapi.error import QAPIParseError, QAPISemError 22from qapi.source import QAPISourceInfo 23 24 25class QAPISchemaParser: 26 27 def __init__(self, fname, previously_included=None, incl_info=None): 28 previously_included = previously_included or set() 29 previously_included.add(os.path.abspath(fname)) 30 31 try: 32 fp = open(fname, 'r', encoding='utf-8') 33 self.src = fp.read() 34 except IOError as e: 35 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), 36 "can't read %s file '%s': %s" 37 % ("include" if incl_info else "schema", 38 fname, 39 e.strerror)) 40 41 if self.src == '' or self.src[-1] != '\n': 42 self.src += '\n' 43 self.cursor = 0 44 self.info = QAPISourceInfo(fname, 1, incl_info) 45 self.line_pos = 0 46 self.exprs = [] 47 self.docs = [] 48 self.accept() 49 cur_doc = None 50 51 while self.tok is not None: 52 info = self.info 53 if self.tok == '#': 54 self.reject_expr_doc(cur_doc) 55 cur_doc = self.get_doc(info) 56 self.docs.append(cur_doc) 57 continue 58 59 expr = self.get_expr(False) 60 if 'include' in expr: 61 self.reject_expr_doc(cur_doc) 62 if len(expr) != 1: 63 raise QAPISemError(info, "invalid 'include' directive") 64 include = expr['include'] 65 if not isinstance(include, str): 66 raise QAPISemError(info, 67 "value of 'include' must be a string") 68 incl_fname = os.path.join(os.path.dirname(fname), 69 include) 70 self.exprs.append({'expr': {'include': incl_fname}, 71 'info': info}) 72 exprs_include = self._include(include, info, incl_fname, 73 previously_included) 74 if exprs_include: 75 self.exprs.extend(exprs_include.exprs) 76 self.docs.extend(exprs_include.docs) 77 elif "pragma" in expr: 78 self.reject_expr_doc(cur_doc) 79 if len(expr) != 1: 80 raise QAPISemError(info, "invalid 'pragma' directive") 81 pragma = expr['pragma'] 82 if not isinstance(pragma, dict): 83 raise QAPISemError( 84 info, "value of 'pragma' must be an object") 85 for name, value in pragma.items(): 86 self._pragma(name, value, info) 87 else: 88 expr_elem = {'expr': expr, 89 'info': info} 90 if cur_doc: 91 if not cur_doc.symbol: 92 raise QAPISemError( 93 cur_doc.info, "definition documentation required") 94 expr_elem['doc'] = cur_doc 95 self.exprs.append(expr_elem) 96 cur_doc = None 97 self.reject_expr_doc(cur_doc) 98 99 @staticmethod 100 def reject_expr_doc(doc): 101 if doc and doc.symbol: 102 raise QAPISemError( 103 doc.info, 104 "documentation for '%s' is not followed by the definition" 105 % doc.symbol) 106 107 def _include(self, include, info, incl_fname, previously_included): 108 incl_abs_fname = os.path.abspath(incl_fname) 109 # catch inclusion cycle 110 inf = info 111 while inf: 112 if incl_abs_fname == os.path.abspath(inf.fname): 113 raise QAPISemError(info, "inclusion loop for %s" % include) 114 inf = inf.parent 115 116 # skip multiple include of the same file 117 if incl_abs_fname in previously_included: 118 return None 119 120 return QAPISchemaParser(incl_fname, previously_included, info) 121 122 def _pragma(self, name, value, info): 123 if name == 'doc-required': 124 if not isinstance(value, bool): 125 raise QAPISemError(info, 126 "pragma 'doc-required' must be boolean") 127 info.pragma.doc_required = value 128 elif name == 'returns-whitelist': 129 if (not isinstance(value, list) 130 or any([not isinstance(elt, str) for elt in value])): 131 raise QAPISemError( 132 info, 133 "pragma returns-whitelist must be a list of strings") 134 info.pragma.returns_whitelist = value 135 elif name == 'name-case-whitelist': 136 if (not isinstance(value, list) 137 or any([not isinstance(elt, str) for elt in value])): 138 raise QAPISemError( 139 info, 140 "pragma name-case-whitelist must be a list of strings") 141 info.pragma.name_case_whitelist = value 142 else: 143 raise QAPISemError(info, "unknown pragma '%s'" % name) 144 145 def accept(self, skip_comment=True): 146 while True: 147 self.tok = self.src[self.cursor] 148 self.pos = self.cursor 149 self.cursor += 1 150 self.val = None 151 152 if self.tok == '#': 153 if self.src[self.cursor] == '#': 154 # Start of doc comment 155 skip_comment = False 156 self.cursor = self.src.find('\n', self.cursor) 157 if not skip_comment: 158 self.val = self.src[self.pos:self.cursor] 159 return 160 elif self.tok in '{}:,[]': 161 return 162 elif self.tok == "'": 163 # Note: we accept only printable ASCII 164 string = '' 165 esc = False 166 while True: 167 ch = self.src[self.cursor] 168 self.cursor += 1 169 if ch == '\n': 170 raise QAPIParseError(self, "missing terminating \"'\"") 171 if esc: 172 # Note: we recognize only \\ because we have 173 # no use for funny characters in strings 174 if ch != '\\': 175 raise QAPIParseError(self, 176 "unknown escape \\%s" % ch) 177 esc = False 178 elif ch == '\\': 179 esc = True 180 continue 181 elif ch == "'": 182 self.val = string 183 return 184 if ord(ch) < 32 or ord(ch) >= 127: 185 raise QAPIParseError( 186 self, "funny character in string") 187 string += ch 188 elif self.src.startswith('true', self.pos): 189 self.val = True 190 self.cursor += 3 191 return 192 elif self.src.startswith('false', self.pos): 193 self.val = False 194 self.cursor += 4 195 return 196 elif self.tok == '\n': 197 if self.cursor == len(self.src): 198 self.tok = None 199 return 200 self.info = self.info.next_line() 201 self.line_pos = self.cursor 202 elif not self.tok.isspace(): 203 # Show up to next structural, whitespace or quote 204 # character 205 match = re.match('[^[\\]{}:,\\s\'"]+', 206 self.src[self.cursor-1:]) 207 raise QAPIParseError(self, "stray '%s'" % match.group(0)) 208 209 def get_members(self): 210 expr = OrderedDict() 211 if self.tok == '}': 212 self.accept() 213 return expr 214 if self.tok != "'": 215 raise QAPIParseError(self, "expected string or '}'") 216 while True: 217 key = self.val 218 self.accept() 219 if self.tok != ':': 220 raise QAPIParseError(self, "expected ':'") 221 self.accept() 222 if key in expr: 223 raise QAPIParseError(self, "duplicate key '%s'" % key) 224 expr[key] = self.get_expr(True) 225 if self.tok == '}': 226 self.accept() 227 return expr 228 if self.tok != ',': 229 raise QAPIParseError(self, "expected ',' or '}'") 230 self.accept() 231 if self.tok != "'": 232 raise QAPIParseError(self, "expected string") 233 234 def get_values(self): 235 expr = [] 236 if self.tok == ']': 237 self.accept() 238 return expr 239 if self.tok not in "{['tfn": 240 raise QAPIParseError( 241 self, "expected '{', '[', ']', string, boolean or 'null'") 242 while True: 243 expr.append(self.get_expr(True)) 244 if self.tok == ']': 245 self.accept() 246 return expr 247 if self.tok != ',': 248 raise QAPIParseError(self, "expected ',' or ']'") 249 self.accept() 250 251 def get_expr(self, nested): 252 if self.tok != '{' and not nested: 253 raise QAPIParseError(self, "expected '{'") 254 if self.tok == '{': 255 self.accept() 256 expr = self.get_members() 257 elif self.tok == '[': 258 self.accept() 259 expr = self.get_values() 260 elif self.tok in "'tfn": 261 expr = self.val 262 self.accept() 263 else: 264 raise QAPIParseError( 265 self, "expected '{', '[', string, boolean or 'null'") 266 return expr 267 268 def get_doc(self, info): 269 if self.val != '##': 270 raise QAPIParseError( 271 self, "junk after '##' at start of documentation comment") 272 273 doc = QAPIDoc(self, info) 274 self.accept(False) 275 while self.tok == '#': 276 if self.val.startswith('##'): 277 # End of doc comment 278 if self.val != '##': 279 raise QAPIParseError( 280 self, 281 "junk after '##' at end of documentation comment") 282 doc.end_comment() 283 self.accept() 284 return doc 285 doc.append(self.val) 286 self.accept(False) 287 288 raise QAPIParseError(self, "documentation comment must end with '##'") 289 290 291class QAPIDoc: 292 """ 293 A documentation comment block, either definition or free-form 294 295 Definition documentation blocks consist of 296 297 * a body section: one line naming the definition, followed by an 298 overview (any number of lines) 299 300 * argument sections: a description of each argument (for commands 301 and events) or member (for structs, unions and alternates) 302 303 * features sections: a description of each feature flag 304 305 * additional (non-argument) sections, possibly tagged 306 307 Free-form documentation blocks consist only of a body section. 308 """ 309 310 class Section: 311 def __init__(self, name=None): 312 # optional section name (argument/member or section name) 313 self.name = name 314 # the list of lines for this section 315 self.text = '' 316 317 def append(self, line): 318 self.text += line.rstrip() + '\n' 319 320 class ArgSection(Section): 321 def __init__(self, name): 322 super().__init__(name) 323 self.member = None 324 325 def connect(self, member): 326 self.member = member 327 328 def __init__(self, parser, info): 329 # self._parser is used to report errors with QAPIParseError. The 330 # resulting error position depends on the state of the parser. 331 # It happens to be the beginning of the comment. More or less 332 # servicable, but action at a distance. 333 self._parser = parser 334 self.info = info 335 self.symbol = None 336 self.body = QAPIDoc.Section() 337 # dict mapping parameter name to ArgSection 338 self.args = OrderedDict() 339 self.features = OrderedDict() 340 # a list of Section 341 self.sections = [] 342 # the current section 343 self._section = self.body 344 self._append_line = self._append_body_line 345 346 def has_section(self, name): 347 """Return True if we have a section with this name.""" 348 for i in self.sections: 349 if i.name == name: 350 return True 351 return False 352 353 def append(self, line): 354 """ 355 Parse a comment line and add it to the documentation. 356 357 The way that the line is dealt with depends on which part of 358 the documentation we're parsing right now: 359 * The body section: ._append_line is ._append_body_line 360 * An argument section: ._append_line is ._append_args_line 361 * A features section: ._append_line is ._append_features_line 362 * An additional section: ._append_line is ._append_various_line 363 """ 364 line = line[1:] 365 if not line: 366 self._append_freeform(line) 367 return 368 369 if line[0] != ' ': 370 raise QAPIParseError(self._parser, "missing space after #") 371 line = line[1:] 372 self._append_line(line) 373 374 def end_comment(self): 375 self._end_section() 376 377 @staticmethod 378 def _is_section_tag(name): 379 return name in ('Returns:', 'Since:', 380 # those are often singular or plural 381 'Note:', 'Notes:', 382 'Example:', 'Examples:', 383 'TODO:') 384 385 def _append_body_line(self, line): 386 """ 387 Process a line of documentation text in the body section. 388 389 If this a symbol line and it is the section's first line, this 390 is a definition documentation block for that symbol. 391 392 If it's a definition documentation block, another symbol line 393 begins the argument section for the argument named by it, and 394 a section tag begins an additional section. Start that 395 section and append the line to it. 396 397 Else, append the line to the current section. 398 """ 399 name = line.split(' ', 1)[0] 400 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't 401 # recognized, and get silently treated as ordinary text 402 if not self.symbol and not self.body.text and line.startswith('@'): 403 if not line.endswith(':'): 404 raise QAPIParseError(self._parser, "line should end with ':'") 405 self.symbol = line[1:-1] 406 # FIXME invalid names other than the empty string aren't flagged 407 if not self.symbol: 408 raise QAPIParseError(self._parser, "invalid name") 409 elif self.symbol: 410 # This is a definition documentation block 411 if name.startswith('@') and name.endswith(':'): 412 self._append_line = self._append_args_line 413 self._append_args_line(line) 414 elif line == 'Features:': 415 self._append_line = self._append_features_line 416 elif self._is_section_tag(name): 417 self._append_line = self._append_various_line 418 self._append_various_line(line) 419 else: 420 self._append_freeform(line.strip()) 421 else: 422 # This is a free-form documentation block 423 self._append_freeform(line.strip()) 424 425 def _append_args_line(self, line): 426 """ 427 Process a line of documentation text in an argument section. 428 429 A symbol line begins the next argument section, a section tag 430 section or a non-indented line after a blank line begins an 431 additional section. Start that section and append the line to 432 it. 433 434 Else, append the line to the current section. 435 436 """ 437 name = line.split(' ', 1)[0] 438 439 if name.startswith('@') and name.endswith(':'): 440 line = line[len(name)+1:] 441 self._start_args_section(name[1:-1]) 442 elif self._is_section_tag(name): 443 self._append_line = self._append_various_line 444 self._append_various_line(line) 445 return 446 elif (self._section.text.endswith('\n\n') 447 and line and not line[0].isspace()): 448 if line == 'Features:': 449 self._append_line = self._append_features_line 450 else: 451 self._start_section() 452 self._append_line = self._append_various_line 453 self._append_various_line(line) 454 return 455 456 self._append_freeform(line.strip()) 457 458 def _append_features_line(self, line): 459 name = line.split(' ', 1)[0] 460 461 if name.startswith('@') and name.endswith(':'): 462 line = line[len(name)+1:] 463 self._start_features_section(name[1:-1]) 464 elif self._is_section_tag(name): 465 self._append_line = self._append_various_line 466 self._append_various_line(line) 467 return 468 elif (self._section.text.endswith('\n\n') 469 and line and not line[0].isspace()): 470 self._start_section() 471 self._append_line = self._append_various_line 472 self._append_various_line(line) 473 return 474 475 self._append_freeform(line.strip()) 476 477 def _append_various_line(self, line): 478 """ 479 Process a line of documentation text in an additional section. 480 481 A symbol line is an error. 482 483 A section tag begins an additional section. Start that 484 section and append the line to it. 485 486 Else, append the line to the current section. 487 """ 488 name = line.split(' ', 1)[0] 489 490 if name.startswith('@') and name.endswith(':'): 491 raise QAPIParseError(self._parser, 492 "'%s' can't follow '%s' section" 493 % (name, self.sections[0].name)) 494 if self._is_section_tag(name): 495 line = line[len(name)+1:] 496 self._start_section(name[:-1]) 497 498 if (not self._section.name or 499 not self._section.name.startswith('Example')): 500 line = line.strip() 501 502 self._append_freeform(line) 503 504 def _start_symbol_section(self, symbols_dict, name): 505 # FIXME invalid names other than the empty string aren't flagged 506 if not name: 507 raise QAPIParseError(self._parser, "invalid parameter name") 508 if name in symbols_dict: 509 raise QAPIParseError(self._parser, 510 "'%s' parameter name duplicated" % name) 511 assert not self.sections 512 self._end_section() 513 self._section = QAPIDoc.ArgSection(name) 514 symbols_dict[name] = self._section 515 516 def _start_args_section(self, name): 517 self._start_symbol_section(self.args, name) 518 519 def _start_features_section(self, name): 520 self._start_symbol_section(self.features, name) 521 522 def _start_section(self, name=None): 523 if name in ('Returns', 'Since') and self.has_section(name): 524 raise QAPIParseError(self._parser, 525 "duplicated '%s' section" % name) 526 self._end_section() 527 self._section = QAPIDoc.Section(name) 528 self.sections.append(self._section) 529 530 def _end_section(self): 531 if self._section: 532 text = self._section.text = self._section.text.strip() 533 if self._section.name and (not text or text.isspace()): 534 raise QAPIParseError( 535 self._parser, 536 "empty doc section '%s'" % self._section.name) 537 self._section = None 538 539 def _append_freeform(self, line): 540 match = re.match(r'(@\S+:)', line) 541 if match: 542 raise QAPIParseError(self._parser, 543 "'%s' not allowed in free-form documentation" 544 % match.group(1)) 545 self._section.append(line) 546 547 def connect_member(self, member): 548 if member.name not in self.args: 549 # Undocumented TODO outlaw 550 self.args[member.name] = QAPIDoc.ArgSection(member.name) 551 self.args[member.name].connect(member) 552 553 def connect_feature(self, feature): 554 if feature.name not in self.features: 555 raise QAPISemError(feature.info, 556 "feature '%s' lacks documentation" 557 % feature.name) 558 self.features[feature.name].connect(feature) 559 560 def check_expr(self, expr): 561 if self.has_section('Returns') and 'command' not in expr: 562 raise QAPISemError(self.info, 563 "'Returns:' is only valid for commands") 564 565 def check(self): 566 567 def check_args_section(args, info, what): 568 bogus = [name for name, section in args.items() 569 if not section.member] 570 if bogus: 571 raise QAPISemError( 572 self.info, 573 "documented member%s '%s' %s not exist" 574 % ("s" if len(bogus) > 1 else "", 575 "', '".join(bogus), 576 "do" if len(bogus) > 1 else "does")) 577 578 check_args_section(self.args, self.info, 'members') 579 check_args_section(self.features, self.info, 'features') 580