1# -*- coding: utf-8 -*- 2# 3# QAPI schema parser 4# 5# Copyright IBM, Corp. 2011 6# Copyright (c) 2013-2019 Red Hat Inc. 7# 8# Authors: 9# Anthony Liguori <aliguori@us.ibm.com> 10# Markus Armbruster <armbru@redhat.com> 11# Marc-André Lureau <marcandre.lureau@redhat.com> 12# Kevin Wolf <kwolf@redhat.com> 13# 14# This work is licensed under the terms of the GNU GPL, version 2. 15# See the COPYING file in the top-level directory. 16 17import os 18import re 19import sys 20from collections import OrderedDict 21 22from qapi.error import QAPIParseError, QAPISemError 23from qapi.source import QAPISourceInfo 24 25 26class QAPISchemaParser(object): 27 28 def __init__(self, fname, previously_included=None, incl_info=None): 29 previously_included = previously_included or set() 30 previously_included.add(os.path.abspath(fname)) 31 32 try: 33 if sys.version_info[0] >= 3: 34 fp = open(fname, 'r', encoding='utf-8') 35 else: 36 fp = open(fname, 'r') 37 self.src = fp.read() 38 except IOError as e: 39 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), 40 "can't read %s file '%s': %s" 41 % ("include" if incl_info else "schema", 42 fname, 43 e.strerror)) 44 45 if self.src == '' or self.src[-1] != '\n': 46 self.src += '\n' 47 self.cursor = 0 48 self.info = QAPISourceInfo(fname, 1, incl_info) 49 self.line_pos = 0 50 self.exprs = [] 51 self.docs = [] 52 self.accept() 53 cur_doc = None 54 55 while self.tok is not None: 56 info = self.info 57 if self.tok == '#': 58 self.reject_expr_doc(cur_doc) 59 cur_doc = self.get_doc(info) 60 self.docs.append(cur_doc) 61 continue 62 63 expr = self.get_expr(False) 64 if 'include' in expr: 65 self.reject_expr_doc(cur_doc) 66 if len(expr) != 1: 67 raise QAPISemError(info, "invalid 'include' directive") 68 include = expr['include'] 69 if not isinstance(include, str): 70 raise QAPISemError(info, 71 "value of 'include' must be a string") 72 incl_fname = os.path.join(os.path.dirname(fname), 73 include) 74 self.exprs.append({'expr': {'include': incl_fname}, 75 'info': info}) 76 exprs_include = self._include(include, info, incl_fname, 77 previously_included) 78 if exprs_include: 79 self.exprs.extend(exprs_include.exprs) 80 self.docs.extend(exprs_include.docs) 81 elif "pragma" in expr: 82 self.reject_expr_doc(cur_doc) 83 if len(expr) != 1: 84 raise QAPISemError(info, "invalid 'pragma' directive") 85 pragma = expr['pragma'] 86 if not isinstance(pragma, dict): 87 raise QAPISemError( 88 info, "value of 'pragma' must be an object") 89 for name, value in pragma.items(): 90 self._pragma(name, value, info) 91 else: 92 expr_elem = {'expr': expr, 93 'info': info} 94 if cur_doc: 95 if not cur_doc.symbol: 96 raise QAPISemError( 97 cur_doc.info, "definition documentation required") 98 expr_elem['doc'] = cur_doc 99 self.exprs.append(expr_elem) 100 cur_doc = None 101 self.reject_expr_doc(cur_doc) 102 103 @staticmethod 104 def reject_expr_doc(doc): 105 if doc and doc.symbol: 106 raise QAPISemError( 107 doc.info, 108 "documentation for '%s' is not followed by the definition" 109 % doc.symbol) 110 111 def _include(self, include, info, incl_fname, previously_included): 112 incl_abs_fname = os.path.abspath(incl_fname) 113 # catch inclusion cycle 114 inf = info 115 while inf: 116 if incl_abs_fname == os.path.abspath(inf.fname): 117 raise QAPISemError(info, "inclusion loop for %s" % include) 118 inf = inf.parent 119 120 # skip multiple include of the same file 121 if incl_abs_fname in previously_included: 122 return None 123 124 return QAPISchemaParser(incl_fname, previously_included, info) 125 126 def _pragma(self, name, value, info): 127 if name == 'doc-required': 128 if not isinstance(value, bool): 129 raise QAPISemError(info, 130 "pragma 'doc-required' must be boolean") 131 info.pragma.doc_required = value 132 elif name == 'returns-whitelist': 133 if (not isinstance(value, list) 134 or any([not isinstance(elt, str) for elt in value])): 135 raise QAPISemError( 136 info, 137 "pragma returns-whitelist must be a list of strings") 138 info.pragma.returns_whitelist = value 139 elif name == 'name-case-whitelist': 140 if (not isinstance(value, list) 141 or any([not isinstance(elt, str) for elt in value])): 142 raise QAPISemError( 143 info, 144 "pragma name-case-whitelist must be a list of strings") 145 info.pragma.name_case_whitelist = value 146 else: 147 raise QAPISemError(info, "unknown pragma '%s'" % name) 148 149 def accept(self, skip_comment=True): 150 while True: 151 self.tok = self.src[self.cursor] 152 self.pos = self.cursor 153 self.cursor += 1 154 self.val = None 155 156 if self.tok == '#': 157 if self.src[self.cursor] == '#': 158 # Start of doc comment 159 skip_comment = False 160 self.cursor = self.src.find('\n', self.cursor) 161 if not skip_comment: 162 self.val = self.src[self.pos:self.cursor] 163 return 164 elif self.tok in '{}:,[]': 165 return 166 elif self.tok == "'": 167 # Note: we accept only printable ASCII 168 string = '' 169 esc = False 170 while True: 171 ch = self.src[self.cursor] 172 self.cursor += 1 173 if ch == '\n': 174 raise QAPIParseError(self, "missing terminating \"'\"") 175 if esc: 176 # Note: we recognize only \\ because we have 177 # no use for funny characters in strings 178 if ch != '\\': 179 raise QAPIParseError(self, 180 "unknown escape \\%s" % ch) 181 esc = False 182 elif ch == '\\': 183 esc = True 184 continue 185 elif ch == "'": 186 self.val = string 187 return 188 if ord(ch) < 32 or ord(ch) >= 127: 189 raise QAPIParseError( 190 self, "funny character in string") 191 string += ch 192 elif self.src.startswith('true', self.pos): 193 self.val = True 194 self.cursor += 3 195 return 196 elif self.src.startswith('false', self.pos): 197 self.val = False 198 self.cursor += 4 199 return 200 elif self.tok == '\n': 201 if self.cursor == len(self.src): 202 self.tok = None 203 return 204 self.info = self.info.next_line() 205 self.line_pos = self.cursor 206 elif not self.tok.isspace(): 207 # Show up to next structural, whitespace or quote 208 # character 209 match = re.match('[^[\\]{}:,\\s\'"]+', 210 self.src[self.cursor-1:]) 211 raise QAPIParseError(self, "stray '%s'" % match.group(0)) 212 213 def get_members(self): 214 expr = OrderedDict() 215 if self.tok == '}': 216 self.accept() 217 return expr 218 if self.tok != "'": 219 raise QAPIParseError(self, "expected string or '}'") 220 while True: 221 key = self.val 222 self.accept() 223 if self.tok != ':': 224 raise QAPIParseError(self, "expected ':'") 225 self.accept() 226 if key in expr: 227 raise QAPIParseError(self, "duplicate key '%s'" % key) 228 expr[key] = self.get_expr(True) 229 if self.tok == '}': 230 self.accept() 231 return expr 232 if self.tok != ',': 233 raise QAPIParseError(self, "expected ',' or '}'") 234 self.accept() 235 if self.tok != "'": 236 raise QAPIParseError(self, "expected string") 237 238 def get_values(self): 239 expr = [] 240 if self.tok == ']': 241 self.accept() 242 return expr 243 if self.tok not in "{['tfn": 244 raise QAPIParseError( 245 self, "expected '{', '[', ']', string, boolean or 'null'") 246 while True: 247 expr.append(self.get_expr(True)) 248 if self.tok == ']': 249 self.accept() 250 return expr 251 if self.tok != ',': 252 raise QAPIParseError(self, "expected ',' or ']'") 253 self.accept() 254 255 def get_expr(self, nested): 256 if self.tok != '{' and not nested: 257 raise QAPIParseError(self, "expected '{'") 258 if self.tok == '{': 259 self.accept() 260 expr = self.get_members() 261 elif self.tok == '[': 262 self.accept() 263 expr = self.get_values() 264 elif self.tok in "'tfn": 265 expr = self.val 266 self.accept() 267 else: 268 raise QAPIParseError( 269 self, "expected '{', '[', string, boolean or 'null'") 270 return expr 271 272 def get_doc(self, info): 273 if self.val != '##': 274 raise QAPIParseError( 275 self, "junk after '##' at start of documentation comment") 276 277 doc = QAPIDoc(self, info) 278 self.accept(False) 279 while self.tok == '#': 280 if self.val.startswith('##'): 281 # End of doc comment 282 if self.val != '##': 283 raise QAPIParseError( 284 self, 285 "junk after '##' at end of documentation comment") 286 doc.end_comment() 287 self.accept() 288 return doc 289 else: 290 doc.append(self.val) 291 self.accept(False) 292 293 raise QAPIParseError(self, "documentation comment must end with '##'") 294 295 296class QAPIDoc(object): 297 """ 298 A documentation comment block, either definition or free-form 299 300 Definition documentation blocks consist of 301 302 * a body section: one line naming the definition, followed by an 303 overview (any number of lines) 304 305 * argument sections: a description of each argument (for commands 306 and events) or member (for structs, unions and alternates) 307 308 * features sections: a description of each feature flag 309 310 * additional (non-argument) sections, possibly tagged 311 312 Free-form documentation blocks consist only of a body section. 313 """ 314 315 class Section(object): 316 def __init__(self, name=None): 317 # optional section name (argument/member or section name) 318 self.name = name 319 # the list of lines for this section 320 self.text = '' 321 322 def append(self, line): 323 self.text += line.rstrip() + '\n' 324 325 class ArgSection(Section): 326 def __init__(self, name): 327 QAPIDoc.Section.__init__(self, name) 328 self.member = None 329 330 def connect(self, member): 331 self.member = member 332 333 def __init__(self, parser, info): 334 # self._parser is used to report errors with QAPIParseError. The 335 # resulting error position depends on the state of the parser. 336 # It happens to be the beginning of the comment. More or less 337 # servicable, but action at a distance. 338 self._parser = parser 339 self.info = info 340 self.symbol = None 341 self.body = QAPIDoc.Section() 342 # dict mapping parameter name to ArgSection 343 self.args = OrderedDict() 344 self.features = OrderedDict() 345 # a list of Section 346 self.sections = [] 347 # the current section 348 self._section = self.body 349 self._append_line = self._append_body_line 350 351 def has_section(self, name): 352 """Return True if we have a section with this name.""" 353 for i in self.sections: 354 if i.name == name: 355 return True 356 return False 357 358 def append(self, line): 359 """ 360 Parse a comment line and add it to the documentation. 361 362 The way that the line is dealt with depends on which part of 363 the documentation we're parsing right now: 364 * The body section: ._append_line is ._append_body_line 365 * An argument section: ._append_line is ._append_args_line 366 * A features section: ._append_line is ._append_features_line 367 * An additional section: ._append_line is ._append_various_line 368 """ 369 line = line[1:] 370 if not line: 371 self._append_freeform(line) 372 return 373 374 if line[0] != ' ': 375 raise QAPIParseError(self._parser, "missing space after #") 376 line = line[1:] 377 self._append_line(line) 378 379 def end_comment(self): 380 self._end_section() 381 382 @staticmethod 383 def _is_section_tag(name): 384 return name in ('Returns:', 'Since:', 385 # those are often singular or plural 386 'Note:', 'Notes:', 387 'Example:', 'Examples:', 388 'TODO:') 389 390 def _append_body_line(self, line): 391 """ 392 Process a line of documentation text in the body section. 393 394 If this a symbol line and it is the section's first line, this 395 is a definition documentation block for that symbol. 396 397 If it's a definition documentation block, another symbol line 398 begins the argument section for the argument named by it, and 399 a section tag begins an additional section. Start that 400 section and append the line to it. 401 402 Else, append the line to the current section. 403 """ 404 name = line.split(' ', 1)[0] 405 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't 406 # recognized, and get silently treated as ordinary text 407 if not self.symbol and not self.body.text and line.startswith('@'): 408 if not line.endswith(':'): 409 raise QAPIParseError(self._parser, "line should end with ':'") 410 self.symbol = line[1:-1] 411 # FIXME invalid names other than the empty string aren't flagged 412 if not self.symbol: 413 raise QAPIParseError(self._parser, "invalid name") 414 elif self.symbol: 415 # This is a definition documentation block 416 if name.startswith('@') and name.endswith(':'): 417 self._append_line = self._append_args_line 418 self._append_args_line(line) 419 elif line == 'Features:': 420 self._append_line = self._append_features_line 421 elif self._is_section_tag(name): 422 self._append_line = self._append_various_line 423 self._append_various_line(line) 424 else: 425 self._append_freeform(line.strip()) 426 else: 427 # This is a free-form documentation block 428 self._append_freeform(line.strip()) 429 430 def _append_args_line(self, line): 431 """ 432 Process a line of documentation text in an argument section. 433 434 A symbol line begins the next argument section, a section tag 435 section or a non-indented line after a blank line begins an 436 additional section. Start that section and append the line to 437 it. 438 439 Else, append the line to the current section. 440 441 """ 442 name = line.split(' ', 1)[0] 443 444 if name.startswith('@') and name.endswith(':'): 445 line = line[len(name)+1:] 446 self._start_args_section(name[1:-1]) 447 elif self._is_section_tag(name): 448 self._append_line = self._append_various_line 449 self._append_various_line(line) 450 return 451 elif (self._section.text.endswith('\n\n') 452 and line and not line[0].isspace()): 453 if line == 'Features:': 454 self._append_line = self._append_features_line 455 else: 456 self._start_section() 457 self._append_line = self._append_various_line 458 self._append_various_line(line) 459 return 460 461 self._append_freeform(line.strip()) 462 463 def _append_features_line(self, line): 464 name = line.split(' ', 1)[0] 465 466 if name.startswith('@') and name.endswith(':'): 467 line = line[len(name)+1:] 468 self._start_features_section(name[1:-1]) 469 elif self._is_section_tag(name): 470 self._append_line = self._append_various_line 471 self._append_various_line(line) 472 return 473 elif (self._section.text.endswith('\n\n') 474 and line and not line[0].isspace()): 475 self._start_section() 476 self._append_line = self._append_various_line 477 self._append_various_line(line) 478 return 479 480 self._append_freeform(line.strip()) 481 482 def _append_various_line(self, line): 483 """ 484 Process a line of documentation text in an additional section. 485 486 A symbol line is an error. 487 488 A section tag begins an additional section. Start that 489 section and append the line to it. 490 491 Else, append the line to the current section. 492 """ 493 name = line.split(' ', 1)[0] 494 495 if name.startswith('@') and name.endswith(':'): 496 raise QAPIParseError(self._parser, 497 "'%s' can't follow '%s' section" 498 % (name, self.sections[0].name)) 499 elif self._is_section_tag(name): 500 line = line[len(name)+1:] 501 self._start_section(name[:-1]) 502 503 if (not self._section.name or 504 not self._section.name.startswith('Example')): 505 line = line.strip() 506 507 self._append_freeform(line) 508 509 def _start_symbol_section(self, symbols_dict, name): 510 # FIXME invalid names other than the empty string aren't flagged 511 if not name: 512 raise QAPIParseError(self._parser, "invalid parameter name") 513 if name in symbols_dict: 514 raise QAPIParseError(self._parser, 515 "'%s' parameter name duplicated" % name) 516 assert not self.sections 517 self._end_section() 518 self._section = QAPIDoc.ArgSection(name) 519 symbols_dict[name] = self._section 520 521 def _start_args_section(self, name): 522 self._start_symbol_section(self.args, name) 523 524 def _start_features_section(self, name): 525 self._start_symbol_section(self.features, name) 526 527 def _start_section(self, name=None): 528 if name in ('Returns', 'Since') and self.has_section(name): 529 raise QAPIParseError(self._parser, 530 "duplicated '%s' section" % name) 531 self._end_section() 532 self._section = QAPIDoc.Section(name) 533 self.sections.append(self._section) 534 535 def _end_section(self): 536 if self._section: 537 text = self._section.text = self._section.text.strip() 538 if self._section.name and (not text or text.isspace()): 539 raise QAPIParseError( 540 self._parser, 541 "empty doc section '%s'" % self._section.name) 542 self._section = None 543 544 def _append_freeform(self, line): 545 match = re.match(r'(@\S+:)', line) 546 if match: 547 raise QAPIParseError(self._parser, 548 "'%s' not allowed in free-form documentation" 549 % match.group(1)) 550 self._section.append(line) 551 552 def connect_member(self, member): 553 if member.name not in self.args: 554 # Undocumented TODO outlaw 555 self.args[member.name] = QAPIDoc.ArgSection(member.name) 556 self.args[member.name].connect(member) 557 558 def connect_feature(self, feature): 559 if feature.name not in self.features: 560 raise QAPISemError(feature.info, 561 "feature '%s' lacks documentation" 562 % feature.name) 563 self.features[feature.name] = QAPIDoc.ArgSection(feature.name) 564 self.features[feature.name].connect(feature) 565 566 def check_expr(self, expr): 567 if self.has_section('Returns') and 'command' not in expr: 568 raise QAPISemError(self.info, 569 "'Returns:' is only valid for commands") 570 571 def check(self): 572 573 def check_args_section(args, info, what): 574 bogus = [name for name, section in args.items() 575 if not section.member] 576 if bogus: 577 raise QAPISemError( 578 self.info, 579 "documented member%s '%s' %s not exist" 580 % ("s" if len(bogus) > 1 else "", 581 "', '".join(bogus), 582 "do" if len(bogus) > 1 else "does")) 583 584 check_args_section(self.args, self.info, 'members') 585 check_args_section(self.features, self.info, 'features') 586