1# -*- coding: utf-8 -*- 2# 3# QAPI schema parser 4# 5# Copyright IBM, Corp. 2011 6# Copyright (c) 2013-2019 Red Hat Inc. 7# 8# Authors: 9# Anthony Liguori <aliguori@us.ibm.com> 10# Markus Armbruster <armbru@redhat.com> 11# Marc-André Lureau <marcandre.lureau@redhat.com> 12# Kevin Wolf <kwolf@redhat.com> 13# 14# This work is licensed under the terms of the GNU GPL, version 2. 15# See the COPYING file in the top-level directory. 16 17import os 18import re 19from collections import OrderedDict 20 21from qapi.error import QAPIParseError, QAPISemError 22from qapi.source import QAPISourceInfo 23 24 25class QAPISchemaParser: 26 27 def __init__(self, fname, previously_included=None, incl_info=None): 28 previously_included = previously_included or set() 29 previously_included.add(os.path.abspath(fname)) 30 31 try: 32 fp = open(fname, 'r', encoding='utf-8') 33 self.src = fp.read() 34 except IOError as e: 35 raise QAPISemError(incl_info or QAPISourceInfo(None, None, None), 36 "can't read %s file '%s': %s" 37 % ("include" if incl_info else "schema", 38 fname, 39 e.strerror)) 40 41 if self.src == '' or self.src[-1] != '\n': 42 self.src += '\n' 43 self.cursor = 0 44 self.info = QAPISourceInfo(fname, 1, incl_info) 45 self.line_pos = 0 46 self.exprs = [] 47 self.docs = [] 48 self.accept() 49 cur_doc = None 50 51 while self.tok is not None: 52 info = self.info 53 if self.tok == '#': 54 self.reject_expr_doc(cur_doc) 55 cur_doc = self.get_doc(info) 56 self.docs.append(cur_doc) 57 continue 58 59 expr = self.get_expr(False) 60 if 'include' in expr: 61 self.reject_expr_doc(cur_doc) 62 if len(expr) != 1: 63 raise QAPISemError(info, "invalid 'include' directive") 64 include = expr['include'] 65 if not isinstance(include, str): 66 raise QAPISemError(info, 67 "value of 'include' must be a string") 68 incl_fname = os.path.join(os.path.dirname(fname), 69 include) 70 self.exprs.append({'expr': {'include': incl_fname}, 71 'info': info}) 72 exprs_include = self._include(include, info, incl_fname, 73 previously_included) 74 if exprs_include: 75 self.exprs.extend(exprs_include.exprs) 76 self.docs.extend(exprs_include.docs) 77 elif "pragma" in expr: 78 self.reject_expr_doc(cur_doc) 79 if len(expr) != 1: 80 raise QAPISemError(info, "invalid 'pragma' directive") 81 pragma = expr['pragma'] 82 if not isinstance(pragma, dict): 83 raise QAPISemError( 84 info, "value of 'pragma' must be an object") 85 for name, value in pragma.items(): 86 self._pragma(name, value, info) 87 else: 88 expr_elem = {'expr': expr, 89 'info': info} 90 if cur_doc: 91 if not cur_doc.symbol: 92 raise QAPISemError( 93 cur_doc.info, "definition documentation required") 94 expr_elem['doc'] = cur_doc 95 self.exprs.append(expr_elem) 96 cur_doc = None 97 self.reject_expr_doc(cur_doc) 98 99 @staticmethod 100 def reject_expr_doc(doc): 101 if doc and doc.symbol: 102 raise QAPISemError( 103 doc.info, 104 "documentation for '%s' is not followed by the definition" 105 % doc.symbol) 106 107 def _include(self, include, info, incl_fname, previously_included): 108 incl_abs_fname = os.path.abspath(incl_fname) 109 # catch inclusion cycle 110 inf = info 111 while inf: 112 if incl_abs_fname == os.path.abspath(inf.fname): 113 raise QAPISemError(info, "inclusion loop for %s" % include) 114 inf = inf.parent 115 116 # skip multiple include of the same file 117 if incl_abs_fname in previously_included: 118 return None 119 120 return QAPISchemaParser(incl_fname, previously_included, info) 121 122 def _pragma(self, name, value, info): 123 if name == 'doc-required': 124 if not isinstance(value, bool): 125 raise QAPISemError(info, 126 "pragma 'doc-required' must be boolean") 127 info.pragma.doc_required = value 128 elif name == 'returns-whitelist': 129 if (not isinstance(value, list) 130 or any([not isinstance(elt, str) for elt in value])): 131 raise QAPISemError( 132 info, 133 "pragma returns-whitelist must be a list of strings") 134 info.pragma.returns_whitelist = value 135 elif name == 'name-case-whitelist': 136 if (not isinstance(value, list) 137 or any([not isinstance(elt, str) for elt in value])): 138 raise QAPISemError( 139 info, 140 "pragma name-case-whitelist must be a list of strings") 141 info.pragma.name_case_whitelist = value 142 else: 143 raise QAPISemError(info, "unknown pragma '%s'" % name) 144 145 def accept(self, skip_comment=True): 146 while True: 147 self.tok = self.src[self.cursor] 148 self.pos = self.cursor 149 self.cursor += 1 150 self.val = None 151 152 if self.tok == '#': 153 if self.src[self.cursor] == '#': 154 # Start of doc comment 155 skip_comment = False 156 self.cursor = self.src.find('\n', self.cursor) 157 if not skip_comment: 158 self.val = self.src[self.pos:self.cursor] 159 return 160 elif self.tok in '{}:,[]': 161 return 162 elif self.tok == "'": 163 # Note: we accept only printable ASCII 164 string = '' 165 esc = False 166 while True: 167 ch = self.src[self.cursor] 168 self.cursor += 1 169 if ch == '\n': 170 raise QAPIParseError(self, "missing terminating \"'\"") 171 if esc: 172 # Note: we recognize only \\ because we have 173 # no use for funny characters in strings 174 if ch != '\\': 175 raise QAPIParseError(self, 176 "unknown escape \\%s" % ch) 177 esc = False 178 elif ch == '\\': 179 esc = True 180 continue 181 elif ch == "'": 182 self.val = string 183 return 184 if ord(ch) < 32 or ord(ch) >= 127: 185 raise QAPIParseError( 186 self, "funny character in string") 187 string += ch 188 elif self.src.startswith('true', self.pos): 189 self.val = True 190 self.cursor += 3 191 return 192 elif self.src.startswith('false', self.pos): 193 self.val = False 194 self.cursor += 4 195 return 196 elif self.tok == '\n': 197 if self.cursor == len(self.src): 198 self.tok = None 199 return 200 self.info = self.info.next_line() 201 self.line_pos = self.cursor 202 elif not self.tok.isspace(): 203 # Show up to next structural, whitespace or quote 204 # character 205 match = re.match('[^[\\]{}:,\\s\'"]+', 206 self.src[self.cursor-1:]) 207 raise QAPIParseError(self, "stray '%s'" % match.group(0)) 208 209 def get_members(self): 210 expr = OrderedDict() 211 if self.tok == '}': 212 self.accept() 213 return expr 214 if self.tok != "'": 215 raise QAPIParseError(self, "expected string or '}'") 216 while True: 217 key = self.val 218 self.accept() 219 if self.tok != ':': 220 raise QAPIParseError(self, "expected ':'") 221 self.accept() 222 if key in expr: 223 raise QAPIParseError(self, "duplicate key '%s'" % key) 224 expr[key] = self.get_expr(True) 225 if self.tok == '}': 226 self.accept() 227 return expr 228 if self.tok != ',': 229 raise QAPIParseError(self, "expected ',' or '}'") 230 self.accept() 231 if self.tok != "'": 232 raise QAPIParseError(self, "expected string") 233 234 def get_values(self): 235 expr = [] 236 if self.tok == ']': 237 self.accept() 238 return expr 239 if self.tok not in "{['tfn": 240 raise QAPIParseError( 241 self, "expected '{', '[', ']', string, boolean or 'null'") 242 while True: 243 expr.append(self.get_expr(True)) 244 if self.tok == ']': 245 self.accept() 246 return expr 247 if self.tok != ',': 248 raise QAPIParseError(self, "expected ',' or ']'") 249 self.accept() 250 251 def get_expr(self, nested): 252 if self.tok != '{' and not nested: 253 raise QAPIParseError(self, "expected '{'") 254 if self.tok == '{': 255 self.accept() 256 expr = self.get_members() 257 elif self.tok == '[': 258 self.accept() 259 expr = self.get_values() 260 elif self.tok in "'tfn": 261 expr = self.val 262 self.accept() 263 else: 264 raise QAPIParseError( 265 self, "expected '{', '[', string, boolean or 'null'") 266 return expr 267 268 def get_doc(self, info): 269 if self.val != '##': 270 raise QAPIParseError( 271 self, "junk after '##' at start of documentation comment") 272 273 doc = QAPIDoc(self, info) 274 self.accept(False) 275 while self.tok == '#': 276 if self.val.startswith('##'): 277 # End of doc comment 278 if self.val != '##': 279 raise QAPIParseError( 280 self, 281 "junk after '##' at end of documentation comment") 282 doc.end_comment() 283 self.accept() 284 return doc 285 else: 286 doc.append(self.val) 287 self.accept(False) 288 289 raise QAPIParseError(self, "documentation comment must end with '##'") 290 291 292class QAPIDoc: 293 """ 294 A documentation comment block, either definition or free-form 295 296 Definition documentation blocks consist of 297 298 * a body section: one line naming the definition, followed by an 299 overview (any number of lines) 300 301 * argument sections: a description of each argument (for commands 302 and events) or member (for structs, unions and alternates) 303 304 * features sections: a description of each feature flag 305 306 * additional (non-argument) sections, possibly tagged 307 308 Free-form documentation blocks consist only of a body section. 309 """ 310 311 class Section: 312 def __init__(self, name=None): 313 # optional section name (argument/member or section name) 314 self.name = name 315 # the list of lines for this section 316 self.text = '' 317 318 def append(self, line): 319 self.text += line.rstrip() + '\n' 320 321 class ArgSection(Section): 322 def __init__(self, name): 323 QAPIDoc.Section.__init__(self, name) 324 self.member = None 325 326 def connect(self, member): 327 self.member = member 328 329 def __init__(self, parser, info): 330 # self._parser is used to report errors with QAPIParseError. The 331 # resulting error position depends on the state of the parser. 332 # It happens to be the beginning of the comment. More or less 333 # servicable, but action at a distance. 334 self._parser = parser 335 self.info = info 336 self.symbol = None 337 self.body = QAPIDoc.Section() 338 # dict mapping parameter name to ArgSection 339 self.args = OrderedDict() 340 self.features = OrderedDict() 341 # a list of Section 342 self.sections = [] 343 # the current section 344 self._section = self.body 345 self._append_line = self._append_body_line 346 347 def has_section(self, name): 348 """Return True if we have a section with this name.""" 349 for i in self.sections: 350 if i.name == name: 351 return True 352 return False 353 354 def append(self, line): 355 """ 356 Parse a comment line and add it to the documentation. 357 358 The way that the line is dealt with depends on which part of 359 the documentation we're parsing right now: 360 * The body section: ._append_line is ._append_body_line 361 * An argument section: ._append_line is ._append_args_line 362 * A features section: ._append_line is ._append_features_line 363 * An additional section: ._append_line is ._append_various_line 364 """ 365 line = line[1:] 366 if not line: 367 self._append_freeform(line) 368 return 369 370 if line[0] != ' ': 371 raise QAPIParseError(self._parser, "missing space after #") 372 line = line[1:] 373 self._append_line(line) 374 375 def end_comment(self): 376 self._end_section() 377 378 @staticmethod 379 def _is_section_tag(name): 380 return name in ('Returns:', 'Since:', 381 # those are often singular or plural 382 'Note:', 'Notes:', 383 'Example:', 'Examples:', 384 'TODO:') 385 386 def _append_body_line(self, line): 387 """ 388 Process a line of documentation text in the body section. 389 390 If this a symbol line and it is the section's first line, this 391 is a definition documentation block for that symbol. 392 393 If it's a definition documentation block, another symbol line 394 begins the argument section for the argument named by it, and 395 a section tag begins an additional section. Start that 396 section and append the line to it. 397 398 Else, append the line to the current section. 399 """ 400 name = line.split(' ', 1)[0] 401 # FIXME not nice: things like '# @foo:' and '# @foo: ' aren't 402 # recognized, and get silently treated as ordinary text 403 if not self.symbol and not self.body.text and line.startswith('@'): 404 if not line.endswith(':'): 405 raise QAPIParseError(self._parser, "line should end with ':'") 406 self.symbol = line[1:-1] 407 # FIXME invalid names other than the empty string aren't flagged 408 if not self.symbol: 409 raise QAPIParseError(self._parser, "invalid name") 410 elif self.symbol: 411 # This is a definition documentation block 412 if name.startswith('@') and name.endswith(':'): 413 self._append_line = self._append_args_line 414 self._append_args_line(line) 415 elif line == 'Features:': 416 self._append_line = self._append_features_line 417 elif self._is_section_tag(name): 418 self._append_line = self._append_various_line 419 self._append_various_line(line) 420 else: 421 self._append_freeform(line.strip()) 422 else: 423 # This is a free-form documentation block 424 self._append_freeform(line.strip()) 425 426 def _append_args_line(self, line): 427 """ 428 Process a line of documentation text in an argument section. 429 430 A symbol line begins the next argument section, a section tag 431 section or a non-indented line after a blank line begins an 432 additional section. Start that section and append the line to 433 it. 434 435 Else, append the line to the current section. 436 437 """ 438 name = line.split(' ', 1)[0] 439 440 if name.startswith('@') and name.endswith(':'): 441 line = line[len(name)+1:] 442 self._start_args_section(name[1:-1]) 443 elif self._is_section_tag(name): 444 self._append_line = self._append_various_line 445 self._append_various_line(line) 446 return 447 elif (self._section.text.endswith('\n\n') 448 and line and not line[0].isspace()): 449 if line == 'Features:': 450 self._append_line = self._append_features_line 451 else: 452 self._start_section() 453 self._append_line = self._append_various_line 454 self._append_various_line(line) 455 return 456 457 self._append_freeform(line.strip()) 458 459 def _append_features_line(self, line): 460 name = line.split(' ', 1)[0] 461 462 if name.startswith('@') and name.endswith(':'): 463 line = line[len(name)+1:] 464 self._start_features_section(name[1:-1]) 465 elif self._is_section_tag(name): 466 self._append_line = self._append_various_line 467 self._append_various_line(line) 468 return 469 elif (self._section.text.endswith('\n\n') 470 and line and not line[0].isspace()): 471 self._start_section() 472 self._append_line = self._append_various_line 473 self._append_various_line(line) 474 return 475 476 self._append_freeform(line.strip()) 477 478 def _append_various_line(self, line): 479 """ 480 Process a line of documentation text in an additional section. 481 482 A symbol line is an error. 483 484 A section tag begins an additional section. Start that 485 section and append the line to it. 486 487 Else, append the line to the current section. 488 """ 489 name = line.split(' ', 1)[0] 490 491 if name.startswith('@') and name.endswith(':'): 492 raise QAPIParseError(self._parser, 493 "'%s' can't follow '%s' section" 494 % (name, self.sections[0].name)) 495 elif self._is_section_tag(name): 496 line = line[len(name)+1:] 497 self._start_section(name[:-1]) 498 499 if (not self._section.name or 500 not self._section.name.startswith('Example')): 501 line = line.strip() 502 503 self._append_freeform(line) 504 505 def _start_symbol_section(self, symbols_dict, name): 506 # FIXME invalid names other than the empty string aren't flagged 507 if not name: 508 raise QAPIParseError(self._parser, "invalid parameter name") 509 if name in symbols_dict: 510 raise QAPIParseError(self._parser, 511 "'%s' parameter name duplicated" % name) 512 assert not self.sections 513 self._end_section() 514 self._section = QAPIDoc.ArgSection(name) 515 symbols_dict[name] = self._section 516 517 def _start_args_section(self, name): 518 self._start_symbol_section(self.args, name) 519 520 def _start_features_section(self, name): 521 self._start_symbol_section(self.features, name) 522 523 def _start_section(self, name=None): 524 if name in ('Returns', 'Since') and self.has_section(name): 525 raise QAPIParseError(self._parser, 526 "duplicated '%s' section" % name) 527 self._end_section() 528 self._section = QAPIDoc.Section(name) 529 self.sections.append(self._section) 530 531 def _end_section(self): 532 if self._section: 533 text = self._section.text = self._section.text.strip() 534 if self._section.name and (not text or text.isspace()): 535 raise QAPIParseError( 536 self._parser, 537 "empty doc section '%s'" % self._section.name) 538 self._section = None 539 540 def _append_freeform(self, line): 541 match = re.match(r'(@\S+:)', line) 542 if match: 543 raise QAPIParseError(self._parser, 544 "'%s' not allowed in free-form documentation" 545 % match.group(1)) 546 self._section.append(line) 547 548 def connect_member(self, member): 549 if member.name not in self.args: 550 # Undocumented TODO outlaw 551 self.args[member.name] = QAPIDoc.ArgSection(member.name) 552 self.args[member.name].connect(member) 553 554 def connect_feature(self, feature): 555 if feature.name not in self.features: 556 raise QAPISemError(feature.info, 557 "feature '%s' lacks documentation" 558 % feature.name) 559 self.features[feature.name] = QAPIDoc.ArgSection(feature.name) 560 self.features[feature.name].connect(feature) 561 562 def check_expr(self, expr): 563 if self.has_section('Returns') and 'command' not in expr: 564 raise QAPISemError(self.info, 565 "'Returns:' is only valid for commands") 566 567 def check(self): 568 569 def check_args_section(args, info, what): 570 bogus = [name for name, section in args.items() 571 if not section.member] 572 if bogus: 573 raise QAPISemError( 574 self.info, 575 "documented member%s '%s' %s not exist" 576 % ("s" if len(bogus) > 1 else "", 577 "', '".join(bogus), 578 "do" if len(bogus) > 1 else "does")) 579 580 check_args_section(self.args, self.info, 'members') 581 check_args_section(self.features, self.info, 'features') 582