1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2020 Linaro 6# 7# This work is licensed under the terms of the GNU GPLv2 or later. 8# See the COPYING file in the top-level directory. 9 10""" 11qapidoc is a Sphinx extension that implements the qapi-doc directive 12 13The purpose of this extension is to read the documentation comments 14in QAPI schema files, and insert them all into the current document. 15 16It implements one new rST directive, "qapi-doc::". 17Each qapi-doc:: directive takes one argument, which is the 18pathname of the schema file to process, relative to the source tree. 19 20The docs/conf.py file must set the qapidoc_srctree config value to 21the root of the QEMU source tree. 22 23The Sphinx documentation on writing extensions is at: 24https://www.sphinx-doc.org/en/master/development/index.html 25""" 26 27import os 28import re 29import sys 30import textwrap 31from typing import List 32 33from docutils import nodes 34from docutils.parsers.rst import Directive, directives 35from docutils.statemachine import ViewList 36from qapi.error import QAPIError, QAPISemError 37from qapi.gen import QAPISchemaVisitor 38from qapi.parser import QAPIDoc 39from qapi.schema import QAPISchema 40 41from sphinx import addnodes 42from sphinx.directives.code import CodeBlock 43from sphinx.errors import ExtensionError 44from sphinx.util.docutils import switch_source_input 45from sphinx.util.nodes import nested_parse_with_titles 46 47 48__version__ = "1.0" 49 50 51def dedent(text: str) -> str: 52 # Adjust indentation to make description text parse as paragraph. 53 54 lines = text.splitlines(True) 55 if re.match(r"\s+", lines[0]): 56 # First line is indented; description started on the line after 57 # the name. dedent the whole block. 58 return textwrap.dedent(text) 59 60 # Descr started on same line. Dedent line 2+. 61 return lines[0] + textwrap.dedent("".join(lines[1:])) 62 63 64# Disable black auto-formatter until re-enabled: 65# fmt: off 66 67 68class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): 69 """A QAPI schema visitor which generates docutils/Sphinx nodes 70 71 This class builds up a tree of docutils/Sphinx nodes corresponding 72 to documentation for the various QAPI objects. To use it, first 73 create a QAPISchemaGenRSTVisitor object, and call its 74 visit_begin() method. Then you can call one of the two methods 75 'freeform' (to add documentation for a freeform documentation 76 chunk) or 'symbol' (to add documentation for a QAPI symbol). These 77 will cause the visitor to build up the tree of document 78 nodes. Once you've added all the documentation via 'freeform' and 79 'symbol' method calls, you can call 'get_document_nodes' to get 80 the final list of document nodes (in a form suitable for returning 81 from a Sphinx directive's 'run' method). 82 """ 83 def __init__(self, sphinx_directive): 84 self._cur_doc = None 85 self._sphinx_directive = sphinx_directive 86 self._top_node = nodes.section() 87 self._active_headings = [self._top_node] 88 89 def _make_dlitem(self, term, defn): 90 """Return a dlitem node with the specified term and definition. 91 92 term should be a list of Text and literal nodes. 93 defn should be one of: 94 - a string, which will be handed to _parse_text_into_node 95 - a list of Text and literal nodes, which will be put into 96 a paragraph node 97 """ 98 dlitem = nodes.definition_list_item() 99 dlterm = nodes.term('', '', *term) 100 dlitem += dlterm 101 if defn: 102 dldef = nodes.definition() 103 if isinstance(defn, list): 104 dldef += nodes.paragraph('', '', *defn) 105 else: 106 self._parse_text_into_node(defn, dldef) 107 dlitem += dldef 108 return dlitem 109 110 def _make_section(self, title): 111 """Return a section node with optional title""" 112 section = nodes.section(ids=[self._sphinx_directive.new_serialno()]) 113 if title: 114 section += nodes.title(title, title) 115 return section 116 117 def _nodes_for_ifcond(self, ifcond, with_if=True): 118 """Return list of Text, literal nodes for the ifcond 119 120 Return a list which gives text like ' (If: condition)'. 121 If with_if is False, we don't return the "(If: " and ")". 122 """ 123 124 doc = ifcond.docgen() 125 if not doc: 126 return [] 127 doc = nodes.literal('', doc) 128 if not with_if: 129 return [doc] 130 131 nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')] 132 nodelist.append(doc) 133 nodelist.append(nodes.Text(')')) 134 return nodelist 135 136 def _nodes_for_one_member(self, member): 137 """Return list of Text, literal nodes for this member 138 139 Return a list of doctree nodes which give text like 140 'name: type (optional) (If: ...)' suitable for use as the 141 'term' part of a definition list item. 142 """ 143 term = [nodes.literal('', member.name)] 144 if member.type.doc_type(): 145 term.append(nodes.Text(': ')) 146 term.append(nodes.literal('', member.type.doc_type())) 147 if member.optional: 148 term.append(nodes.Text(' (optional)')) 149 if member.ifcond.is_present(): 150 term.extend(self._nodes_for_ifcond(member.ifcond)) 151 return term 152 153 def _nodes_for_variant_when(self, branches, variant): 154 """Return list of Text, literal nodes for variant 'when' clause 155 156 Return a list of doctree nodes which give text like 157 'when tagname is variant (If: ...)' suitable for use in 158 the 'branches' part of a definition list. 159 """ 160 term = [nodes.Text(' when '), 161 nodes.literal('', branches.tag_member.name), 162 nodes.Text(' is '), 163 nodes.literal('', '"%s"' % variant.name)] 164 if variant.ifcond.is_present(): 165 term.extend(self._nodes_for_ifcond(variant.ifcond)) 166 return term 167 168 def _nodes_for_members(self, doc, what, base=None, branches=None): 169 """Return list of doctree nodes for the table of members""" 170 dlnode = nodes.definition_list() 171 for section in doc.args.values(): 172 term = self._nodes_for_one_member(section.member) 173 # TODO drop fallbacks when undocumented members are outlawed 174 if section.text: 175 defn = dedent(section.text) 176 else: 177 defn = [nodes.Text('Not documented')] 178 179 dlnode += self._make_dlitem(term, defn) 180 181 if base: 182 dlnode += self._make_dlitem([nodes.Text('The members of '), 183 nodes.literal('', base.doc_type())], 184 None) 185 186 if branches: 187 for v in branches.variants: 188 if v.type.name == 'q_empty': 189 continue 190 assert not v.type.is_implicit() 191 term = [nodes.Text('The members of '), 192 nodes.literal('', v.type.doc_type())] 193 term.extend(self._nodes_for_variant_when(branches, v)) 194 dlnode += self._make_dlitem(term, None) 195 196 if not dlnode.children: 197 return [] 198 199 section = self._make_section(what) 200 section += dlnode 201 return [section] 202 203 def _nodes_for_enum_values(self, doc): 204 """Return list of doctree nodes for the table of enum values""" 205 seen_item = False 206 dlnode = nodes.definition_list() 207 for section in doc.args.values(): 208 termtext = [nodes.literal('', section.member.name)] 209 if section.member.ifcond.is_present(): 210 termtext.extend(self._nodes_for_ifcond(section.member.ifcond)) 211 # TODO drop fallbacks when undocumented members are outlawed 212 if section.text: 213 defn = dedent(section.text) 214 else: 215 defn = [nodes.Text('Not documented')] 216 217 dlnode += self._make_dlitem(termtext, defn) 218 seen_item = True 219 220 if not seen_item: 221 return [] 222 223 section = self._make_section('Values') 224 section += dlnode 225 return [section] 226 227 def _nodes_for_arguments(self, doc, arg_type): 228 """Return list of doctree nodes for the arguments section""" 229 if arg_type and not arg_type.is_implicit(): 230 assert not doc.args 231 section = self._make_section('Arguments') 232 dlnode = nodes.definition_list() 233 dlnode += self._make_dlitem( 234 [nodes.Text('The members of '), 235 nodes.literal('', arg_type.name)], 236 None) 237 section += dlnode 238 return [section] 239 240 return self._nodes_for_members(doc, 'Arguments') 241 242 def _nodes_for_features(self, doc): 243 """Return list of doctree nodes for the table of features""" 244 seen_item = False 245 dlnode = nodes.definition_list() 246 for section in doc.features.values(): 247 dlnode += self._make_dlitem( 248 [nodes.literal('', section.member.name)], dedent(section.text)) 249 seen_item = True 250 251 if not seen_item: 252 return [] 253 254 section = self._make_section('Features') 255 section += dlnode 256 return [section] 257 258 def _nodes_for_sections(self, doc): 259 """Return list of doctree nodes for additional sections""" 260 nodelist = [] 261 for section in doc.sections: 262 if section.kind == QAPIDoc.Kind.TODO: 263 # Hide TODO: sections 264 continue 265 266 if section.kind == QAPIDoc.Kind.PLAIN: 267 # Sphinx cannot handle sectionless titles; 268 # Instead, just append the results to the prior section. 269 container = nodes.container() 270 self._parse_text_into_node(section.text, container) 271 nodelist += container.children 272 continue 273 274 snode = self._make_section(section.kind.name.title()) 275 self._parse_text_into_node(dedent(section.text), snode) 276 nodelist.append(snode) 277 return nodelist 278 279 def _nodes_for_if_section(self, ifcond): 280 """Return list of doctree nodes for the "If" section""" 281 nodelist = [] 282 if ifcond.is_present(): 283 snode = self._make_section('If') 284 snode += nodes.paragraph( 285 '', '', *self._nodes_for_ifcond(ifcond, with_if=False) 286 ) 287 nodelist.append(snode) 288 return nodelist 289 290 def _add_doc(self, typ, sections): 291 """Add documentation for a command/object/enum... 292 293 We assume we're documenting the thing defined in self._cur_doc. 294 typ is the type of thing being added ("Command", "Object", etc) 295 296 sections is a list of nodes for sections to add to the definition. 297 """ 298 299 doc = self._cur_doc 300 snode = nodes.section(ids=[self._sphinx_directive.new_serialno()]) 301 snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol), 302 nodes.Text(' (' + typ + ')')]) 303 self._parse_text_into_node(doc.body.text, snode) 304 for s in sections: 305 if s is not None: 306 snode += s 307 self._add_node_to_current_heading(snode) 308 309 def visit_enum_type(self, name, info, ifcond, features, members, prefix): 310 doc = self._cur_doc 311 self._add_doc('Enum', 312 self._nodes_for_enum_values(doc) 313 + self._nodes_for_features(doc) 314 + self._nodes_for_sections(doc) 315 + self._nodes_for_if_section(ifcond)) 316 317 def visit_object_type(self, name, info, ifcond, features, 318 base, members, branches): 319 doc = self._cur_doc 320 if base and base.is_implicit(): 321 base = None 322 self._add_doc('Object', 323 self._nodes_for_members(doc, 'Members', base, branches) 324 + self._nodes_for_features(doc) 325 + self._nodes_for_sections(doc) 326 + self._nodes_for_if_section(ifcond)) 327 328 def visit_alternate_type(self, name, info, ifcond, features, 329 alternatives): 330 doc = self._cur_doc 331 self._add_doc('Alternate', 332 self._nodes_for_members(doc, 'Members') 333 + self._nodes_for_features(doc) 334 + self._nodes_for_sections(doc) 335 + self._nodes_for_if_section(ifcond)) 336 337 def visit_command(self, name, info, ifcond, features, arg_type, 338 ret_type, gen, success_response, boxed, allow_oob, 339 allow_preconfig, coroutine): 340 doc = self._cur_doc 341 self._add_doc('Command', 342 self._nodes_for_arguments(doc, arg_type) 343 + self._nodes_for_features(doc) 344 + self._nodes_for_sections(doc) 345 + self._nodes_for_if_section(ifcond)) 346 347 def visit_event(self, name, info, ifcond, features, arg_type, boxed): 348 doc = self._cur_doc 349 self._add_doc('Event', 350 self._nodes_for_arguments(doc, arg_type) 351 + self._nodes_for_features(doc) 352 + self._nodes_for_sections(doc) 353 + self._nodes_for_if_section(ifcond)) 354 355 def symbol(self, doc, entity): 356 """Add documentation for one symbol to the document tree 357 358 This is the main entry point which causes us to add documentation 359 nodes for a symbol (which could be a 'command', 'object', 'event', 360 etc). We do this by calling 'visit' on the schema entity, which 361 will then call back into one of our visit_* methods, depending 362 on what kind of thing this symbol is. 363 """ 364 self._cur_doc = doc 365 entity.visit(self) 366 self._cur_doc = None 367 368 def _start_new_heading(self, heading, level): 369 """Start a new heading at the specified heading level 370 371 Create a new section whose title is 'heading' and which is placed 372 in the docutils node tree as a child of the most recent level-1 373 heading. Subsequent document sections (commands, freeform doc chunks, 374 etc) will be placed as children of this new heading section. 375 """ 376 if len(self._active_headings) < level: 377 raise QAPISemError(self._cur_doc.info, 378 'Level %d subheading found outside a ' 379 'level %d heading' 380 % (level, level - 1)) 381 snode = self._make_section(heading) 382 self._active_headings[level - 1] += snode 383 self._active_headings = self._active_headings[:level] 384 self._active_headings.append(snode) 385 return snode 386 387 def _add_node_to_current_heading(self, node): 388 """Add the node to whatever the current active heading is""" 389 self._active_headings[-1] += node 390 391 def freeform(self, doc): 392 """Add a piece of 'freeform' documentation to the document tree 393 394 A 'freeform' document chunk doesn't relate to any particular 395 symbol (for instance, it could be an introduction). 396 397 If the freeform document starts with a line of the form 398 '= Heading text', this is a section or subsection heading, with 399 the heading level indicated by the number of '=' signs. 400 """ 401 402 # QAPIDoc documentation says free-form documentation blocks 403 # must have only a body section, nothing else. 404 assert not doc.sections 405 assert not doc.args 406 assert not doc.features 407 self._cur_doc = doc 408 409 text = doc.body.text 410 if re.match(r'=+ ', text): 411 # Section/subsection heading (if present, will always be 412 # the first line of the block) 413 (heading, _, text) = text.partition('\n') 414 (leader, _, heading) = heading.partition(' ') 415 node = self._start_new_heading(heading, len(leader)) 416 if text == '': 417 return 418 else: 419 node = nodes.container() 420 421 self._parse_text_into_node(text, node) 422 self._cur_doc = None 423 424 def _parse_text_into_node(self, doctext, node): 425 """Parse a chunk of QAPI-doc-format text into the node 426 427 The doc comment can contain most inline rST markup, including 428 bulleted and enumerated lists. 429 As an extra permitted piece of markup, @var will be turned 430 into ``var``. 431 """ 432 433 # Handle the "@var means ``var`` case 434 doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext) 435 436 rstlist = ViewList() 437 for line in doctext.splitlines(): 438 # The reported line number will always be that of the start line 439 # of the doc comment, rather than the actual location of the error. 440 # Being more precise would require overhaul of the QAPIDoc class 441 # to track lines more exactly within all the sub-parts of the doc 442 # comment, as well as counting lines here. 443 rstlist.append(line, self._cur_doc.info.fname, 444 self._cur_doc.info.line) 445 # Append a blank line -- in some cases rST syntax errors get 446 # attributed to the line after one with actual text, and if there 447 # isn't anything in the ViewList corresponding to that then Sphinx 448 # 1.6's AutodocReporter will then misidentify the source/line location 449 # in the error message (usually attributing it to the top-level 450 # .rst file rather than the offending .json file). The extra blank 451 # line won't affect the rendered output. 452 rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) 453 self._sphinx_directive.do_parse(rstlist, node) 454 455 def get_document_nodes(self): 456 """Return the list of docutils nodes which make up the document""" 457 return self._top_node.children 458 459 460# Turn the black formatter on for the rest of the file. 461# fmt: on 462 463 464class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 465 """A QAPI schema visitor which adds Sphinx dependencies each module 466 467 This class calls the Sphinx note_dependency() function to tell Sphinx 468 that the generated documentation output depends on the input 469 schema file associated with each module in the QAPI input. 470 """ 471 472 def __init__(self, env, qapidir): 473 self._env = env 474 self._qapidir = qapidir 475 476 def visit_module(self, name): 477 if name != "./builtin": 478 qapifile = self._qapidir + "/" + name 479 self._env.note_dependency(os.path.abspath(qapifile)) 480 super().visit_module(name) 481 482 483class NestedDirective(Directive): 484 def run(self): 485 raise NotImplementedError 486 487 def do_parse(self, rstlist, node): 488 """ 489 Parse rST source lines and add them to the specified node 490 491 Take the list of rST source lines rstlist, parse them as 492 rST, and add the resulting docutils nodes as children of node. 493 The nodes are parsed in a way that allows them to include 494 subheadings (titles) without confusing the rendering of 495 anything else. 496 """ 497 with switch_source_input(self.state, rstlist): 498 nested_parse_with_titles(self.state, rstlist, node) 499 500 501class QAPIDocDirective(NestedDirective): 502 """Extract documentation from the specified QAPI .json file""" 503 504 required_argument = 1 505 optional_arguments = 1 506 option_spec = {"qapifile": directives.unchanged_required} 507 has_content = False 508 509 def new_serialno(self): 510 """Return a unique new ID string suitable for use as a node's ID""" 511 env = self.state.document.settings.env 512 return "qapidoc-%d" % env.new_serialno("qapidoc") 513 514 def run(self): 515 env = self.state.document.settings.env 516 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 517 qapidir = os.path.dirname(qapifile) 518 519 try: 520 schema = QAPISchema(qapifile) 521 522 # First tell Sphinx about all the schema files that the 523 # output documentation depends on (including 'qapifile' itself) 524 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 525 526 vis = QAPISchemaGenRSTVisitor(self) 527 vis.visit_begin(schema) 528 for doc in schema.docs: 529 if doc.symbol: 530 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 531 else: 532 vis.freeform(doc) 533 return vis.get_document_nodes() 534 except QAPIError as err: 535 # Launder QAPI parse errors into Sphinx extension errors 536 # so they are displayed nicely to the user 537 raise ExtensionError(str(err)) from err 538 539 540class QMPExample(CodeBlock, NestedDirective): 541 """ 542 Custom admonition for QMP code examples. 543 544 When the :annotated: option is present, the body of this directive 545 is parsed as normal rST, but with any '::' code blocks set to use 546 the QMP lexer. Code blocks must be explicitly written by the user, 547 but this allows for intermingling explanatory paragraphs with 548 arbitrary rST syntax and code blocks for more involved examples. 549 550 When :annotated: is absent, the directive body is treated as a 551 simple standalone QMP code block literal. 552 """ 553 554 required_argument = 0 555 optional_arguments = 0 556 has_content = True 557 option_spec = { 558 "annotated": directives.flag, 559 "title": directives.unchanged, 560 } 561 562 def _highlightlang(self) -> addnodes.highlightlang: 563 """Return the current highlightlang setting for the document""" 564 node = None 565 doc = self.state.document 566 567 if hasattr(doc, "findall"): 568 # docutils >= 0.18.1 569 for node in doc.findall(addnodes.highlightlang): 570 pass 571 else: 572 for elem in doc.traverse(): 573 if isinstance(elem, addnodes.highlightlang): 574 node = elem 575 576 if node: 577 return node 578 579 # No explicit directive found, use defaults 580 node = addnodes.highlightlang( 581 lang=self.env.config.highlight_language, 582 force=False, 583 # Yes, Sphinx uses this value to effectively disable line 584 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 585 linenothreshold=sys.maxsize, 586 ) 587 return node 588 589 def admonition_wrap(self, *content) -> List[nodes.Node]: 590 title = "Example:" 591 if "title" in self.options: 592 title = f"{title} {self.options['title']}" 593 594 admon = nodes.admonition( 595 "", 596 nodes.title("", title), 597 *content, 598 classes=["admonition", "admonition-example"], 599 ) 600 return [admon] 601 602 def run_annotated(self) -> List[nodes.Node]: 603 lang_node = self._highlightlang() 604 605 content_node: nodes.Element = nodes.section() 606 607 # Configure QMP highlighting for "::" blocks, if needed 608 if lang_node["lang"] != "QMP": 609 content_node += addnodes.highlightlang( 610 lang="QMP", 611 force=False, # "True" ignores lexing errors 612 linenothreshold=lang_node["linenothreshold"], 613 ) 614 615 self.do_parse(self.content, content_node) 616 617 # Restore prior language highlighting, if needed 618 if lang_node["lang"] != "QMP": 619 content_node += addnodes.highlightlang(**lang_node.attributes) 620 621 return content_node.children 622 623 def run(self) -> List[nodes.Node]: 624 annotated = "annotated" in self.options 625 626 if annotated: 627 content_nodes = self.run_annotated() 628 else: 629 self.arguments = ["QMP"] 630 content_nodes = super().run() 631 632 return self.admonition_wrap(*content_nodes) 633 634 635def setup(app): 636 """Register qapi-doc directive with Sphinx""" 637 app.add_config_value("qapidoc_srctree", None, "env") 638 app.add_directive("qapi-doc", QAPIDocDirective) 639 app.add_directive("qmp-example", QMPExample) 640 641 return { 642 "version": __version__, 643 "parallel_read_safe": True, 644 "parallel_write_safe": True, 645 } 646