1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2020 Linaro 6# 7# This work is licensed under the terms of the GNU GPLv2 or later. 8# See the COPYING file in the top-level directory. 9 10""" 11qapidoc is a Sphinx extension that implements the qapi-doc directive 12 13The purpose of this extension is to read the documentation comments 14in QAPI schema files, and insert them all into the current document. 15 16It implements one new rST directive, "qapi-doc::". 17Each qapi-doc:: directive takes one argument, which is the 18pathname of the schema file to process, relative to the source tree. 19 20The docs/conf.py file must set the qapidoc_srctree config value to 21the root of the QEMU source tree. 22 23The Sphinx documentation on writing extensions is at: 24https://www.sphinx-doc.org/en/master/development/index.html 25""" 26 27import os 28import re 29import textwrap 30 31from docutils import nodes 32from docutils.parsers.rst import Directive, directives 33from docutils.statemachine import ViewList 34from qapi.error import QAPIError, QAPISemError 35from qapi.gen import QAPISchemaVisitor 36from qapi.schema import QAPISchema 37 38import sphinx 39from sphinx.errors import ExtensionError 40from sphinx.util.nodes import nested_parse_with_titles 41 42 43# Sphinx up to 1.6 uses AutodocReporter; 1.7 and later 44# use switch_source_input. Check borrowed from kerneldoc.py. 45USE_SSI = sphinx.__version__[:3] >= "1.7" 46if USE_SSI: 47 from sphinx.util.docutils import switch_source_input 48else: 49 from sphinx.ext.autodoc import ( # pylint: disable=no-name-in-module 50 AutodocReporter, 51 ) 52 53 54__version__ = "1.0" 55 56 57def dedent(text: str) -> str: 58 # Adjust indentation to make description text parse as paragraph. 59 60 lines = text.splitlines(True) 61 if re.match(r"\s+", lines[0]): 62 # First line is indented; description started on the line after 63 # the name. dedent the whole block. 64 return textwrap.dedent(text) 65 66 # Descr started on same line. Dedent line 2+. 67 return lines[0] + textwrap.dedent("".join(lines[1:])) 68 69 70# Disable black auto-formatter until re-enabled: 71# fmt: off 72 73 74class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): 75 """A QAPI schema visitor which generates docutils/Sphinx nodes 76 77 This class builds up a tree of docutils/Sphinx nodes corresponding 78 to documentation for the various QAPI objects. To use it, first 79 create a QAPISchemaGenRSTVisitor object, and call its 80 visit_begin() method. Then you can call one of the two methods 81 'freeform' (to add documentation for a freeform documentation 82 chunk) or 'symbol' (to add documentation for a QAPI symbol). These 83 will cause the visitor to build up the tree of document 84 nodes. Once you've added all the documentation via 'freeform' and 85 'symbol' method calls, you can call 'get_document_nodes' to get 86 the final list of document nodes (in a form suitable for returning 87 from a Sphinx directive's 'run' method). 88 """ 89 def __init__(self, sphinx_directive): 90 self._cur_doc = None 91 self._sphinx_directive = sphinx_directive 92 self._top_node = nodes.section() 93 self._active_headings = [self._top_node] 94 95 def _make_dlitem(self, term, defn): 96 """Return a dlitem node with the specified term and definition. 97 98 term should be a list of Text and literal nodes. 99 defn should be one of: 100 - a string, which will be handed to _parse_text_into_node 101 - a list of Text and literal nodes, which will be put into 102 a paragraph node 103 """ 104 dlitem = nodes.definition_list_item() 105 dlterm = nodes.term('', '', *term) 106 dlitem += dlterm 107 if defn: 108 dldef = nodes.definition() 109 if isinstance(defn, list): 110 dldef += nodes.paragraph('', '', *defn) 111 else: 112 self._parse_text_into_node(defn, dldef) 113 dlitem += dldef 114 return dlitem 115 116 def _make_section(self, title): 117 """Return a section node with optional title""" 118 section = nodes.section(ids=[self._sphinx_directive.new_serialno()]) 119 if title: 120 section += nodes.title(title, title) 121 return section 122 123 def _nodes_for_ifcond(self, ifcond, with_if=True): 124 """Return list of Text, literal nodes for the ifcond 125 126 Return a list which gives text like ' (If: condition)'. 127 If with_if is False, we don't return the "(If: " and ")". 128 """ 129 130 doc = ifcond.docgen() 131 if not doc: 132 return [] 133 doc = nodes.literal('', doc) 134 if not with_if: 135 return [doc] 136 137 nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')] 138 nodelist.append(doc) 139 nodelist.append(nodes.Text(')')) 140 return nodelist 141 142 def _nodes_for_one_member(self, member): 143 """Return list of Text, literal nodes for this member 144 145 Return a list of doctree nodes which give text like 146 'name: type (optional) (If: ...)' suitable for use as the 147 'term' part of a definition list item. 148 """ 149 term = [nodes.literal('', member.name)] 150 if member.type.doc_type(): 151 term.append(nodes.Text(': ')) 152 term.append(nodes.literal('', member.type.doc_type())) 153 if member.optional: 154 term.append(nodes.Text(' (optional)')) 155 if member.ifcond.is_present(): 156 term.extend(self._nodes_for_ifcond(member.ifcond)) 157 return term 158 159 def _nodes_for_variant_when(self, branches, variant): 160 """Return list of Text, literal nodes for variant 'when' clause 161 162 Return a list of doctree nodes which give text like 163 'when tagname is variant (If: ...)' suitable for use in 164 the 'branches' part of a definition list. 165 """ 166 term = [nodes.Text(' when '), 167 nodes.literal('', branches.tag_member.name), 168 nodes.Text(' is '), 169 nodes.literal('', '"%s"' % variant.name)] 170 if variant.ifcond.is_present(): 171 term.extend(self._nodes_for_ifcond(variant.ifcond)) 172 return term 173 174 def _nodes_for_members(self, doc, what, base=None, branches=None): 175 """Return list of doctree nodes for the table of members""" 176 dlnode = nodes.definition_list() 177 for section in doc.args.values(): 178 term = self._nodes_for_one_member(section.member) 179 # TODO drop fallbacks when undocumented members are outlawed 180 if section.text: 181 defn = dedent(section.text) 182 else: 183 defn = [nodes.Text('Not documented')] 184 185 dlnode += self._make_dlitem(term, defn) 186 187 if base: 188 dlnode += self._make_dlitem([nodes.Text('The members of '), 189 nodes.literal('', base.doc_type())], 190 None) 191 192 if branches: 193 for v in branches.variants: 194 if v.type.name == 'q_empty': 195 continue 196 assert not v.type.is_implicit() 197 term = [nodes.Text('The members of '), 198 nodes.literal('', v.type.doc_type())] 199 term.extend(self._nodes_for_variant_when(branches, v)) 200 dlnode += self._make_dlitem(term, None) 201 202 if not dlnode.children: 203 return [] 204 205 section = self._make_section(what) 206 section += dlnode 207 return [section] 208 209 def _nodes_for_enum_values(self, doc): 210 """Return list of doctree nodes for the table of enum values""" 211 seen_item = False 212 dlnode = nodes.definition_list() 213 for section in doc.args.values(): 214 termtext = [nodes.literal('', section.member.name)] 215 if section.member.ifcond.is_present(): 216 termtext.extend(self._nodes_for_ifcond(section.member.ifcond)) 217 # TODO drop fallbacks when undocumented members are outlawed 218 if section.text: 219 defn = dedent(section.text) 220 else: 221 defn = [nodes.Text('Not documented')] 222 223 dlnode += self._make_dlitem(termtext, defn) 224 seen_item = True 225 226 if not seen_item: 227 return [] 228 229 section = self._make_section('Values') 230 section += dlnode 231 return [section] 232 233 def _nodes_for_arguments(self, doc, arg_type): 234 """Return list of doctree nodes for the arguments section""" 235 if arg_type and not arg_type.is_implicit(): 236 assert not doc.args 237 section = self._make_section('Arguments') 238 dlnode = nodes.definition_list() 239 dlnode += self._make_dlitem( 240 [nodes.Text('The members of '), 241 nodes.literal('', arg_type.name)], 242 None) 243 section += dlnode 244 return [section] 245 246 return self._nodes_for_members(doc, 'Arguments') 247 248 def _nodes_for_features(self, doc): 249 """Return list of doctree nodes for the table of features""" 250 seen_item = False 251 dlnode = nodes.definition_list() 252 for section in doc.features.values(): 253 dlnode += self._make_dlitem( 254 [nodes.literal('', section.member.name)], dedent(section.text)) 255 seen_item = True 256 257 if not seen_item: 258 return [] 259 260 section = self._make_section('Features') 261 section += dlnode 262 return [section] 263 264 def _nodes_for_example(self, exampletext): 265 """Return list of doctree nodes for a code example snippet""" 266 return [nodes.literal_block(exampletext, exampletext)] 267 268 def _nodes_for_sections(self, doc): 269 """Return list of doctree nodes for additional sections""" 270 nodelist = [] 271 for section in doc.sections: 272 if section.tag and section.tag == 'TODO': 273 # Hide TODO: sections 274 continue 275 276 if not section.tag: 277 # Sphinx cannot handle sectionless titles; 278 # Instead, just append the results to the prior section. 279 container = nodes.container() 280 self._parse_text_into_node(section.text, container) 281 nodelist += container.children 282 continue 283 284 snode = self._make_section(section.tag) 285 if section.tag.startswith('Example'): 286 snode += self._nodes_for_example(dedent(section.text)) 287 else: 288 self._parse_text_into_node(dedent(section.text), snode) 289 nodelist.append(snode) 290 return nodelist 291 292 def _nodes_for_if_section(self, ifcond): 293 """Return list of doctree nodes for the "If" section""" 294 nodelist = [] 295 if ifcond.is_present(): 296 snode = self._make_section('If') 297 snode += nodes.paragraph( 298 '', '', *self._nodes_for_ifcond(ifcond, with_if=False) 299 ) 300 nodelist.append(snode) 301 return nodelist 302 303 def _add_doc(self, typ, sections): 304 """Add documentation for a command/object/enum... 305 306 We assume we're documenting the thing defined in self._cur_doc. 307 typ is the type of thing being added ("Command", "Object", etc) 308 309 sections is a list of nodes for sections to add to the definition. 310 """ 311 312 doc = self._cur_doc 313 snode = nodes.section(ids=[self._sphinx_directive.new_serialno()]) 314 snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol), 315 nodes.Text(' (' + typ + ')')]) 316 self._parse_text_into_node(doc.body.text, snode) 317 for s in sections: 318 if s is not None: 319 snode += s 320 self._add_node_to_current_heading(snode) 321 322 def visit_enum_type(self, name, info, ifcond, features, members, prefix): 323 doc = self._cur_doc 324 self._add_doc('Enum', 325 self._nodes_for_enum_values(doc) 326 + self._nodes_for_features(doc) 327 + self._nodes_for_sections(doc) 328 + self._nodes_for_if_section(ifcond)) 329 330 def visit_object_type(self, name, info, ifcond, features, 331 base, members, branches): 332 doc = self._cur_doc 333 if base and base.is_implicit(): 334 base = None 335 self._add_doc('Object', 336 self._nodes_for_members(doc, 'Members', base, branches) 337 + self._nodes_for_features(doc) 338 + self._nodes_for_sections(doc) 339 + self._nodes_for_if_section(ifcond)) 340 341 def visit_alternate_type(self, name, info, ifcond, features, 342 alternatives): 343 doc = self._cur_doc 344 self._add_doc('Alternate', 345 self._nodes_for_members(doc, 'Members') 346 + self._nodes_for_features(doc) 347 + self._nodes_for_sections(doc) 348 + self._nodes_for_if_section(ifcond)) 349 350 def visit_command(self, name, info, ifcond, features, arg_type, 351 ret_type, gen, success_response, boxed, allow_oob, 352 allow_preconfig, coroutine): 353 doc = self._cur_doc 354 self._add_doc('Command', 355 self._nodes_for_arguments(doc, arg_type) 356 + self._nodes_for_features(doc) 357 + self._nodes_for_sections(doc) 358 + self._nodes_for_if_section(ifcond)) 359 360 def visit_event(self, name, info, ifcond, features, arg_type, boxed): 361 doc = self._cur_doc 362 self._add_doc('Event', 363 self._nodes_for_arguments(doc, arg_type) 364 + self._nodes_for_features(doc) 365 + self._nodes_for_sections(doc) 366 + self._nodes_for_if_section(ifcond)) 367 368 def symbol(self, doc, entity): 369 """Add documentation for one symbol to the document tree 370 371 This is the main entry point which causes us to add documentation 372 nodes for a symbol (which could be a 'command', 'object', 'event', 373 etc). We do this by calling 'visit' on the schema entity, which 374 will then call back into one of our visit_* methods, depending 375 on what kind of thing this symbol is. 376 """ 377 self._cur_doc = doc 378 entity.visit(self) 379 self._cur_doc = None 380 381 def _start_new_heading(self, heading, level): 382 """Start a new heading at the specified heading level 383 384 Create a new section whose title is 'heading' and which is placed 385 in the docutils node tree as a child of the most recent level-1 386 heading. Subsequent document sections (commands, freeform doc chunks, 387 etc) will be placed as children of this new heading section. 388 """ 389 if len(self._active_headings) < level: 390 raise QAPISemError(self._cur_doc.info, 391 'Level %d subheading found outside a ' 392 'level %d heading' 393 % (level, level - 1)) 394 snode = self._make_section(heading) 395 self._active_headings[level - 1] += snode 396 self._active_headings = self._active_headings[:level] 397 self._active_headings.append(snode) 398 399 def _add_node_to_current_heading(self, node): 400 """Add the node to whatever the current active heading is""" 401 self._active_headings[-1] += node 402 403 def freeform(self, doc): 404 """Add a piece of 'freeform' documentation to the document tree 405 406 A 'freeform' document chunk doesn't relate to any particular 407 symbol (for instance, it could be an introduction). 408 409 If the freeform document starts with a line of the form 410 '= Heading text', this is a section or subsection heading, with 411 the heading level indicated by the number of '=' signs. 412 """ 413 414 # QAPIDoc documentation says free-form documentation blocks 415 # must have only a body section, nothing else. 416 assert not doc.sections 417 assert not doc.args 418 assert not doc.features 419 self._cur_doc = doc 420 421 text = doc.body.text 422 if re.match(r'=+ ', text): 423 # Section/subsection heading (if present, will always be 424 # the first line of the block) 425 (heading, _, text) = text.partition('\n') 426 (leader, _, heading) = heading.partition(' ') 427 self._start_new_heading(heading, len(leader)) 428 if text == '': 429 return 430 431 node = self._make_section(None) 432 self._parse_text_into_node(text, node) 433 self._add_node_to_current_heading(node) 434 self._cur_doc = None 435 436 def _parse_text_into_node(self, doctext, node): 437 """Parse a chunk of QAPI-doc-format text into the node 438 439 The doc comment can contain most inline rST markup, including 440 bulleted and enumerated lists. 441 As an extra permitted piece of markup, @var will be turned 442 into ``var``. 443 """ 444 445 # Handle the "@var means ``var`` case 446 doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext) 447 448 rstlist = ViewList() 449 for line in doctext.splitlines(): 450 # The reported line number will always be that of the start line 451 # of the doc comment, rather than the actual location of the error. 452 # Being more precise would require overhaul of the QAPIDoc class 453 # to track lines more exactly within all the sub-parts of the doc 454 # comment, as well as counting lines here. 455 rstlist.append(line, self._cur_doc.info.fname, 456 self._cur_doc.info.line) 457 # Append a blank line -- in some cases rST syntax errors get 458 # attributed to the line after one with actual text, and if there 459 # isn't anything in the ViewList corresponding to that then Sphinx 460 # 1.6's AutodocReporter will then misidentify the source/line location 461 # in the error message (usually attributing it to the top-level 462 # .rst file rather than the offending .json file). The extra blank 463 # line won't affect the rendered output. 464 rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) 465 self._sphinx_directive.do_parse(rstlist, node) 466 467 def get_document_nodes(self): 468 """Return the list of docutils nodes which make up the document""" 469 return self._top_node.children 470 471 472# Turn the black formatter on for the rest of the file. 473# fmt: on 474 475 476class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 477 """A QAPI schema visitor which adds Sphinx dependencies each module 478 479 This class calls the Sphinx note_dependency() function to tell Sphinx 480 that the generated documentation output depends on the input 481 schema file associated with each module in the QAPI input. 482 """ 483 484 def __init__(self, env, qapidir): 485 self._env = env 486 self._qapidir = qapidir 487 488 def visit_module(self, name): 489 if name != "./builtin": 490 qapifile = self._qapidir + "/" + name 491 self._env.note_dependency(os.path.abspath(qapifile)) 492 super().visit_module(name) 493 494 495class QAPIDocDirective(Directive): 496 """Extract documentation from the specified QAPI .json file""" 497 498 required_argument = 1 499 optional_arguments = 1 500 option_spec = {"qapifile": directives.unchanged_required} 501 has_content = False 502 503 def new_serialno(self): 504 """Return a unique new ID string suitable for use as a node's ID""" 505 env = self.state.document.settings.env 506 return "qapidoc-%d" % env.new_serialno("qapidoc") 507 508 def run(self): 509 env = self.state.document.settings.env 510 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 511 qapidir = os.path.dirname(qapifile) 512 513 try: 514 schema = QAPISchema(qapifile) 515 516 # First tell Sphinx about all the schema files that the 517 # output documentation depends on (including 'qapifile' itself) 518 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 519 520 vis = QAPISchemaGenRSTVisitor(self) 521 vis.visit_begin(schema) 522 for doc in schema.docs: 523 if doc.symbol: 524 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 525 else: 526 vis.freeform(doc) 527 return vis.get_document_nodes() 528 except QAPIError as err: 529 # Launder QAPI parse errors into Sphinx extension errors 530 # so they are displayed nicely to the user 531 raise ExtensionError(str(err)) from err 532 533 def do_parse(self, rstlist, node): 534 """Parse rST source lines and add them to the specified node 535 536 Take the list of rST source lines rstlist, parse them as 537 rST, and add the resulting docutils nodes as children of node. 538 The nodes are parsed in a way that allows them to include 539 subheadings (titles) without confusing the rendering of 540 anything else. 541 """ 542 # This is from kerneldoc.py -- it works around an API change in 543 # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use 544 # sphinx.util.nodes.nested_parse_with_titles() rather than the 545 # plain self.state.nested_parse(), and so we can drop the saving 546 # of title_styles and section_level that kerneldoc.py does, 547 # because nested_parse_with_titles() does that for us. 548 if USE_SSI: 549 with switch_source_input(self.state, rstlist): 550 nested_parse_with_titles(self.state, rstlist, node) 551 else: 552 save = self.state.memo.reporter 553 self.state.memo.reporter = AutodocReporter( 554 rstlist, self.state.memo.reporter 555 ) 556 try: 557 nested_parse_with_titles(self.state, rstlist, node) 558 finally: 559 self.state.memo.reporter = save 560 561 562def setup(app): 563 """Register qapi-doc directive with Sphinx""" 564 app.add_config_value("qapidoc_srctree", None, "env") 565 app.add_directive("qapi-doc", QAPIDocDirective) 566 567 return { 568 "version": __version__, 569 "parallel_read_safe": True, 570 "parallel_write_safe": True, 571 } 572