1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30__version__ = "2.0" 31 32from contextlib import contextmanager 33import os 34from pathlib import Path 35import re 36import sys 37from typing import TYPE_CHECKING 38 39from docutils import nodes 40from docutils.parsers.rst import directives 41from docutils.statemachine import StringList 42from qapi.error import QAPIError 43from qapi.parser import QAPIDoc 44from qapi.schema import ( 45 QAPISchema, 46 QAPISchemaArrayType, 47 QAPISchemaCommand, 48 QAPISchemaDefinition, 49 QAPISchemaEnumMember, 50 QAPISchemaEvent, 51 QAPISchemaFeature, 52 QAPISchemaMember, 53 QAPISchemaObjectType, 54 QAPISchemaObjectTypeMember, 55 QAPISchemaType, 56 QAPISchemaVisitor, 57) 58from qapi.source import QAPISourceInfo 59 60from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore 61from sphinx import addnodes 62from sphinx.directives.code import CodeBlock 63from sphinx.errors import ExtensionError 64from sphinx.util import logging 65from sphinx.util.docutils import SphinxDirective, switch_source_input 66from sphinx.util.nodes import nested_parse_with_titles 67 68 69if TYPE_CHECKING: 70 from typing import ( 71 Any, 72 Generator, 73 List, 74 Optional, 75 Sequence, 76 Union, 77 ) 78 79 from sphinx.application import Sphinx 80 from sphinx.util.typing import ExtensionMetadata 81 82 83logger = logging.getLogger(__name__) 84 85 86class Transmogrifier: 87 # pylint: disable=too-many-public-methods 88 89 # Field names used for different entity types: 90 field_types = { 91 "enum": "value", 92 "struct": "memb", 93 "union": "memb", 94 "event": "memb", 95 "command": "arg", 96 "alternate": "alt", 97 } 98 99 def __init__(self) -> None: 100 self._curr_ent: Optional[QAPISchemaDefinition] = None 101 self._result = StringList() 102 self.indent = 0 103 104 @property 105 def result(self) -> StringList: 106 return self._result 107 108 @property 109 def entity(self) -> QAPISchemaDefinition: 110 assert self._curr_ent is not None 111 return self._curr_ent 112 113 @property 114 def member_field_type(self) -> str: 115 return self.field_types[self.entity.meta] 116 117 # General-purpose rST generation functions 118 119 def get_indent(self) -> str: 120 return " " * self.indent 121 122 @contextmanager 123 def indented(self) -> Generator[None]: 124 self.indent += 1 125 try: 126 yield 127 finally: 128 self.indent -= 1 129 130 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 131 """Append one line of generated reST to the output.""" 132 133 # NB: Sphinx uses zero-indexed lines; subtract one. 134 lineno = tuple((n - 1 for n in lineno)) 135 136 if line.strip(): 137 # not a blank line 138 self._result.append( 139 self.get_indent() + line.rstrip("\n"), source, *lineno 140 ) 141 else: 142 self._result.append("", source, *lineno) 143 144 def add_line(self, content: str, info: QAPISourceInfo) -> None: 145 # NB: We *require* an info object; this works out OK because we 146 # don't document built-in objects that don't have 147 # one. Everything else should. 148 self.add_line_raw(content, info.fname, info.line) 149 150 def add_lines( 151 self, 152 content: str, 153 info: QAPISourceInfo, 154 ) -> None: 155 lines = content.splitlines(True) 156 for i, line in enumerate(lines): 157 self.add_line_raw(line, info.fname, info.line + i) 158 159 def ensure_blank_line(self) -> None: 160 # Empty document -- no blank line required. 161 if not self._result: 162 return 163 164 # Last line isn't blank, add one. 165 if self._result[-1].strip(): # pylint: disable=no-member 166 fname, line = self._result.info(-1) 167 assert isinstance(line, int) 168 # New blank line is credited to one-after the current last line. 169 # +2: correct for zero/one index, then increment by one. 170 self.add_line_raw("", fname, line + 2) 171 172 def add_field( 173 self, 174 kind: str, 175 name: str, 176 body: str, 177 info: QAPISourceInfo, 178 typ: Optional[str] = None, 179 ) -> None: 180 if typ: 181 text = f":{kind} {typ} {name}: {body}" 182 else: 183 text = f":{kind} {name}: {body}" 184 self.add_lines(text, info) 185 186 def format_type( 187 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 188 ) -> Optional[str]: 189 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 190 return None 191 192 qapi_type = ent 193 optional = False 194 if isinstance(ent, QAPISchemaObjectTypeMember): 195 qapi_type = ent.type 196 optional = ent.optional 197 198 if isinstance(qapi_type, QAPISchemaArrayType): 199 ret = f"[{qapi_type.element_type.doc_type()}]" 200 else: 201 assert isinstance(qapi_type, QAPISchemaType) 202 tmp = qapi_type.doc_type() 203 assert tmp 204 ret = tmp 205 if optional: 206 ret += "?" 207 208 return ret 209 210 def generate_field( 211 self, 212 kind: str, 213 member: QAPISchemaMember, 214 body: str, 215 info: QAPISourceInfo, 216 ) -> None: 217 typ = self.format_type(member) 218 self.add_field(kind, member.name, body, info, typ) 219 220 # Transmogrification helpers 221 222 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 223 # Squelch empty paragraphs. 224 if not section.text: 225 return 226 227 self.ensure_blank_line() 228 self.add_lines(section.text, section.info) 229 self.ensure_blank_line() 230 231 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 232 # FIXME: ifcond for members 233 # TODO: features for members (documented at entity-level, 234 # but sometimes defined per-member. Should we add such 235 # information to member descriptions when we can?) 236 assert section.member 237 self.generate_field( 238 self.member_field_type, 239 section.member, 240 # TODO drop fallbacks when undocumented members are outlawed 241 section.text if section.text else "Not documented", 242 section.info, 243 ) 244 245 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 246 # FIXME - ifcond for features is not handled at all yet! 247 # Proposal: decorate the right-hand column with some graphical 248 # element to indicate conditional availability? 249 assert section.text # Guaranteed by parser.py 250 assert section.member 251 252 self.generate_field("feat", section.member, section.text, section.info) 253 254 def visit_returns(self, section: QAPIDoc.Section) -> None: 255 assert isinstance(self.entity, QAPISchemaCommand) 256 rtype = self.entity.ret_type 257 # q_empty can produce None, but we won't be documenting anything 258 # without an explicit return statement in the doc block, and we 259 # should not have any such explicit statements when there is no 260 # return value. 261 assert rtype 262 263 typ = self.format_type(rtype) 264 assert typ 265 assert section.text 266 self.add_field("return", typ, section.text, section.info) 267 268 def visit_errors(self, section: QAPIDoc.Section) -> None: 269 # FIXME: the formatting for errors may be inconsistent and may 270 # or may not require different newline placement to ensure 271 # proper rendering as a nested list. 272 self.add_lines(f":error:\n{section.text}", section.info) 273 274 def preamble(self, ent: QAPISchemaDefinition) -> None: 275 """ 276 Generate option lines for QAPI entity directives. 277 """ 278 if ent.doc and ent.doc.since: 279 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 280 # Generated from the entity's docblock; info location is exact. 281 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 282 283 if ent.ifcond.is_present(): 284 doc = ent.ifcond.docgen() 285 assert ent.info 286 # Generated from entity definition; info location is approximate. 287 self.add_line(f":ifcond: {doc}", ent.info) 288 289 # Hoist special features such as :deprecated: and :unstable: 290 # into the options block for the entity. If, in the future, new 291 # special features are added, qapi-domain will chirp about 292 # unrecognized options and fail until they are handled in 293 # qapi-domain. 294 for feat in ent.features: 295 if feat.is_special(): 296 # FIXME: handle ifcond if present. How to display that 297 # information is TBD. 298 # Generated from entity def; info location is approximate. 299 assert feat.info 300 self.add_line(f":{feat.name}:", feat.info) 301 302 self.ensure_blank_line() 303 304 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 305 306 def _get_target( 307 ent: QAPISchemaDefinition, 308 ) -> Optional[QAPISchemaDefinition]: 309 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 310 return ent.arg_type 311 if isinstance(ent, QAPISchemaObjectType): 312 return ent.base 313 return None 314 315 target = _get_target(ent) 316 if target is not None and not target.is_implicit(): 317 assert ent.info 318 self.add_field( 319 self.member_field_type, 320 "q_dummy", 321 f"The members of :qapi:type:`{target.name}`.", 322 ent.info, 323 "q_dummy", 324 ) 325 326 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 327 for variant in ent.branches.variants: 328 if variant.type.name == "q_empty": 329 continue 330 assert ent.info 331 self.add_field( 332 self.member_field_type, 333 "q_dummy", 334 f" When ``{ent.branches.tag_member.name}`` is " 335 f"``{variant.name}``: " 336 f"The members of :qapi:type:`{variant.type.name}`.", 337 ent.info, 338 "q_dummy", 339 ) 340 341 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 342 sections = ent.doc.all_sections if ent.doc else [] 343 344 # Determine the index location at which we should generate 345 # documentation for "The members of ..." pointers. This should 346 # go at the end of the members section(s) if any. Note that 347 # index 0 is assumed to be a plain intro section, even if it is 348 # empty; and that a members section if present will always 349 # immediately follow the opening PLAIN section. 350 gen_index = 1 351 if len(sections) > 1: 352 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 353 gen_index += 1 354 if gen_index >= len(sections): 355 break 356 357 # Add sections in source order: 358 for i, section in enumerate(sections): 359 # @var is translated to ``var``: 360 section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) 361 362 if section.kind == QAPIDoc.Kind.PLAIN: 363 self.visit_paragraph(section) 364 elif section.kind == QAPIDoc.Kind.MEMBER: 365 assert isinstance(section, QAPIDoc.ArgSection) 366 self.visit_member(section) 367 elif section.kind == QAPIDoc.Kind.FEATURE: 368 assert isinstance(section, QAPIDoc.ArgSection) 369 self.visit_feature(section) 370 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 371 # Since is handled in preamble, TODO is skipped intentionally. 372 pass 373 elif section.kind == QAPIDoc.Kind.RETURNS: 374 self.visit_returns(section) 375 elif section.kind == QAPIDoc.Kind.ERRORS: 376 self.visit_errors(section) 377 else: 378 assert False 379 380 # Generate "The members of ..." entries if necessary: 381 if i == gen_index - 1: 382 self._insert_member_pointer(ent) 383 384 self.ensure_blank_line() 385 386 # Transmogrification core methods 387 388 def visit_module(self, path: str) -> None: 389 name = Path(path).stem 390 # module directives are credited to the first line of a module file. 391 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 392 self.ensure_blank_line() 393 394 def visit_freeform(self, doc: QAPIDoc) -> None: 395 # TODO: Once the old qapidoc transformer is deprecated, freeform 396 # sections can be updated to pure rST, and this transformed removed. 397 # 398 # For now, translate our micro-format into rST. Code adapted 399 # from Peter Maydell's freeform(). 400 401 assert len(doc.all_sections) == 1, doc.all_sections 402 body = doc.all_sections[0] 403 text = body.text 404 info = doc.info 405 406 if re.match(r"=+ ", text): 407 # Section/subsection heading (if present, will always be the 408 # first line of the block) 409 (heading, _, text) = text.partition("\n") 410 (leader, _, heading) = heading.partition(" ") 411 # Implicit +1 for heading in the containing .rst doc 412 level = len(leader) + 1 413 414 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 415 markers = ' #*=_^"' 416 overline = level <= 2 417 marker = markers[level] 418 419 self.ensure_blank_line() 420 # This credits all 2 or 3 lines to the single source line. 421 if overline: 422 self.add_line(marker * len(heading), info) 423 self.add_line(heading, info) 424 self.add_line(marker * len(heading), info) 425 self.ensure_blank_line() 426 427 # Eat blank line(s) and advance info 428 trimmed = text.lstrip("\n") 429 text = trimmed 430 info = info.next_line(len(text) - len(trimmed) + 1) 431 432 self.add_lines(text, info) 433 self.ensure_blank_line() 434 435 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 436 assert ent.info 437 438 try: 439 self._curr_ent = ent 440 441 # Squish structs and unions together into an "object" directive. 442 meta = ent.meta 443 if meta in ("struct", "union"): 444 meta = "object" 445 446 # This line gets credited to the start of the /definition/. 447 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 448 with self.indented(): 449 self.preamble(ent) 450 self.visit_sections(ent) 451 finally: 452 self._curr_ent = None 453 454 455class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 456 """A QAPI schema visitor which adds Sphinx dependencies each module 457 458 This class calls the Sphinx note_dependency() function to tell Sphinx 459 that the generated documentation output depends on the input 460 schema file associated with each module in the QAPI input. 461 """ 462 463 def __init__(self, env: Any, qapidir: str) -> None: 464 self._env = env 465 self._qapidir = qapidir 466 467 def visit_module(self, name: str) -> None: 468 if name != "./builtin": 469 qapifile = self._qapidir + "/" + name 470 self._env.note_dependency(os.path.abspath(qapifile)) 471 super().visit_module(name) 472 473 474class NestedDirective(SphinxDirective): 475 def run(self) -> Sequence[nodes.Node]: 476 raise NotImplementedError 477 478 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 479 """ 480 Parse rST source lines and add them to the specified node 481 482 Take the list of rST source lines rstlist, parse them as 483 rST, and add the resulting docutils nodes as children of node. 484 The nodes are parsed in a way that allows them to include 485 subheadings (titles) without confusing the rendering of 486 anything else. 487 """ 488 with switch_source_input(self.state, rstlist): 489 nested_parse_with_titles(self.state, rstlist, node) 490 491 492class QAPIDocDirective(NestedDirective): 493 """Extract documentation from the specified QAPI .json file""" 494 495 required_argument = 1 496 optional_arguments = 1 497 option_spec = { 498 "qapifile": directives.unchanged_required, 499 "transmogrify": directives.flag, 500 } 501 has_content = False 502 503 def new_serialno(self) -> str: 504 """Return a unique new ID string suitable for use as a node's ID""" 505 env = self.state.document.settings.env 506 return "qapidoc-%d" % env.new_serialno("qapidoc") 507 508 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 509 logger.info("Transmogrifying QAPI to rST ...") 510 vis = Transmogrifier() 511 modules = set() 512 513 for doc in schema.docs: 514 module_source = doc.info.fname 515 if module_source not in modules: 516 vis.visit_module(module_source) 517 modules.add(module_source) 518 519 if doc.symbol: 520 ent = schema.lookup_entity(doc.symbol) 521 assert isinstance(ent, QAPISchemaDefinition) 522 vis.visit_entity(ent) 523 else: 524 vis.visit_freeform(doc) 525 526 logger.info("Transmogrification complete.") 527 528 contentnode = nodes.section() 529 content = vis.result 530 titles_allowed = True 531 532 logger.info("Transmogrifier running nested parse ...") 533 with switch_source_input(self.state, content): 534 if titles_allowed: 535 node: nodes.Element = nodes.section() 536 node.document = self.state.document 537 nested_parse_with_titles(self.state, content, contentnode) 538 else: 539 node = nodes.paragraph() 540 node.document = self.state.document 541 self.state.nested_parse(content, 0, contentnode) 542 logger.info("Transmogrifier's nested parse completed.") 543 544 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 545 argname = "_".join(Path(self.arguments[0]).parts) 546 name = Path(argname).stem + ".ir" 547 self.write_intermediate(content, name) 548 549 sys.stdout.flush() 550 return contentnode 551 552 def write_intermediate(self, content: StringList, filename: str) -> None: 553 logger.info( 554 "writing intermediate rST for '%s' to '%s'", 555 self.arguments[0], 556 filename, 557 ) 558 559 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 560 outlines = [] 561 lcol_width = 0 562 563 for i, line in enumerate(content): 564 src, lineno = content.info(i) 565 srcpath = Path(src).resolve() 566 srcpath = srcpath.relative_to(srctree) 567 568 lcol = f"{srcpath}:{lineno:04d}" 569 lcol_width = max(lcol_width, len(lcol)) 570 outlines.append((lcol, line)) 571 572 with open(filename, "w", encoding="UTF-8") as outfile: 573 for lcol, rcol in outlines: 574 outfile.write(lcol.rjust(lcol_width)) 575 outfile.write(" |") 576 if rcol: 577 outfile.write(f" {rcol}") 578 outfile.write("\n") 579 580 def legacy(self, schema: QAPISchema) -> nodes.Element: 581 vis = QAPISchemaGenRSTVisitor(self) 582 vis.visit_begin(schema) 583 for doc in schema.docs: 584 if doc.symbol: 585 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 586 else: 587 vis.freeform(doc) 588 return vis.get_document_node() # type: ignore 589 590 def run(self) -> Sequence[nodes.Node]: 591 env = self.state.document.settings.env 592 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 593 qapidir = os.path.dirname(qapifile) 594 transmogrify = "transmogrify" in self.options 595 596 try: 597 schema = QAPISchema(qapifile) 598 599 # First tell Sphinx about all the schema files that the 600 # output documentation depends on (including 'qapifile' itself) 601 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 602 except QAPIError as err: 603 # Launder QAPI parse errors into Sphinx extension errors 604 # so they are displayed nicely to the user 605 raise ExtensionError(str(err)) from err 606 607 if transmogrify: 608 contentnode = self.transmogrify(schema) 609 else: 610 contentnode = self.legacy(schema) 611 612 return contentnode.children 613 614 615class QMPExample(CodeBlock, NestedDirective): 616 """ 617 Custom admonition for QMP code examples. 618 619 When the :annotated: option is present, the body of this directive 620 is parsed as normal rST, but with any '::' code blocks set to use 621 the QMP lexer. Code blocks must be explicitly written by the user, 622 but this allows for intermingling explanatory paragraphs with 623 arbitrary rST syntax and code blocks for more involved examples. 624 625 When :annotated: is absent, the directive body is treated as a 626 simple standalone QMP code block literal. 627 """ 628 629 required_argument = 0 630 optional_arguments = 0 631 has_content = True 632 option_spec = { 633 "annotated": directives.flag, 634 "title": directives.unchanged, 635 } 636 637 def _highlightlang(self) -> addnodes.highlightlang: 638 """Return the current highlightlang setting for the document""" 639 node = None 640 doc = self.state.document 641 642 if hasattr(doc, "findall"): 643 # docutils >= 0.18.1 644 for node in doc.findall(addnodes.highlightlang): 645 pass 646 else: 647 for elem in doc.traverse(): 648 if isinstance(elem, addnodes.highlightlang): 649 node = elem 650 651 if node: 652 return node 653 654 # No explicit directive found, use defaults 655 node = addnodes.highlightlang( 656 lang=self.env.config.highlight_language, 657 force=False, 658 # Yes, Sphinx uses this value to effectively disable line 659 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 660 linenothreshold=sys.maxsize, 661 ) 662 return node 663 664 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 665 title = "Example:" 666 if "title" in self.options: 667 title = f"{title} {self.options['title']}" 668 669 admon = nodes.admonition( 670 "", 671 nodes.title("", title), 672 *content, 673 classes=["admonition", "admonition-example"], 674 ) 675 return [admon] 676 677 def run_annotated(self) -> List[nodes.Node]: 678 lang_node = self._highlightlang() 679 680 content_node: nodes.Element = nodes.section() 681 682 # Configure QMP highlighting for "::" blocks, if needed 683 if lang_node["lang"] != "QMP": 684 content_node += addnodes.highlightlang( 685 lang="QMP", 686 force=False, # "True" ignores lexing errors 687 linenothreshold=lang_node["linenothreshold"], 688 ) 689 690 self.do_parse(self.content, content_node) 691 692 # Restore prior language highlighting, if needed 693 if lang_node["lang"] != "QMP": 694 content_node += addnodes.highlightlang(**lang_node.attributes) 695 696 return content_node.children 697 698 def run(self) -> List[nodes.Node]: 699 annotated = "annotated" in self.options 700 701 if annotated: 702 content_nodes = self.run_annotated() 703 else: 704 self.arguments = ["QMP"] 705 content_nodes = super().run() 706 707 return self.admonition_wrap(*content_nodes) 708 709 710def setup(app: Sphinx) -> ExtensionMetadata: 711 """Register qapi-doc directive with Sphinx""" 712 app.setup_extension("qapi_domain") 713 app.add_config_value("qapidoc_srctree", None, "env") 714 app.add_directive("qapi-doc", QAPIDocDirective) 715 app.add_directive("qmp-example", QMPExample) 716 717 return { 718 "version": __version__, 719 "parallel_read_safe": True, 720 "parallel_write_safe": True, 721 } 722