1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30 31__version__ = "2.0" 32 33from contextlib import contextmanager 34import os 35from pathlib import Path 36import re 37import sys 38from typing import TYPE_CHECKING 39 40from docutils import nodes 41from docutils.parsers.rst import directives 42from docutils.statemachine import StringList 43from qapi.error import QAPIError 44from qapi.parser import QAPIDoc 45from qapi.schema import ( 46 QAPISchema, 47 QAPISchemaArrayType, 48 QAPISchemaCommand, 49 QAPISchemaDefinition, 50 QAPISchemaEnumMember, 51 QAPISchemaEvent, 52 QAPISchemaFeature, 53 QAPISchemaMember, 54 QAPISchemaObjectType, 55 QAPISchemaObjectTypeMember, 56 QAPISchemaType, 57 QAPISchemaVisitor, 58) 59from qapi.source import QAPISourceInfo 60from sphinx import addnodes 61from sphinx.directives.code import CodeBlock 62from sphinx.errors import ExtensionError 63from sphinx.util import logging 64from sphinx.util.docutils import SphinxDirective, switch_source_input 65from sphinx.util.nodes import nested_parse_with_titles 66 67from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore 68 69 70if TYPE_CHECKING: 71 from typing import ( 72 Any, 73 Generator, 74 List, 75 Optional, 76 Sequence, 77 Union, 78 ) 79 80 from sphinx.application import Sphinx 81 from sphinx.util.typing import ExtensionMetadata 82 83 84logger = logging.getLogger(__name__) 85 86 87class Transmogrifier: 88 # pylint: disable=too-many-public-methods 89 90 # Field names used for different entity types: 91 field_types = { 92 "enum": "value", 93 "struct": "memb", 94 "union": "memb", 95 "event": "memb", 96 "command": "arg", 97 "alternate": "alt", 98 } 99 100 def __init__(self) -> None: 101 self._curr_ent: Optional[QAPISchemaDefinition] = None 102 self._result = StringList() 103 self.indent = 0 104 105 @property 106 def result(self) -> StringList: 107 return self._result 108 109 @property 110 def entity(self) -> QAPISchemaDefinition: 111 assert self._curr_ent is not None 112 return self._curr_ent 113 114 @property 115 def member_field_type(self) -> str: 116 return self.field_types[self.entity.meta] 117 118 # General-purpose rST generation functions 119 120 def get_indent(self) -> str: 121 return " " * self.indent 122 123 @contextmanager 124 def indented(self) -> Generator[None]: 125 self.indent += 1 126 try: 127 yield 128 finally: 129 self.indent -= 1 130 131 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 132 """Append one line of generated reST to the output.""" 133 134 # NB: Sphinx uses zero-indexed lines; subtract one. 135 lineno = tuple((n - 1 for n in lineno)) 136 137 if line.strip(): 138 # not a blank line 139 self._result.append( 140 self.get_indent() + line.rstrip("\n"), source, *lineno 141 ) 142 else: 143 self._result.append("", source, *lineno) 144 145 def add_line(self, content: str, info: QAPISourceInfo) -> None: 146 # NB: We *require* an info object; this works out OK because we 147 # don't document built-in objects that don't have 148 # one. Everything else should. 149 self.add_line_raw(content, info.fname, info.line) 150 151 def add_lines( 152 self, 153 content: str, 154 info: QAPISourceInfo, 155 ) -> None: 156 lines = content.splitlines(True) 157 for i, line in enumerate(lines): 158 self.add_line_raw(line, info.fname, info.line + i) 159 160 def ensure_blank_line(self) -> None: 161 # Empty document -- no blank line required. 162 if not self._result: 163 return 164 165 # Last line isn't blank, add one. 166 if self._result[-1].strip(): # pylint: disable=no-member 167 fname, line = self._result.info(-1) 168 assert isinstance(line, int) 169 # New blank line is credited to one-after the current last line. 170 # +2: correct for zero/one index, then increment by one. 171 self.add_line_raw("", fname, line + 2) 172 173 def add_field( 174 self, 175 kind: str, 176 name: str, 177 body: str, 178 info: QAPISourceInfo, 179 typ: Optional[str] = None, 180 ) -> None: 181 if typ: 182 text = f":{kind} {typ} {name}: {body}" 183 else: 184 text = f":{kind} {name}: {body}" 185 self.add_lines(text, info) 186 187 def format_type( 188 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 189 ) -> Optional[str]: 190 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 191 return None 192 193 qapi_type = ent 194 optional = False 195 if isinstance(ent, QAPISchemaObjectTypeMember): 196 qapi_type = ent.type 197 optional = ent.optional 198 199 if isinstance(qapi_type, QAPISchemaArrayType): 200 ret = f"[{qapi_type.element_type.doc_type()}]" 201 else: 202 assert isinstance(qapi_type, QAPISchemaType) 203 tmp = qapi_type.doc_type() 204 assert tmp 205 ret = tmp 206 if optional: 207 ret += "?" 208 209 return ret 210 211 def generate_field( 212 self, 213 kind: str, 214 member: QAPISchemaMember, 215 body: str, 216 info: QAPISourceInfo, 217 ) -> None: 218 typ = self.format_type(member) 219 self.add_field(kind, member.name, body, info, typ) 220 221 # Transmogrification helpers 222 223 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 224 # Squelch empty paragraphs. 225 if not section.text: 226 return 227 228 self.ensure_blank_line() 229 self.add_lines(section.text, section.info) 230 self.ensure_blank_line() 231 232 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 233 # FIXME: ifcond for members 234 # TODO: features for members (documented at entity-level, 235 # but sometimes defined per-member. Should we add such 236 # information to member descriptions when we can?) 237 assert section.member 238 self.generate_field( 239 self.member_field_type, 240 section.member, 241 # TODO drop fallbacks when undocumented members are outlawed 242 section.text if section.text else "Not documented", 243 section.info, 244 ) 245 246 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 247 # FIXME - ifcond for features is not handled at all yet! 248 # Proposal: decorate the right-hand column with some graphical 249 # element to indicate conditional availability? 250 assert section.text # Guaranteed by parser.py 251 assert section.member 252 253 self.generate_field("feat", section.member, section.text, section.info) 254 255 def visit_returns(self, section: QAPIDoc.Section) -> None: 256 assert isinstance(self.entity, QAPISchemaCommand) 257 rtype = self.entity.ret_type 258 # q_empty can produce None, but we won't be documenting anything 259 # without an explicit return statement in the doc block, and we 260 # should not have any such explicit statements when there is no 261 # return value. 262 assert rtype 263 264 typ = self.format_type(rtype) 265 assert typ 266 assert section.text 267 self.add_field("return", typ, section.text, section.info) 268 269 def visit_errors(self, section: QAPIDoc.Section) -> None: 270 # If the section text does not start with a space, it means text 271 # began on the same line as the "Error:" string and we should 272 # not insert a newline in this case. 273 if section.text[0].isspace(): 274 text = f":error:\n{section.text}" 275 else: 276 text = f":error: {section.text}" 277 self.add_lines(text, section.info) 278 279 def preamble(self, ent: QAPISchemaDefinition) -> None: 280 """ 281 Generate option lines for QAPI entity directives. 282 """ 283 if ent.doc and ent.doc.since: 284 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 285 # Generated from the entity's docblock; info location is exact. 286 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 287 288 if ent.ifcond.is_present(): 289 doc = ent.ifcond.docgen() 290 assert ent.info 291 # Generated from entity definition; info location is approximate. 292 self.add_line(f":ifcond: {doc}", ent.info) 293 294 # Hoist special features such as :deprecated: and :unstable: 295 # into the options block for the entity. If, in the future, new 296 # special features are added, qapi-domain will chirp about 297 # unrecognized options and fail until they are handled in 298 # qapi-domain. 299 for feat in ent.features: 300 if feat.is_special(): 301 # FIXME: handle ifcond if present. How to display that 302 # information is TBD. 303 # Generated from entity def; info location is approximate. 304 assert feat.info 305 self.add_line(f":{feat.name}:", feat.info) 306 307 self.ensure_blank_line() 308 309 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 310 311 def _get_target( 312 ent: QAPISchemaDefinition, 313 ) -> Optional[QAPISchemaDefinition]: 314 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 315 return ent.arg_type 316 if isinstance(ent, QAPISchemaObjectType): 317 return ent.base 318 return None 319 320 target = _get_target(ent) 321 if target is not None and not target.is_implicit(): 322 assert ent.info 323 self.add_field( 324 self.member_field_type, 325 "q_dummy", 326 f"The members of :qapi:type:`{target.name}`.", 327 ent.info, 328 "q_dummy", 329 ) 330 331 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 332 for variant in ent.branches.variants: 333 if variant.type.name == "q_empty": 334 continue 335 assert ent.info 336 self.add_field( 337 self.member_field_type, 338 "q_dummy", 339 f" When ``{ent.branches.tag_member.name}`` is " 340 f"``{variant.name}``: " 341 f"The members of :qapi:type:`{variant.type.name}`.", 342 ent.info, 343 "q_dummy", 344 ) 345 346 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 347 sections = ent.doc.all_sections if ent.doc else [] 348 349 # Determine the index location at which we should generate 350 # documentation for "The members of ..." pointers. This should 351 # go at the end of the members section(s) if any. Note that 352 # index 0 is assumed to be a plain intro section, even if it is 353 # empty; and that a members section if present will always 354 # immediately follow the opening PLAIN section. 355 gen_index = 1 356 if len(sections) > 1: 357 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 358 gen_index += 1 359 if gen_index >= len(sections): 360 break 361 362 # Add sections in source order: 363 for i, section in enumerate(sections): 364 # @var is translated to ``var``: 365 section.text = re.sub(r"@([\w-]+)", r"``\1``", section.text) 366 367 if section.kind == QAPIDoc.Kind.PLAIN: 368 self.visit_paragraph(section) 369 elif section.kind == QAPIDoc.Kind.MEMBER: 370 assert isinstance(section, QAPIDoc.ArgSection) 371 self.visit_member(section) 372 elif section.kind == QAPIDoc.Kind.FEATURE: 373 assert isinstance(section, QAPIDoc.ArgSection) 374 self.visit_feature(section) 375 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 376 # Since is handled in preamble, TODO is skipped intentionally. 377 pass 378 elif section.kind == QAPIDoc.Kind.RETURNS: 379 self.visit_returns(section) 380 elif section.kind == QAPIDoc.Kind.ERRORS: 381 self.visit_errors(section) 382 else: 383 assert False 384 385 # Generate "The members of ..." entries if necessary: 386 if i == gen_index - 1: 387 self._insert_member_pointer(ent) 388 389 self.ensure_blank_line() 390 391 # Transmogrification core methods 392 393 def visit_module(self, path: str) -> None: 394 name = Path(path).stem 395 # module directives are credited to the first line of a module file. 396 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 397 self.ensure_blank_line() 398 399 def visit_freeform(self, doc: QAPIDoc) -> None: 400 # TODO: Once the old qapidoc transformer is deprecated, freeform 401 # sections can be updated to pure rST, and this transformed removed. 402 # 403 # For now, translate our micro-format into rST. Code adapted 404 # from Peter Maydell's freeform(). 405 406 assert len(doc.all_sections) == 1, doc.all_sections 407 body = doc.all_sections[0] 408 text = body.text 409 info = doc.info 410 411 if re.match(r"=+ ", text): 412 # Section/subsection heading (if present, will always be the 413 # first line of the block) 414 (heading, _, text) = text.partition("\n") 415 (leader, _, heading) = heading.partition(" ") 416 # Implicit +1 for heading in the containing .rst doc 417 level = len(leader) + 1 418 419 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 420 markers = ' #*=_^"' 421 overline = level <= 2 422 marker = markers[level] 423 424 self.ensure_blank_line() 425 # This credits all 2 or 3 lines to the single source line. 426 if overline: 427 self.add_line(marker * len(heading), info) 428 self.add_line(heading, info) 429 self.add_line(marker * len(heading), info) 430 self.ensure_blank_line() 431 432 # Eat blank line(s) and advance info 433 trimmed = text.lstrip("\n") 434 text = trimmed 435 info = info.next_line(len(text) - len(trimmed) + 1) 436 437 self.add_lines(text, info) 438 self.ensure_blank_line() 439 440 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 441 assert ent.info 442 443 try: 444 self._curr_ent = ent 445 446 # Squish structs and unions together into an "object" directive. 447 meta = ent.meta 448 if meta in ("struct", "union"): 449 meta = "object" 450 451 # This line gets credited to the start of the /definition/. 452 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 453 with self.indented(): 454 self.preamble(ent) 455 self.visit_sections(ent) 456 finally: 457 self._curr_ent = None 458 459 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 460 self.add_line_raw( 461 f".. qapi:namespace:: {namespace}", source, lineno + 1 462 ) 463 self.ensure_blank_line() 464 465 466class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 467 """A QAPI schema visitor which adds Sphinx dependencies each module 468 469 This class calls the Sphinx note_dependency() function to tell Sphinx 470 that the generated documentation output depends on the input 471 schema file associated with each module in the QAPI input. 472 """ 473 474 def __init__(self, env: Any, qapidir: str) -> None: 475 self._env = env 476 self._qapidir = qapidir 477 478 def visit_module(self, name: str) -> None: 479 if name != "./builtin": 480 qapifile = self._qapidir + "/" + name 481 self._env.note_dependency(os.path.abspath(qapifile)) 482 super().visit_module(name) 483 484 485class NestedDirective(SphinxDirective): 486 def run(self) -> Sequence[nodes.Node]: 487 raise NotImplementedError 488 489 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 490 """ 491 Parse rST source lines and add them to the specified node 492 493 Take the list of rST source lines rstlist, parse them as 494 rST, and add the resulting docutils nodes as children of node. 495 The nodes are parsed in a way that allows them to include 496 subheadings (titles) without confusing the rendering of 497 anything else. 498 """ 499 with switch_source_input(self.state, rstlist): 500 nested_parse_with_titles(self.state, rstlist, node) 501 502 503class QAPIDocDirective(NestedDirective): 504 """Extract documentation from the specified QAPI .json file""" 505 506 required_argument = 1 507 optional_arguments = 1 508 option_spec = { 509 "qapifile": directives.unchanged_required, 510 "namespace": directives.unchanged, 511 "transmogrify": directives.flag, 512 } 513 has_content = False 514 515 def new_serialno(self) -> str: 516 """Return a unique new ID string suitable for use as a node's ID""" 517 env = self.state.document.settings.env 518 return "qapidoc-%d" % env.new_serialno("qapidoc") 519 520 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 521 logger.info("Transmogrifying QAPI to rST ...") 522 vis = Transmogrifier() 523 modules = set() 524 525 if "namespace" in self.options: 526 vis.set_namespace( 527 self.options["namespace"], *self.get_source_info() 528 ) 529 530 for doc in schema.docs: 531 module_source = doc.info.fname 532 if module_source not in modules: 533 vis.visit_module(module_source) 534 modules.add(module_source) 535 536 if doc.symbol: 537 ent = schema.lookup_entity(doc.symbol) 538 assert isinstance(ent, QAPISchemaDefinition) 539 vis.visit_entity(ent) 540 else: 541 vis.visit_freeform(doc) 542 543 logger.info("Transmogrification complete.") 544 545 contentnode = nodes.section() 546 content = vis.result 547 titles_allowed = True 548 549 logger.info("Transmogrifier running nested parse ...") 550 with switch_source_input(self.state, content): 551 if titles_allowed: 552 node: nodes.Element = nodes.section() 553 node.document = self.state.document 554 nested_parse_with_titles(self.state, content, contentnode) 555 else: 556 node = nodes.paragraph() 557 node.document = self.state.document 558 self.state.nested_parse(content, 0, contentnode) 559 logger.info("Transmogrifier's nested parse completed.") 560 561 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 562 argname = "_".join(Path(self.arguments[0]).parts) 563 name = Path(argname).stem + ".ir" 564 self.write_intermediate(content, name) 565 566 sys.stdout.flush() 567 return contentnode 568 569 def write_intermediate(self, content: StringList, filename: str) -> None: 570 logger.info( 571 "writing intermediate rST for '%s' to '%s'", 572 self.arguments[0], 573 filename, 574 ) 575 576 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 577 outlines = [] 578 lcol_width = 0 579 580 for i, line in enumerate(content): 581 src, lineno = content.info(i) 582 srcpath = Path(src).resolve() 583 srcpath = srcpath.relative_to(srctree) 584 585 lcol = f"{srcpath}:{lineno:04d}" 586 lcol_width = max(lcol_width, len(lcol)) 587 outlines.append((lcol, line)) 588 589 with open(filename, "w", encoding="UTF-8") as outfile: 590 for lcol, rcol in outlines: 591 outfile.write(lcol.rjust(lcol_width)) 592 outfile.write(" |") 593 if rcol: 594 outfile.write(f" {rcol}") 595 outfile.write("\n") 596 597 def legacy(self, schema: QAPISchema) -> nodes.Element: 598 vis = QAPISchemaGenRSTVisitor(self) 599 vis.visit_begin(schema) 600 for doc in schema.docs: 601 if doc.symbol: 602 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 603 else: 604 vis.freeform(doc) 605 return vis.get_document_node() # type: ignore 606 607 def run(self) -> Sequence[nodes.Node]: 608 env = self.state.document.settings.env 609 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 610 qapidir = os.path.dirname(qapifile) 611 transmogrify = "transmogrify" in self.options 612 613 try: 614 schema = QAPISchema(qapifile) 615 616 # First tell Sphinx about all the schema files that the 617 # output documentation depends on (including 'qapifile' itself) 618 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 619 except QAPIError as err: 620 # Launder QAPI parse errors into Sphinx extension errors 621 # so they are displayed nicely to the user 622 raise ExtensionError(str(err)) from err 623 624 if transmogrify: 625 contentnode = self.transmogrify(schema) 626 else: 627 contentnode = self.legacy(schema) 628 629 return contentnode.children 630 631 632class QMPExample(CodeBlock, NestedDirective): 633 """ 634 Custom admonition for QMP code examples. 635 636 When the :annotated: option is present, the body of this directive 637 is parsed as normal rST, but with any '::' code blocks set to use 638 the QMP lexer. Code blocks must be explicitly written by the user, 639 but this allows for intermingling explanatory paragraphs with 640 arbitrary rST syntax and code blocks for more involved examples. 641 642 When :annotated: is absent, the directive body is treated as a 643 simple standalone QMP code block literal. 644 """ 645 646 required_argument = 0 647 optional_arguments = 0 648 has_content = True 649 option_spec = { 650 "annotated": directives.flag, 651 "title": directives.unchanged, 652 } 653 654 def _highlightlang(self) -> addnodes.highlightlang: 655 """Return the current highlightlang setting for the document""" 656 node = None 657 doc = self.state.document 658 659 if hasattr(doc, "findall"): 660 # docutils >= 0.18.1 661 for node in doc.findall(addnodes.highlightlang): 662 pass 663 else: 664 for elem in doc.traverse(): 665 if isinstance(elem, addnodes.highlightlang): 666 node = elem 667 668 if node: 669 return node 670 671 # No explicit directive found, use defaults 672 node = addnodes.highlightlang( 673 lang=self.env.config.highlight_language, 674 force=False, 675 # Yes, Sphinx uses this value to effectively disable line 676 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 677 linenothreshold=sys.maxsize, 678 ) 679 return node 680 681 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 682 title = "Example:" 683 if "title" in self.options: 684 title = f"{title} {self.options['title']}" 685 686 admon = nodes.admonition( 687 "", 688 nodes.title("", title), 689 *content, 690 classes=["admonition", "admonition-example"], 691 ) 692 return [admon] 693 694 def run_annotated(self) -> List[nodes.Node]: 695 lang_node = self._highlightlang() 696 697 content_node: nodes.Element = nodes.section() 698 699 # Configure QMP highlighting for "::" blocks, if needed 700 if lang_node["lang"] != "QMP": 701 content_node += addnodes.highlightlang( 702 lang="QMP", 703 force=False, # "True" ignores lexing errors 704 linenothreshold=lang_node["linenothreshold"], 705 ) 706 707 self.do_parse(self.content, content_node) 708 709 # Restore prior language highlighting, if needed 710 if lang_node["lang"] != "QMP": 711 content_node += addnodes.highlightlang(**lang_node.attributes) 712 713 return content_node.children 714 715 def run(self) -> List[nodes.Node]: 716 annotated = "annotated" in self.options 717 718 if annotated: 719 content_nodes = self.run_annotated() 720 else: 721 self.arguments = ["QMP"] 722 content_nodes = super().run() 723 724 return self.admonition_wrap(*content_nodes) 725 726 727def setup(app: Sphinx) -> ExtensionMetadata: 728 """Register qapi-doc directive with Sphinx""" 729 app.setup_extension("qapi_domain") 730 app.add_config_value("qapidoc_srctree", None, "env") 731 app.add_directive("qapi-doc", QAPIDocDirective) 732 app.add_directive("qmp-example", QMPExample) 733 734 return { 735 "version": __version__, 736 "parallel_read_safe": True, 737 "parallel_write_safe": True, 738 } 739