1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30 31__version__ = "2.0" 32 33from contextlib import contextmanager 34import os 35from pathlib import Path 36import re 37import sys 38from typing import TYPE_CHECKING 39 40from docutils import nodes 41from docutils.parsers.rst import directives 42from docutils.statemachine import StringList 43from qapi.error import QAPIError 44from qapi.parser import QAPIDoc 45from qapi.schema import ( 46 QAPISchema, 47 QAPISchemaArrayType, 48 QAPISchemaCommand, 49 QAPISchemaDefinition, 50 QAPISchemaEnumMember, 51 QAPISchemaEvent, 52 QAPISchemaFeature, 53 QAPISchemaMember, 54 QAPISchemaObjectType, 55 QAPISchemaObjectTypeMember, 56 QAPISchemaType, 57 QAPISchemaVisitor, 58) 59from qapi.source import QAPISourceInfo 60from sphinx import addnodes 61from sphinx.directives.code import CodeBlock 62from sphinx.errors import ExtensionError 63from sphinx.util import logging 64from sphinx.util.docutils import SphinxDirective, switch_source_input 65from sphinx.util.nodes import nested_parse_with_titles 66 67 68if TYPE_CHECKING: 69 from typing import ( 70 Any, 71 Generator, 72 List, 73 Optional, 74 Sequence, 75 Union, 76 ) 77 78 from sphinx.application import Sphinx 79 from sphinx.util.typing import ExtensionMetadata 80 81 82logger = logging.getLogger(__name__) 83 84 85class Transmogrifier: 86 # pylint: disable=too-many-public-methods 87 88 # Field names used for different entity types: 89 field_types = { 90 "enum": "value", 91 "struct": "memb", 92 "union": "memb", 93 "event": "memb", 94 "command": "arg", 95 "alternate": "alt", 96 } 97 98 def __init__(self) -> None: 99 self._curr_ent: Optional[QAPISchemaDefinition] = None 100 self._result = StringList() 101 self.indent = 0 102 103 @property 104 def result(self) -> StringList: 105 return self._result 106 107 @property 108 def entity(self) -> QAPISchemaDefinition: 109 assert self._curr_ent is not None 110 return self._curr_ent 111 112 @property 113 def member_field_type(self) -> str: 114 return self.field_types[self.entity.meta] 115 116 # General-purpose rST generation functions 117 118 def get_indent(self) -> str: 119 return " " * self.indent 120 121 @contextmanager 122 def indented(self) -> Generator[None]: 123 self.indent += 1 124 try: 125 yield 126 finally: 127 self.indent -= 1 128 129 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 130 """Append one line of generated reST to the output.""" 131 132 # NB: Sphinx uses zero-indexed lines; subtract one. 133 lineno = tuple((n - 1 for n in lineno)) 134 135 if line.strip(): 136 # not a blank line 137 self._result.append( 138 self.get_indent() + line.rstrip("\n"), source, *lineno 139 ) 140 else: 141 self._result.append("", source, *lineno) 142 143 def add_line(self, content: str, info: QAPISourceInfo) -> None: 144 # NB: We *require* an info object; this works out OK because we 145 # don't document built-in objects that don't have 146 # one. Everything else should. 147 self.add_line_raw(content, info.fname, info.line) 148 149 def add_lines( 150 self, 151 content: str, 152 info: QAPISourceInfo, 153 ) -> None: 154 lines = content.splitlines(True) 155 for i, line in enumerate(lines): 156 self.add_line_raw(line, info.fname, info.line + i) 157 158 def ensure_blank_line(self) -> None: 159 # Empty document -- no blank line required. 160 if not self._result: 161 return 162 163 # Last line isn't blank, add one. 164 if self._result[-1].strip(): # pylint: disable=no-member 165 fname, line = self._result.info(-1) 166 assert isinstance(line, int) 167 # New blank line is credited to one-after the current last line. 168 # +2: correct for zero/one index, then increment by one. 169 self.add_line_raw("", fname, line + 2) 170 171 def add_field( 172 self, 173 kind: str, 174 name: str, 175 body: str, 176 info: QAPISourceInfo, 177 typ: Optional[str] = None, 178 ) -> None: 179 if typ: 180 text = f":{kind} {typ} {name}: {body}" 181 else: 182 text = f":{kind} {name}: {body}" 183 self.add_lines(text, info) 184 185 def format_type( 186 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 187 ) -> Optional[str]: 188 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 189 return None 190 191 qapi_type = ent 192 optional = False 193 if isinstance(ent, QAPISchemaObjectTypeMember): 194 qapi_type = ent.type 195 optional = ent.optional 196 197 if isinstance(qapi_type, QAPISchemaArrayType): 198 ret = f"[{qapi_type.element_type.doc_type()}]" 199 else: 200 assert isinstance(qapi_type, QAPISchemaType) 201 tmp = qapi_type.doc_type() 202 assert tmp 203 ret = tmp 204 if optional: 205 ret += "?" 206 207 return ret 208 209 def generate_field( 210 self, 211 kind: str, 212 member: QAPISchemaMember, 213 body: str, 214 info: QAPISourceInfo, 215 ) -> None: 216 typ = self.format_type(member) 217 self.add_field(kind, member.name, body, info, typ) 218 219 @staticmethod 220 def reformat_arobase(text: str) -> str: 221 """ reformats @var to ``var`` """ 222 return re.sub(r"@([\w-]+)", r"``\1``", text) 223 224 # Transmogrification helpers 225 226 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 227 # Squelch empty paragraphs. 228 if not section.text: 229 return 230 231 self.ensure_blank_line() 232 self.add_lines(section.text, section.info) 233 self.ensure_blank_line() 234 235 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 236 # FIXME: ifcond for members 237 # TODO: features for members (documented at entity-level, 238 # but sometimes defined per-member. Should we add such 239 # information to member descriptions when we can?) 240 assert section.member 241 self.generate_field( 242 self.member_field_type, 243 section.member, 244 # TODO drop fallbacks when undocumented members are outlawed 245 section.text if section.text else "Not documented", 246 section.info, 247 ) 248 249 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 250 # FIXME - ifcond for features is not handled at all yet! 251 # Proposal: decorate the right-hand column with some graphical 252 # element to indicate conditional availability? 253 assert section.text # Guaranteed by parser.py 254 assert section.member 255 256 self.generate_field("feat", section.member, section.text, section.info) 257 258 def visit_returns(self, section: QAPIDoc.Section) -> None: 259 assert isinstance(self.entity, QAPISchemaCommand) 260 rtype = self.entity.ret_type 261 # return statements will not be present (and won't be 262 # autogenerated) for any command that doesn't return 263 # *something*, so rtype will always be defined here. 264 assert rtype 265 266 typ = self.format_type(rtype) 267 assert typ 268 269 if section.text: 270 self.add_field("return", typ, section.text, section.info) 271 else: 272 self.add_lines(f":return-nodesc: {typ}", section.info) 273 274 def visit_errors(self, section: QAPIDoc.Section) -> None: 275 # If the section text does not start with a space, it means text 276 # began on the same line as the "Error:" string and we should 277 # not insert a newline in this case. 278 if section.text[0].isspace(): 279 text = f":error:\n{section.text}" 280 else: 281 text = f":error: {section.text}" 282 self.add_lines(text, section.info) 283 284 def preamble(self, ent: QAPISchemaDefinition) -> None: 285 """ 286 Generate option lines for QAPI entity directives. 287 """ 288 if ent.doc and ent.doc.since: 289 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 290 # Generated from the entity's docblock; info location is exact. 291 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 292 293 if ent.ifcond.is_present(): 294 doc = ent.ifcond.docgen() 295 assert ent.info 296 # Generated from entity definition; info location is approximate. 297 self.add_line(f":ifcond: {doc}", ent.info) 298 299 # Hoist special features such as :deprecated: and :unstable: 300 # into the options block for the entity. If, in the future, new 301 # special features are added, qapi-domain will chirp about 302 # unrecognized options and fail until they are handled in 303 # qapi-domain. 304 for feat in ent.features: 305 if feat.is_special(): 306 # FIXME: handle ifcond if present. How to display that 307 # information is TBD. 308 # Generated from entity def; info location is approximate. 309 assert feat.info 310 self.add_line(f":{feat.name}:", feat.info) 311 312 self.ensure_blank_line() 313 314 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 315 316 def _get_target( 317 ent: QAPISchemaDefinition, 318 ) -> Optional[QAPISchemaDefinition]: 319 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 320 return ent.arg_type 321 if isinstance(ent, QAPISchemaObjectType): 322 return ent.base 323 return None 324 325 target = _get_target(ent) 326 if target is not None and not target.is_implicit(): 327 assert ent.info 328 self.add_field( 329 self.member_field_type, 330 "q_dummy", 331 f"The members of :qapi:type:`{target.name}`.", 332 ent.info, 333 "q_dummy", 334 ) 335 336 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 337 for variant in ent.branches.variants: 338 if variant.type.name == "q_empty": 339 continue 340 assert ent.info 341 self.add_field( 342 self.member_field_type, 343 "q_dummy", 344 f" When ``{ent.branches.tag_member.name}`` is " 345 f"``{variant.name}``: " 346 f"The members of :qapi:type:`{variant.type.name}`.", 347 ent.info, 348 "q_dummy", 349 ) 350 351 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 352 sections = ent.doc.all_sections if ent.doc else [] 353 354 # Determine the index location at which we should generate 355 # documentation for "The members of ..." pointers. This should 356 # go at the end of the members section(s) if any. Note that 357 # index 0 is assumed to be a plain intro section, even if it is 358 # empty; and that a members section if present will always 359 # immediately follow the opening PLAIN section. 360 gen_index = 1 361 if len(sections) > 1: 362 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 363 gen_index += 1 364 if gen_index >= len(sections): 365 break 366 367 # Add sections in source order: 368 for i, section in enumerate(sections): 369 section.text = self.reformat_arobase(section.text) 370 371 if section.kind == QAPIDoc.Kind.PLAIN: 372 self.visit_paragraph(section) 373 elif section.kind == QAPIDoc.Kind.MEMBER: 374 assert isinstance(section, QAPIDoc.ArgSection) 375 self.visit_member(section) 376 elif section.kind == QAPIDoc.Kind.FEATURE: 377 assert isinstance(section, QAPIDoc.ArgSection) 378 self.visit_feature(section) 379 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 380 # Since is handled in preamble, TODO is skipped intentionally. 381 pass 382 elif section.kind == QAPIDoc.Kind.RETURNS: 383 self.visit_returns(section) 384 elif section.kind == QAPIDoc.Kind.ERRORS: 385 self.visit_errors(section) 386 else: 387 assert False 388 389 # Generate "The members of ..." entries if necessary: 390 if i == gen_index - 1: 391 self._insert_member_pointer(ent) 392 393 self.ensure_blank_line() 394 395 # Transmogrification core methods 396 397 def visit_module(self, path: str) -> None: 398 name = Path(path).stem 399 # module directives are credited to the first line of a module file. 400 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 401 self.ensure_blank_line() 402 403 def visit_freeform(self, doc: QAPIDoc) -> None: 404 assert len(doc.all_sections) == 1, doc.all_sections 405 body = doc.all_sections[0] 406 self.add_lines(self.reformat_arobase(body.text), doc.info) 407 self.ensure_blank_line() 408 409 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 410 assert ent.info 411 412 try: 413 self._curr_ent = ent 414 415 # Squish structs and unions together into an "object" directive. 416 meta = ent.meta 417 if meta in ("struct", "union"): 418 meta = "object" 419 420 # This line gets credited to the start of the /definition/. 421 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 422 with self.indented(): 423 self.preamble(ent) 424 self.visit_sections(ent) 425 finally: 426 self._curr_ent = None 427 428 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 429 self.add_line_raw( 430 f".. qapi:namespace:: {namespace}", source, lineno + 1 431 ) 432 self.ensure_blank_line() 433 434 435class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 436 """A QAPI schema visitor which adds Sphinx dependencies each module 437 438 This class calls the Sphinx note_dependency() function to tell Sphinx 439 that the generated documentation output depends on the input 440 schema file associated with each module in the QAPI input. 441 """ 442 443 def __init__(self, env: Any, qapidir: str) -> None: 444 self._env = env 445 self._qapidir = qapidir 446 447 def visit_module(self, name: str) -> None: 448 if name != "./builtin": 449 qapifile = self._qapidir + "/" + name 450 self._env.note_dependency(os.path.abspath(qapifile)) 451 super().visit_module(name) 452 453 454class NestedDirective(SphinxDirective): 455 def run(self) -> Sequence[nodes.Node]: 456 raise NotImplementedError 457 458 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 459 """ 460 Parse rST source lines and add them to the specified node 461 462 Take the list of rST source lines rstlist, parse them as 463 rST, and add the resulting docutils nodes as children of node. 464 The nodes are parsed in a way that allows them to include 465 subheadings (titles) without confusing the rendering of 466 anything else. 467 """ 468 with switch_source_input(self.state, rstlist): 469 nested_parse_with_titles(self.state, rstlist, node) 470 471 472class QAPIDocDirective(NestedDirective): 473 """Extract documentation from the specified QAPI .json file""" 474 475 required_argument = 1 476 optional_arguments = 1 477 option_spec = { 478 "qapifile": directives.unchanged_required, 479 "namespace": directives.unchanged, 480 } 481 has_content = False 482 483 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 484 logger.info("Transmogrifying QAPI to rST ...") 485 vis = Transmogrifier() 486 modules = set() 487 488 if "namespace" in self.options: 489 vis.set_namespace( 490 self.options["namespace"], *self.get_source_info() 491 ) 492 493 for doc in schema.docs: 494 module_source = doc.info.fname 495 if module_source not in modules: 496 vis.visit_module(module_source) 497 modules.add(module_source) 498 499 if doc.symbol: 500 ent = schema.lookup_entity(doc.symbol) 501 assert isinstance(ent, QAPISchemaDefinition) 502 vis.visit_entity(ent) 503 else: 504 vis.visit_freeform(doc) 505 506 logger.info("Transmogrification complete.") 507 508 contentnode = nodes.section() 509 content = vis.result 510 titles_allowed = True 511 512 logger.info("Transmogrifier running nested parse ...") 513 with switch_source_input(self.state, content): 514 if titles_allowed: 515 node: nodes.Element = nodes.section() 516 node.document = self.state.document 517 nested_parse_with_titles(self.state, content, contentnode) 518 else: 519 node = nodes.paragraph() 520 node.document = self.state.document 521 self.state.nested_parse(content, 0, contentnode) 522 logger.info("Transmogrifier's nested parse completed.") 523 524 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 525 argname = "_".join(Path(self.arguments[0]).parts) 526 name = Path(argname).stem + ".ir" 527 self.write_intermediate(content, name) 528 529 sys.stdout.flush() 530 return contentnode 531 532 def write_intermediate(self, content: StringList, filename: str) -> None: 533 logger.info( 534 "writing intermediate rST for '%s' to '%s'", 535 self.arguments[0], 536 filename, 537 ) 538 539 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 540 outlines = [] 541 lcol_width = 0 542 543 for i, line in enumerate(content): 544 src, lineno = content.info(i) 545 srcpath = Path(src).resolve() 546 srcpath = srcpath.relative_to(srctree) 547 548 lcol = f"{srcpath}:{lineno:04d}" 549 lcol_width = max(lcol_width, len(lcol)) 550 outlines.append((lcol, line)) 551 552 with open(filename, "w", encoding="UTF-8") as outfile: 553 for lcol, rcol in outlines: 554 outfile.write(lcol.rjust(lcol_width)) 555 outfile.write(" |") 556 if rcol: 557 outfile.write(f" {rcol}") 558 outfile.write("\n") 559 560 def run(self) -> Sequence[nodes.Node]: 561 env = self.state.document.settings.env 562 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 563 qapidir = os.path.dirname(qapifile) 564 565 try: 566 schema = QAPISchema(qapifile) 567 568 # First tell Sphinx about all the schema files that the 569 # output documentation depends on (including 'qapifile' itself) 570 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 571 except QAPIError as err: 572 # Launder QAPI parse errors into Sphinx extension errors 573 # so they are displayed nicely to the user 574 raise ExtensionError(str(err)) from err 575 576 contentnode = self.transmogrify(schema) 577 return contentnode.children 578 579 580class QMPExample(CodeBlock, NestedDirective): 581 """ 582 Custom admonition for QMP code examples. 583 584 When the :annotated: option is present, the body of this directive 585 is parsed as normal rST, but with any '::' code blocks set to use 586 the QMP lexer. Code blocks must be explicitly written by the user, 587 but this allows for intermingling explanatory paragraphs with 588 arbitrary rST syntax and code blocks for more involved examples. 589 590 When :annotated: is absent, the directive body is treated as a 591 simple standalone QMP code block literal. 592 """ 593 594 required_argument = 0 595 optional_arguments = 0 596 has_content = True 597 option_spec = { 598 "annotated": directives.flag, 599 "title": directives.unchanged, 600 } 601 602 def _highlightlang(self) -> addnodes.highlightlang: 603 """Return the current highlightlang setting for the document""" 604 node = None 605 doc = self.state.document 606 607 if hasattr(doc, "findall"): 608 # docutils >= 0.18.1 609 for node in doc.findall(addnodes.highlightlang): 610 pass 611 else: 612 for elem in doc.traverse(): 613 if isinstance(elem, addnodes.highlightlang): 614 node = elem 615 616 if node: 617 return node 618 619 # No explicit directive found, use defaults 620 node = addnodes.highlightlang( 621 lang=self.env.config.highlight_language, 622 force=False, 623 # Yes, Sphinx uses this value to effectively disable line 624 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 625 linenothreshold=sys.maxsize, 626 ) 627 return node 628 629 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 630 title = "Example:" 631 if "title" in self.options: 632 title = f"{title} {self.options['title']}" 633 634 admon = nodes.admonition( 635 "", 636 nodes.title("", title), 637 *content, 638 classes=["admonition", "admonition-example"], 639 ) 640 return [admon] 641 642 def run_annotated(self) -> List[nodes.Node]: 643 lang_node = self._highlightlang() 644 645 content_node: nodes.Element = nodes.section() 646 647 # Configure QMP highlighting for "::" blocks, if needed 648 if lang_node["lang"] != "QMP": 649 content_node += addnodes.highlightlang( 650 lang="QMP", 651 force=False, # "True" ignores lexing errors 652 linenothreshold=lang_node["linenothreshold"], 653 ) 654 655 self.do_parse(self.content, content_node) 656 657 # Restore prior language highlighting, if needed 658 if lang_node["lang"] != "QMP": 659 content_node += addnodes.highlightlang(**lang_node.attributes) 660 661 return content_node.children 662 663 def run(self) -> List[nodes.Node]: 664 annotated = "annotated" in self.options 665 666 if annotated: 667 content_nodes = self.run_annotated() 668 else: 669 self.arguments = ["QMP"] 670 content_nodes = super().run() 671 672 return self.admonition_wrap(*content_nodes) 673 674 675def setup(app: Sphinx) -> ExtensionMetadata: 676 """Register qapi-doc directive with Sphinx""" 677 app.setup_extension("qapi_domain") 678 app.add_config_value("qapidoc_srctree", None, "env") 679 app.add_directive("qapi-doc", QAPIDocDirective) 680 app.add_directive("qmp-example", QMPExample) 681 682 return { 683 "version": __version__, 684 "parallel_read_safe": True, 685 "parallel_write_safe": True, 686 } 687