1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30 31__version__ = "2.0" 32 33from contextlib import contextmanager 34import os 35from pathlib import Path 36import re 37import sys 38from typing import TYPE_CHECKING 39 40from docutils import nodes 41from docutils.parsers.rst import directives 42from docutils.statemachine import StringList 43from qapi.error import QAPIError 44from qapi.parser import QAPIDoc 45from qapi.schema import ( 46 QAPISchema, 47 QAPISchemaArrayType, 48 QAPISchemaCommand, 49 QAPISchemaDefinition, 50 QAPISchemaEnumMember, 51 QAPISchemaEvent, 52 QAPISchemaFeature, 53 QAPISchemaMember, 54 QAPISchemaObjectType, 55 QAPISchemaObjectTypeMember, 56 QAPISchemaType, 57 QAPISchemaVisitor, 58) 59from qapi.source import QAPISourceInfo 60from sphinx import addnodes 61from sphinx.directives.code import CodeBlock 62from sphinx.errors import ExtensionError 63from sphinx.util import logging 64from sphinx.util.docutils import SphinxDirective, switch_source_input 65from sphinx.util.nodes import nested_parse_with_titles 66 67 68if TYPE_CHECKING: 69 from typing import ( 70 Any, 71 Generator, 72 List, 73 Optional, 74 Sequence, 75 Union, 76 ) 77 78 from sphinx.application import Sphinx 79 from sphinx.util.typing import ExtensionMetadata 80 81 82logger = logging.getLogger(__name__) 83 84 85class Transmogrifier: 86 # pylint: disable=too-many-public-methods 87 88 # Field names used for different entity types: 89 field_types = { 90 "enum": "value", 91 "struct": "memb", 92 "union": "memb", 93 "event": "memb", 94 "command": "arg", 95 "alternate": "alt", 96 } 97 98 def __init__(self) -> None: 99 self._curr_ent: Optional[QAPISchemaDefinition] = None 100 self._result = StringList() 101 self.indent = 0 102 103 @property 104 def result(self) -> StringList: 105 return self._result 106 107 @property 108 def entity(self) -> QAPISchemaDefinition: 109 assert self._curr_ent is not None 110 return self._curr_ent 111 112 @property 113 def member_field_type(self) -> str: 114 return self.field_types[self.entity.meta] 115 116 # General-purpose rST generation functions 117 118 def get_indent(self) -> str: 119 return " " * self.indent 120 121 @contextmanager 122 def indented(self) -> Generator[None]: 123 self.indent += 1 124 try: 125 yield 126 finally: 127 self.indent -= 1 128 129 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 130 """Append one line of generated reST to the output.""" 131 132 # NB: Sphinx uses zero-indexed lines; subtract one. 133 lineno = tuple((n - 1 for n in lineno)) 134 135 if line.strip(): 136 # not a blank line 137 self._result.append( 138 self.get_indent() + line.rstrip("\n"), source, *lineno 139 ) 140 else: 141 self._result.append("", source, *lineno) 142 143 def add_line(self, content: str, info: QAPISourceInfo) -> None: 144 # NB: We *require* an info object; this works out OK because we 145 # don't document built-in objects that don't have 146 # one. Everything else should. 147 self.add_line_raw(content, info.fname, info.line) 148 149 def add_lines( 150 self, 151 content: str, 152 info: QAPISourceInfo, 153 ) -> None: 154 lines = content.splitlines(True) 155 for i, line in enumerate(lines): 156 self.add_line_raw(line, info.fname, info.line + i) 157 158 def ensure_blank_line(self) -> None: 159 # Empty document -- no blank line required. 160 if not self._result: 161 return 162 163 # Last line isn't blank, add one. 164 if self._result[-1].strip(): # pylint: disable=no-member 165 fname, line = self._result.info(-1) 166 assert isinstance(line, int) 167 # New blank line is credited to one-after the current last line. 168 # +2: correct for zero/one index, then increment by one. 169 self.add_line_raw("", fname, line + 2) 170 171 def add_field( 172 self, 173 kind: str, 174 name: str, 175 body: str, 176 info: QAPISourceInfo, 177 typ: Optional[str] = None, 178 ) -> None: 179 if typ: 180 text = f":{kind} {typ} {name}: {body}" 181 else: 182 text = f":{kind} {name}: {body}" 183 self.add_lines(text, info) 184 185 def format_type( 186 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 187 ) -> Optional[str]: 188 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 189 return None 190 191 qapi_type = ent 192 optional = False 193 if isinstance(ent, QAPISchemaObjectTypeMember): 194 qapi_type = ent.type 195 optional = ent.optional 196 197 if isinstance(qapi_type, QAPISchemaArrayType): 198 ret = f"[{qapi_type.element_type.doc_type()}]" 199 else: 200 assert isinstance(qapi_type, QAPISchemaType) 201 tmp = qapi_type.doc_type() 202 assert tmp 203 ret = tmp 204 if optional: 205 ret += "?" 206 207 return ret 208 209 def generate_field( 210 self, 211 kind: str, 212 member: QAPISchemaMember, 213 body: str, 214 info: QAPISourceInfo, 215 ) -> None: 216 typ = self.format_type(member) 217 self.add_field(kind, member.name, body, info, typ) 218 219 @staticmethod 220 def reformat_arobase(text: str) -> str: 221 """ reformats @var to ``var`` """ 222 return re.sub(r"@([\w-]+)", r"``\1``", text) 223 224 # Transmogrification helpers 225 226 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 227 # Squelch empty paragraphs. 228 if not section.text: 229 return 230 231 self.ensure_blank_line() 232 self.add_lines(section.text, section.info) 233 self.ensure_blank_line() 234 235 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 236 # FIXME: ifcond for members 237 # TODO: features for members (documented at entity-level, 238 # but sometimes defined per-member. Should we add such 239 # information to member descriptions when we can?) 240 assert section.member 241 self.generate_field( 242 self.member_field_type, 243 section.member, 244 # TODO drop fallbacks when undocumented members are outlawed 245 section.text if section.text else "Not documented", 246 section.info, 247 ) 248 249 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 250 # FIXME - ifcond for features is not handled at all yet! 251 # Proposal: decorate the right-hand column with some graphical 252 # element to indicate conditional availability? 253 assert section.text # Guaranteed by parser.py 254 assert section.member 255 256 self.generate_field("feat", section.member, section.text, section.info) 257 258 def visit_returns(self, section: QAPIDoc.Section) -> None: 259 assert isinstance(self.entity, QAPISchemaCommand) 260 rtype = self.entity.ret_type 261 # q_empty can produce None, but we won't be documenting anything 262 # without an explicit return statement in the doc block, and we 263 # should not have any such explicit statements when there is no 264 # return value. 265 assert rtype 266 267 typ = self.format_type(rtype) 268 assert typ 269 assert section.text 270 self.add_field("return", typ, section.text, section.info) 271 272 def visit_errors(self, section: QAPIDoc.Section) -> None: 273 # If the section text does not start with a space, it means text 274 # began on the same line as the "Error:" string and we should 275 # not insert a newline in this case. 276 if section.text[0].isspace(): 277 text = f":error:\n{section.text}" 278 else: 279 text = f":error: {section.text}" 280 self.add_lines(text, section.info) 281 282 def preamble(self, ent: QAPISchemaDefinition) -> None: 283 """ 284 Generate option lines for QAPI entity directives. 285 """ 286 if ent.doc and ent.doc.since: 287 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 288 # Generated from the entity's docblock; info location is exact. 289 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 290 291 if ent.ifcond.is_present(): 292 doc = ent.ifcond.docgen() 293 assert ent.info 294 # Generated from entity definition; info location is approximate. 295 self.add_line(f":ifcond: {doc}", ent.info) 296 297 # Hoist special features such as :deprecated: and :unstable: 298 # into the options block for the entity. If, in the future, new 299 # special features are added, qapi-domain will chirp about 300 # unrecognized options and fail until they are handled in 301 # qapi-domain. 302 for feat in ent.features: 303 if feat.is_special(): 304 # FIXME: handle ifcond if present. How to display that 305 # information is TBD. 306 # Generated from entity def; info location is approximate. 307 assert feat.info 308 self.add_line(f":{feat.name}:", feat.info) 309 310 self.ensure_blank_line() 311 312 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 313 314 def _get_target( 315 ent: QAPISchemaDefinition, 316 ) -> Optional[QAPISchemaDefinition]: 317 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 318 return ent.arg_type 319 if isinstance(ent, QAPISchemaObjectType): 320 return ent.base 321 return None 322 323 target = _get_target(ent) 324 if target is not None and not target.is_implicit(): 325 assert ent.info 326 self.add_field( 327 self.member_field_type, 328 "q_dummy", 329 f"The members of :qapi:type:`{target.name}`.", 330 ent.info, 331 "q_dummy", 332 ) 333 334 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 335 for variant in ent.branches.variants: 336 if variant.type.name == "q_empty": 337 continue 338 assert ent.info 339 self.add_field( 340 self.member_field_type, 341 "q_dummy", 342 f" When ``{ent.branches.tag_member.name}`` is " 343 f"``{variant.name}``: " 344 f"The members of :qapi:type:`{variant.type.name}`.", 345 ent.info, 346 "q_dummy", 347 ) 348 349 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 350 sections = ent.doc.all_sections if ent.doc else [] 351 352 # Determine the index location at which we should generate 353 # documentation for "The members of ..." pointers. This should 354 # go at the end of the members section(s) if any. Note that 355 # index 0 is assumed to be a plain intro section, even if it is 356 # empty; and that a members section if present will always 357 # immediately follow the opening PLAIN section. 358 gen_index = 1 359 if len(sections) > 1: 360 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 361 gen_index += 1 362 if gen_index >= len(sections): 363 break 364 365 # Add sections in source order: 366 for i, section in enumerate(sections): 367 section.text = self.reformat_arobase(section.text) 368 369 if section.kind == QAPIDoc.Kind.PLAIN: 370 self.visit_paragraph(section) 371 elif section.kind == QAPIDoc.Kind.MEMBER: 372 assert isinstance(section, QAPIDoc.ArgSection) 373 self.visit_member(section) 374 elif section.kind == QAPIDoc.Kind.FEATURE: 375 assert isinstance(section, QAPIDoc.ArgSection) 376 self.visit_feature(section) 377 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 378 # Since is handled in preamble, TODO is skipped intentionally. 379 pass 380 elif section.kind == QAPIDoc.Kind.RETURNS: 381 self.visit_returns(section) 382 elif section.kind == QAPIDoc.Kind.ERRORS: 383 self.visit_errors(section) 384 else: 385 assert False 386 387 # Generate "The members of ..." entries if necessary: 388 if i == gen_index - 1: 389 self._insert_member_pointer(ent) 390 391 self.ensure_blank_line() 392 393 # Transmogrification core methods 394 395 def visit_module(self, path: str) -> None: 396 name = Path(path).stem 397 # module directives are credited to the first line of a module file. 398 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 399 self.ensure_blank_line() 400 401 def visit_freeform(self, doc: QAPIDoc) -> None: 402 # TODO: Once the old qapidoc transformer is deprecated, freeform 403 # sections can be updated to pure rST, and this transformed removed. 404 # 405 # For now, translate our micro-format into rST. Code adapted 406 # from Peter Maydell's freeform(). 407 408 assert len(doc.all_sections) == 1, doc.all_sections 409 body = doc.all_sections[0] 410 text = self.reformat_arobase(body.text) 411 info = doc.info 412 413 if re.match(r"=+ ", text): 414 # Section/subsection heading (if present, will always be the 415 # first line of the block) 416 (heading, _, text) = text.partition("\n") 417 (leader, _, heading) = heading.partition(" ") 418 # Implicit +1 for heading in the containing .rst doc 419 level = len(leader) + 1 420 421 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 422 markers = ' #*=_^"' 423 overline = level <= 2 424 marker = markers[level] 425 426 self.ensure_blank_line() 427 # This credits all 2 or 3 lines to the single source line. 428 if overline: 429 self.add_line(marker * len(heading), info) 430 self.add_line(heading, info) 431 self.add_line(marker * len(heading), info) 432 self.ensure_blank_line() 433 434 # Eat blank line(s) and advance info 435 trimmed = text.lstrip("\n") 436 text = trimmed 437 info = info.next_line(len(text) - len(trimmed) + 1) 438 439 self.add_lines(text, info) 440 self.ensure_blank_line() 441 442 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 443 assert ent.info 444 445 try: 446 self._curr_ent = ent 447 448 # Squish structs and unions together into an "object" directive. 449 meta = ent.meta 450 if meta in ("struct", "union"): 451 meta = "object" 452 453 # This line gets credited to the start of the /definition/. 454 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 455 with self.indented(): 456 self.preamble(ent) 457 self.visit_sections(ent) 458 finally: 459 self._curr_ent = None 460 461 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 462 self.add_line_raw( 463 f".. qapi:namespace:: {namespace}", source, lineno + 1 464 ) 465 self.ensure_blank_line() 466 467 468class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 469 """A QAPI schema visitor which adds Sphinx dependencies each module 470 471 This class calls the Sphinx note_dependency() function to tell Sphinx 472 that the generated documentation output depends on the input 473 schema file associated with each module in the QAPI input. 474 """ 475 476 def __init__(self, env: Any, qapidir: str) -> None: 477 self._env = env 478 self._qapidir = qapidir 479 480 def visit_module(self, name: str) -> None: 481 if name != "./builtin": 482 qapifile = self._qapidir + "/" + name 483 self._env.note_dependency(os.path.abspath(qapifile)) 484 super().visit_module(name) 485 486 487class NestedDirective(SphinxDirective): 488 def run(self) -> Sequence[nodes.Node]: 489 raise NotImplementedError 490 491 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 492 """ 493 Parse rST source lines and add them to the specified node 494 495 Take the list of rST source lines rstlist, parse them as 496 rST, and add the resulting docutils nodes as children of node. 497 The nodes are parsed in a way that allows them to include 498 subheadings (titles) without confusing the rendering of 499 anything else. 500 """ 501 with switch_source_input(self.state, rstlist): 502 nested_parse_with_titles(self.state, rstlist, node) 503 504 505class QAPIDocDirective(NestedDirective): 506 """Extract documentation from the specified QAPI .json file""" 507 508 required_argument = 1 509 optional_arguments = 1 510 option_spec = { 511 "qapifile": directives.unchanged_required, 512 "namespace": directives.unchanged, 513 } 514 has_content = False 515 516 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 517 logger.info("Transmogrifying QAPI to rST ...") 518 vis = Transmogrifier() 519 modules = set() 520 521 if "namespace" in self.options: 522 vis.set_namespace( 523 self.options["namespace"], *self.get_source_info() 524 ) 525 526 for doc in schema.docs: 527 module_source = doc.info.fname 528 if module_source not in modules: 529 vis.visit_module(module_source) 530 modules.add(module_source) 531 532 if doc.symbol: 533 ent = schema.lookup_entity(doc.symbol) 534 assert isinstance(ent, QAPISchemaDefinition) 535 vis.visit_entity(ent) 536 else: 537 vis.visit_freeform(doc) 538 539 logger.info("Transmogrification complete.") 540 541 contentnode = nodes.section() 542 content = vis.result 543 titles_allowed = True 544 545 logger.info("Transmogrifier running nested parse ...") 546 with switch_source_input(self.state, content): 547 if titles_allowed: 548 node: nodes.Element = nodes.section() 549 node.document = self.state.document 550 nested_parse_with_titles(self.state, content, contentnode) 551 else: 552 node = nodes.paragraph() 553 node.document = self.state.document 554 self.state.nested_parse(content, 0, contentnode) 555 logger.info("Transmogrifier's nested parse completed.") 556 557 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 558 argname = "_".join(Path(self.arguments[0]).parts) 559 name = Path(argname).stem + ".ir" 560 self.write_intermediate(content, name) 561 562 sys.stdout.flush() 563 return contentnode 564 565 def write_intermediate(self, content: StringList, filename: str) -> None: 566 logger.info( 567 "writing intermediate rST for '%s' to '%s'", 568 self.arguments[0], 569 filename, 570 ) 571 572 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 573 outlines = [] 574 lcol_width = 0 575 576 for i, line in enumerate(content): 577 src, lineno = content.info(i) 578 srcpath = Path(src).resolve() 579 srcpath = srcpath.relative_to(srctree) 580 581 lcol = f"{srcpath}:{lineno:04d}" 582 lcol_width = max(lcol_width, len(lcol)) 583 outlines.append((lcol, line)) 584 585 with open(filename, "w", encoding="UTF-8") as outfile: 586 for lcol, rcol in outlines: 587 outfile.write(lcol.rjust(lcol_width)) 588 outfile.write(" |") 589 if rcol: 590 outfile.write(f" {rcol}") 591 outfile.write("\n") 592 593 def run(self) -> Sequence[nodes.Node]: 594 env = self.state.document.settings.env 595 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 596 qapidir = os.path.dirname(qapifile) 597 598 try: 599 schema = QAPISchema(qapifile) 600 601 # First tell Sphinx about all the schema files that the 602 # output documentation depends on (including 'qapifile' itself) 603 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 604 except QAPIError as err: 605 # Launder QAPI parse errors into Sphinx extension errors 606 # so they are displayed nicely to the user 607 raise ExtensionError(str(err)) from err 608 609 contentnode = self.transmogrify(schema) 610 return contentnode.children 611 612 613class QMPExample(CodeBlock, NestedDirective): 614 """ 615 Custom admonition for QMP code examples. 616 617 When the :annotated: option is present, the body of this directive 618 is parsed as normal rST, but with any '::' code blocks set to use 619 the QMP lexer. Code blocks must be explicitly written by the user, 620 but this allows for intermingling explanatory paragraphs with 621 arbitrary rST syntax and code blocks for more involved examples. 622 623 When :annotated: is absent, the directive body is treated as a 624 simple standalone QMP code block literal. 625 """ 626 627 required_argument = 0 628 optional_arguments = 0 629 has_content = True 630 option_spec = { 631 "annotated": directives.flag, 632 "title": directives.unchanged, 633 } 634 635 def _highlightlang(self) -> addnodes.highlightlang: 636 """Return the current highlightlang setting for the document""" 637 node = None 638 doc = self.state.document 639 640 if hasattr(doc, "findall"): 641 # docutils >= 0.18.1 642 for node in doc.findall(addnodes.highlightlang): 643 pass 644 else: 645 for elem in doc.traverse(): 646 if isinstance(elem, addnodes.highlightlang): 647 node = elem 648 649 if node: 650 return node 651 652 # No explicit directive found, use defaults 653 node = addnodes.highlightlang( 654 lang=self.env.config.highlight_language, 655 force=False, 656 # Yes, Sphinx uses this value to effectively disable line 657 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 658 linenothreshold=sys.maxsize, 659 ) 660 return node 661 662 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 663 title = "Example:" 664 if "title" in self.options: 665 title = f"{title} {self.options['title']}" 666 667 admon = nodes.admonition( 668 "", 669 nodes.title("", title), 670 *content, 671 classes=["admonition", "admonition-example"], 672 ) 673 return [admon] 674 675 def run_annotated(self) -> List[nodes.Node]: 676 lang_node = self._highlightlang() 677 678 content_node: nodes.Element = nodes.section() 679 680 # Configure QMP highlighting for "::" blocks, if needed 681 if lang_node["lang"] != "QMP": 682 content_node += addnodes.highlightlang( 683 lang="QMP", 684 force=False, # "True" ignores lexing errors 685 linenothreshold=lang_node["linenothreshold"], 686 ) 687 688 self.do_parse(self.content, content_node) 689 690 # Restore prior language highlighting, if needed 691 if lang_node["lang"] != "QMP": 692 content_node += addnodes.highlightlang(**lang_node.attributes) 693 694 return content_node.children 695 696 def run(self) -> List[nodes.Node]: 697 annotated = "annotated" in self.options 698 699 if annotated: 700 content_nodes = self.run_annotated() 701 else: 702 self.arguments = ["QMP"] 703 content_nodes = super().run() 704 705 return self.admonition_wrap(*content_nodes) 706 707 708def setup(app: Sphinx) -> ExtensionMetadata: 709 """Register qapi-doc directive with Sphinx""" 710 app.setup_extension("qapi_domain") 711 app.add_config_value("qapidoc_srctree", None, "env") 712 app.add_directive("qapi-doc", QAPIDocDirective) 713 app.add_directive("qmp-example", QMPExample) 714 715 return { 716 "version": __version__, 717 "parallel_read_safe": True, 718 "parallel_write_safe": True, 719 } 720