1# coding=utf-8 2# 3# QEMU qapidoc QAPI file parsing extension 4# 5# Copyright (c) 2024-2025 Red Hat 6# Copyright (c) 2020 Linaro 7# 8# This work is licensed under the terms of the GNU GPLv2 or later. 9# See the COPYING file in the top-level directory. 10 11""" 12qapidoc is a Sphinx extension that implements the qapi-doc directive 13 14The purpose of this extension is to read the documentation comments 15in QAPI schema files, and insert them all into the current document. 16 17It implements one new rST directive, "qapi-doc::". 18Each qapi-doc:: directive takes one argument, which is the 19pathname of the schema file to process, relative to the source tree. 20 21The docs/conf.py file must set the qapidoc_srctree config value to 22the root of the QEMU source tree. 23 24The Sphinx documentation on writing extensions is at: 25https://www.sphinx-doc.org/en/master/development/index.html 26""" 27 28from __future__ import annotations 29 30 31__version__ = "2.0" 32 33from contextlib import contextmanager 34import os 35from pathlib import Path 36import re 37import sys 38from typing import TYPE_CHECKING 39 40from docutils import nodes 41from docutils.parsers.rst import directives 42from docutils.statemachine import StringList 43from qapi.error import QAPIError 44from qapi.parser import QAPIDoc 45from qapi.schema import ( 46 QAPISchema, 47 QAPISchemaArrayType, 48 QAPISchemaCommand, 49 QAPISchemaDefinition, 50 QAPISchemaEnumMember, 51 QAPISchemaEvent, 52 QAPISchemaFeature, 53 QAPISchemaMember, 54 QAPISchemaObjectType, 55 QAPISchemaObjectTypeMember, 56 QAPISchemaType, 57 QAPISchemaVisitor, 58) 59from qapi.source import QAPISourceInfo 60from sphinx import addnodes 61from sphinx.directives.code import CodeBlock 62from sphinx.errors import ExtensionError 63from sphinx.util import logging 64from sphinx.util.docutils import SphinxDirective, switch_source_input 65from sphinx.util.nodes import nested_parse_with_titles 66 67from qapidoc_legacy import QAPISchemaGenRSTVisitor # type: ignore 68 69 70if TYPE_CHECKING: 71 from typing import ( 72 Any, 73 Generator, 74 List, 75 Optional, 76 Sequence, 77 Union, 78 ) 79 80 from sphinx.application import Sphinx 81 from sphinx.util.typing import ExtensionMetadata 82 83 84logger = logging.getLogger(__name__) 85 86 87class Transmogrifier: 88 # pylint: disable=too-many-public-methods 89 90 # Field names used for different entity types: 91 field_types = { 92 "enum": "value", 93 "struct": "memb", 94 "union": "memb", 95 "event": "memb", 96 "command": "arg", 97 "alternate": "alt", 98 } 99 100 def __init__(self) -> None: 101 self._curr_ent: Optional[QAPISchemaDefinition] = None 102 self._result = StringList() 103 self.indent = 0 104 105 @property 106 def result(self) -> StringList: 107 return self._result 108 109 @property 110 def entity(self) -> QAPISchemaDefinition: 111 assert self._curr_ent is not None 112 return self._curr_ent 113 114 @property 115 def member_field_type(self) -> str: 116 return self.field_types[self.entity.meta] 117 118 # General-purpose rST generation functions 119 120 def get_indent(self) -> str: 121 return " " * self.indent 122 123 @contextmanager 124 def indented(self) -> Generator[None]: 125 self.indent += 1 126 try: 127 yield 128 finally: 129 self.indent -= 1 130 131 def add_line_raw(self, line: str, source: str, *lineno: int) -> None: 132 """Append one line of generated reST to the output.""" 133 134 # NB: Sphinx uses zero-indexed lines; subtract one. 135 lineno = tuple((n - 1 for n in lineno)) 136 137 if line.strip(): 138 # not a blank line 139 self._result.append( 140 self.get_indent() + line.rstrip("\n"), source, *lineno 141 ) 142 else: 143 self._result.append("", source, *lineno) 144 145 def add_line(self, content: str, info: QAPISourceInfo) -> None: 146 # NB: We *require* an info object; this works out OK because we 147 # don't document built-in objects that don't have 148 # one. Everything else should. 149 self.add_line_raw(content, info.fname, info.line) 150 151 def add_lines( 152 self, 153 content: str, 154 info: QAPISourceInfo, 155 ) -> None: 156 lines = content.splitlines(True) 157 for i, line in enumerate(lines): 158 self.add_line_raw(line, info.fname, info.line + i) 159 160 def ensure_blank_line(self) -> None: 161 # Empty document -- no blank line required. 162 if not self._result: 163 return 164 165 # Last line isn't blank, add one. 166 if self._result[-1].strip(): # pylint: disable=no-member 167 fname, line = self._result.info(-1) 168 assert isinstance(line, int) 169 # New blank line is credited to one-after the current last line. 170 # +2: correct for zero/one index, then increment by one. 171 self.add_line_raw("", fname, line + 2) 172 173 def add_field( 174 self, 175 kind: str, 176 name: str, 177 body: str, 178 info: QAPISourceInfo, 179 typ: Optional[str] = None, 180 ) -> None: 181 if typ: 182 text = f":{kind} {typ} {name}: {body}" 183 else: 184 text = f":{kind} {name}: {body}" 185 self.add_lines(text, info) 186 187 def format_type( 188 self, ent: Union[QAPISchemaDefinition | QAPISchemaMember] 189 ) -> Optional[str]: 190 if isinstance(ent, (QAPISchemaEnumMember, QAPISchemaFeature)): 191 return None 192 193 qapi_type = ent 194 optional = False 195 if isinstance(ent, QAPISchemaObjectTypeMember): 196 qapi_type = ent.type 197 optional = ent.optional 198 199 if isinstance(qapi_type, QAPISchemaArrayType): 200 ret = f"[{qapi_type.element_type.doc_type()}]" 201 else: 202 assert isinstance(qapi_type, QAPISchemaType) 203 tmp = qapi_type.doc_type() 204 assert tmp 205 ret = tmp 206 if optional: 207 ret += "?" 208 209 return ret 210 211 def generate_field( 212 self, 213 kind: str, 214 member: QAPISchemaMember, 215 body: str, 216 info: QAPISourceInfo, 217 ) -> None: 218 typ = self.format_type(member) 219 self.add_field(kind, member.name, body, info, typ) 220 221 @staticmethod 222 def reformat_arobase(text: str) -> str: 223 """ reformats @var to ``var`` """ 224 return re.sub(r"@([\w-]+)", r"``\1``", text) 225 226 # Transmogrification helpers 227 228 def visit_paragraph(self, section: QAPIDoc.Section) -> None: 229 # Squelch empty paragraphs. 230 if not section.text: 231 return 232 233 self.ensure_blank_line() 234 self.add_lines(section.text, section.info) 235 self.ensure_blank_line() 236 237 def visit_member(self, section: QAPIDoc.ArgSection) -> None: 238 # FIXME: ifcond for members 239 # TODO: features for members (documented at entity-level, 240 # but sometimes defined per-member. Should we add such 241 # information to member descriptions when we can?) 242 assert section.member 243 self.generate_field( 244 self.member_field_type, 245 section.member, 246 # TODO drop fallbacks when undocumented members are outlawed 247 section.text if section.text else "Not documented", 248 section.info, 249 ) 250 251 def visit_feature(self, section: QAPIDoc.ArgSection) -> None: 252 # FIXME - ifcond for features is not handled at all yet! 253 # Proposal: decorate the right-hand column with some graphical 254 # element to indicate conditional availability? 255 assert section.text # Guaranteed by parser.py 256 assert section.member 257 258 self.generate_field("feat", section.member, section.text, section.info) 259 260 def visit_returns(self, section: QAPIDoc.Section) -> None: 261 assert isinstance(self.entity, QAPISchemaCommand) 262 rtype = self.entity.ret_type 263 # q_empty can produce None, but we won't be documenting anything 264 # without an explicit return statement in the doc block, and we 265 # should not have any such explicit statements when there is no 266 # return value. 267 assert rtype 268 269 typ = self.format_type(rtype) 270 assert typ 271 assert section.text 272 self.add_field("return", typ, section.text, section.info) 273 274 def visit_errors(self, section: QAPIDoc.Section) -> None: 275 # If the section text does not start with a space, it means text 276 # began on the same line as the "Error:" string and we should 277 # not insert a newline in this case. 278 if section.text[0].isspace(): 279 text = f":error:\n{section.text}" 280 else: 281 text = f":error: {section.text}" 282 self.add_lines(text, section.info) 283 284 def preamble(self, ent: QAPISchemaDefinition) -> None: 285 """ 286 Generate option lines for QAPI entity directives. 287 """ 288 if ent.doc and ent.doc.since: 289 assert ent.doc.since.kind == QAPIDoc.Kind.SINCE 290 # Generated from the entity's docblock; info location is exact. 291 self.add_line(f":since: {ent.doc.since.text}", ent.doc.since.info) 292 293 if ent.ifcond.is_present(): 294 doc = ent.ifcond.docgen() 295 assert ent.info 296 # Generated from entity definition; info location is approximate. 297 self.add_line(f":ifcond: {doc}", ent.info) 298 299 # Hoist special features such as :deprecated: and :unstable: 300 # into the options block for the entity. If, in the future, new 301 # special features are added, qapi-domain will chirp about 302 # unrecognized options and fail until they are handled in 303 # qapi-domain. 304 for feat in ent.features: 305 if feat.is_special(): 306 # FIXME: handle ifcond if present. How to display that 307 # information is TBD. 308 # Generated from entity def; info location is approximate. 309 assert feat.info 310 self.add_line(f":{feat.name}:", feat.info) 311 312 self.ensure_blank_line() 313 314 def _insert_member_pointer(self, ent: QAPISchemaDefinition) -> None: 315 316 def _get_target( 317 ent: QAPISchemaDefinition, 318 ) -> Optional[QAPISchemaDefinition]: 319 if isinstance(ent, (QAPISchemaCommand, QAPISchemaEvent)): 320 return ent.arg_type 321 if isinstance(ent, QAPISchemaObjectType): 322 return ent.base 323 return None 324 325 target = _get_target(ent) 326 if target is not None and not target.is_implicit(): 327 assert ent.info 328 self.add_field( 329 self.member_field_type, 330 "q_dummy", 331 f"The members of :qapi:type:`{target.name}`.", 332 ent.info, 333 "q_dummy", 334 ) 335 336 if isinstance(ent, QAPISchemaObjectType) and ent.branches is not None: 337 for variant in ent.branches.variants: 338 if variant.type.name == "q_empty": 339 continue 340 assert ent.info 341 self.add_field( 342 self.member_field_type, 343 "q_dummy", 344 f" When ``{ent.branches.tag_member.name}`` is " 345 f"``{variant.name}``: " 346 f"The members of :qapi:type:`{variant.type.name}`.", 347 ent.info, 348 "q_dummy", 349 ) 350 351 def visit_sections(self, ent: QAPISchemaDefinition) -> None: 352 sections = ent.doc.all_sections if ent.doc else [] 353 354 # Determine the index location at which we should generate 355 # documentation for "The members of ..." pointers. This should 356 # go at the end of the members section(s) if any. Note that 357 # index 0 is assumed to be a plain intro section, even if it is 358 # empty; and that a members section if present will always 359 # immediately follow the opening PLAIN section. 360 gen_index = 1 361 if len(sections) > 1: 362 while sections[gen_index].kind == QAPIDoc.Kind.MEMBER: 363 gen_index += 1 364 if gen_index >= len(sections): 365 break 366 367 # Add sections in source order: 368 for i, section in enumerate(sections): 369 section.text = self.reformat_arobase(section.text) 370 371 if section.kind == QAPIDoc.Kind.PLAIN: 372 self.visit_paragraph(section) 373 elif section.kind == QAPIDoc.Kind.MEMBER: 374 assert isinstance(section, QAPIDoc.ArgSection) 375 self.visit_member(section) 376 elif section.kind == QAPIDoc.Kind.FEATURE: 377 assert isinstance(section, QAPIDoc.ArgSection) 378 self.visit_feature(section) 379 elif section.kind in (QAPIDoc.Kind.SINCE, QAPIDoc.Kind.TODO): 380 # Since is handled in preamble, TODO is skipped intentionally. 381 pass 382 elif section.kind == QAPIDoc.Kind.RETURNS: 383 self.visit_returns(section) 384 elif section.kind == QAPIDoc.Kind.ERRORS: 385 self.visit_errors(section) 386 else: 387 assert False 388 389 # Generate "The members of ..." entries if necessary: 390 if i == gen_index - 1: 391 self._insert_member_pointer(ent) 392 393 self.ensure_blank_line() 394 395 # Transmogrification core methods 396 397 def visit_module(self, path: str) -> None: 398 name = Path(path).stem 399 # module directives are credited to the first line of a module file. 400 self.add_line_raw(f".. qapi:module:: {name}", path, 1) 401 self.ensure_blank_line() 402 403 def visit_freeform(self, doc: QAPIDoc) -> None: 404 # TODO: Once the old qapidoc transformer is deprecated, freeform 405 # sections can be updated to pure rST, and this transformed removed. 406 # 407 # For now, translate our micro-format into rST. Code adapted 408 # from Peter Maydell's freeform(). 409 410 assert len(doc.all_sections) == 1, doc.all_sections 411 body = doc.all_sections[0] 412 text = self.reformat_arobase(body.text) 413 info = doc.info 414 415 if re.match(r"=+ ", text): 416 # Section/subsection heading (if present, will always be the 417 # first line of the block) 418 (heading, _, text) = text.partition("\n") 419 (leader, _, heading) = heading.partition(" ") 420 # Implicit +1 for heading in the containing .rst doc 421 level = len(leader) + 1 422 423 # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#sections 424 markers = ' #*=_^"' 425 overline = level <= 2 426 marker = markers[level] 427 428 self.ensure_blank_line() 429 # This credits all 2 or 3 lines to the single source line. 430 if overline: 431 self.add_line(marker * len(heading), info) 432 self.add_line(heading, info) 433 self.add_line(marker * len(heading), info) 434 self.ensure_blank_line() 435 436 # Eat blank line(s) and advance info 437 trimmed = text.lstrip("\n") 438 text = trimmed 439 info = info.next_line(len(text) - len(trimmed) + 1) 440 441 self.add_lines(text, info) 442 self.ensure_blank_line() 443 444 def visit_entity(self, ent: QAPISchemaDefinition) -> None: 445 assert ent.info 446 447 try: 448 self._curr_ent = ent 449 450 # Squish structs and unions together into an "object" directive. 451 meta = ent.meta 452 if meta in ("struct", "union"): 453 meta = "object" 454 455 # This line gets credited to the start of the /definition/. 456 self.add_line(f".. qapi:{meta}:: {ent.name}", ent.info) 457 with self.indented(): 458 self.preamble(ent) 459 self.visit_sections(ent) 460 finally: 461 self._curr_ent = None 462 463 def set_namespace(self, namespace: str, source: str, lineno: int) -> None: 464 self.add_line_raw( 465 f".. qapi:namespace:: {namespace}", source, lineno + 1 466 ) 467 self.ensure_blank_line() 468 469 470class QAPISchemaGenDepVisitor(QAPISchemaVisitor): 471 """A QAPI schema visitor which adds Sphinx dependencies each module 472 473 This class calls the Sphinx note_dependency() function to tell Sphinx 474 that the generated documentation output depends on the input 475 schema file associated with each module in the QAPI input. 476 """ 477 478 def __init__(self, env: Any, qapidir: str) -> None: 479 self._env = env 480 self._qapidir = qapidir 481 482 def visit_module(self, name: str) -> None: 483 if name != "./builtin": 484 qapifile = self._qapidir + "/" + name 485 self._env.note_dependency(os.path.abspath(qapifile)) 486 super().visit_module(name) 487 488 489class NestedDirective(SphinxDirective): 490 def run(self) -> Sequence[nodes.Node]: 491 raise NotImplementedError 492 493 def do_parse(self, rstlist: StringList, node: nodes.Node) -> None: 494 """ 495 Parse rST source lines and add them to the specified node 496 497 Take the list of rST source lines rstlist, parse them as 498 rST, and add the resulting docutils nodes as children of node. 499 The nodes are parsed in a way that allows them to include 500 subheadings (titles) without confusing the rendering of 501 anything else. 502 """ 503 with switch_source_input(self.state, rstlist): 504 nested_parse_with_titles(self.state, rstlist, node) 505 506 507class QAPIDocDirective(NestedDirective): 508 """Extract documentation from the specified QAPI .json file""" 509 510 required_argument = 1 511 optional_arguments = 1 512 option_spec = { 513 "qapifile": directives.unchanged_required, 514 "namespace": directives.unchanged, 515 "transmogrify": directives.flag, 516 } 517 has_content = False 518 519 def new_serialno(self) -> str: 520 """Return a unique new ID string suitable for use as a node's ID""" 521 env = self.state.document.settings.env 522 return "qapidoc-%d" % env.new_serialno("qapidoc") 523 524 def transmogrify(self, schema: QAPISchema) -> nodes.Element: 525 logger.info("Transmogrifying QAPI to rST ...") 526 vis = Transmogrifier() 527 modules = set() 528 529 if "namespace" in self.options: 530 vis.set_namespace( 531 self.options["namespace"], *self.get_source_info() 532 ) 533 534 for doc in schema.docs: 535 module_source = doc.info.fname 536 if module_source not in modules: 537 vis.visit_module(module_source) 538 modules.add(module_source) 539 540 if doc.symbol: 541 ent = schema.lookup_entity(doc.symbol) 542 assert isinstance(ent, QAPISchemaDefinition) 543 vis.visit_entity(ent) 544 else: 545 vis.visit_freeform(doc) 546 547 logger.info("Transmogrification complete.") 548 549 contentnode = nodes.section() 550 content = vis.result 551 titles_allowed = True 552 553 logger.info("Transmogrifier running nested parse ...") 554 with switch_source_input(self.state, content): 555 if titles_allowed: 556 node: nodes.Element = nodes.section() 557 node.document = self.state.document 558 nested_parse_with_titles(self.state, content, contentnode) 559 else: 560 node = nodes.paragraph() 561 node.document = self.state.document 562 self.state.nested_parse(content, 0, contentnode) 563 logger.info("Transmogrifier's nested parse completed.") 564 565 if self.env.app.verbosity >= 2 or os.environ.get("DEBUG"): 566 argname = "_".join(Path(self.arguments[0]).parts) 567 name = Path(argname).stem + ".ir" 568 self.write_intermediate(content, name) 569 570 sys.stdout.flush() 571 return contentnode 572 573 def write_intermediate(self, content: StringList, filename: str) -> None: 574 logger.info( 575 "writing intermediate rST for '%s' to '%s'", 576 self.arguments[0], 577 filename, 578 ) 579 580 srctree = Path(self.env.app.config.qapidoc_srctree).resolve() 581 outlines = [] 582 lcol_width = 0 583 584 for i, line in enumerate(content): 585 src, lineno = content.info(i) 586 srcpath = Path(src).resolve() 587 srcpath = srcpath.relative_to(srctree) 588 589 lcol = f"{srcpath}:{lineno:04d}" 590 lcol_width = max(lcol_width, len(lcol)) 591 outlines.append((lcol, line)) 592 593 with open(filename, "w", encoding="UTF-8") as outfile: 594 for lcol, rcol in outlines: 595 outfile.write(lcol.rjust(lcol_width)) 596 outfile.write(" |") 597 if rcol: 598 outfile.write(f" {rcol}") 599 outfile.write("\n") 600 601 def legacy(self, schema: QAPISchema) -> nodes.Element: 602 vis = QAPISchemaGenRSTVisitor(self) 603 vis.visit_begin(schema) 604 for doc in schema.docs: 605 if doc.symbol: 606 vis.symbol(doc, schema.lookup_entity(doc.symbol)) 607 else: 608 vis.freeform(doc) 609 return vis.get_document_node() # type: ignore 610 611 def run(self) -> Sequence[nodes.Node]: 612 env = self.state.document.settings.env 613 qapifile = env.config.qapidoc_srctree + "/" + self.arguments[0] 614 qapidir = os.path.dirname(qapifile) 615 transmogrify = "transmogrify" in self.options 616 617 try: 618 schema = QAPISchema(qapifile) 619 620 # First tell Sphinx about all the schema files that the 621 # output documentation depends on (including 'qapifile' itself) 622 schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) 623 except QAPIError as err: 624 # Launder QAPI parse errors into Sphinx extension errors 625 # so they are displayed nicely to the user 626 raise ExtensionError(str(err)) from err 627 628 if transmogrify: 629 contentnode = self.transmogrify(schema) 630 else: 631 contentnode = self.legacy(schema) 632 633 return contentnode.children 634 635 636class QMPExample(CodeBlock, NestedDirective): 637 """ 638 Custom admonition for QMP code examples. 639 640 When the :annotated: option is present, the body of this directive 641 is parsed as normal rST, but with any '::' code blocks set to use 642 the QMP lexer. Code blocks must be explicitly written by the user, 643 but this allows for intermingling explanatory paragraphs with 644 arbitrary rST syntax and code blocks for more involved examples. 645 646 When :annotated: is absent, the directive body is treated as a 647 simple standalone QMP code block literal. 648 """ 649 650 required_argument = 0 651 optional_arguments = 0 652 has_content = True 653 option_spec = { 654 "annotated": directives.flag, 655 "title": directives.unchanged, 656 } 657 658 def _highlightlang(self) -> addnodes.highlightlang: 659 """Return the current highlightlang setting for the document""" 660 node = None 661 doc = self.state.document 662 663 if hasattr(doc, "findall"): 664 # docutils >= 0.18.1 665 for node in doc.findall(addnodes.highlightlang): 666 pass 667 else: 668 for elem in doc.traverse(): 669 if isinstance(elem, addnodes.highlightlang): 670 node = elem 671 672 if node: 673 return node 674 675 # No explicit directive found, use defaults 676 node = addnodes.highlightlang( 677 lang=self.env.config.highlight_language, 678 force=False, 679 # Yes, Sphinx uses this value to effectively disable line 680 # numbers and not 0 or None or -1 or something. ¯\_(ツ)_/¯ 681 linenothreshold=sys.maxsize, 682 ) 683 return node 684 685 def admonition_wrap(self, *content: nodes.Node) -> List[nodes.Node]: 686 title = "Example:" 687 if "title" in self.options: 688 title = f"{title} {self.options['title']}" 689 690 admon = nodes.admonition( 691 "", 692 nodes.title("", title), 693 *content, 694 classes=["admonition", "admonition-example"], 695 ) 696 return [admon] 697 698 def run_annotated(self) -> List[nodes.Node]: 699 lang_node = self._highlightlang() 700 701 content_node: nodes.Element = nodes.section() 702 703 # Configure QMP highlighting for "::" blocks, if needed 704 if lang_node["lang"] != "QMP": 705 content_node += addnodes.highlightlang( 706 lang="QMP", 707 force=False, # "True" ignores lexing errors 708 linenothreshold=lang_node["linenothreshold"], 709 ) 710 711 self.do_parse(self.content, content_node) 712 713 # Restore prior language highlighting, if needed 714 if lang_node["lang"] != "QMP": 715 content_node += addnodes.highlightlang(**lang_node.attributes) 716 717 return content_node.children 718 719 def run(self) -> List[nodes.Node]: 720 annotated = "annotated" in self.options 721 722 if annotated: 723 content_nodes = self.run_annotated() 724 else: 725 self.arguments = ["QMP"] 726 content_nodes = super().run() 727 728 return self.admonition_wrap(*content_nodes) 729 730 731def setup(app: Sphinx) -> ExtensionMetadata: 732 """Register qapi-doc directive with Sphinx""" 733 app.setup_extension("qapi_domain") 734 app.add_config_value("qapidoc_srctree", None, "env") 735 app.add_directive("qapi-doc", QAPIDocDirective) 736 app.add_directive("qmp-example", QMPExample) 737 738 return { 739 "version": __version__, 740 "parallel_read_safe": True, 741 "parallel_write_safe": True, 742 } 743