xref: /openbmc/qemu/docs/sphinx/qapidoc.py (revision 64c6e744)
1# coding=utf-8
2#
3# QEMU qapidoc QAPI file parsing extension
4#
5# Copyright (c) 2020 Linaro
6#
7# This work is licensed under the terms of the GNU GPLv2 or later.
8# See the COPYING file in the top-level directory.
9
10"""
11qapidoc is a Sphinx extension that implements the qapi-doc directive
12
13The purpose of this extension is to read the documentation comments
14in QAPI schema files, and insert them all into the current document.
15
16It implements one new rST directive, "qapi-doc::".
17Each qapi-doc:: directive takes one argument, which is the
18pathname of the schema file to process, relative to the source tree.
19
20The docs/conf.py file must set the qapidoc_srctree config value to
21the root of the QEMU source tree.
22
23The Sphinx documentation on writing extensions is at:
24https://www.sphinx-doc.org/en/master/development/index.html
25"""
26
27import os
28import re
29
30from docutils import nodes
31from docutils.statemachine import ViewList
32from docutils.parsers.rst import directives, Directive
33from sphinx.errors import ExtensionError
34from sphinx.util.nodes import nested_parse_with_titles
35import sphinx
36from qapi.gen import QAPISchemaVisitor
37from qapi.error import QAPIError, QAPISemError
38from qapi.schema import QAPISchema
39
40
41# Sphinx up to 1.6 uses AutodocReporter; 1.7 and later
42# use switch_source_input. Check borrowed from kerneldoc.py.
43Use_SSI = sphinx.__version__[:3] >= '1.7'
44if Use_SSI:
45    from sphinx.util.docutils import switch_source_input
46else:
47    from sphinx.ext.autodoc import AutodocReporter
48
49
50__version__ = '1.0'
51
52
53# Function borrowed from pydash, which is under the MIT license
54def intersperse(iterable, separator):
55    """Yield the members of *iterable* interspersed with *separator*."""
56    iterable = iter(iterable)
57    yield next(iterable)
58    for item in iterable:
59        yield separator
60        yield item
61
62
63class QAPISchemaGenRSTVisitor(QAPISchemaVisitor):
64    """A QAPI schema visitor which generates docutils/Sphinx nodes
65
66    This class builds up a tree of docutils/Sphinx nodes corresponding
67    to documentation for the various QAPI objects. To use it, first
68    create a QAPISchemaGenRSTVisitor object, and call its
69    visit_begin() method.  Then you can call one of the two methods
70    'freeform' (to add documentation for a freeform documentation
71    chunk) or 'symbol' (to add documentation for a QAPI symbol). These
72    will cause the visitor to build up the tree of document
73    nodes. Once you've added all the documentation via 'freeform' and
74    'symbol' method calls, you can call 'get_document_nodes' to get
75    the final list of document nodes (in a form suitable for returning
76    from a Sphinx directive's 'run' method).
77    """
78    def __init__(self, sphinx_directive):
79        self._cur_doc = None
80        self._sphinx_directive = sphinx_directive
81        self._top_node = nodes.section()
82        self._active_headings = [self._top_node]
83
84    def _make_dlitem(self, term, defn):
85        """Return a dlitem node with the specified term and definition.
86
87        term should be a list of Text and literal nodes.
88        defn should be one of:
89        - a string, which will be handed to _parse_text_into_node
90        - a list of Text and literal nodes, which will be put into
91          a paragraph node
92        """
93        dlitem = nodes.definition_list_item()
94        dlterm = nodes.term('', '', *term)
95        dlitem += dlterm
96        if defn:
97            dldef = nodes.definition()
98            if isinstance(defn, list):
99                dldef += nodes.paragraph('', '', *defn)
100            else:
101                self._parse_text_into_node(defn, dldef)
102            dlitem += dldef
103        return dlitem
104
105    def _make_section(self, title):
106        """Return a section node with optional title"""
107        section = nodes.section(ids=[self._sphinx_directive.new_serialno()])
108        if title:
109            section += nodes.title(title, title)
110        return section
111
112    def _nodes_for_ifcond(self, ifcond, with_if=True):
113        """Return list of Text, literal nodes for the ifcond
114
115        Return a list which gives text like ' (If: condition)'.
116        If with_if is False, we don't return the "(If: " and ")".
117        """
118
119        doc = ifcond.docgen()
120        if not doc:
121            return []
122        doc = nodes.literal('', doc)
123        if not with_if:
124            return [doc]
125
126        nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')]
127        nodelist.append(doc)
128        nodelist.append(nodes.Text(')'))
129        return nodelist
130
131    def _nodes_for_one_member(self, member):
132        """Return list of Text, literal nodes for this member
133
134        Return a list of doctree nodes which give text like
135        'name: type (optional) (If: ...)' suitable for use as the
136        'term' part of a definition list item.
137        """
138        term = [nodes.literal('', member.name)]
139        if member.type.doc_type():
140            term.append(nodes.Text(': '))
141            term.append(nodes.literal('', member.type.doc_type()))
142        if member.optional:
143            term.append(nodes.Text(' (optional)'))
144        if member.ifcond.is_present():
145            term.extend(self._nodes_for_ifcond(member.ifcond))
146        return term
147
148    def _nodes_for_variant_when(self, variants, variant):
149        """Return list of Text, literal nodes for variant 'when' clause
150
151        Return a list of doctree nodes which give text like
152        'when tagname is variant (If: ...)' suitable for use in
153        the 'variants' part of a definition list.
154        """
155        term = [nodes.Text(' when '),
156                nodes.literal('', variants.tag_member.name),
157                nodes.Text(' is '),
158                nodes.literal('', '"%s"' % variant.name)]
159        if variant.ifcond.is_present():
160            term.extend(self._nodes_for_ifcond(variant.ifcond))
161        return term
162
163    def _nodes_for_members(self, doc, what, base=None, variants=None):
164        """Return list of doctree nodes for the table of members"""
165        dlnode = nodes.definition_list()
166        for section in doc.args.values():
167            term = self._nodes_for_one_member(section.member)
168            # TODO drop fallbacks when undocumented members are outlawed
169            if section.text:
170                defn = section.text
171            else:
172                defn = [nodes.Text('Not documented')]
173
174            dlnode += self._make_dlitem(term, defn)
175
176        if base:
177            dlnode += self._make_dlitem([nodes.Text('The members of '),
178                                         nodes.literal('', base.doc_type())],
179                                        None)
180
181        if variants:
182            for v in variants.variants:
183                if v.type.is_implicit():
184                    assert not v.type.base and not v.type.variants
185                    for m in v.type.local_members:
186                        term = self._nodes_for_one_member(m)
187                        term.extend(self._nodes_for_variant_when(variants, v))
188                        dlnode += self._make_dlitem(term, None)
189                else:
190                    term = [nodes.Text('The members of '),
191                            nodes.literal('', v.type.doc_type())]
192                    term.extend(self._nodes_for_variant_when(variants, v))
193                    dlnode += self._make_dlitem(term, None)
194
195        if not dlnode.children:
196            return []
197
198        section = self._make_section(what)
199        section += dlnode
200        return [section]
201
202    def _nodes_for_enum_values(self, doc):
203        """Return list of doctree nodes for the table of enum values"""
204        seen_item = False
205        dlnode = nodes.definition_list()
206        for section in doc.args.values():
207            termtext = [nodes.literal('', section.member.name)]
208            if section.member.ifcond.is_present():
209                termtext.extend(self._nodes_for_ifcond(section.member.ifcond))
210            # TODO drop fallbacks when undocumented members are outlawed
211            if section.text:
212                defn = section.text
213            else:
214                defn = [nodes.Text('Not documented')]
215
216            dlnode += self._make_dlitem(termtext, defn)
217            seen_item = True
218
219        if not seen_item:
220            return []
221
222        section = self._make_section('Values')
223        section += dlnode
224        return [section]
225
226    def _nodes_for_arguments(self, doc, boxed_arg_type):
227        """Return list of doctree nodes for the arguments section"""
228        if boxed_arg_type:
229            assert not doc.args
230            section = self._make_section('Arguments')
231            dlnode = nodes.definition_list()
232            dlnode += self._make_dlitem(
233                [nodes.Text('The members of '),
234                 nodes.literal('', boxed_arg_type.name)],
235                None)
236            section += dlnode
237            return [section]
238
239        return self._nodes_for_members(doc, 'Arguments')
240
241    def _nodes_for_features(self, doc):
242        """Return list of doctree nodes for the table of features"""
243        seen_item = False
244        dlnode = nodes.definition_list()
245        for section in doc.features.values():
246            dlnode += self._make_dlitem([nodes.literal('', section.name)],
247                                        section.text)
248            seen_item = True
249
250        if not seen_item:
251            return []
252
253        section = self._make_section('Features')
254        section += dlnode
255        return [section]
256
257    def _nodes_for_example(self, exampletext):
258        """Return list of doctree nodes for a code example snippet"""
259        return [nodes.literal_block(exampletext, exampletext)]
260
261    def _nodes_for_sections(self, doc):
262        """Return list of doctree nodes for additional sections"""
263        nodelist = []
264        for section in doc.sections:
265            if section.name and section.name == 'TODO':
266                # Hide TODO: sections
267                continue
268            snode = self._make_section(section.name)
269            if section.name and section.name.startswith('Example'):
270                snode += self._nodes_for_example(section.text)
271            else:
272                self._parse_text_into_node(section.text, snode)
273            nodelist.append(snode)
274        return nodelist
275
276    def _nodes_for_if_section(self, ifcond):
277        """Return list of doctree nodes for the "If" section"""
278        nodelist = []
279        if ifcond.is_present():
280            snode = self._make_section('If')
281            snode += nodes.paragraph(
282                '', '', *self._nodes_for_ifcond(ifcond, with_if=False)
283            )
284            nodelist.append(snode)
285        return nodelist
286
287    def _add_doc(self, typ, sections):
288        """Add documentation for a command/object/enum...
289
290        We assume we're documenting the thing defined in self._cur_doc.
291        typ is the type of thing being added ("Command", "Object", etc)
292
293        sections is a list of nodes for sections to add to the definition.
294        """
295
296        doc = self._cur_doc
297        snode = nodes.section(ids=[self._sphinx_directive.new_serialno()])
298        snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol),
299                                       nodes.Text(' (' + typ + ')')])
300        self._parse_text_into_node(doc.body.text, snode)
301        for s in sections:
302            if s is not None:
303                snode += s
304        self._add_node_to_current_heading(snode)
305
306    def visit_enum_type(self, name, info, ifcond, features, members, prefix):
307        doc = self._cur_doc
308        self._add_doc('Enum',
309                      self._nodes_for_enum_values(doc)
310                      + self._nodes_for_features(doc)
311                      + self._nodes_for_sections(doc)
312                      + self._nodes_for_if_section(ifcond))
313
314    def visit_object_type(self, name, info, ifcond, features,
315                          base, members, variants):
316        doc = self._cur_doc
317        if base and base.is_implicit():
318            base = None
319        self._add_doc('Object',
320                      self._nodes_for_members(doc, 'Members', base, variants)
321                      + self._nodes_for_features(doc)
322                      + self._nodes_for_sections(doc)
323                      + self._nodes_for_if_section(ifcond))
324
325    def visit_alternate_type(self, name, info, ifcond, features, variants):
326        doc = self._cur_doc
327        self._add_doc('Alternate',
328                      self._nodes_for_members(doc, 'Members')
329                      + self._nodes_for_features(doc)
330                      + self._nodes_for_sections(doc)
331                      + self._nodes_for_if_section(ifcond))
332
333    def visit_command(self, name, info, ifcond, features, arg_type,
334                      ret_type, gen, success_response, boxed, allow_oob,
335                      allow_preconfig, coroutine):
336        doc = self._cur_doc
337        self._add_doc('Command',
338                      self._nodes_for_arguments(doc,
339                                                arg_type if boxed else None)
340                      + self._nodes_for_features(doc)
341                      + self._nodes_for_sections(doc)
342                      + self._nodes_for_if_section(ifcond))
343
344    def visit_event(self, name, info, ifcond, features, arg_type, boxed):
345        doc = self._cur_doc
346        self._add_doc('Event',
347                      self._nodes_for_arguments(doc,
348                                                arg_type if boxed else None)
349                      + self._nodes_for_features(doc)
350                      + self._nodes_for_sections(doc)
351                      + self._nodes_for_if_section(ifcond))
352
353    def symbol(self, doc, entity):
354        """Add documentation for one symbol to the document tree
355
356        This is the main entry point which causes us to add documentation
357        nodes for a symbol (which could be a 'command', 'object', 'event',
358        etc). We do this by calling 'visit' on the schema entity, which
359        will then call back into one of our visit_* methods, depending
360        on what kind of thing this symbol is.
361        """
362        self._cur_doc = doc
363        entity.visit(self)
364        self._cur_doc = None
365
366    def _start_new_heading(self, heading, level):
367        """Start a new heading at the specified heading level
368
369        Create a new section whose title is 'heading' and which is placed
370        in the docutils node tree as a child of the most recent level-1
371        heading. Subsequent document sections (commands, freeform doc chunks,
372        etc) will be placed as children of this new heading section.
373        """
374        if len(self._active_headings) < level:
375            raise QAPISemError(self._cur_doc.info,
376                               'Level %d subheading found outside a '
377                               'level %d heading'
378                               % (level, level - 1))
379        snode = self._make_section(heading)
380        self._active_headings[level - 1] += snode
381        self._active_headings = self._active_headings[:level]
382        self._active_headings.append(snode)
383
384    def _add_node_to_current_heading(self, node):
385        """Add the node to whatever the current active heading is"""
386        self._active_headings[-1] += node
387
388    def freeform(self, doc):
389        """Add a piece of 'freeform' documentation to the document tree
390
391        A 'freeform' document chunk doesn't relate to any particular
392        symbol (for instance, it could be an introduction).
393
394        If the freeform document starts with a line of the form
395        '= Heading text', this is a section or subsection heading, with
396        the heading level indicated by the number of '=' signs.
397        """
398
399        # QAPIDoc documentation says free-form documentation blocks
400        # must have only a body section, nothing else.
401        assert not doc.sections
402        assert not doc.args
403        assert not doc.features
404        self._cur_doc = doc
405
406        text = doc.body.text
407        if re.match(r'=+ ', text):
408            # Section/subsection heading (if present, will always be
409            # the first line of the block)
410            (heading, _, text) = text.partition('\n')
411            (leader, _, heading) = heading.partition(' ')
412            self._start_new_heading(heading, len(leader))
413            if text == '':
414                return
415
416        node = self._make_section(None)
417        self._parse_text_into_node(text, node)
418        self._add_node_to_current_heading(node)
419        self._cur_doc = None
420
421    def _parse_text_into_node(self, doctext, node):
422        """Parse a chunk of QAPI-doc-format text into the node
423
424        The doc comment can contain most inline rST markup, including
425        bulleted and enumerated lists.
426        As an extra permitted piece of markup, @var will be turned
427        into ``var``.
428        """
429
430        # Handle the "@var means ``var`` case
431        doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext)
432
433        rstlist = ViewList()
434        for line in doctext.splitlines():
435            # The reported line number will always be that of the start line
436            # of the doc comment, rather than the actual location of the error.
437            # Being more precise would require overhaul of the QAPIDoc class
438            # to track lines more exactly within all the sub-parts of the doc
439            # comment, as well as counting lines here.
440            rstlist.append(line, self._cur_doc.info.fname,
441                           self._cur_doc.info.line)
442        # Append a blank line -- in some cases rST syntax errors get
443        # attributed to the line after one with actual text, and if there
444        # isn't anything in the ViewList corresponding to that then Sphinx
445        # 1.6's AutodocReporter will then misidentify the source/line location
446        # in the error message (usually attributing it to the top-level
447        # .rst file rather than the offending .json file). The extra blank
448        # line won't affect the rendered output.
449        rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line)
450        self._sphinx_directive.do_parse(rstlist, node)
451
452    def get_document_nodes(self):
453        """Return the list of docutils nodes which make up the document"""
454        return self._top_node.children
455
456
457class QAPISchemaGenDepVisitor(QAPISchemaVisitor):
458    """A QAPI schema visitor which adds Sphinx dependencies each module
459
460    This class calls the Sphinx note_dependency() function to tell Sphinx
461    that the generated documentation output depends on the input
462    schema file associated with each module in the QAPI input.
463    """
464    def __init__(self, env, qapidir):
465        self._env = env
466        self._qapidir = qapidir
467
468    def visit_module(self, name):
469        if name != "./builtin":
470            qapifile = self._qapidir + '/' + name
471            self._env.note_dependency(os.path.abspath(qapifile))
472        super().visit_module(name)
473
474
475class QAPIDocDirective(Directive):
476    """Extract documentation from the specified QAPI .json file"""
477    required_argument = 1
478    optional_arguments = 1
479    option_spec = {
480        'qapifile': directives.unchanged_required
481    }
482    has_content = False
483
484    def new_serialno(self):
485        """Return a unique new ID string suitable for use as a node's ID"""
486        env = self.state.document.settings.env
487        return 'qapidoc-%d' % env.new_serialno('qapidoc')
488
489    def run(self):
490        env = self.state.document.settings.env
491        qapifile = env.config.qapidoc_srctree + '/' + self.arguments[0]
492        qapidir = os.path.dirname(qapifile)
493
494        try:
495            schema = QAPISchema(qapifile)
496
497            # First tell Sphinx about all the schema files that the
498            # output documentation depends on (including 'qapifile' itself)
499            schema.visit(QAPISchemaGenDepVisitor(env, qapidir))
500
501            vis = QAPISchemaGenRSTVisitor(self)
502            vis.visit_begin(schema)
503            for doc in schema.docs:
504                if doc.symbol:
505                    vis.symbol(doc, schema.lookup_entity(doc.symbol))
506                else:
507                    vis.freeform(doc)
508            return vis.get_document_nodes()
509        except QAPIError as err:
510            # Launder QAPI parse errors into Sphinx extension errors
511            # so they are displayed nicely to the user
512            raise ExtensionError(str(err)) from err
513
514    def do_parse(self, rstlist, node):
515        """Parse rST source lines and add them to the specified node
516
517        Take the list of rST source lines rstlist, parse them as
518        rST, and add the resulting docutils nodes as children of node.
519        The nodes are parsed in a way that allows them to include
520        subheadings (titles) without confusing the rendering of
521        anything else.
522        """
523        # This is from kerneldoc.py -- it works around an API change in
524        # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use
525        # sphinx.util.nodes.nested_parse_with_titles() rather than the
526        # plain self.state.nested_parse(), and so we can drop the saving
527        # of title_styles and section_level that kerneldoc.py does,
528        # because nested_parse_with_titles() does that for us.
529        if Use_SSI:
530            with switch_source_input(self.state, rstlist):
531                nested_parse_with_titles(self.state, rstlist, node)
532        else:
533            save = self.state.memo.reporter
534            self.state.memo.reporter = AutodocReporter(
535                rstlist, self.state.memo.reporter)
536            try:
537                nested_parse_with_titles(self.state, rstlist, node)
538            finally:
539                self.state.memo.reporter = save
540
541
542def setup(app):
543    """ Register qapi-doc directive with Sphinx"""
544    app.add_config_value('qapidoc_srctree', None, 'env')
545    app.add_directive('qapi-doc', QAPIDocDirective)
546
547    return dict(
548        version=__version__,
549        parallel_read_safe=True,
550        parallel_write_safe=True
551    )
552