xref: /openbmc/qemu/docs/sphinx/qapidoc.py (revision 0e2b9edf)
1# coding=utf-8
2#
3# QEMU qapidoc QAPI file parsing extension
4#
5# Copyright (c) 2020 Linaro
6#
7# This work is licensed under the terms of the GNU GPLv2 or later.
8# See the COPYING file in the top-level directory.
9
10"""
11qapidoc is a Sphinx extension that implements the qapi-doc directive
12
13The purpose of this extension is to read the documentation comments
14in QAPI schema files, and insert them all into the current document.
15
16It implements one new rST directive, "qapi-doc::".
17Each qapi-doc:: directive takes one argument, which is the
18pathname of the schema file to process, relative to the source tree.
19
20The docs/conf.py file must set the qapidoc_srctree config value to
21the root of the QEMU source tree.
22
23The Sphinx documentation on writing extensions is at:
24https://www.sphinx-doc.org/en/master/development/index.html
25"""
26
27import os
28import re
29
30from docutils import nodes
31from docutils.statemachine import ViewList
32from docutils.parsers.rst import directives, Directive
33from sphinx.errors import ExtensionError
34from sphinx.util.nodes import nested_parse_with_titles
35import sphinx
36from qapi.gen import QAPISchemaVisitor
37from qapi.error import QAPIError, QAPISemError
38from qapi.schema import QAPISchema
39
40
41# Sphinx up to 1.6 uses AutodocReporter; 1.7 and later
42# use switch_source_input. Check borrowed from kerneldoc.py.
43Use_SSI = sphinx.__version__[:3] >= '1.7'
44if Use_SSI:
45    from sphinx.util.docutils import switch_source_input
46else:
47    from sphinx.ext.autodoc import AutodocReporter
48
49
50__version__ = '1.0'
51
52
53# Function borrowed from pydash, which is under the MIT license
54def intersperse(iterable, separator):
55    """Yield the members of *iterable* interspersed with *separator*."""
56    iterable = iter(iterable)
57    yield next(iterable)
58    for item in iterable:
59        yield separator
60        yield item
61
62
63class QAPISchemaGenRSTVisitor(QAPISchemaVisitor):
64    """A QAPI schema visitor which generates docutils/Sphinx nodes
65
66    This class builds up a tree of docutils/Sphinx nodes corresponding
67    to documentation for the various QAPI objects. To use it, first
68    create a QAPISchemaGenRSTVisitor object, and call its
69    visit_begin() method.  Then you can call one of the two methods
70    'freeform' (to add documentation for a freeform documentation
71    chunk) or 'symbol' (to add documentation for a QAPI symbol). These
72    will cause the visitor to build up the tree of document
73    nodes. Once you've added all the documentation via 'freeform' and
74    'symbol' method calls, you can call 'get_document_nodes' to get
75    the final list of document nodes (in a form suitable for returning
76    from a Sphinx directive's 'run' method).
77    """
78    def __init__(self, sphinx_directive):
79        self._cur_doc = None
80        self._sphinx_directive = sphinx_directive
81        self._top_node = nodes.section()
82        self._active_headings = [self._top_node]
83
84    def _make_dlitem(self, term, defn):
85        """Return a dlitem node with the specified term and definition.
86
87        term should be a list of Text and literal nodes.
88        defn should be one of:
89        - a string, which will be handed to _parse_text_into_node
90        - a list of Text and literal nodes, which will be put into
91          a paragraph node
92        """
93        dlitem = nodes.definition_list_item()
94        dlterm = nodes.term('', '', *term)
95        dlitem += dlterm
96        if defn:
97            dldef = nodes.definition()
98            if isinstance(defn, list):
99                dldef += nodes.paragraph('', '', *defn)
100            else:
101                self._parse_text_into_node(defn, dldef)
102            dlitem += dldef
103        return dlitem
104
105    def _make_section(self, title):
106        """Return a section node with optional title"""
107        section = nodes.section(ids=[self._sphinx_directive.new_serialno()])
108        if title:
109            section += nodes.title(title, title)
110        return section
111
112    def _nodes_for_ifcond(self, ifcond, with_if=True):
113        """Return list of Text, literal nodes for the ifcond
114
115        Return a list which gives text like ' (If: condition)'.
116        If with_if is False, we don't return the "(If: " and ")".
117        """
118
119        doc = ifcond.docgen()
120        if not doc:
121            return []
122        doc = nodes.literal('', doc)
123        if not with_if:
124            return [doc]
125
126        nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')]
127        nodelist.append(doc)
128        nodelist.append(nodes.Text(')'))
129        return nodelist
130
131    def _nodes_for_one_member(self, member):
132        """Return list of Text, literal nodes for this member
133
134        Return a list of doctree nodes which give text like
135        'name: type (optional) (If: ...)' suitable for use as the
136        'term' part of a definition list item.
137        """
138        term = [nodes.literal('', member.name)]
139        if member.type.doc_type():
140            term.append(nodes.Text(': '))
141            term.append(nodes.literal('', member.type.doc_type()))
142        if member.optional:
143            term.append(nodes.Text(' (optional)'))
144        if member.ifcond.is_present():
145            term.extend(self._nodes_for_ifcond(member.ifcond))
146        return term
147
148    def _nodes_for_variant_when(self, branches, variant):
149        """Return list of Text, literal nodes for variant 'when' clause
150
151        Return a list of doctree nodes which give text like
152        'when tagname is variant (If: ...)' suitable for use in
153        the 'branches' part of a definition list.
154        """
155        term = [nodes.Text(' when '),
156                nodes.literal('', branches.tag_member.name),
157                nodes.Text(' is '),
158                nodes.literal('', '"%s"' % variant.name)]
159        if variant.ifcond.is_present():
160            term.extend(self._nodes_for_ifcond(variant.ifcond))
161        return term
162
163    def _nodes_for_members(self, doc, what, base=None, branches=None):
164        """Return list of doctree nodes for the table of members"""
165        dlnode = nodes.definition_list()
166        for section in doc.args.values():
167            term = self._nodes_for_one_member(section.member)
168            # TODO drop fallbacks when undocumented members are outlawed
169            if section.text:
170                defn = section.text
171            else:
172                defn = [nodes.Text('Not documented')]
173
174            dlnode += self._make_dlitem(term, defn)
175
176        if base:
177            dlnode += self._make_dlitem([nodes.Text('The members of '),
178                                         nodes.literal('', base.doc_type())],
179                                        None)
180
181        if branches:
182            for v in branches.variants:
183                if v.type.name == 'q_empty':
184                    continue
185                assert not v.type.is_implicit()
186                term = [nodes.Text('The members of '),
187                        nodes.literal('', v.type.doc_type())]
188                term.extend(self._nodes_for_variant_when(branches, v))
189                dlnode += self._make_dlitem(term, None)
190
191        if not dlnode.children:
192            return []
193
194        section = self._make_section(what)
195        section += dlnode
196        return [section]
197
198    def _nodes_for_enum_values(self, doc):
199        """Return list of doctree nodes for the table of enum values"""
200        seen_item = False
201        dlnode = nodes.definition_list()
202        for section in doc.args.values():
203            termtext = [nodes.literal('', section.member.name)]
204            if section.member.ifcond.is_present():
205                termtext.extend(self._nodes_for_ifcond(section.member.ifcond))
206            # TODO drop fallbacks when undocumented members are outlawed
207            if section.text:
208                defn = section.text
209            else:
210                defn = [nodes.Text('Not documented')]
211
212            dlnode += self._make_dlitem(termtext, defn)
213            seen_item = True
214
215        if not seen_item:
216            return []
217
218        section = self._make_section('Values')
219        section += dlnode
220        return [section]
221
222    def _nodes_for_arguments(self, doc, boxed_arg_type):
223        """Return list of doctree nodes for the arguments section"""
224        if boxed_arg_type:
225            assert not doc.args
226            section = self._make_section('Arguments')
227            dlnode = nodes.definition_list()
228            dlnode += self._make_dlitem(
229                [nodes.Text('The members of '),
230                 nodes.literal('', boxed_arg_type.name)],
231                None)
232            section += dlnode
233            return [section]
234
235        return self._nodes_for_members(doc, 'Arguments')
236
237    def _nodes_for_features(self, doc):
238        """Return list of doctree nodes for the table of features"""
239        seen_item = False
240        dlnode = nodes.definition_list()
241        for section in doc.features.values():
242            dlnode += self._make_dlitem(
243                [nodes.literal('', section.member.name)], section.text)
244            seen_item = True
245
246        if not seen_item:
247            return []
248
249        section = self._make_section('Features')
250        section += dlnode
251        return [section]
252
253    def _nodes_for_example(self, exampletext):
254        """Return list of doctree nodes for a code example snippet"""
255        return [nodes.literal_block(exampletext, exampletext)]
256
257    def _nodes_for_sections(self, doc):
258        """Return list of doctree nodes for additional sections"""
259        nodelist = []
260        for section in doc.sections:
261            if section.tag and section.tag == 'TODO':
262                # Hide TODO: sections
263                continue
264            snode = self._make_section(section.tag)
265            if section.tag and section.tag.startswith('Example'):
266                snode += self._nodes_for_example(section.text)
267            else:
268                self._parse_text_into_node(section.text, snode)
269            nodelist.append(snode)
270        return nodelist
271
272    def _nodes_for_if_section(self, ifcond):
273        """Return list of doctree nodes for the "If" section"""
274        nodelist = []
275        if ifcond.is_present():
276            snode = self._make_section('If')
277            snode += nodes.paragraph(
278                '', '', *self._nodes_for_ifcond(ifcond, with_if=False)
279            )
280            nodelist.append(snode)
281        return nodelist
282
283    def _add_doc(self, typ, sections):
284        """Add documentation for a command/object/enum...
285
286        We assume we're documenting the thing defined in self._cur_doc.
287        typ is the type of thing being added ("Command", "Object", etc)
288
289        sections is a list of nodes for sections to add to the definition.
290        """
291
292        doc = self._cur_doc
293        snode = nodes.section(ids=[self._sphinx_directive.new_serialno()])
294        snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol),
295                                       nodes.Text(' (' + typ + ')')])
296        self._parse_text_into_node(doc.body.text, snode)
297        for s in sections:
298            if s is not None:
299                snode += s
300        self._add_node_to_current_heading(snode)
301
302    def visit_enum_type(self, name, info, ifcond, features, members, prefix):
303        doc = self._cur_doc
304        self._add_doc('Enum',
305                      self._nodes_for_enum_values(doc)
306                      + self._nodes_for_features(doc)
307                      + self._nodes_for_sections(doc)
308                      + self._nodes_for_if_section(ifcond))
309
310    def visit_object_type(self, name, info, ifcond, features,
311                          base, members, branches):
312        doc = self._cur_doc
313        if base and base.is_implicit():
314            base = None
315        self._add_doc('Object',
316                      self._nodes_for_members(doc, 'Members', base, branches)
317                      + self._nodes_for_features(doc)
318                      + self._nodes_for_sections(doc)
319                      + self._nodes_for_if_section(ifcond))
320
321    def visit_alternate_type(self, name, info, ifcond, features,
322                             alternatives):
323        doc = self._cur_doc
324        self._add_doc('Alternate',
325                      self._nodes_for_members(doc, 'Members')
326                      + self._nodes_for_features(doc)
327                      + self._nodes_for_sections(doc)
328                      + self._nodes_for_if_section(ifcond))
329
330    def visit_command(self, name, info, ifcond, features, arg_type,
331                      ret_type, gen, success_response, boxed, allow_oob,
332                      allow_preconfig, coroutine):
333        doc = self._cur_doc
334        self._add_doc('Command',
335                      self._nodes_for_arguments(doc,
336                                                arg_type if boxed else None)
337                      + self._nodes_for_features(doc)
338                      + self._nodes_for_sections(doc)
339                      + self._nodes_for_if_section(ifcond))
340
341    def visit_event(self, name, info, ifcond, features, arg_type, boxed):
342        doc = self._cur_doc
343        self._add_doc('Event',
344                      self._nodes_for_arguments(doc,
345                                                arg_type if boxed else None)
346                      + self._nodes_for_features(doc)
347                      + self._nodes_for_sections(doc)
348                      + self._nodes_for_if_section(ifcond))
349
350    def symbol(self, doc, entity):
351        """Add documentation for one symbol to the document tree
352
353        This is the main entry point which causes us to add documentation
354        nodes for a symbol (which could be a 'command', 'object', 'event',
355        etc). We do this by calling 'visit' on the schema entity, which
356        will then call back into one of our visit_* methods, depending
357        on what kind of thing this symbol is.
358        """
359        self._cur_doc = doc
360        entity.visit(self)
361        self._cur_doc = None
362
363    def _start_new_heading(self, heading, level):
364        """Start a new heading at the specified heading level
365
366        Create a new section whose title is 'heading' and which is placed
367        in the docutils node tree as a child of the most recent level-1
368        heading. Subsequent document sections (commands, freeform doc chunks,
369        etc) will be placed as children of this new heading section.
370        """
371        if len(self._active_headings) < level:
372            raise QAPISemError(self._cur_doc.info,
373                               'Level %d subheading found outside a '
374                               'level %d heading'
375                               % (level, level - 1))
376        snode = self._make_section(heading)
377        self._active_headings[level - 1] += snode
378        self._active_headings = self._active_headings[:level]
379        self._active_headings.append(snode)
380
381    def _add_node_to_current_heading(self, node):
382        """Add the node to whatever the current active heading is"""
383        self._active_headings[-1] += node
384
385    def freeform(self, doc):
386        """Add a piece of 'freeform' documentation to the document tree
387
388        A 'freeform' document chunk doesn't relate to any particular
389        symbol (for instance, it could be an introduction).
390
391        If the freeform document starts with a line of the form
392        '= Heading text', this is a section or subsection heading, with
393        the heading level indicated by the number of '=' signs.
394        """
395
396        # QAPIDoc documentation says free-form documentation blocks
397        # must have only a body section, nothing else.
398        assert not doc.sections
399        assert not doc.args
400        assert not doc.features
401        self._cur_doc = doc
402
403        text = doc.body.text
404        if re.match(r'=+ ', text):
405            # Section/subsection heading (if present, will always be
406            # the first line of the block)
407            (heading, _, text) = text.partition('\n')
408            (leader, _, heading) = heading.partition(' ')
409            self._start_new_heading(heading, len(leader))
410            if text == '':
411                return
412
413        node = self._make_section(None)
414        self._parse_text_into_node(text, node)
415        self._add_node_to_current_heading(node)
416        self._cur_doc = None
417
418    def _parse_text_into_node(self, doctext, node):
419        """Parse a chunk of QAPI-doc-format text into the node
420
421        The doc comment can contain most inline rST markup, including
422        bulleted and enumerated lists.
423        As an extra permitted piece of markup, @var will be turned
424        into ``var``.
425        """
426
427        # Handle the "@var means ``var`` case
428        doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext)
429
430        rstlist = ViewList()
431        for line in doctext.splitlines():
432            # The reported line number will always be that of the start line
433            # of the doc comment, rather than the actual location of the error.
434            # Being more precise would require overhaul of the QAPIDoc class
435            # to track lines more exactly within all the sub-parts of the doc
436            # comment, as well as counting lines here.
437            rstlist.append(line, self._cur_doc.info.fname,
438                           self._cur_doc.info.line)
439        # Append a blank line -- in some cases rST syntax errors get
440        # attributed to the line after one with actual text, and if there
441        # isn't anything in the ViewList corresponding to that then Sphinx
442        # 1.6's AutodocReporter will then misidentify the source/line location
443        # in the error message (usually attributing it to the top-level
444        # .rst file rather than the offending .json file). The extra blank
445        # line won't affect the rendered output.
446        rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line)
447        self._sphinx_directive.do_parse(rstlist, node)
448
449    def get_document_nodes(self):
450        """Return the list of docutils nodes which make up the document"""
451        return self._top_node.children
452
453
454class QAPISchemaGenDepVisitor(QAPISchemaVisitor):
455    """A QAPI schema visitor which adds Sphinx dependencies each module
456
457    This class calls the Sphinx note_dependency() function to tell Sphinx
458    that the generated documentation output depends on the input
459    schema file associated with each module in the QAPI input.
460    """
461    def __init__(self, env, qapidir):
462        self._env = env
463        self._qapidir = qapidir
464
465    def visit_module(self, name):
466        if name != "./builtin":
467            qapifile = self._qapidir + '/' + name
468            self._env.note_dependency(os.path.abspath(qapifile))
469        super().visit_module(name)
470
471
472class QAPIDocDirective(Directive):
473    """Extract documentation from the specified QAPI .json file"""
474    required_argument = 1
475    optional_arguments = 1
476    option_spec = {
477        'qapifile': directives.unchanged_required
478    }
479    has_content = False
480
481    def new_serialno(self):
482        """Return a unique new ID string suitable for use as a node's ID"""
483        env = self.state.document.settings.env
484        return 'qapidoc-%d' % env.new_serialno('qapidoc')
485
486    def run(self):
487        env = self.state.document.settings.env
488        qapifile = env.config.qapidoc_srctree + '/' + self.arguments[0]
489        qapidir = os.path.dirname(qapifile)
490
491        try:
492            schema = QAPISchema(qapifile)
493
494            # First tell Sphinx about all the schema files that the
495            # output documentation depends on (including 'qapifile' itself)
496            schema.visit(QAPISchemaGenDepVisitor(env, qapidir))
497
498            vis = QAPISchemaGenRSTVisitor(self)
499            vis.visit_begin(schema)
500            for doc in schema.docs:
501                if doc.symbol:
502                    vis.symbol(doc, schema.lookup_entity(doc.symbol))
503                else:
504                    vis.freeform(doc)
505            return vis.get_document_nodes()
506        except QAPIError as err:
507            # Launder QAPI parse errors into Sphinx extension errors
508            # so they are displayed nicely to the user
509            raise ExtensionError(str(err)) from err
510
511    def do_parse(self, rstlist, node):
512        """Parse rST source lines and add them to the specified node
513
514        Take the list of rST source lines rstlist, parse them as
515        rST, and add the resulting docutils nodes as children of node.
516        The nodes are parsed in a way that allows them to include
517        subheadings (titles) without confusing the rendering of
518        anything else.
519        """
520        # This is from kerneldoc.py -- it works around an API change in
521        # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use
522        # sphinx.util.nodes.nested_parse_with_titles() rather than the
523        # plain self.state.nested_parse(), and so we can drop the saving
524        # of title_styles and section_level that kerneldoc.py does,
525        # because nested_parse_with_titles() does that for us.
526        if Use_SSI:
527            with switch_source_input(self.state, rstlist):
528                nested_parse_with_titles(self.state, rstlist, node)
529        else:
530            save = self.state.memo.reporter
531            self.state.memo.reporter = AutodocReporter(
532                rstlist, self.state.memo.reporter)
533            try:
534                nested_parse_with_titles(self.state, rstlist, node)
535            finally:
536                self.state.memo.reporter = save
537
538
539def setup(app):
540    """ Register qapi-doc directive with Sphinx"""
541    app.add_config_value('qapidoc_srctree', None, 'env')
542    app.add_directive('qapi-doc', QAPIDocDirective)
543
544    return dict(
545        version=__version__,
546        parallel_read_safe=True,
547        parallel_write_safe=True
548    )
549