1# -*- coding: utf-8; mode: python -*-
2# pylint: disable=C0103, R0903, R0912, R0915
3u"""
4    scalable figure and image handling
5    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
6
7    Sphinx extension which implements scalable image handling.
8
9    :copyright:  Copyright (C) 2016  Markus Heiser
10    :license:    GPL Version 2, June 1991 see Linux/COPYING for details.
11
12    The build for image formats depend on image's source format and output's
13    destination format. This extension implement methods to simplify image
14    handling from the author's POV. Directives like ``kernel-figure`` implement
15    methods *to* always get the best output-format even if some tools are not
16    installed. For more details take a look at ``convert_image(...)`` which is
17    the core of all conversions.
18
19    * ``.. kernel-image``: for image handling / a ``.. image::`` replacement
20
21    * ``.. kernel-figure``: for figure handling / a ``.. figure::`` replacement
22
23    * ``.. kernel-render``: for render markup / a concept to embed *render*
24      markups (or languages). Supported markups (see ``RENDER_MARKUP_EXT``)
25
26      - ``DOT``: render embedded Graphviz's **DOC**
27      - ``SVG``: render embedded Scalable Vector Graphics (**SVG**)
28      - ... *developable*
29
30    Used tools:
31
32    * ``dot(1)``: Graphviz (https://www.graphviz.org). If Graphviz is not
33      available, the DOT language is inserted as literal-block.
34      For conversion to PDF, ``rsvg-convert(1)`` of librsvg
35      (https://gitlab.gnome.org/GNOME/librsvg) is used when available.
36
37    * SVG to PDF: To generate PDF, you need at least one of this tools:
38
39      - ``convert(1)``: ImageMagick (https://www.imagemagick.org)
40      - ``inkscape(1)``: Inkscape (https://inkscape.org/)
41
42    List of customizations:
43
44    * generate PDF from SVG / used by PDF (LaTeX) builder
45
46    * generate SVG (html-builder) and PDF (latex-builder) from DOT files.
47      DOT: see https://www.graphviz.org/content/dot-language
48
49    """
50
51import os
52from os import path
53import subprocess
54from hashlib import sha1
55import re
56from docutils import nodes
57from docutils.statemachine import ViewList
58from docutils.parsers.rst import directives
59from docutils.parsers.rst.directives import images
60import sphinx
61from sphinx.util.nodes import clean_astext
62import kernellog
63
64# Get Sphinx version
65major, minor, patch = sphinx.version_info[:3]
66if major == 1 and minor > 3:
67    # patches.Figure only landed in Sphinx 1.4
68    from sphinx.directives.patches import Figure  # pylint: disable=C0413
69else:
70    Figure = images.Figure
71
72__version__  = '1.0.0'
73
74# simple helper
75# -------------
76
77def which(cmd):
78    """Searches the ``cmd`` in the ``PATH`` environment.
79
80    This *which* searches the PATH for executable ``cmd`` . First match is
81    returned, if nothing is found, ``None` is returned.
82    """
83    envpath = os.environ.get('PATH', None) or os.defpath
84    for folder in envpath.split(os.pathsep):
85        fname = folder + os.sep + cmd
86        if path.isfile(fname):
87            return fname
88
89def mkdir(folder, mode=0o775):
90    if not path.isdir(folder):
91        os.makedirs(folder, mode)
92
93def file2literal(fname):
94    with open(fname, "r") as src:
95        data = src.read()
96        node = nodes.literal_block(data, data)
97    return node
98
99def isNewer(path1, path2):
100    """Returns True if ``path1`` is newer than ``path2``
101
102    If ``path1`` exists and is newer than ``path2`` the function returns
103    ``True`` is returned otherwise ``False``
104    """
105    return (path.exists(path1)
106            and os.stat(path1).st_ctime > os.stat(path2).st_ctime)
107
108def pass_handle(self, node):           # pylint: disable=W0613
109    pass
110
111# setup conversion tools and sphinx extension
112# -------------------------------------------
113
114# Graphviz's dot(1) support
115dot_cmd = None
116# dot(1) -Tpdf should be used
117dot_Tpdf = False
118
119# ImageMagick' convert(1) support
120convert_cmd = None
121
122# librsvg's rsvg-convert(1) support
123rsvg_convert_cmd = None
124
125# Inkscape's inkscape(1) support
126inkscape_cmd = None
127# Inkscape prior to 1.0 uses different command options
128inkscape_ver_one = False
129
130
131def setup(app):
132    # check toolchain first
133    app.connect('builder-inited', setupTools)
134
135    # image handling
136    app.add_directive("kernel-image",  KernelImage)
137    app.add_node(kernel_image,
138                 html    = (visit_kernel_image, pass_handle),
139                 latex   = (visit_kernel_image, pass_handle),
140                 texinfo = (visit_kernel_image, pass_handle),
141                 text    = (visit_kernel_image, pass_handle),
142                 man     = (visit_kernel_image, pass_handle), )
143
144    # figure handling
145    app.add_directive("kernel-figure", KernelFigure)
146    app.add_node(kernel_figure,
147                 html    = (visit_kernel_figure, pass_handle),
148                 latex   = (visit_kernel_figure, pass_handle),
149                 texinfo = (visit_kernel_figure, pass_handle),
150                 text    = (visit_kernel_figure, pass_handle),
151                 man     = (visit_kernel_figure, pass_handle), )
152
153    # render handling
154    app.add_directive('kernel-render', KernelRender)
155    app.add_node(kernel_render,
156                 html    = (visit_kernel_render, pass_handle),
157                 latex   = (visit_kernel_render, pass_handle),
158                 texinfo = (visit_kernel_render, pass_handle),
159                 text    = (visit_kernel_render, pass_handle),
160                 man     = (visit_kernel_render, pass_handle), )
161
162    app.connect('doctree-read', add_kernel_figure_to_std_domain)
163
164    return dict(
165        version = __version__,
166        parallel_read_safe = True,
167        parallel_write_safe = True
168    )
169
170
171def setupTools(app):
172    u"""
173    Check available build tools and log some *verbose* messages.
174
175    This function is called once, when the builder is initiated.
176    """
177    global dot_cmd, dot_Tpdf, convert_cmd, rsvg_convert_cmd   # pylint: disable=W0603
178    global inkscape_cmd, inkscape_ver_one  # pylint: disable=W0603
179    kernellog.verbose(app, "kfigure: check installed tools ...")
180
181    dot_cmd = which('dot')
182    convert_cmd = which('convert')
183    rsvg_convert_cmd = which('rsvg-convert')
184    inkscape_cmd = which('inkscape')
185
186    if dot_cmd:
187        kernellog.verbose(app, "use dot(1) from: " + dot_cmd)
188
189        try:
190            dot_Thelp_list = subprocess.check_output([dot_cmd, '-Thelp'],
191                                    stderr=subprocess.STDOUT)
192        except subprocess.CalledProcessError as err:
193            dot_Thelp_list = err.output
194            pass
195
196        dot_Tpdf_ptn = b'pdf'
197        dot_Tpdf = re.search(dot_Tpdf_ptn, dot_Thelp_list)
198    else:
199        kernellog.warn(app, "dot(1) not found, for better output quality install "
200                       "graphviz from https://www.graphviz.org")
201    if inkscape_cmd:
202        kernellog.verbose(app, "use inkscape(1) from: " + inkscape_cmd)
203        inkscape_ver = subprocess.check_output([inkscape_cmd, '--version'],
204                                               stderr=subprocess.DEVNULL)
205        ver_one_ptn = b'Inkscape 1'
206        inkscape_ver_one = re.search(ver_one_ptn, inkscape_ver)
207        convert_cmd = None
208        rsvg_convert_cmd = None
209        dot_Tpdf = False
210
211    else:
212        if convert_cmd:
213            kernellog.verbose(app, "use convert(1) from: " + convert_cmd)
214        else:
215            kernellog.verbose(app,
216                "Neither inkscape(1) nor convert(1) found.\n"
217                "For SVG to PDF conversion, "
218                "install either Inkscape (https://inkscape.org/) (preferred) or\n"
219                "ImageMagick (https://www.imagemagick.org)")
220
221        if rsvg_convert_cmd:
222            kernellog.verbose(app, "use rsvg-convert(1) from: " + rsvg_convert_cmd)
223            kernellog.verbose(app, "use 'dot -Tsvg' and rsvg-convert(1) for DOT -> PDF conversion")
224            dot_Tpdf = False
225        else:
226            kernellog.verbose(app,
227                "rsvg-convert(1) not found.\n"
228                "  SVG rendering of convert(1) is done by ImageMagick-native renderer.")
229            if dot_Tpdf:
230                kernellog.verbose(app, "use 'dot -Tpdf' for DOT -> PDF conversion")
231            else:
232                kernellog.verbose(app, "use 'dot -Tsvg' and convert(1) for DOT -> PDF conversion")
233
234
235# integrate conversion tools
236# --------------------------
237
238RENDER_MARKUP_EXT = {
239    # The '.ext' must be handled by convert_image(..) function's *in_ext* input.
240    # <name> : <.ext>
241    'DOT' : '.dot',
242    'SVG' : '.svg'
243}
244
245def convert_image(img_node, translator, src_fname=None):
246    """Convert a image node for the builder.
247
248    Different builder prefer different image formats, e.g. *latex* builder
249    prefer PDF while *html* builder prefer SVG format for images.
250
251    This function handles output image formats in dependence of source the
252    format (of the image) and the translator's output format.
253    """
254    app = translator.builder.app
255
256    fname, in_ext = path.splitext(path.basename(img_node['uri']))
257    if src_fname is None:
258        src_fname = path.join(translator.builder.srcdir, img_node['uri'])
259        if not path.exists(src_fname):
260            src_fname = path.join(translator.builder.outdir, img_node['uri'])
261
262    dst_fname = None
263
264    # in kernel builds, use 'make SPHINXOPTS=-v' to see verbose messages
265
266    kernellog.verbose(app, 'assert best format for: ' + img_node['uri'])
267
268    if in_ext == '.dot':
269
270        if not dot_cmd:
271            kernellog.verbose(app,
272                              "dot from graphviz not available / include DOT raw.")
273            img_node.replace_self(file2literal(src_fname))
274
275        elif translator.builder.format == 'latex':
276            dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
277            img_node['uri'] = fname + '.pdf'
278            img_node['candidates'] = {'*': fname + '.pdf'}
279
280
281        elif translator.builder.format == 'html':
282            dst_fname = path.join(
283                translator.builder.outdir,
284                translator.builder.imagedir,
285                fname + '.svg')
286            img_node['uri'] = path.join(
287                translator.builder.imgpath, fname + '.svg')
288            img_node['candidates'] = {
289                '*': path.join(translator.builder.imgpath, fname + '.svg')}
290
291        else:
292            # all other builder formats will include DOT as raw
293            img_node.replace_self(file2literal(src_fname))
294
295    elif in_ext == '.svg':
296
297        if translator.builder.format == 'latex':
298            if not inkscape_cmd and convert_cmd is None:
299                kernellog.warn(app,
300                                  "no SVG to PDF conversion available / include SVG raw."
301                                  "\nIncluding large raw SVGs can cause xelatex error."
302                                  "\nInstall Inkscape (preferred) or ImageMagick.")
303                img_node.replace_self(file2literal(src_fname))
304            else:
305                dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
306                img_node['uri'] = fname + '.pdf'
307                img_node['candidates'] = {'*': fname + '.pdf'}
308
309    if dst_fname:
310        # the builder needs not to copy one more time, so pop it if exists.
311        translator.builder.images.pop(img_node['uri'], None)
312        _name = dst_fname[len(translator.builder.outdir) + 1:]
313
314        if isNewer(dst_fname, src_fname):
315            kernellog.verbose(app,
316                              "convert: {out}/%s already exists and is newer" % _name)
317
318        else:
319            ok = False
320            mkdir(path.dirname(dst_fname))
321
322            if in_ext == '.dot':
323                kernellog.verbose(app, 'convert DOT to: {out}/' + _name)
324                if translator.builder.format == 'latex' and not dot_Tpdf:
325                    svg_fname = path.join(translator.builder.outdir, fname + '.svg')
326                    ok1 = dot2format(app, src_fname, svg_fname)
327                    ok2 = svg2pdf_by_rsvg(app, svg_fname, dst_fname)
328                    ok = ok1 and ok2
329
330                else:
331                    ok = dot2format(app, src_fname, dst_fname)
332
333            elif in_ext == '.svg':
334                kernellog.verbose(app, 'convert SVG to: {out}/' + _name)
335                ok = svg2pdf(app, src_fname, dst_fname)
336
337            if not ok:
338                img_node.replace_self(file2literal(src_fname))
339
340
341def dot2format(app, dot_fname, out_fname):
342    """Converts DOT file to ``out_fname`` using ``dot(1)``.
343
344    * ``dot_fname`` pathname of the input DOT file, including extension ``.dot``
345    * ``out_fname`` pathname of the output file, including format extension
346
347    The *format extension* depends on the ``dot`` command (see ``man dot``
348    option ``-Txxx``). Normally you will use one of the following extensions:
349
350    - ``.ps`` for PostScript,
351    - ``.svg`` or ``svgz`` for Structured Vector Graphics,
352    - ``.fig`` for XFIG graphics and
353    - ``.png`` or ``gif`` for common bitmap graphics.
354
355    """
356    out_format = path.splitext(out_fname)[1][1:]
357    cmd = [dot_cmd, '-T%s' % out_format, dot_fname]
358    exit_code = 42
359
360    with open(out_fname, "w") as out:
361        exit_code = subprocess.call(cmd, stdout = out)
362        if exit_code != 0:
363            kernellog.warn(app,
364                          "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
365    return bool(exit_code == 0)
366
367def svg2pdf(app, svg_fname, pdf_fname):
368    """Converts SVG to PDF with ``inkscape(1)`` or ``convert(1)`` command.
369
370    Uses ``inkscape(1)`` from Inkscape (https://inkscape.org/) or ``convert(1)``
371    from ImageMagick (https://www.imagemagick.org) for conversion.
372    Returns ``True`` on success and ``False`` if an error occurred.
373
374    * ``svg_fname`` pathname of the input SVG file with extension (``.svg``)
375    * ``pdf_name``  pathname of the output PDF file with extension (``.pdf``)
376
377    """
378    cmd = [convert_cmd, svg_fname, pdf_fname]
379    cmd_name = 'convert(1)'
380
381    if inkscape_cmd:
382        cmd_name = 'inkscape(1)'
383        if inkscape_ver_one:
384            cmd = [inkscape_cmd, '-o', pdf_fname, svg_fname]
385        else:
386            cmd = [inkscape_cmd, '-z', '--export-pdf=%s' % pdf_fname, svg_fname]
387
388    try:
389        warning_msg = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
390        exit_code = 0
391    except subprocess.CalledProcessError as err:
392        warning_msg = err.output
393        exit_code = err.returncode
394        pass
395
396    if exit_code != 0:
397        kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
398        if warning_msg:
399            kernellog.warn(app, "Warning msg from %s: %s"
400                           % (cmd_name, str(warning_msg, 'utf-8')))
401    elif warning_msg:
402        kernellog.verbose(app, "Warning msg from %s (likely harmless):\n%s"
403                          % (cmd_name, str(warning_msg, 'utf-8')))
404
405    return bool(exit_code == 0)
406
407def svg2pdf_by_rsvg(app, svg_fname, pdf_fname):
408    """Convert SVG to PDF with ``rsvg-convert(1)`` command.
409
410    * ``svg_fname`` pathname of input SVG file, including extension ``.svg``
411    * ``pdf_fname`` pathname of output PDF file, including extension ``.pdf``
412
413    Input SVG file should be the one generated by ``dot2format()``.
414    SVG -> PDF conversion is done by ``rsvg-convert(1)``.
415
416    If ``rsvg-convert(1)`` is unavailable, fall back to ``svg2pdf()``.
417
418    """
419
420    if rsvg_convert_cmd is None:
421        ok = svg2pdf(app, svg_fname, pdf_fname)
422    else:
423        cmd = [rsvg_convert_cmd, '--format=pdf', '-o', pdf_fname, svg_fname]
424        # use stdout and stderr from parent
425        exit_code = subprocess.call(cmd)
426        if exit_code != 0:
427            kernellog.warn(app, "Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
428        ok = bool(exit_code == 0)
429
430    return ok
431
432
433# image handling
434# ---------------------
435
436def visit_kernel_image(self, node):    # pylint: disable=W0613
437    """Visitor of the ``kernel_image`` Node.
438
439    Handles the ``image`` child-node with the ``convert_image(...)``.
440    """
441    img_node = node[0]
442    convert_image(img_node, self)
443
444class kernel_image(nodes.image):
445    """Node for ``kernel-image`` directive."""
446    pass
447
448class KernelImage(images.Image):
449    u"""KernelImage directive
450
451    Earns everything from ``.. image::`` directive, except *remote URI* and
452    *glob* pattern. The KernelImage wraps a image node into a
453    kernel_image node. See ``visit_kernel_image``.
454    """
455
456    def run(self):
457        uri = self.arguments[0]
458        if uri.endswith('.*') or uri.find('://') != -1:
459            raise self.severe(
460                'Error in "%s: %s": glob pattern and remote images are not allowed'
461                % (self.name, uri))
462        result = images.Image.run(self)
463        if len(result) == 2 or isinstance(result[0], nodes.system_message):
464            return result
465        (image_node,) = result
466        # wrap image node into a kernel_image node / see visitors
467        node = kernel_image('', image_node)
468        return [node]
469
470# figure handling
471# ---------------------
472
473def visit_kernel_figure(self, node):   # pylint: disable=W0613
474    """Visitor of the ``kernel_figure`` Node.
475
476    Handles the ``image`` child-node with the ``convert_image(...)``.
477    """
478    img_node = node[0][0]
479    convert_image(img_node, self)
480
481class kernel_figure(nodes.figure):
482    """Node for ``kernel-figure`` directive."""
483
484class KernelFigure(Figure):
485    u"""KernelImage directive
486
487    Earns everything from ``.. figure::`` directive, except *remote URI* and
488    *glob* pattern.  The KernelFigure wraps a figure node into a kernel_figure
489    node. See ``visit_kernel_figure``.
490    """
491
492    def run(self):
493        uri = self.arguments[0]
494        if uri.endswith('.*') or uri.find('://') != -1:
495            raise self.severe(
496                'Error in "%s: %s":'
497                ' glob pattern and remote images are not allowed'
498                % (self.name, uri))
499        result = Figure.run(self)
500        if len(result) == 2 or isinstance(result[0], nodes.system_message):
501            return result
502        (figure_node,) = result
503        # wrap figure node into a kernel_figure node / see visitors
504        node = kernel_figure('', figure_node)
505        return [node]
506
507
508# render handling
509# ---------------------
510
511def visit_kernel_render(self, node):
512    """Visitor of the ``kernel_render`` Node.
513
514    If rendering tools available, save the markup of the ``literal_block`` child
515    node into a file and replace the ``literal_block`` node with a new created
516    ``image`` node, pointing to the saved markup file. Afterwards, handle the
517    image child-node with the ``convert_image(...)``.
518    """
519    app = self.builder.app
520    srclang = node.get('srclang')
521
522    kernellog.verbose(app, 'visit kernel-render node lang: "%s"' % (srclang))
523
524    tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)
525    if tmp_ext is None:
526        kernellog.warn(app, 'kernel-render: "%s" unknown / include raw.' % (srclang))
527        return
528
529    if not dot_cmd and tmp_ext == '.dot':
530        kernellog.verbose(app, "dot from graphviz not available / include raw.")
531        return
532
533    literal_block = node[0]
534
535    code      = literal_block.astext()
536    hashobj   = code.encode('utf-8') #  str(node.attributes)
537    fname     = path.join('%s-%s' % (srclang, sha1(hashobj).hexdigest()))
538
539    tmp_fname = path.join(
540        self.builder.outdir, self.builder.imagedir, fname + tmp_ext)
541
542    if not path.isfile(tmp_fname):
543        mkdir(path.dirname(tmp_fname))
544        with open(tmp_fname, "w") as out:
545            out.write(code)
546
547    img_node = nodes.image(node.rawsource, **node.attributes)
548    img_node['uri'] = path.join(self.builder.imgpath, fname + tmp_ext)
549    img_node['candidates'] = {
550        '*': path.join(self.builder.imgpath, fname + tmp_ext)}
551
552    literal_block.replace_self(img_node)
553    convert_image(img_node, self, tmp_fname)
554
555
556class kernel_render(nodes.General, nodes.Inline, nodes.Element):
557    """Node for ``kernel-render`` directive."""
558    pass
559
560class KernelRender(Figure):
561    u"""KernelRender directive
562
563    Render content by external tool.  Has all the options known from the
564    *figure*  directive, plus option ``caption``.  If ``caption`` has a
565    value, a figure node with the *caption* is inserted. If not, a image node is
566    inserted.
567
568    The KernelRender directive wraps the text of the directive into a
569    literal_block node and wraps it into a kernel_render node. See
570    ``visit_kernel_render``.
571    """
572    has_content = True
573    required_arguments = 1
574    optional_arguments = 0
575    final_argument_whitespace = False
576
577    # earn options from 'figure'
578    option_spec = Figure.option_spec.copy()
579    option_spec['caption'] = directives.unchanged
580
581    def run(self):
582        return [self.build_node()]
583
584    def build_node(self):
585
586        srclang = self.arguments[0].strip()
587        if srclang not in RENDER_MARKUP_EXT.keys():
588            return [self.state_machine.reporter.warning(
589                'Unknown source language "%s", use one of: %s.' % (
590                    srclang, ",".join(RENDER_MARKUP_EXT.keys())),
591                line=self.lineno)]
592
593        code = '\n'.join(self.content)
594        if not code.strip():
595            return [self.state_machine.reporter.warning(
596                'Ignoring "%s" directive without content.' % (
597                    self.name),
598                line=self.lineno)]
599
600        node = kernel_render()
601        node['alt'] = self.options.get('alt','')
602        node['srclang'] = srclang
603        literal_node = nodes.literal_block(code, code)
604        node += literal_node
605
606        caption = self.options.get('caption')
607        if caption:
608            # parse caption's content
609            parsed = nodes.Element()
610            self.state.nested_parse(
611                ViewList([caption], source=''), self.content_offset, parsed)
612            caption_node = nodes.caption(
613                parsed[0].rawsource, '', *parsed[0].children)
614            caption_node.source = parsed[0].source
615            caption_node.line = parsed[0].line
616
617            figure_node = nodes.figure('', node)
618            for k,v in self.options.items():
619                figure_node[k] = v
620            figure_node += caption_node
621
622            node = figure_node
623
624        return node
625
626def add_kernel_figure_to_std_domain(app, doctree):
627    """Add kernel-figure anchors to 'std' domain.
628
629    The ``StandardDomain.process_doc(..)`` method does not know how to resolve
630    the caption (label) of ``kernel-figure`` directive (it only knows about
631    standard nodes, e.g. table, figure etc.). Without any additional handling
632    this will result in a 'undefined label' for kernel-figures.
633
634    This handle adds labels of kernel-figure to the 'std' domain labels.
635    """
636
637    std = app.env.domains["std"]
638    docname = app.env.docname
639    labels = std.data["labels"]
640
641    for name, explicit in doctree.nametypes.items():
642        if not explicit:
643            continue
644        labelid = doctree.nameids[name]
645        if labelid is None:
646            continue
647        node = doctree.ids[labelid]
648
649        if node.tagname == 'kernel_figure':
650            for n in node.next_node():
651                if n.tagname == 'caption':
652                    sectname = clean_astext(n)
653                    # add label to std domain
654                    labels[name] = docname, labelid, sectname
655                    break
656