1# -*- coding: utf-8; mode: python -*-
2# pylint: disable=C0103, R0903, R0912, R0915
3u"""
4    scalable figure and image handling
5    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
6
7    Sphinx extension which implements scalable image handling.
8
9    :copyright:  Copyright (C) 2016  Markus Heiser
10    :license:    GPL Version 2, June 1991 see Linux/COPYING for details.
11
12    The build for image formats depend on image's source format and output's
13    destination format. This extension implement methods to simplify image
14    handling from the author's POV. Directives like ``kernel-figure`` implement
15    methods *to* always get the best output-format even if some tools are not
16    installed. For more details take a look at ``convert_image(...)`` which is
17    the core of all conversions.
18
19    * ``.. kernel-image``: for image handling / a ``.. image::`` replacement
20
21    * ``.. kernel-figure``: for figure handling / a ``.. figure::`` replacement
22
23    * ``.. kernel-render``: for render markup / a concept to embed *render*
24      markups (or languages). Supported markups (see ``RENDER_MARKUP_EXT``)
25
26      - ``DOT``: render embedded Graphviz's **DOC**
27      - ``SVG``: render embedded Scalable Vector Graphics (**SVG**)
28      - ... *developable*
29
30    Used tools:
31
32    * ``dot(1)``: Graphviz (http://www.graphviz.org). If Graphviz is not
33      available, the DOT language is inserted as literal-block.
34
35    * SVG to PDF: To generate PDF, you need at least one of this tools:
36
37      - ``convert(1)``: ImageMagick (https://www.imagemagick.org)
38
39    List of customizations:
40
41    * generate PDF from SVG / used by PDF (LaTeX) builder
42
43    * generate SVG (html-builder) and PDF (latex-builder) from DOT files.
44      DOT: see http://www.graphviz.org/content/dot-language
45
46    """
47
48import os
49from os import path
50import subprocess
51from hashlib import sha1
52import sys
53
54from docutils import nodes
55from docutils.statemachine import ViewList
56from docutils.parsers.rst import directives
57from docutils.parsers.rst.directives import images
58import sphinx
59
60from sphinx.util.nodes import clean_astext
61from six import iteritems
62
63PY3 = sys.version_info[0] == 3
64
65if PY3:
66    _unicode = str
67else:
68    _unicode = unicode
69
70# Get Sphinx version
71major, minor, patch = sphinx.version_info[:3]
72if major == 1 and minor > 3:
73    # patches.Figure only landed in Sphinx 1.4
74    from sphinx.directives.patches import Figure  # pylint: disable=C0413
75else:
76    Figure = images.Figure
77
78__version__  = '1.0.0'
79
80# simple helper
81# -------------
82
83def which(cmd):
84    """Searches the ``cmd`` in the ``PATH`` environment.
85
86    This *which* searches the PATH for executable ``cmd`` . First match is
87    returned, if nothing is found, ``None` is returned.
88    """
89    envpath = os.environ.get('PATH', None) or os.defpath
90    for folder in envpath.split(os.pathsep):
91        fname = folder + os.sep + cmd
92        if path.isfile(fname):
93            return fname
94
95def mkdir(folder, mode=0o775):
96    if not path.isdir(folder):
97        os.makedirs(folder, mode)
98
99def file2literal(fname):
100    with open(fname, "r") as src:
101        data = src.read()
102        node = nodes.literal_block(data, data)
103    return node
104
105def isNewer(path1, path2):
106    """Returns True if ``path1`` is newer than ``path2``
107
108    If ``path1`` exists and is newer than ``path2`` the function returns
109    ``True`` is returned otherwise ``False``
110    """
111    return (path.exists(path1)
112            and os.stat(path1).st_ctime > os.stat(path2).st_ctime)
113
114def pass_handle(self, node):           # pylint: disable=W0613
115    pass
116
117# setup conversion tools and sphinx extension
118# -------------------------------------------
119
120# Graphviz's dot(1) support
121dot_cmd = None
122
123# ImageMagick' convert(1) support
124convert_cmd = None
125
126
127def setup(app):
128    # check toolchain first
129    app.connect('builder-inited', setupTools)
130
131    # image handling
132    app.add_directive("kernel-image",  KernelImage)
133    app.add_node(kernel_image,
134                 html    = (visit_kernel_image, pass_handle),
135                 latex   = (visit_kernel_image, pass_handle),
136                 texinfo = (visit_kernel_image, pass_handle),
137                 text    = (visit_kernel_image, pass_handle),
138                 man     = (visit_kernel_image, pass_handle), )
139
140    # figure handling
141    app.add_directive("kernel-figure", KernelFigure)
142    app.add_node(kernel_figure,
143                 html    = (visit_kernel_figure, pass_handle),
144                 latex   = (visit_kernel_figure, pass_handle),
145                 texinfo = (visit_kernel_figure, pass_handle),
146                 text    = (visit_kernel_figure, pass_handle),
147                 man     = (visit_kernel_figure, pass_handle), )
148
149    # render handling
150    app.add_directive('kernel-render', KernelRender)
151    app.add_node(kernel_render,
152                 html    = (visit_kernel_render, pass_handle),
153                 latex   = (visit_kernel_render, pass_handle),
154                 texinfo = (visit_kernel_render, pass_handle),
155                 text    = (visit_kernel_render, pass_handle),
156                 man     = (visit_kernel_render, pass_handle), )
157
158    app.connect('doctree-read', add_kernel_figure_to_std_domain)
159
160    return dict(
161        version = __version__,
162        parallel_read_safe = True,
163        parallel_write_safe = True
164    )
165
166
167def setupTools(app):
168    u"""
169    Check available build tools and log some *verbose* messages.
170
171    This function is called once, when the builder is initiated.
172    """
173    global dot_cmd, convert_cmd   # pylint: disable=W0603
174    app.verbose("kfigure: check installed tools ...")
175
176    dot_cmd = which('dot')
177    convert_cmd = which('convert')
178
179    if dot_cmd:
180        app.verbose("use dot(1) from: " + dot_cmd)
181    else:
182        app.warn("dot(1) not found, for better output quality install "
183                 "graphviz from http://www.graphviz.org")
184    if convert_cmd:
185        app.verbose("use convert(1) from: " + convert_cmd)
186    else:
187        app.warn(
188            "convert(1) not found, for SVG to PDF conversion install "
189            "ImageMagick (https://www.imagemagick.org)")
190
191
192# integrate conversion tools
193# --------------------------
194
195RENDER_MARKUP_EXT = {
196    # The '.ext' must be handled by convert_image(..) function's *in_ext* input.
197    # <name> : <.ext>
198    'DOT' : '.dot',
199    'SVG' : '.svg'
200}
201
202def convert_image(img_node, translator, src_fname=None):
203    """Convert a image node for the builder.
204
205    Different builder prefer different image formats, e.g. *latex* builder
206    prefer PDF while *html* builder prefer SVG format for images.
207
208    This function handles output image formats in dependence of source the
209    format (of the image) and the translator's output format.
210    """
211    app = translator.builder.app
212
213    fname, in_ext = path.splitext(path.basename(img_node['uri']))
214    if src_fname is None:
215        src_fname = path.join(translator.builder.srcdir, img_node['uri'])
216        if not path.exists(src_fname):
217            src_fname = path.join(translator.builder.outdir, img_node['uri'])
218
219    dst_fname = None
220
221    # in kernel builds, use 'make SPHINXOPTS=-v' to see verbose messages
222
223    app.verbose('assert best format for: ' + img_node['uri'])
224
225    if in_ext == '.dot':
226
227        if not dot_cmd:
228            app.verbose("dot from graphviz not available / include DOT raw.")
229            img_node.replace_self(file2literal(src_fname))
230
231        elif translator.builder.format == 'latex':
232            dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
233            img_node['uri'] = fname + '.pdf'
234            img_node['candidates'] = {'*': fname + '.pdf'}
235
236
237        elif translator.builder.format == 'html':
238            dst_fname = path.join(
239                translator.builder.outdir,
240                translator.builder.imagedir,
241                fname + '.svg')
242            img_node['uri'] = path.join(
243                translator.builder.imgpath, fname + '.svg')
244            img_node['candidates'] = {
245                '*': path.join(translator.builder.imgpath, fname + '.svg')}
246
247        else:
248            # all other builder formats will include DOT as raw
249            img_node.replace_self(file2literal(src_fname))
250
251    elif in_ext == '.svg':
252
253        if translator.builder.format == 'latex':
254            if convert_cmd is None:
255                app.verbose("no SVG to PDF conversion available / include SVG raw.")
256                img_node.replace_self(file2literal(src_fname))
257            else:
258                dst_fname = path.join(translator.builder.outdir, fname + '.pdf')
259                img_node['uri'] = fname + '.pdf'
260                img_node['candidates'] = {'*': fname + '.pdf'}
261
262    if dst_fname:
263        # the builder needs not to copy one more time, so pop it if exists.
264        translator.builder.images.pop(img_node['uri'], None)
265        _name = dst_fname[len(translator.builder.outdir) + 1:]
266
267        if isNewer(dst_fname, src_fname):
268            app.verbose("convert: {out}/%s already exists and is newer" % _name)
269
270        else:
271            ok = False
272            mkdir(path.dirname(dst_fname))
273
274            if in_ext == '.dot':
275                app.verbose('convert DOT to: {out}/' + _name)
276                ok = dot2format(app, src_fname, dst_fname)
277
278            elif in_ext == '.svg':
279                app.verbose('convert SVG to: {out}/' + _name)
280                ok = svg2pdf(app, src_fname, dst_fname)
281
282            if not ok:
283                img_node.replace_self(file2literal(src_fname))
284
285
286def dot2format(app, dot_fname, out_fname):
287    """Converts DOT file to ``out_fname`` using ``dot(1)``.
288
289    * ``dot_fname`` pathname of the input DOT file, including extension ``.dot``
290    * ``out_fname`` pathname of the output file, including format extension
291
292    The *format extension* depends on the ``dot`` command (see ``man dot``
293    option ``-Txxx``). Normally you will use one of the following extensions:
294
295    - ``.ps`` for PostScript,
296    - ``.svg`` or ``svgz`` for Structured Vector Graphics,
297    - ``.fig`` for XFIG graphics and
298    - ``.png`` or ``gif`` for common bitmap graphics.
299
300    """
301    out_format = path.splitext(out_fname)[1][1:]
302    cmd = [dot_cmd, '-T%s' % out_format, dot_fname]
303    exit_code = 42
304
305    with open(out_fname, "w") as out:
306        exit_code = subprocess.call(cmd, stdout = out)
307        if exit_code != 0:
308            app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
309    return bool(exit_code == 0)
310
311def svg2pdf(app, svg_fname, pdf_fname):
312    """Converts SVG to PDF with ``convert(1)`` command.
313
314    Uses ``convert(1)`` from ImageMagick (https://www.imagemagick.org) for
315    conversion.  Returns ``True`` on success and ``False`` if an error occurred.
316
317    * ``svg_fname`` pathname of the input SVG file with extension (``.svg``)
318    * ``pdf_name``  pathname of the output PDF file with extension (``.pdf``)
319
320    """
321    cmd = [convert_cmd, svg_fname, pdf_fname]
322    # use stdout and stderr from parent
323    exit_code = subprocess.call(cmd)
324    if exit_code != 0:
325        app.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
326    return bool(exit_code == 0)
327
328
329# image handling
330# ---------------------
331
332def visit_kernel_image(self, node):    # pylint: disable=W0613
333    """Visitor of the ``kernel_image`` Node.
334
335    Handles the ``image`` child-node with the ``convert_image(...)``.
336    """
337    img_node = node[0]
338    convert_image(img_node, self)
339
340class kernel_image(nodes.image):
341    """Node for ``kernel-image`` directive."""
342    pass
343
344class KernelImage(images.Image):
345    u"""KernelImage directive
346
347    Earns everything from ``.. image::`` directive, except *remote URI* and
348    *glob* pattern. The KernelImage wraps a image node into a
349    kernel_image node. See ``visit_kernel_image``.
350    """
351
352    def run(self):
353        uri = self.arguments[0]
354        if uri.endswith('.*') or uri.find('://') != -1:
355            raise self.severe(
356                'Error in "%s: %s": glob pattern and remote images are not allowed'
357                % (self.name, uri))
358        result = images.Image.run(self)
359        if len(result) == 2 or isinstance(result[0], nodes.system_message):
360            return result
361        (image_node,) = result
362        # wrap image node into a kernel_image node / see visitors
363        node = kernel_image('', image_node)
364        return [node]
365
366# figure handling
367# ---------------------
368
369def visit_kernel_figure(self, node):   # pylint: disable=W0613
370    """Visitor of the ``kernel_figure`` Node.
371
372    Handles the ``image`` child-node with the ``convert_image(...)``.
373    """
374    img_node = node[0][0]
375    convert_image(img_node, self)
376
377class kernel_figure(nodes.figure):
378    """Node for ``kernel-figure`` directive."""
379
380class KernelFigure(Figure):
381    u"""KernelImage directive
382
383    Earns everything from ``.. figure::`` directive, except *remote URI* and
384    *glob* pattern.  The KernelFigure wraps a figure node into a kernel_figure
385    node. See ``visit_kernel_figure``.
386    """
387
388    def run(self):
389        uri = self.arguments[0]
390        if uri.endswith('.*') or uri.find('://') != -1:
391            raise self.severe(
392                'Error in "%s: %s":'
393                ' glob pattern and remote images are not allowed'
394                % (self.name, uri))
395        result = Figure.run(self)
396        if len(result) == 2 or isinstance(result[0], nodes.system_message):
397            return result
398        (figure_node,) = result
399        # wrap figure node into a kernel_figure node / see visitors
400        node = kernel_figure('', figure_node)
401        return [node]
402
403
404# render handling
405# ---------------------
406
407def visit_kernel_render(self, node):
408    """Visitor of the ``kernel_render`` Node.
409
410    If rendering tools available, save the markup of the ``literal_block`` child
411    node into a file and replace the ``literal_block`` node with a new created
412    ``image`` node, pointing to the saved markup file. Afterwards, handle the
413    image child-node with the ``convert_image(...)``.
414    """
415    app = self.builder.app
416    srclang = node.get('srclang')
417
418    app.verbose('visit kernel-render node lang: "%s"' % (srclang))
419
420    tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)
421    if tmp_ext is None:
422        app.warn('kernel-render: "%s" unknown / include raw.' % (srclang))
423        return
424
425    if not dot_cmd and tmp_ext == '.dot':
426        app.verbose("dot from graphviz not available / include raw.")
427        return
428
429    literal_block = node[0]
430
431    code      = literal_block.astext()
432    hashobj   = code.encode('utf-8') #  str(node.attributes)
433    fname     = path.join('%s-%s' % (srclang, sha1(hashobj).hexdigest()))
434
435    tmp_fname = path.join(
436        self.builder.outdir, self.builder.imagedir, fname + tmp_ext)
437
438    if not path.isfile(tmp_fname):
439        mkdir(path.dirname(tmp_fname))
440        with open(tmp_fname, "w") as out:
441            out.write(code)
442
443    img_node = nodes.image(node.rawsource, **node.attributes)
444    img_node['uri'] = path.join(self.builder.imgpath, fname + tmp_ext)
445    img_node['candidates'] = {
446        '*': path.join(self.builder.imgpath, fname + tmp_ext)}
447
448    literal_block.replace_self(img_node)
449    convert_image(img_node, self, tmp_fname)
450
451
452class kernel_render(nodes.General, nodes.Inline, nodes.Element):
453    """Node for ``kernel-render`` directive."""
454    pass
455
456class KernelRender(Figure):
457    u"""KernelRender directive
458
459    Render content by external tool.  Has all the options known from the
460    *figure*  directive, plus option ``caption``.  If ``caption`` has a
461    value, a figure node with the *caption* is inserted. If not, a image node is
462    inserted.
463
464    The KernelRender directive wraps the text of the directive into a
465    literal_block node and wraps it into a kernel_render node. See
466    ``visit_kernel_render``.
467    """
468    has_content = True
469    required_arguments = 1
470    optional_arguments = 0
471    final_argument_whitespace = False
472
473    # earn options from 'figure'
474    option_spec = Figure.option_spec.copy()
475    option_spec['caption'] = directives.unchanged
476
477    def run(self):
478        return [self.build_node()]
479
480    def build_node(self):
481
482        srclang = self.arguments[0].strip()
483        if srclang not in RENDER_MARKUP_EXT.keys():
484            return [self.state_machine.reporter.warning(
485                'Unknown source language "%s", use one of: %s.' % (
486                    srclang, ",".join(RENDER_MARKUP_EXT.keys())),
487                line=self.lineno)]
488
489        code = '\n'.join(self.content)
490        if not code.strip():
491            return [self.state_machine.reporter.warning(
492                'Ignoring "%s" directive without content.' % (
493                    self.name),
494                line=self.lineno)]
495
496        node = kernel_render()
497        node['alt'] = self.options.get('alt','')
498        node['srclang'] = srclang
499        literal_node = nodes.literal_block(code, code)
500        node += literal_node
501
502        caption = self.options.get('caption')
503        if caption:
504            # parse caption's content
505            parsed = nodes.Element()
506            self.state.nested_parse(
507                ViewList([caption], source=''), self.content_offset, parsed)
508            caption_node = nodes.caption(
509                parsed[0].rawsource, '', *parsed[0].children)
510            caption_node.source = parsed[0].source
511            caption_node.line = parsed[0].line
512
513            figure_node = nodes.figure('', node)
514            for k,v in self.options.items():
515                figure_node[k] = v
516            figure_node += caption_node
517
518            node = figure_node
519
520        return node
521
522def add_kernel_figure_to_std_domain(app, doctree):
523    """Add kernel-figure anchors to 'std' domain.
524
525    The ``StandardDomain.process_doc(..)`` method does not know how to resolve
526    the caption (label) of ``kernel-figure`` directive (it only knows about
527    standard nodes, e.g. table, figure etc.). Without any additional handling
528    this will result in a 'undefined label' for kernel-figures.
529
530    This handle adds labels of kernel-figure to the 'std' domain labels.
531    """
532
533    std = app.env.domains["std"]
534    docname = app.env.docname
535    labels = std.data["labels"]
536
537    for name, explicit in iteritems(doctree.nametypes):
538        if not explicit:
539            continue
540        labelid = doctree.nameids[name]
541        if labelid is None:
542            continue
543        node = doctree.ids[labelid]
544
545        if node.tagname == 'kernel_figure':
546            for n in node.next_node():
547                if n.tagname == 'caption':
548                    sectname = clean_astext(n)
549                    # add label to std domain
550                    labels[name] = docname, labelid, sectname
551                    break
552