xref: /openbmc/qemu/tests/docker/docker.py (revision 12a6c15e) 
1#!/usr/bin/env python2
2#
3# Docker controlling module
4#
5# Copyright (c) 2016 Red Hat Inc.
6#
7# Authors:
8#  Fam Zheng <famz@redhat.com>
9#
10# This work is licensed under the terms of the GNU GPL, version 2
11# or (at your option) any later version. See the COPYING file in
12# the top-level directory.
13
14import os
15import sys
16sys.path.append(os.path.join(os.path.dirname(__file__),
17                             '..', '..', 'scripts'))
18import argparse
19import subprocess
20import json
21import hashlib
22import atexit
23import uuid
24import tempfile
25import re
26import signal
27from tarfile import TarFile, TarInfo
28from StringIO import StringIO
29from shutil import copy, rmtree
30from pwd import getpwuid
31
32
33FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
34
35
36DEVNULL = open(os.devnull, 'wb')
37
38
39def _text_checksum(text):
40    """Calculate a digest string unique to the text content"""
41    return hashlib.sha1(text).hexdigest()
42
43def _file_checksum(filename):
44    return _text_checksum(open(filename, 'rb').read())
45
46def _guess_docker_command():
47    """ Guess a working docker command or raise exception if not found"""
48    commands = [["docker"], ["sudo", "-n", "docker"]]
49    for cmd in commands:
50        try:
51            if subprocess.call(cmd + ["images"],
52                               stdout=DEVNULL, stderr=DEVNULL) == 0:
53                return cmd
54        except OSError:
55            pass
56    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
57    raise Exception("Cannot find working docker command. Tried:\n%s" % \
58                    commands_txt)
59
60def _copy_with_mkdir(src, root_dir, sub_path='.'):
61    """Copy src into root_dir, creating sub_path as needed."""
62    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
63    try:
64        os.makedirs(dest_dir)
65    except OSError:
66        # we can safely ignore already created directories
67        pass
68
69    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
70    copy(src, dest_file)
71
72
73def _get_so_libs(executable):
74    """Return a list of libraries associated with an executable.
75
76    The paths may be symbolic links which would need to be resolved to
77    ensure theright data is copied."""
78
79    libs = []
80    ldd_re = re.compile(r"(/.*/)(\S*)")
81    try:
82        ldd_output = subprocess.check_output(["ldd", executable])
83        for line in ldd_output.split("\n"):
84            search = ldd_re.search(line)
85            if search and len(search.groups()) == 2:
86                so_path = search.groups()[0]
87                so_lib = search.groups()[1]
88                libs.append("%s/%s" % (so_path, so_lib))
89    except subprocess.CalledProcessError:
90        print "%s had no associated libraries (static build?)" % (executable)
91
92    return libs
93
94def _copy_binary_with_libs(src, dest_dir):
95    """Copy a binary executable and all its dependant libraries.
96
97    This does rely on the host file-system being fairly multi-arch
98    aware so the file don't clash with the guests layout."""
99
100    _copy_with_mkdir(src, dest_dir, "/usr/bin")
101
102    libs = _get_so_libs(src)
103    if libs:
104        for l in libs:
105            so_path = os.path.dirname(l)
106            _copy_with_mkdir(l , dest_dir, so_path)
107
108class Docker(object):
109    """ Running Docker commands """
110    def __init__(self):
111        self._command = _guess_docker_command()
112        self._instances = []
113        atexit.register(self._kill_instances)
114        signal.signal(signal.SIGTERM, self._kill_instances)
115        signal.signal(signal.SIGHUP, self._kill_instances)
116
117    def _do(self, cmd, quiet=True, **kwargs):
118        if quiet:
119            kwargs["stdout"] = DEVNULL
120        return subprocess.call(self._command + cmd, **kwargs)
121
122    def _do_check(self, cmd, quiet=True, **kwargs):
123        if quiet:
124            kwargs["stdout"] = DEVNULL
125        return subprocess.check_call(self._command + cmd, **kwargs)
126
127    def _do_kill_instances(self, only_known, only_active=True):
128        cmd = ["ps", "-q"]
129        if not only_active:
130            cmd.append("-a")
131        for i in self._output(cmd).split():
132            resp = self._output(["inspect", i])
133            labels = json.loads(resp)[0]["Config"]["Labels"]
134            active = json.loads(resp)[0]["State"]["Running"]
135            if not labels:
136                continue
137            instance_uuid = labels.get("com.qemu.instance.uuid", None)
138            if not instance_uuid:
139                continue
140            if only_known and instance_uuid not in self._instances:
141                continue
142            print "Terminating", i
143            if active:
144                self._do(["kill", i])
145            self._do(["rm", i])
146
147    def clean(self):
148        self._do_kill_instances(False, False)
149        return 0
150
151    def _kill_instances(self, *args, **kwargs):
152        return self._do_kill_instances(True)
153
154    def _output(self, cmd, **kwargs):
155        return subprocess.check_output(self._command + cmd,
156                                       stderr=subprocess.STDOUT,
157                                       **kwargs)
158
159    def get_image_dockerfile_checksum(self, tag):
160        resp = self._output(["inspect", tag])
161        labels = json.loads(resp)[0]["Config"].get("Labels", {})
162        return labels.get("com.qemu.dockerfile-checksum", "")
163
164    def build_image(self, tag, docker_dir, dockerfile,
165                    quiet=True, user=False, argv=None, extra_files_cksum=[]):
166        if argv == None:
167            argv = []
168
169        tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
170        tmp_df.write(dockerfile)
171
172        if user:
173            uid = os.getuid()
174            uname = getpwuid(uid).pw_name
175            tmp_df.write("\n")
176            tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
177                         (uname, uid, uname))
178
179        tmp_df.write("\n")
180        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
181                     _text_checksum("\n".join([dockerfile] +
182                                    extra_files_cksum)))
183        tmp_df.flush()
184
185        self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
186                       [docker_dir],
187                       quiet=quiet)
188
189    def update_image(self, tag, tarball, quiet=True):
190        "Update a tagged image using "
191
192        self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
193
194    def image_matches_dockerfile(self, tag, dockerfile):
195        try:
196            checksum = self.get_image_dockerfile_checksum(tag)
197        except Exception:
198            return False
199        return checksum == _text_checksum(dockerfile)
200
201    def run(self, cmd, keep, quiet):
202        label = uuid.uuid1().hex
203        if not keep:
204            self._instances.append(label)
205        ret = self._do_check(["run", "--label",
206                             "com.qemu.instance.uuid=" + label] + cmd,
207                             quiet=quiet)
208        if not keep:
209            self._instances.remove(label)
210        return ret
211
212    def command(self, cmd, argv, quiet):
213        return self._do([cmd] + argv, quiet=quiet)
214
215class SubCommand(object):
216    """A SubCommand template base class"""
217    name = None # Subcommand name
218    def shared_args(self, parser):
219        parser.add_argument("--quiet", action="store_true",
220                            help="Run quietly unless an error occured")
221
222    def args(self, parser):
223        """Setup argument parser"""
224        pass
225    def run(self, args, argv):
226        """Run command.
227        args: parsed argument by argument parser.
228        argv: remaining arguments from sys.argv.
229        """
230        pass
231
232class RunCommand(SubCommand):
233    """Invoke docker run and take care of cleaning up"""
234    name = "run"
235    def args(self, parser):
236        parser.add_argument("--keep", action="store_true",
237                            help="Don't remove image when command completes")
238    def run(self, args, argv):
239        return Docker().run(argv, args.keep, quiet=args.quiet)
240
241class BuildCommand(SubCommand):
242    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
243    name = "build"
244    def args(self, parser):
245        parser.add_argument("--include-executable", "-e",
246                            help="""Specify a binary that will be copied to the
247                            container together with all its dependent
248                            libraries""")
249        parser.add_argument("--extra-files", "-f", nargs='*',
250                            help="""Specify files that will be copied in the
251                            Docker image, fulfilling the ADD directive from the
252                            Dockerfile""")
253        parser.add_argument("--add-current-user", "-u", dest="user",
254                            action="store_true",
255                            help="Add the current user to image's passwd")
256        parser.add_argument("tag",
257                            help="Image Tag")
258        parser.add_argument("dockerfile",
259                            help="Dockerfile name")
260
261    def run(self, args, argv):
262        dockerfile = open(args.dockerfile, "rb").read()
263        tag = args.tag
264
265        dkr = Docker()
266        if dkr.image_matches_dockerfile(tag, dockerfile):
267            if not args.quiet:
268                print "Image is up to date."
269        else:
270            # Create a docker context directory for the build
271            docker_dir = tempfile.mkdtemp(prefix="docker_build")
272
273            # Is there a .pre file to run in the build context?
274            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
275            if os.path.exists(docker_pre):
276                stdout = DEVNULL if args.quiet else None
277                rc = subprocess.call(os.path.realpath(docker_pre),
278                                     cwd=docker_dir, stdout=stdout)
279                if rc == 3:
280                    print "Skip"
281                    return 0
282                elif rc != 0:
283                    print "%s exited with code %d" % (docker_pre, rc)
284                    return 1
285
286            # Copy any extra files into the Docker context. These can be
287            # included by the use of the ADD directive in the Dockerfile.
288            cksum = []
289            if args.include_executable:
290                # FIXME: there is no checksum of this executable and the linked
291                # libraries, once the image built any change of this executable
292                # or any library won't trigger another build.
293                _copy_binary_with_libs(args.include_executable, docker_dir)
294            for filename in args.extra_files or []:
295                _copy_with_mkdir(filename, docker_dir)
296                cksum += [_file_checksum(filename)]
297
298            argv += ["--build-arg=" + k.lower() + "=" + v
299                        for k, v in os.environ.iteritems()
300                        if k.lower() in FILTERED_ENV_NAMES]
301            dkr.build_image(tag, docker_dir, dockerfile,
302                            quiet=args.quiet, user=args.user, argv=argv,
303                            extra_files_cksum=cksum)
304
305            rmtree(docker_dir)
306
307        return 0
308
309class UpdateCommand(SubCommand):
310    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
311    name = "update"
312    def args(self, parser):
313        parser.add_argument("tag",
314                            help="Image Tag")
315        parser.add_argument("executable",
316                            help="Executable to copy")
317
318    def run(self, args, argv):
319        # Create a temporary tarball with our whole build context and
320        # dockerfile for the update
321        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
322        tmp_tar = TarFile(fileobj=tmp, mode='w')
323
324        # Add the executable to the tarball
325        bn = os.path.basename(args.executable)
326        ff = "/usr/bin/%s" % bn
327        tmp_tar.add(args.executable, arcname=ff)
328
329        # Add any associated libraries
330        libs = _get_so_libs(args.executable)
331        if libs:
332            for l in libs:
333                tmp_tar.add(os.path.realpath(l), arcname=l)
334
335        # Create a Docker buildfile
336        df = StringIO()
337        df.write("FROM %s\n" % args.tag)
338        df.write("ADD . /\n")
339        df.seek(0)
340
341        df_tar = TarInfo(name="Dockerfile")
342        df_tar.size = len(df.buf)
343        tmp_tar.addfile(df_tar, fileobj=df)
344
345        tmp_tar.close()
346
347        # reset the file pointers
348        tmp.flush()
349        tmp.seek(0)
350
351        # Run the build with our tarball context
352        dkr = Docker()
353        dkr.update_image(args.tag, tmp, quiet=args.quiet)
354
355        return 0
356
357class CleanCommand(SubCommand):
358    """Clean up docker instances"""
359    name = "clean"
360    def run(self, args, argv):
361        Docker().clean()
362        return 0
363
364class ImagesCommand(SubCommand):
365    """Run "docker images" command"""
366    name = "images"
367    def run(self, args, argv):
368        return Docker().command("images", argv, args.quiet)
369
370def main():
371    parser = argparse.ArgumentParser(description="A Docker helper",
372            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
373    subparsers = parser.add_subparsers(title="subcommands", help=None)
374    for cls in SubCommand.__subclasses__():
375        cmd = cls()
376        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
377        cmd.shared_args(subp)
378        cmd.args(subp)
379        subp.set_defaults(cmdobj=cmd)
380    args, argv = parser.parse_known_args()
381    return args.cmdobj.run(args, argv)
382
383if __name__ == "__main__":
384    sys.exit(main())
385