xref: /openbmc/qemu/tests/docker/docker.py (revision c574e161) 
1#!/usr/bin/env python3
2#
3# Docker controlling module
4#
5# Copyright (c) 2016 Red Hat Inc.
6#
7# Authors:
8#  Fam Zheng <famz@redhat.com>
9#
10# This work is licensed under the terms of the GNU GPL, version 2
11# or (at your option) any later version. See the COPYING file in
12# the top-level directory.
13
14import os
15import sys
16import subprocess
17import json
18import hashlib
19import atexit
20import uuid
21import argparse
22import enum
23import tempfile
24import re
25import signal
26from tarfile import TarFile, TarInfo
27from io import StringIO, BytesIO
28from shutil import copy, rmtree
29from pwd import getpwuid
30from datetime import datetime, timedelta
31
32
33FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
34
35
36DEVNULL = open(os.devnull, 'wb')
37
38class EngineEnum(enum.IntEnum):
39    AUTO = 1
40    DOCKER = 2
41    PODMAN = 3
42
43    def __str__(self):
44        return self.name.lower()
45
46    def __repr__(self):
47        return str(self)
48
49    @staticmethod
50    def argparse(s):
51        try:
52            return EngineEnum[s.upper()]
53        except KeyError:
54            return s
55
56
57USE_ENGINE = EngineEnum.AUTO
58
59def _bytes_checksum(bytes):
60    """Calculate a digest string unique to the text content"""
61    return hashlib.sha1(bytes).hexdigest()
62
63def _text_checksum(text):
64    """Calculate a digest string unique to the text content"""
65    return _bytes_checksum(text.encode('utf-8'))
66
67def _read_dockerfile(path):
68    return open(path, 'rt', encoding='utf-8').read()
69
70def _file_checksum(filename):
71    return _bytes_checksum(open(filename, 'rb').read())
72
73
74def _guess_engine_command():
75    """ Guess a working engine command or raise exception if not found"""
76    commands = []
77
78    if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
79        commands += [["podman"]]
80    if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
81        commands += [["docker"], ["sudo", "-n", "docker"]]
82    for cmd in commands:
83        try:
84            # docker version will return the client details in stdout
85            # but still report a status of 1 if it can't contact the daemon
86            if subprocess.call(cmd + ["version"],
87                               stdout=DEVNULL, stderr=DEVNULL) == 0:
88                return cmd
89        except OSError:
90            pass
91    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
92    raise Exception("Cannot find working engine command. Tried:\n%s" %
93                    commands_txt)
94
95
96def _copy_with_mkdir(src, root_dir, sub_path='.'):
97    """Copy src into root_dir, creating sub_path as needed."""
98    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
99    try:
100        os.makedirs(dest_dir)
101    except OSError:
102        # we can safely ignore already created directories
103        pass
104
105    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
106    copy(src, dest_file)
107
108
109def _get_so_libs(executable):
110    """Return a list of libraries associated with an executable.
111
112    The paths may be symbolic links which would need to be resolved to
113    ensure the right data is copied."""
114
115    libs = []
116    ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
117    try:
118        ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
119        for line in ldd_output.split("\n"):
120            search = ldd_re.search(line)
121            if search:
122                try:
123                    libs.append(s.group(1))
124                except IndexError:
125                    pass
126    except subprocess.CalledProcessError:
127        print("%s had no associated libraries (static build?)" % (executable))
128
129    return libs
130
131
132def _copy_binary_with_libs(src, bin_dest, dest_dir):
133    """Maybe copy a binary and all its dependent libraries.
134
135    If bin_dest isn't set we only copy the support libraries because
136    we don't need qemu in the docker path to run (due to persistent
137    mapping). Indeed users may get confused if we aren't running what
138    is in the image.
139
140    This does rely on the host file-system being fairly multi-arch
141    aware so the file don't clash with the guests layout.
142    """
143
144    if bin_dest:
145        _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
146    else:
147        print("only copying support libraries for %s" % (src))
148
149    libs = _get_so_libs(src)
150    if libs:
151        for l in libs:
152            so_path = os.path.dirname(l)
153            real_l = os.path.realpath(l)
154            _copy_with_mkdir(real_l, dest_dir, so_path)
155
156
157def _check_binfmt_misc(executable):
158    """Check binfmt_misc has entry for executable in the right place.
159
160    The details of setting up binfmt_misc are outside the scope of
161    this script but we should at least fail early with a useful
162    message if it won't work.
163
164    Returns the configured binfmt path and a valid flag. For
165    persistent configurations we will still want to copy and dependent
166    libraries.
167    """
168
169    binary = os.path.basename(executable)
170    binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
171
172    if not os.path.exists(binfmt_entry):
173        print ("No binfmt_misc entry for %s" % (binary))
174        return None, False
175
176    with open(binfmt_entry) as x: entry = x.read()
177
178    if re.search("flags:.*F.*\n", entry):
179        print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
180              (binary))
181        return None, True
182
183    m = re.search("interpreter (\S+)\n", entry)
184    interp = m.group(1)
185    if interp and interp != executable:
186        print("binfmt_misc for %s does not point to %s, using %s" %
187              (binary, executable, interp))
188
189    return interp, True
190
191
192def _read_qemu_dockerfile(img_name):
193    # special case for Debian linux-user images
194    if img_name.startswith("debian") and img_name.endswith("user"):
195        img_name = "debian-bootstrap"
196
197    df = os.path.join(os.path.dirname(__file__), "dockerfiles",
198                      img_name + ".docker")
199    return _read_dockerfile(df)
200
201
202def _dockerfile_preprocess(df):
203    out = ""
204    for l in df.splitlines():
205        if len(l.strip()) == 0 or l.startswith("#"):
206            continue
207        from_pref = "FROM qemu/"
208        if l.startswith(from_pref):
209            # TODO: Alternatively we could replace this line with "FROM $ID"
210            # where $ID is the image's hex id obtained with
211            #    $ docker images $IMAGE --format="{{.Id}}"
212            # but unfortunately that's not supported by RHEL 7.
213            inlining = _read_qemu_dockerfile(l[len(from_pref):])
214            out += _dockerfile_preprocess(inlining)
215            continue
216        out += l + "\n"
217    return out
218
219
220class Docker(object):
221    """ Running Docker commands """
222    def __init__(self):
223        self._command = _guess_engine_command()
224
225        if "docker" in self._command and "TRAVIS" not in os.environ:
226            os.environ["DOCKER_BUILDKIT"] = "1"
227            self._buildkit = True
228        else:
229            self._buildkit = False
230
231        self._instance = None
232        atexit.register(self._kill_instances)
233        signal.signal(signal.SIGTERM, self._kill_instances)
234        signal.signal(signal.SIGHUP, self._kill_instances)
235
236    def _do(self, cmd, quiet=True, **kwargs):
237        if quiet:
238            kwargs["stdout"] = DEVNULL
239        return subprocess.call(self._command + cmd, **kwargs)
240
241    def _do_check(self, cmd, quiet=True, **kwargs):
242        if quiet:
243            kwargs["stdout"] = DEVNULL
244        return subprocess.check_call(self._command + cmd, **kwargs)
245
246    def _do_kill_instances(self, only_known, only_active=True):
247        cmd = ["ps", "-q"]
248        if not only_active:
249            cmd.append("-a")
250
251        filter = "--filter=label=com.qemu.instance.uuid"
252        if only_known:
253            if self._instance:
254                filter += "=%s" % (self._instance)
255            else:
256                # no point trying to kill, we finished
257                return
258
259        print("filter=%s" % (filter))
260        cmd.append(filter)
261        for i in self._output(cmd).split():
262            self._do(["rm", "-f", i])
263
264    def clean(self):
265        self._do_kill_instances(False, False)
266        return 0
267
268    def _kill_instances(self, *args, **kwargs):
269        return self._do_kill_instances(True)
270
271    def _output(self, cmd, **kwargs):
272        try:
273            return subprocess.check_output(self._command + cmd,
274                                           stderr=subprocess.STDOUT,
275                                           encoding='utf-8',
276                                           **kwargs)
277        except TypeError:
278            # 'encoding' argument was added in 3.6+
279            return subprocess.check_output(self._command + cmd,
280                                           stderr=subprocess.STDOUT,
281                                           **kwargs).decode('utf-8')
282
283
284    def inspect_tag(self, tag):
285        try:
286            return self._output(["inspect", tag])
287        except subprocess.CalledProcessError:
288            return None
289
290    def get_image_creation_time(self, info):
291        return json.loads(info)[0]["Created"]
292
293    def get_image_dockerfile_checksum(self, tag):
294        resp = self.inspect_tag(tag)
295        labels = json.loads(resp)[0]["Config"].get("Labels", {})
296        return labels.get("com.qemu.dockerfile-checksum", "")
297
298    def build_image(self, tag, docker_dir, dockerfile,
299                    quiet=True, user=False, argv=None, registry=None,
300                    extra_files_cksum=[]):
301        if argv is None:
302            argv = []
303
304        # pre-calculate the docker checksum before any
305        # substitutions we make for caching
306        checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
307
308        if registry is not None:
309            sources = re.findall("FROM qemu\/(.*)", dockerfile)
310            # Fetch any cache layers we can, may fail
311            for s in sources:
312                pull_args = ["pull", "%s/qemu/%s" % (registry, s)]
313                if self._do(pull_args, quiet=quiet) != 0:
314                    registry = None
315                    break
316            # Make substitutions
317            if registry is not None:
318                dockerfile = dockerfile.replace("FROM qemu/",
319                                                "FROM %s/qemu/" %
320                                                (registry))
321
322        tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
323                                             encoding='utf-8',
324                                             dir=docker_dir, suffix=".docker")
325        tmp_df.write(dockerfile)
326
327        if user:
328            uid = os.getuid()
329            uname = getpwuid(uid).pw_name
330            tmp_df.write("\n")
331            tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
332                         (uname, uid, uname))
333
334        tmp_df.write("\n")
335        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum))
336        for f, c in extra_files_cksum:
337            tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
338
339        tmp_df.flush()
340
341        build_args = ["build", "-t", tag, "-f", tmp_df.name]
342        if self._buildkit:
343            build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
344
345        if registry is not None:
346            pull_args = ["pull", "%s/%s" % (registry, tag)]
347            self._do(pull_args, quiet=quiet)
348            cache = "%s/%s" % (registry, tag)
349            build_args += ["--cache-from", cache]
350        build_args += argv
351        build_args += [docker_dir]
352
353        self._do_check(build_args,
354                       quiet=quiet)
355
356    def update_image(self, tag, tarball, quiet=True):
357        "Update a tagged image using "
358
359        self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
360
361    def image_matches_dockerfile(self, tag, dockerfile):
362        try:
363            checksum = self.get_image_dockerfile_checksum(tag)
364        except Exception:
365            return False
366        return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
367
368    def run(self, cmd, keep, quiet, as_user=False):
369        label = uuid.uuid4().hex
370        if not keep:
371            self._instance = label
372
373        if as_user:
374            uid = os.getuid()
375            cmd = [ "-u", str(uid) ] + cmd
376            # podman requires a bit more fiddling
377            if self._command[0] == "podman":
378                cmd.insert(0, '--userns=keep-id')
379
380        ret = self._do_check(["run", "--label",
381                             "com.qemu.instance.uuid=" + label] + cmd,
382                             quiet=quiet)
383        if not keep:
384            self._instance = None
385        return ret
386
387    def command(self, cmd, argv, quiet):
388        return self._do([cmd] + argv, quiet=quiet)
389
390
391class SubCommand(object):
392    """A SubCommand template base class"""
393    name = None  # Subcommand name
394
395    def shared_args(self, parser):
396        parser.add_argument("--quiet", action="store_true",
397                            help="Run quietly unless an error occurred")
398
399    def args(self, parser):
400        """Setup argument parser"""
401        pass
402
403    def run(self, args, argv):
404        """Run command.
405        args: parsed argument by argument parser.
406        argv: remaining arguments from sys.argv.
407        """
408        pass
409
410
411class RunCommand(SubCommand):
412    """Invoke docker run and take care of cleaning up"""
413    name = "run"
414
415    def args(self, parser):
416        parser.add_argument("--keep", action="store_true",
417                            help="Don't remove image when command completes")
418        parser.add_argument("--run-as-current-user", action="store_true",
419                            help="Run container using the current user's uid")
420
421    def run(self, args, argv):
422        return Docker().run(argv, args.keep, quiet=args.quiet,
423                            as_user=args.run_as_current_user)
424
425
426class BuildCommand(SubCommand):
427    """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
428    name = "build"
429
430    def args(self, parser):
431        parser.add_argument("--include-executable", "-e",
432                            help="""Specify a binary that will be copied to the
433                            container together with all its dependent
434                            libraries""")
435        parser.add_argument("--extra-files", nargs='*',
436                            help="""Specify files that will be copied in the
437                            Docker image, fulfilling the ADD directive from the
438                            Dockerfile""")
439        parser.add_argument("--add-current-user", "-u", dest="user",
440                            action="store_true",
441                            help="Add the current user to image's passwd")
442        parser.add_argument("--registry", "-r",
443                            help="cache from docker registry")
444        parser.add_argument("-t", dest="tag",
445                            help="Image Tag")
446        parser.add_argument("-f", dest="dockerfile",
447                            help="Dockerfile name")
448
449    def run(self, args, argv):
450        dockerfile = _read_dockerfile(args.dockerfile)
451        tag = args.tag
452
453        dkr = Docker()
454        if "--no-cache" not in argv and \
455           dkr.image_matches_dockerfile(tag, dockerfile):
456            if not args.quiet:
457                print("Image is up to date.")
458        else:
459            # Create a docker context directory for the build
460            docker_dir = tempfile.mkdtemp(prefix="docker_build")
461
462            # Validate binfmt_misc will work
463            if args.include_executable:
464                qpath, enabled = _check_binfmt_misc(args.include_executable)
465                if not enabled:
466                    return 1
467
468            # Is there a .pre file to run in the build context?
469            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
470            if os.path.exists(docker_pre):
471                stdout = DEVNULL if args.quiet else None
472                rc = subprocess.call(os.path.realpath(docker_pre),
473                                     cwd=docker_dir, stdout=stdout)
474                if rc == 3:
475                    print("Skip")
476                    return 0
477                elif rc != 0:
478                    print("%s exited with code %d" % (docker_pre, rc))
479                    return 1
480
481            # Copy any extra files into the Docker context. These can be
482            # included by the use of the ADD directive in the Dockerfile.
483            cksum = []
484            if args.include_executable:
485                # FIXME: there is no checksum of this executable and the linked
486                # libraries, once the image built any change of this executable
487                # or any library won't trigger another build.
488                _copy_binary_with_libs(args.include_executable,
489                                       qpath, docker_dir)
490
491            for filename in args.extra_files or []:
492                _copy_with_mkdir(filename, docker_dir)
493                cksum += [(filename, _file_checksum(filename))]
494
495            argv += ["--build-arg=" + k.lower() + "=" + v
496                     for k, v in os.environ.items()
497                     if k.lower() in FILTERED_ENV_NAMES]
498            dkr.build_image(tag, docker_dir, dockerfile,
499                            quiet=args.quiet, user=args.user,
500                            argv=argv, registry=args.registry,
501                            extra_files_cksum=cksum)
502
503            rmtree(docker_dir)
504
505        return 0
506
507
508class UpdateCommand(SubCommand):
509    """ Update a docker image with new executables. Args: <tag> <executable>"""
510    name = "update"
511
512    def args(self, parser):
513        parser.add_argument("tag",
514                            help="Image Tag")
515        parser.add_argument("executable",
516                            help="Executable to copy")
517
518    def run(self, args, argv):
519        # Create a temporary tarball with our whole build context and
520        # dockerfile for the update
521        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
522        tmp_tar = TarFile(fileobj=tmp, mode='w')
523
524        # Add the executable to the tarball, using the current
525        # configured binfmt_misc path. If we don't get a path then we
526        # only need the support libraries copied
527        ff, enabled = _check_binfmt_misc(args.executable)
528
529        if not enabled:
530            print("binfmt_misc not enabled, update disabled")
531            return 1
532
533        if ff:
534            tmp_tar.add(args.executable, arcname=ff)
535
536        # Add any associated libraries
537        libs = _get_so_libs(args.executable)
538        if libs:
539            for l in libs:
540                tmp_tar.add(os.path.realpath(l), arcname=l)
541
542        # Create a Docker buildfile
543        df = StringIO()
544        df.write(u"FROM %s\n" % args.tag)
545        df.write(u"ADD . /\n")
546
547        df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
548
549        df_tar = TarInfo(name="Dockerfile")
550        df_tar.size = df_bytes.getbuffer().nbytes
551        tmp_tar.addfile(df_tar, fileobj=df_bytes)
552
553        tmp_tar.close()
554
555        # reset the file pointers
556        tmp.flush()
557        tmp.seek(0)
558
559        # Run the build with our tarball context
560        dkr = Docker()
561        dkr.update_image(args.tag, tmp, quiet=args.quiet)
562
563        return 0
564
565
566class CleanCommand(SubCommand):
567    """Clean up docker instances"""
568    name = "clean"
569
570    def run(self, args, argv):
571        Docker().clean()
572        return 0
573
574
575class ImagesCommand(SubCommand):
576    """Run "docker images" command"""
577    name = "images"
578
579    def run(self, args, argv):
580        return Docker().command("images", argv, args.quiet)
581
582
583class ProbeCommand(SubCommand):
584    """Probe if we can run docker automatically"""
585    name = "probe"
586
587    def run(self, args, argv):
588        try:
589            docker = Docker()
590            if docker._command[0] == "docker":
591                print("docker")
592            elif docker._command[0] == "sudo":
593                print("sudo docker")
594            elif docker._command[0] == "podman":
595                print("podman")
596        except Exception:
597            print("no")
598
599        return
600
601
602class CcCommand(SubCommand):
603    """Compile sources with cc in images"""
604    name = "cc"
605
606    def args(self, parser):
607        parser.add_argument("--image", "-i", required=True,
608                            help="The docker image in which to run cc")
609        parser.add_argument("--cc", default="cc",
610                            help="The compiler executable to call")
611        parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
612                            help="""Extra paths to (ro) mount into container for
613                            reading sources""")
614
615    def run(self, args, argv):
616        if argv and argv[0] == "--":
617            argv = argv[1:]
618        cwd = os.getcwd()
619        cmd = ["--rm", "-w", cwd,
620               "-v", "%s:%s:rw" % (cwd, cwd)]
621        if args.paths:
622            for p in args.paths:
623                cmd += ["-v", "%s:%s:ro,z" % (p, p)]
624        cmd += [args.image, args.cc]
625        cmd += argv
626        return Docker().run(cmd, False, quiet=args.quiet,
627                            as_user=True)
628
629
630class CheckCommand(SubCommand):
631    """Check if we need to re-build a docker image out of a dockerfile.
632    Arguments: <tag> <dockerfile>"""
633    name = "check"
634
635    def args(self, parser):
636        parser.add_argument("tag",
637                            help="Image Tag")
638        parser.add_argument("dockerfile", default=None,
639                            help="Dockerfile name", nargs='?')
640        parser.add_argument("--checktype", choices=["checksum", "age"],
641                            default="checksum", help="check type")
642        parser.add_argument("--olderthan", default=60, type=int,
643                            help="number of minutes")
644
645    def run(self, args, argv):
646        tag = args.tag
647
648        try:
649            dkr = Docker()
650        except subprocess.CalledProcessError:
651            print("Docker not set up")
652            return 1
653
654        info = dkr.inspect_tag(tag)
655        if info is None:
656            print("Image does not exist")
657            return 1
658
659        if args.checktype == "checksum":
660            if not args.dockerfile:
661                print("Need a dockerfile for tag:%s" % (tag))
662                return 1
663
664            dockerfile = _read_dockerfile(args.dockerfile)
665
666            if dkr.image_matches_dockerfile(tag, dockerfile):
667                if not args.quiet:
668                    print("Image is up to date")
669                return 0
670            else:
671                print("Image needs updating")
672                return 1
673        elif args.checktype == "age":
674            timestr = dkr.get_image_creation_time(info).split(".")[0]
675            created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
676            past = datetime.now() - timedelta(minutes=args.olderthan)
677            if created < past:
678                print ("Image created @ %s more than %d minutes old" %
679                       (timestr, args.olderthan))
680                return 1
681            else:
682                if not args.quiet:
683                    print ("Image less than %d minutes old" % (args.olderthan))
684                return 0
685
686
687def main():
688    global USE_ENGINE
689
690    parser = argparse.ArgumentParser(description="A Docker helper",
691                                     usage="%s <subcommand> ..." %
692                                     os.path.basename(sys.argv[0]))
693    parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
694                        help="specify which container engine to use")
695    subparsers = parser.add_subparsers(title="subcommands", help=None)
696    for cls in SubCommand.__subclasses__():
697        cmd = cls()
698        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
699        cmd.shared_args(subp)
700        cmd.args(subp)
701        subp.set_defaults(cmdobj=cmd)
702    args, argv = parser.parse_known_args()
703    if args.engine:
704        USE_ENGINE = args.engine
705    return args.cmdobj.run(args, argv)
706
707
708if __name__ == "__main__":
709    sys.exit(main())
710