1#!/usr/bin/env python2 2# 3# Docker controlling module 4# 5# Copyright (c) 2016 Red Hat Inc. 6# 7# Authors: 8# Fam Zheng <famz@redhat.com> 9# 10# This work is licensed under the terms of the GNU GPL, version 2 11# or (at your option) any later version. See the COPYING file in 12# the top-level directory. 13 14import os 15import sys 16sys.path.append(os.path.join(os.path.dirname(__file__), 17 '..', '..', 'scripts')) 18import argparse 19import subprocess 20import json 21import hashlib 22import atexit 23import uuid 24import tempfile 25import re 26import signal 27from tarfile import TarFile, TarInfo 28from StringIO import StringIO 29from shutil import copy, rmtree 30from pwd import getpwuid 31 32 33FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] 34 35 36DEVNULL = open(os.devnull, 'wb') 37 38 39def _text_checksum(text): 40 """Calculate a digest string unique to the text content""" 41 return hashlib.sha1(text).hexdigest() 42 43def _file_checksum(filename): 44 return _text_checksum(open(filename, 'rb').read()) 45 46def _guess_docker_command(): 47 """ Guess a working docker command or raise exception if not found""" 48 commands = [["docker"], ["sudo", "-n", "docker"]] 49 for cmd in commands: 50 try: 51 if subprocess.call(cmd + ["images"], 52 stdout=DEVNULL, stderr=DEVNULL) == 0: 53 return cmd 54 except OSError: 55 pass 56 commands_txt = "\n".join([" " + " ".join(x) for x in commands]) 57 raise Exception("Cannot find working docker command. Tried:\n%s" % \ 58 commands_txt) 59 60def _copy_with_mkdir(src, root_dir, sub_path='.'): 61 """Copy src into root_dir, creating sub_path as needed.""" 62 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) 63 try: 64 os.makedirs(dest_dir) 65 except OSError: 66 # we can safely ignore already created directories 67 pass 68 69 dest_file = "%s/%s" % (dest_dir, os.path.basename(src)) 70 copy(src, dest_file) 71 72 73def _get_so_libs(executable): 74 """Return a list of libraries associated with an executable. 75 76 The paths may be symbolic links which would need to be resolved to 77 ensure theright data is copied.""" 78 79 libs = [] 80 ldd_re = re.compile(r"(/.*/)(\S*)") 81 try: 82 ldd_output = subprocess.check_output(["ldd", executable]) 83 for line in ldd_output.split("\n"): 84 search = ldd_re.search(line) 85 if search and len(search.groups()) == 2: 86 so_path = search.groups()[0] 87 so_lib = search.groups()[1] 88 libs.append("%s/%s" % (so_path, so_lib)) 89 except subprocess.CalledProcessError: 90 print "%s had no associated libraries (static build?)" % (executable) 91 92 return libs 93 94def _copy_binary_with_libs(src, dest_dir): 95 """Copy a binary executable and all its dependant libraries. 96 97 This does rely on the host file-system being fairly multi-arch 98 aware so the file don't clash with the guests layout.""" 99 100 _copy_with_mkdir(src, dest_dir, "/usr/bin") 101 102 libs = _get_so_libs(src) 103 if libs: 104 for l in libs: 105 so_path = os.path.dirname(l) 106 _copy_with_mkdir(l , dest_dir, so_path) 107 108def _read_qemu_dockerfile(img_name): 109 df = os.path.join(os.path.dirname(__file__), "dockerfiles", 110 img_name + ".docker") 111 return open(df, "r").read() 112 113def _dockerfile_preprocess(df): 114 out = "" 115 for l in df.splitlines(): 116 if len(l.strip()) == 0 or l.startswith("#"): 117 continue 118 from_pref = "FROM qemu:" 119 if l.startswith(from_pref): 120 # TODO: Alternatively we could replace this line with "FROM $ID" 121 # where $ID is the image's hex id obtained with 122 # $ docker images $IMAGE --format="{{.Id}}" 123 # but unfortunately that's not supported by RHEL 7. 124 inlining = _read_qemu_dockerfile(l[len(from_pref):]) 125 out += _dockerfile_preprocess(inlining) 126 continue 127 out += l + "\n" 128 return out 129 130class Docker(object): 131 """ Running Docker commands """ 132 def __init__(self): 133 self._command = _guess_docker_command() 134 self._instances = [] 135 atexit.register(self._kill_instances) 136 signal.signal(signal.SIGTERM, self._kill_instances) 137 signal.signal(signal.SIGHUP, self._kill_instances) 138 139 def _do(self, cmd, quiet=True, **kwargs): 140 if quiet: 141 kwargs["stdout"] = DEVNULL 142 return subprocess.call(self._command + cmd, **kwargs) 143 144 def _do_check(self, cmd, quiet=True, **kwargs): 145 if quiet: 146 kwargs["stdout"] = DEVNULL 147 return subprocess.check_call(self._command + cmd, **kwargs) 148 149 def _do_kill_instances(self, only_known, only_active=True): 150 cmd = ["ps", "-q"] 151 if not only_active: 152 cmd.append("-a") 153 for i in self._output(cmd).split(): 154 resp = self._output(["inspect", i]) 155 labels = json.loads(resp)[0]["Config"]["Labels"] 156 active = json.loads(resp)[0]["State"]["Running"] 157 if not labels: 158 continue 159 instance_uuid = labels.get("com.qemu.instance.uuid", None) 160 if not instance_uuid: 161 continue 162 if only_known and instance_uuid not in self._instances: 163 continue 164 print "Terminating", i 165 if active: 166 self._do(["kill", i]) 167 self._do(["rm", i]) 168 169 def clean(self): 170 self._do_kill_instances(False, False) 171 return 0 172 173 def _kill_instances(self, *args, **kwargs): 174 return self._do_kill_instances(True) 175 176 def _output(self, cmd, **kwargs): 177 return subprocess.check_output(self._command + cmd, 178 stderr=subprocess.STDOUT, 179 **kwargs) 180 181 def get_image_dockerfile_checksum(self, tag): 182 resp = self._output(["inspect", tag]) 183 labels = json.loads(resp)[0]["Config"].get("Labels", {}) 184 return labels.get("com.qemu.dockerfile-checksum", "") 185 186 def build_image(self, tag, docker_dir, dockerfile, 187 quiet=True, user=False, argv=None, extra_files_cksum=[]): 188 if argv == None: 189 argv = [] 190 191 tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") 192 tmp_df.write(dockerfile) 193 194 if user: 195 uid = os.getuid() 196 uname = getpwuid(uid).pw_name 197 tmp_df.write("\n") 198 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % 199 (uname, uid, uname)) 200 201 tmp_df.write("\n") 202 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % 203 _text_checksum("\n".join([dockerfile] + 204 extra_files_cksum))) 205 tmp_df.flush() 206 207 self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \ 208 [docker_dir], 209 quiet=quiet) 210 211 def update_image(self, tag, tarball, quiet=True): 212 "Update a tagged image using " 213 214 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball) 215 216 def image_matches_dockerfile(self, tag, dockerfile): 217 try: 218 checksum = self.get_image_dockerfile_checksum(tag) 219 except Exception: 220 return False 221 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile)) 222 223 def run(self, cmd, keep, quiet): 224 label = uuid.uuid1().hex 225 if not keep: 226 self._instances.append(label) 227 ret = self._do_check(["run", "--label", 228 "com.qemu.instance.uuid=" + label] + cmd, 229 quiet=quiet) 230 if not keep: 231 self._instances.remove(label) 232 return ret 233 234 def command(self, cmd, argv, quiet): 235 return self._do([cmd] + argv, quiet=quiet) 236 237class SubCommand(object): 238 """A SubCommand template base class""" 239 name = None # Subcommand name 240 def shared_args(self, parser): 241 parser.add_argument("--quiet", action="store_true", 242 help="Run quietly unless an error occured") 243 244 def args(self, parser): 245 """Setup argument parser""" 246 pass 247 def run(self, args, argv): 248 """Run command. 249 args: parsed argument by argument parser. 250 argv: remaining arguments from sys.argv. 251 """ 252 pass 253 254class RunCommand(SubCommand): 255 """Invoke docker run and take care of cleaning up""" 256 name = "run" 257 def args(self, parser): 258 parser.add_argument("--keep", action="store_true", 259 help="Don't remove image when command completes") 260 def run(self, args, argv): 261 return Docker().run(argv, args.keep, quiet=args.quiet) 262 263class BuildCommand(SubCommand): 264 """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>""" 265 name = "build" 266 def args(self, parser): 267 parser.add_argument("--include-executable", "-e", 268 help="""Specify a binary that will be copied to the 269 container together with all its dependent 270 libraries""") 271 parser.add_argument("--extra-files", "-f", nargs='*', 272 help="""Specify files that will be copied in the 273 Docker image, fulfilling the ADD directive from the 274 Dockerfile""") 275 parser.add_argument("--add-current-user", "-u", dest="user", 276 action="store_true", 277 help="Add the current user to image's passwd") 278 parser.add_argument("tag", 279 help="Image Tag") 280 parser.add_argument("dockerfile", 281 help="Dockerfile name") 282 283 def run(self, args, argv): 284 dockerfile = open(args.dockerfile, "rb").read() 285 tag = args.tag 286 287 dkr = Docker() 288 if "--no-cache" not in argv and \ 289 dkr.image_matches_dockerfile(tag, dockerfile): 290 if not args.quiet: 291 print "Image is up to date." 292 else: 293 # Create a docker context directory for the build 294 docker_dir = tempfile.mkdtemp(prefix="docker_build") 295 296 # Is there a .pre file to run in the build context? 297 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre" 298 if os.path.exists(docker_pre): 299 stdout = DEVNULL if args.quiet else None 300 rc = subprocess.call(os.path.realpath(docker_pre), 301 cwd=docker_dir, stdout=stdout) 302 if rc == 3: 303 print "Skip" 304 return 0 305 elif rc != 0: 306 print "%s exited with code %d" % (docker_pre, rc) 307 return 1 308 309 # Copy any extra files into the Docker context. These can be 310 # included by the use of the ADD directive in the Dockerfile. 311 cksum = [] 312 if args.include_executable: 313 # FIXME: there is no checksum of this executable and the linked 314 # libraries, once the image built any change of this executable 315 # or any library won't trigger another build. 316 _copy_binary_with_libs(args.include_executable, docker_dir) 317 for filename in args.extra_files or []: 318 _copy_with_mkdir(filename, docker_dir) 319 cksum += [_file_checksum(filename)] 320 321 argv += ["--build-arg=" + k.lower() + "=" + v 322 for k, v in os.environ.iteritems() 323 if k.lower() in FILTERED_ENV_NAMES] 324 dkr.build_image(tag, docker_dir, dockerfile, 325 quiet=args.quiet, user=args.user, argv=argv, 326 extra_files_cksum=cksum) 327 328 rmtree(docker_dir) 329 330 return 0 331 332class UpdateCommand(SubCommand): 333 """ Update a docker image with new executables. Arguments: <tag> <executable>""" 334 name = "update" 335 def args(self, parser): 336 parser.add_argument("tag", 337 help="Image Tag") 338 parser.add_argument("executable", 339 help="Executable to copy") 340 341 def run(self, args, argv): 342 # Create a temporary tarball with our whole build context and 343 # dockerfile for the update 344 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") 345 tmp_tar = TarFile(fileobj=tmp, mode='w') 346 347 # Add the executable to the tarball 348 bn = os.path.basename(args.executable) 349 ff = "/usr/bin/%s" % bn 350 tmp_tar.add(args.executable, arcname=ff) 351 352 # Add any associated libraries 353 libs = _get_so_libs(args.executable) 354 if libs: 355 for l in libs: 356 tmp_tar.add(os.path.realpath(l), arcname=l) 357 358 # Create a Docker buildfile 359 df = StringIO() 360 df.write("FROM %s\n" % args.tag) 361 df.write("ADD . /\n") 362 df.seek(0) 363 364 df_tar = TarInfo(name="Dockerfile") 365 df_tar.size = len(df.buf) 366 tmp_tar.addfile(df_tar, fileobj=df) 367 368 tmp_tar.close() 369 370 # reset the file pointers 371 tmp.flush() 372 tmp.seek(0) 373 374 # Run the build with our tarball context 375 dkr = Docker() 376 dkr.update_image(args.tag, tmp, quiet=args.quiet) 377 378 return 0 379 380class CleanCommand(SubCommand): 381 """Clean up docker instances""" 382 name = "clean" 383 def run(self, args, argv): 384 Docker().clean() 385 return 0 386 387class ImagesCommand(SubCommand): 388 """Run "docker images" command""" 389 name = "images" 390 def run(self, args, argv): 391 return Docker().command("images", argv, args.quiet) 392 393def main(): 394 parser = argparse.ArgumentParser(description="A Docker helper", 395 usage="%s <subcommand> ..." % os.path.basename(sys.argv[0])) 396 subparsers = parser.add_subparsers(title="subcommands", help=None) 397 for cls in SubCommand.__subclasses__(): 398 cmd = cls() 399 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__) 400 cmd.shared_args(subp) 401 cmd.args(subp) 402 subp.set_defaults(cmdobj=cmd) 403 args, argv = parser.parse_known_args() 404 return args.cmdobj.run(args, argv) 405 406if __name__ == "__main__": 407 sys.exit(main()) 408