1# Development tool - deploy/undeploy command plugin 2# 3# Copyright (C) 2014-2016 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool plugin containing the deploy subcommands""" 8 9import logging 10import os 11import shutil 12import subprocess 13import tempfile 14 15import bb.utils 16import argparse_oe 17import oe.types 18 19from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError 20 21logger = logging.getLogger('devtool') 22 23deploylist_path = '/.devtool' 24 25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): 26 """ 27 Prepare a shell script for running on the target to 28 deploy/undeploy files. We have to be careful what we put in this 29 script - only commands that are likely to be available on the 30 target are suitable (the target might be constrained, e.g. using 31 busybox rather than bash with coreutils). 32 """ 33 lines = [] 34 lines.append('#!/bin/sh') 35 lines.append('set -e') 36 if undeployall: 37 # Yes, I know this is crude - but it does work 38 lines.append('for entry in %s/*.list; do' % deploylist_path) 39 lines.append('[ ! -f $entry ] && exit') 40 lines.append('set `basename $entry | sed "s/.list//"`') 41 if dryrun: 42 if not deploy: 43 lines.append('echo "Previously deployed files for $1:"') 44 lines.append('manifest="%s/$1.list"' % deploylist_path) 45 lines.append('preservedir="%s/$1.preserve"' % deploylist_path) 46 lines.append('if [ -f $manifest ] ; then') 47 # Read manifest in reverse and delete files / remove empty dirs 48 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') 49 lines.append(' do') 50 if dryrun: 51 lines.append(' if [ ! -d $file ] ; then') 52 lines.append(' echo $file') 53 lines.append(' fi') 54 else: 55 lines.append(' if [ -d $file ] ; then') 56 # Avoid deleting a preserved directory in case it has special perms 57 lines.append(' if [ ! -d $preservedir/$file ] ; then') 58 lines.append(' rmdir $file > /dev/null 2>&1 || true') 59 lines.append(' fi') 60 lines.append(' else') 61 lines.append(' rm -f $file') 62 lines.append(' fi') 63 lines.append(' done') 64 if not dryrun: 65 lines.append(' rm $manifest') 66 if not deploy and not dryrun: 67 # May as well remove all traces 68 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') 69 lines.append('fi') 70 71 if deploy: 72 if not nocheckspace: 73 # Check for available space 74 # FIXME This doesn't take into account files spread across multiple 75 # partitions, but doing that is non-trivial 76 # Find the part of the destination path that exists 77 lines.append('checkpath="$2"') 78 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') 79 lines.append('do') 80 lines.append(' checkpath=`dirname "$checkpath"`') 81 lines.append('done') 82 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') 83 # First line of the file is the total space 84 lines.append('total=`head -n1 $3`') 85 lines.append('if [ $total -gt $freespace ] ; then') 86 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') 87 lines.append(' exit 1') 88 lines.append('fi') 89 if not nopreserve: 90 # Preserve any files that exist. Note that this will add to the 91 # preserved list with successive deployments if the list of files 92 # deployed changes, but because we've deleted any previously 93 # deployed files at this point it will never preserve anything 94 # that was deployed, only files that existed prior to any deploying 95 # (which makes the most sense) 96 lines.append('cat $3 | sed "1d" | while read file fsize') 97 lines.append('do') 98 lines.append(' if [ -e $file ] ; then') 99 lines.append(' dest="$preservedir/$file"') 100 lines.append(' mkdir -p `dirname $dest`') 101 lines.append(' mv $file $dest') 102 lines.append(' fi') 103 lines.append('done') 104 lines.append('rm $3') 105 lines.append('mkdir -p `dirname $manifest`') 106 lines.append('mkdir -p $2') 107 if verbose: 108 lines.append(' tar xv -C $2 -f - | tee $manifest') 109 else: 110 lines.append(' tar xv -C $2 -f - > $manifest') 111 lines.append('sed -i "s!^./!$2!" $manifest') 112 elif not dryrun: 113 # Put any preserved files back 114 lines.append('if [ -d $preservedir ] ; then') 115 lines.append(' cd $preservedir') 116 # find from busybox might not have -exec, so we don't use that 117 lines.append(' find . -type f | while read file') 118 lines.append(' do') 119 lines.append(' mv $file /$file') 120 lines.append(' done') 121 lines.append(' cd /') 122 lines.append(' rm -rf $preservedir') 123 lines.append('fi') 124 125 if undeployall: 126 if not dryrun: 127 lines.append('echo "NOTE: Successfully undeployed $1"') 128 lines.append('done') 129 130 # Delete the script itself 131 lines.append('rm $0') 132 lines.append('') 133 134 return '\n'.join(lines) 135 136def deploy(args, config, basepath, workspace): 137 """Entry point for the devtool 'deploy' subcommand""" 138 import oe.utils 139 140 check_workspace_recipe(workspace, args.recipename, checksrc=False) 141 142 tinfoil = setup_tinfoil(basepath=basepath) 143 try: 144 try: 145 rd = tinfoil.parse_recipe(args.recipename) 146 except Exception as e: 147 raise DevtoolError('Exception parsing recipe %s: %s' % 148 (args.recipename, e)) 149 150 srcdir = rd.getVar('D') 151 workdir = rd.getVar('WORKDIR') 152 path = rd.getVar('PATH') 153 strip_cmd = rd.getVar('STRIP') 154 libdir = rd.getVar('libdir') 155 base_libdir = rd.getVar('base_libdir') 156 max_process = oe.utils.get_bb_number_threads(rd) 157 fakerootcmd = rd.getVar('FAKEROOTCMD') 158 fakerootenv = rd.getVar('FAKEROOTENV') 159 finally: 160 tinfoil.shutdown() 161 162 return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args) 163 164def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args): 165 import math 166 import oe.package 167 168 try: 169 host, destdir = args.target.split(':') 170 except ValueError: 171 destdir = '/' 172 else: 173 args.target = host 174 if not destdir.endswith('/'): 175 destdir += '/' 176 177 recipe_outdir = srcdir 178 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): 179 raise DevtoolError('No files to deploy - have you built the %s ' 180 'recipe? If so, the install step has not installed ' 181 'any files.' % args.recipename) 182 183 if args.strip and not args.dry_run: 184 # Fakeroot copy to new destination 185 srcdir = recipe_outdir 186 recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped') 187 if os.path.isdir(recipe_outdir): 188 exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True) 189 exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) 190 os.environ['PATH'] = ':'.join([os.environ['PATH'], path or '']) 191 oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process) 192 193 filelist = [] 194 inodes = set({}) 195 ftotalsize = 0 196 for root, _, files in os.walk(recipe_outdir): 197 for fn in files: 198 fstat = os.lstat(os.path.join(root, fn)) 199 # Get the size in kiB (since we'll be comparing it to the output of du -k) 200 # MUST use lstat() here not stat() or getfilesize() since we don't want to 201 # dereference symlinks 202 if fstat.st_ino in inodes: 203 fsize = 0 204 else: 205 fsize = int(math.ceil(float(fstat.st_size)/1024)) 206 inodes.add(fstat.st_ino) 207 ftotalsize += fsize 208 # The path as it would appear on the target 209 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) 210 filelist.append((fpath, fsize)) 211 212 if args.dry_run: 213 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) 214 for item, _ in filelist: 215 print(' %s' % item) 216 return 0 217 218 extraoptions = '' 219 if args.no_host_check: 220 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 221 if not args.show_status: 222 extraoptions += ' -q' 223 224 scp_sshexec = '' 225 ssh_sshexec = 'ssh' 226 if args.ssh_exec: 227 scp_sshexec = "-S %s" % args.ssh_exec 228 ssh_sshexec = args.ssh_exec 229 scp_port = '' 230 ssh_port = '' 231 if args.port: 232 scp_port = "-P %s" % args.port 233 ssh_port = "-p %s" % args.port 234 235 if args.key: 236 extraoptions += ' -i %s' % args.key 237 238 # In order to delete previously deployed files and have the manifest file on 239 # the target, we write out a shell script and then copy it to the target 240 # so we can then run it (piping tar output to it). 241 # (We cannot use scp here, because it doesn't preserve symlinks.) 242 tmpdir = tempfile.mkdtemp(prefix='devtool') 243 try: 244 tmpscript = '/tmp/devtool_deploy.sh' 245 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') 246 shellscript = _prepare_remote_script(deploy=True, 247 verbose=args.show_status, 248 nopreserve=args.no_preserve, 249 nocheckspace=args.no_check_space) 250 # Write out the script to a file 251 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 252 f.write(shellscript) 253 # Write out the file list 254 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: 255 f.write('%d\n' % ftotalsize) 256 for fpath, fsize in filelist: 257 f.write('%s %d\n' % (fpath, fsize)) 258 # Copy them to the target 259 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 260 if ret != 0: 261 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 262 'get a complete error message' % args.target) 263 finally: 264 shutil.rmtree(tmpdir) 265 266 # Now run the script 267 ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) 268 if ret != 0: 269 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 270 'error message') 271 272 logger.info('Successfully deployed %s' % recipe_outdir) 273 274 files_list = [] 275 for root, _, files in os.walk(recipe_outdir): 276 for filename in files: 277 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) 278 files_list.append(os.path.join(destdir, filename)) 279 280 return 0 281 282def undeploy(args, config, basepath, workspace): 283 """Entry point for the devtool 'undeploy' subcommand""" 284 if args.all and args.recipename: 285 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') 286 elif not args.recipename and not args.all: 287 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') 288 289 extraoptions = '' 290 if args.no_host_check: 291 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 292 if not args.show_status: 293 extraoptions += ' -q' 294 295 scp_sshexec = '' 296 ssh_sshexec = 'ssh' 297 if args.ssh_exec: 298 scp_sshexec = "-S %s" % args.ssh_exec 299 ssh_sshexec = args.ssh_exec 300 scp_port = '' 301 ssh_port = '' 302 if args.port: 303 scp_port = "-P %s" % args.port 304 ssh_port = "-p %s" % args.port 305 306 args.target = args.target.split(':')[0] 307 308 tmpdir = tempfile.mkdtemp(prefix='devtool') 309 try: 310 tmpscript = '/tmp/devtool_undeploy.sh' 311 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) 312 # Write out the script to a file 313 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 314 f.write(shellscript) 315 # Copy it to the target 316 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 317 if ret != 0: 318 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 319 'get a complete error message' % args.target) 320 finally: 321 shutil.rmtree(tmpdir) 322 323 # Now run the script 324 ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) 325 if ret != 0: 326 raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 327 'error message') 328 329 if not args.all and not args.dry_run: 330 logger.info('Successfully undeployed %s' % args.recipename) 331 return 0 332 333 334def register_commands(subparsers, context): 335 """Register devtool subcommands from the deploy plugin""" 336 337 parser_deploy = subparsers.add_parser('deploy-target', 338 help='Deploy recipe output files to live target machine', 339 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', 340 group='testbuild') 341 parser_deploy.add_argument('recipename', help='Recipe to deploy') 342 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') 343 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 344 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 345 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') 346 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') 347 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') 348 parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 349 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 350 parser_deploy.add_argument('-I', '--key', 351 help='Specify ssh private key for connection to the target') 352 353 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) 354 strip_opts.add_argument('-S', '--strip', 355 help='Strip executables prior to deploying (default: %(default)s). ' 356 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', 357 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), 358 action='store_true') 359 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') 360 361 parser_deploy.set_defaults(func=deploy) 362 363 parser_undeploy = subparsers.add_parser('undeploy-target', 364 help='Undeploy recipe output files in live target machine', 365 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', 366 group='testbuild') 367 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') 368 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') 369 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 370 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 371 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') 372 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') 373 parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 374 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 375 parser_undeploy.add_argument('-I', '--key', 376 help='Specify ssh private key for connection to the target') 377 378 parser_undeploy.set_defaults(func=undeploy) 379