1# Development tool - deploy/undeploy command plugin 2# 3# Copyright (C) 2014-2016 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool plugin containing the deploy subcommands""" 8 9import logging 10import os 11import shutil 12import subprocess 13import tempfile 14 15import bb.utils 16import argparse_oe 17import oe.types 18 19from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError 20 21logger = logging.getLogger('devtool') 22 23deploylist_path = '/.devtool' 24 25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): 26 """ 27 Prepare a shell script for running on the target to 28 deploy/undeploy files. We have to be careful what we put in this 29 script - only commands that are likely to be available on the 30 target are suitable (the target might be constrained, e.g. using 31 busybox rather than bash with coreutils). 32 """ 33 lines = [] 34 lines.append('#!/bin/sh') 35 lines.append('set -e') 36 if undeployall: 37 # Yes, I know this is crude - but it does work 38 lines.append('for entry in %s/*.list; do' % deploylist_path) 39 lines.append('[ ! -f $entry ] && exit') 40 lines.append('set `basename $entry | sed "s/.list//"`') 41 if dryrun: 42 if not deploy: 43 lines.append('echo "Previously deployed files for $1:"') 44 lines.append('manifest="%s/$1.list"' % deploylist_path) 45 lines.append('preservedir="%s/$1.preserve"' % deploylist_path) 46 lines.append('if [ -f $manifest ] ; then') 47 # Read manifest in reverse and delete files / remove empty dirs 48 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') 49 lines.append(' do') 50 if dryrun: 51 lines.append(' if [ ! -d $file ] ; then') 52 lines.append(' echo $file') 53 lines.append(' fi') 54 else: 55 lines.append(' if [ -d $file ] ; then') 56 # Avoid deleting a preserved directory in case it has special perms 57 lines.append(' if [ ! -d $preservedir/$file ] ; then') 58 lines.append(' rmdir $file > /dev/null 2>&1 || true') 59 lines.append(' fi') 60 lines.append(' else') 61 lines.append(' rm -f $file') 62 lines.append(' fi') 63 lines.append(' done') 64 if not dryrun: 65 lines.append(' rm $manifest') 66 if not deploy and not dryrun: 67 # May as well remove all traces 68 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') 69 lines.append('fi') 70 71 if deploy: 72 if not nocheckspace: 73 # Check for available space 74 # FIXME This doesn't take into account files spread across multiple 75 # partitions, but doing that is non-trivial 76 # Find the part of the destination path that exists 77 lines.append('checkpath="$2"') 78 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') 79 lines.append('do') 80 lines.append(' checkpath=`dirname "$checkpath"`') 81 lines.append('done') 82 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') 83 # First line of the file is the total space 84 lines.append('total=`head -n1 $3`') 85 lines.append('if [ $total -gt $freespace ] ; then') 86 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') 87 lines.append(' exit 1') 88 lines.append('fi') 89 if not nopreserve: 90 # Preserve any files that exist. Note that this will add to the 91 # preserved list with successive deployments if the list of files 92 # deployed changes, but because we've deleted any previously 93 # deployed files at this point it will never preserve anything 94 # that was deployed, only files that existed prior to any deploying 95 # (which makes the most sense) 96 lines.append('cat $3 | sed "1d" | while read file fsize') 97 lines.append('do') 98 lines.append(' if [ -e $file ] ; then') 99 lines.append(' dest="$preservedir/$file"') 100 lines.append(' mkdir -p `dirname $dest`') 101 lines.append(' mv $file $dest') 102 lines.append(' fi') 103 lines.append('done') 104 lines.append('rm $3') 105 lines.append('mkdir -p `dirname $manifest`') 106 lines.append('mkdir -p $2') 107 if verbose: 108 lines.append(' tar xv -C $2 -f - | tee $manifest') 109 else: 110 lines.append(' tar xv -C $2 -f - > $manifest') 111 lines.append('sed -i "s!^./!$2!" $manifest') 112 elif not dryrun: 113 # Put any preserved files back 114 lines.append('if [ -d $preservedir ] ; then') 115 lines.append(' cd $preservedir') 116 # find from busybox might not have -exec, so we don't use that 117 lines.append(' find . -type f | while read file') 118 lines.append(' do') 119 lines.append(' mv $file /$file') 120 lines.append(' done') 121 lines.append(' cd /') 122 lines.append(' rm -rf $preservedir') 123 lines.append('fi') 124 125 if undeployall: 126 if not dryrun: 127 lines.append('echo "NOTE: Successfully undeployed $1"') 128 lines.append('done') 129 130 # Delete the script itself 131 lines.append('rm $0') 132 lines.append('') 133 134 return '\n'.join(lines) 135 136 137 138def deploy(args, config, basepath, workspace): 139 """Entry point for the devtool 'deploy' subcommand""" 140 import math 141 import oe.recipeutils 142 import oe.package 143 import oe.utils 144 145 check_workspace_recipe(workspace, args.recipename, checksrc=False) 146 147 try: 148 host, destdir = args.target.split(':') 149 except ValueError: 150 destdir = '/' 151 else: 152 args.target = host 153 if not destdir.endswith('/'): 154 destdir += '/' 155 156 tinfoil = setup_tinfoil(basepath=basepath) 157 try: 158 try: 159 rd = tinfoil.parse_recipe(args.recipename) 160 except Exception as e: 161 raise DevtoolError('Exception parsing recipe %s: %s' % 162 (args.recipename, e)) 163 recipe_outdir = rd.getVar('D') 164 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): 165 raise DevtoolError('No files to deploy - have you built the %s ' 166 'recipe? If so, the install step has not installed ' 167 'any files.' % args.recipename) 168 169 if args.strip and not args.dry_run: 170 # Fakeroot copy to new destination 171 srcdir = recipe_outdir 172 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') 173 if os.path.isdir(recipe_outdir): 174 exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) 175 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) 176 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) 177 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), 178 rd.getVar('base_libdir'), oe.utils.get_bb_number_threads(rd), rd) 179 180 filelist = [] 181 inodes = set({}) 182 ftotalsize = 0 183 for root, _, files in os.walk(recipe_outdir): 184 for fn in files: 185 fstat = os.lstat(os.path.join(root, fn)) 186 # Get the size in kiB (since we'll be comparing it to the output of du -k) 187 # MUST use lstat() here not stat() or getfilesize() since we don't want to 188 # dereference symlinks 189 if fstat.st_ino in inodes: 190 fsize = 0 191 else: 192 fsize = int(math.ceil(float(fstat.st_size)/1024)) 193 inodes.add(fstat.st_ino) 194 ftotalsize += fsize 195 # The path as it would appear on the target 196 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) 197 filelist.append((fpath, fsize)) 198 199 if args.dry_run: 200 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) 201 for item, _ in filelist: 202 print(' %s' % item) 203 return 0 204 205 extraoptions = '' 206 if args.no_host_check: 207 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 208 if not args.show_status: 209 extraoptions += ' -q' 210 211 scp_sshexec = '' 212 ssh_sshexec = 'ssh' 213 if args.ssh_exec: 214 scp_sshexec = "-S %s" % args.ssh_exec 215 ssh_sshexec = args.ssh_exec 216 scp_port = '' 217 ssh_port = '' 218 if args.port: 219 scp_port = "-P %s" % args.port 220 ssh_port = "-p %s" % args.port 221 222 if args.key: 223 extraoptions += ' -i %s' % args.key 224 225 # In order to delete previously deployed files and have the manifest file on 226 # the target, we write out a shell script and then copy it to the target 227 # so we can then run it (piping tar output to it). 228 # (We cannot use scp here, because it doesn't preserve symlinks.) 229 tmpdir = tempfile.mkdtemp(prefix='devtool') 230 try: 231 tmpscript = '/tmp/devtool_deploy.sh' 232 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') 233 shellscript = _prepare_remote_script(deploy=True, 234 verbose=args.show_status, 235 nopreserve=args.no_preserve, 236 nocheckspace=args.no_check_space) 237 # Write out the script to a file 238 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 239 f.write(shellscript) 240 # Write out the file list 241 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: 242 f.write('%d\n' % ftotalsize) 243 for fpath, fsize in filelist: 244 f.write('%s %d\n' % (fpath, fsize)) 245 # Copy them to the target 246 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 247 if ret != 0: 248 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 249 'get a complete error message' % args.target) 250 finally: 251 shutil.rmtree(tmpdir) 252 253 # Now run the script 254 ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) 255 if ret != 0: 256 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 257 'error message') 258 259 logger.info('Successfully deployed %s' % recipe_outdir) 260 261 files_list = [] 262 for root, _, files in os.walk(recipe_outdir): 263 for filename in files: 264 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) 265 files_list.append(os.path.join(destdir, filename)) 266 finally: 267 tinfoil.shutdown() 268 269 return 0 270 271def undeploy(args, config, basepath, workspace): 272 """Entry point for the devtool 'undeploy' subcommand""" 273 if args.all and args.recipename: 274 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') 275 elif not args.recipename and not args.all: 276 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') 277 278 extraoptions = '' 279 if args.no_host_check: 280 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 281 if not args.show_status: 282 extraoptions += ' -q' 283 284 scp_sshexec = '' 285 ssh_sshexec = 'ssh' 286 if args.ssh_exec: 287 scp_sshexec = "-S %s" % args.ssh_exec 288 ssh_sshexec = args.ssh_exec 289 scp_port = '' 290 ssh_port = '' 291 if args.port: 292 scp_port = "-P %s" % args.port 293 ssh_port = "-p %s" % args.port 294 295 args.target = args.target.split(':')[0] 296 297 tmpdir = tempfile.mkdtemp(prefix='devtool') 298 try: 299 tmpscript = '/tmp/devtool_undeploy.sh' 300 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) 301 # Write out the script to a file 302 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 303 f.write(shellscript) 304 # Copy it to the target 305 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 306 if ret != 0: 307 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 308 'get a complete error message' % args.target) 309 finally: 310 shutil.rmtree(tmpdir) 311 312 # Now run the script 313 ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) 314 if ret != 0: 315 raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 316 'error message') 317 318 if not args.all and not args.dry_run: 319 logger.info('Successfully undeployed %s' % args.recipename) 320 return 0 321 322 323def register_commands(subparsers, context): 324 """Register devtool subcommands from the deploy plugin""" 325 326 parser_deploy = subparsers.add_parser('deploy-target', 327 help='Deploy recipe output files to live target machine', 328 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', 329 group='testbuild') 330 parser_deploy.add_argument('recipename', help='Recipe to deploy') 331 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') 332 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 333 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 334 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') 335 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') 336 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') 337 parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 338 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 339 parser_deploy.add_argument('-I', '--key', 340 help='Specify ssh private key for connection to the target') 341 342 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) 343 strip_opts.add_argument('-S', '--strip', 344 help='Strip executables prior to deploying (default: %(default)s). ' 345 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', 346 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), 347 action='store_true') 348 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') 349 350 parser_deploy.set_defaults(func=deploy) 351 352 parser_undeploy = subparsers.add_parser('undeploy-target', 353 help='Undeploy recipe output files in live target machine', 354 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', 355 group='testbuild') 356 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') 357 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') 358 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 359 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 360 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') 361 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') 362 parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 363 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 364 parser_undeploy.add_argument('-I', '--key', 365 help='Specify ssh private key for connection to the target') 366 367 parser_undeploy.set_defaults(func=undeploy) 368