1# Development tool - deploy/undeploy command plugin 2# 3# Copyright (C) 2014-2016 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool plugin containing the deploy subcommands""" 8 9import logging 10import os 11import shutil 12import subprocess 13import tempfile 14 15import bb.utils 16import argparse_oe 17import oe.types 18 19from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError 20 21logger = logging.getLogger('devtool') 22 23deploylist_path = '/.devtool' 24 25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): 26 """ 27 Prepare a shell script for running on the target to 28 deploy/undeploy files. We have to be careful what we put in this 29 script - only commands that are likely to be available on the 30 target are suitable (the target might be constrained, e.g. using 31 busybox rather than bash with coreutils). 32 """ 33 lines = [] 34 lines.append('#!/bin/sh') 35 lines.append('set -e') 36 if undeployall: 37 # Yes, I know this is crude - but it does work 38 lines.append('for entry in %s/*.list; do' % deploylist_path) 39 lines.append('[ ! -f $entry ] && exit') 40 lines.append('set `basename $entry | sed "s/.list//"`') 41 if dryrun: 42 if not deploy: 43 lines.append('echo "Previously deployed files for $1:"') 44 lines.append('manifest="%s/$1.list"' % deploylist_path) 45 lines.append('preservedir="%s/$1.preserve"' % deploylist_path) 46 lines.append('if [ -f $manifest ] ; then') 47 # Read manifest in reverse and delete files / remove empty dirs 48 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') 49 lines.append(' do') 50 if dryrun: 51 lines.append(' if [ ! -d $file ] ; then') 52 lines.append(' echo $file') 53 lines.append(' fi') 54 else: 55 lines.append(' if [ -d $file ] ; then') 56 # Avoid deleting a preserved directory in case it has special perms 57 lines.append(' if [ ! -d $preservedir/$file ] ; then') 58 lines.append(' rmdir $file > /dev/null 2>&1 || true') 59 lines.append(' fi') 60 lines.append(' else') 61 lines.append(' rm -f $file') 62 lines.append(' fi') 63 lines.append(' done') 64 if not dryrun: 65 lines.append(' rm $manifest') 66 if not deploy and not dryrun: 67 # May as well remove all traces 68 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') 69 lines.append('fi') 70 71 if deploy: 72 if not nocheckspace: 73 # Check for available space 74 # FIXME This doesn't take into account files spread across multiple 75 # partitions, but doing that is non-trivial 76 # Find the part of the destination path that exists 77 lines.append('checkpath="$2"') 78 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') 79 lines.append('do') 80 lines.append(' checkpath=`dirname "$checkpath"`') 81 lines.append('done') 82 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') 83 # First line of the file is the total space 84 lines.append('total=`head -n1 $3`') 85 lines.append('if [ $total -gt $freespace ] ; then') 86 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') 87 lines.append(' exit 1') 88 lines.append('fi') 89 if not nopreserve: 90 # Preserve any files that exist. Note that this will add to the 91 # preserved list with successive deployments if the list of files 92 # deployed changes, but because we've deleted any previously 93 # deployed files at this point it will never preserve anything 94 # that was deployed, only files that existed prior to any deploying 95 # (which makes the most sense) 96 lines.append('cat $3 | sed "1d" | while read file fsize') 97 lines.append('do') 98 lines.append(' if [ -e $file ] ; then') 99 lines.append(' dest="$preservedir/$file"') 100 lines.append(' mkdir -p `dirname $dest`') 101 lines.append(' mv $file $dest') 102 lines.append(' fi') 103 lines.append('done') 104 lines.append('rm $3') 105 lines.append('mkdir -p `dirname $manifest`') 106 lines.append('mkdir -p $2') 107 if verbose: 108 lines.append(' tar xv -C $2 -f - | tee $manifest') 109 else: 110 lines.append(' tar xv -C $2 -f - > $manifest') 111 lines.append('sed -i "s!^./!$2!" $manifest') 112 elif not dryrun: 113 # Put any preserved files back 114 lines.append('if [ -d $preservedir ] ; then') 115 lines.append(' cd $preservedir') 116 # find from busybox might not have -exec, so we don't use that 117 lines.append(' find . -type f | while read file') 118 lines.append(' do') 119 lines.append(' mv $file /$file') 120 lines.append(' done') 121 lines.append(' cd /') 122 lines.append(' rm -rf $preservedir') 123 lines.append('fi') 124 125 if undeployall: 126 if not dryrun: 127 lines.append('echo "NOTE: Successfully undeployed $1"') 128 lines.append('done') 129 130 # Delete the script itself 131 lines.append('rm $0') 132 lines.append('') 133 134 return '\n'.join(lines) 135 136 137 138def deploy(args, config, basepath, workspace): 139 """Entry point for the devtool 'deploy' subcommand""" 140 import math 141 import oe.recipeutils 142 import oe.package 143 144 check_workspace_recipe(workspace, args.recipename, checksrc=False) 145 146 try: 147 host, destdir = args.target.split(':') 148 except ValueError: 149 destdir = '/' 150 else: 151 args.target = host 152 if not destdir.endswith('/'): 153 destdir += '/' 154 155 tinfoil = setup_tinfoil(basepath=basepath) 156 try: 157 try: 158 rd = tinfoil.parse_recipe(args.recipename) 159 except Exception as e: 160 raise DevtoolError('Exception parsing recipe %s: %s' % 161 (args.recipename, e)) 162 recipe_outdir = rd.getVar('D') 163 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): 164 raise DevtoolError('No files to deploy - have you built the %s ' 165 'recipe? If so, the install step has not installed ' 166 'any files.' % args.recipename) 167 168 if args.strip and not args.dry_run: 169 # Fakeroot copy to new destination 170 srcdir = recipe_outdir 171 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped') 172 if os.path.isdir(recipe_outdir): 173 bb.utils.remove(recipe_outdir, True) 174 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) 175 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) 176 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), 177 rd.getVar('base_libdir'), rd) 178 179 filelist = [] 180 inodes = set({}) 181 ftotalsize = 0 182 for root, _, files in os.walk(recipe_outdir): 183 for fn in files: 184 fstat = os.lstat(os.path.join(root, fn)) 185 # Get the size in kiB (since we'll be comparing it to the output of du -k) 186 # MUST use lstat() here not stat() or getfilesize() since we don't want to 187 # dereference symlinks 188 if fstat.st_ino in inodes: 189 fsize = 0 190 else: 191 fsize = int(math.ceil(float(fstat.st_size)/1024)) 192 inodes.add(fstat.st_ino) 193 ftotalsize += fsize 194 # The path as it would appear on the target 195 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) 196 filelist.append((fpath, fsize)) 197 198 if args.dry_run: 199 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) 200 for item, _ in filelist: 201 print(' %s' % item) 202 return 0 203 204 extraoptions = '' 205 if args.no_host_check: 206 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 207 if not args.show_status: 208 extraoptions += ' -q' 209 210 scp_sshexec = '' 211 ssh_sshexec = 'ssh' 212 if args.ssh_exec: 213 scp_sshexec = "-S %s" % args.ssh_exec 214 ssh_sshexec = args.ssh_exec 215 scp_port = '' 216 ssh_port = '' 217 if args.port: 218 scp_port = "-P %s" % args.port 219 ssh_port = "-p %s" % args.port 220 221 if args.key: 222 extraoptions += ' -i %s' % args.key 223 224 # In order to delete previously deployed files and have the manifest file on 225 # the target, we write out a shell script and then copy it to the target 226 # so we can then run it (piping tar output to it). 227 # (We cannot use scp here, because it doesn't preserve symlinks.) 228 tmpdir = tempfile.mkdtemp(prefix='devtool') 229 try: 230 tmpscript = '/tmp/devtool_deploy.sh' 231 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') 232 shellscript = _prepare_remote_script(deploy=True, 233 verbose=args.show_status, 234 nopreserve=args.no_preserve, 235 nocheckspace=args.no_check_space) 236 # Write out the script to a file 237 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 238 f.write(shellscript) 239 # Write out the file list 240 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: 241 f.write('%d\n' % ftotalsize) 242 for fpath, fsize in filelist: 243 f.write('%s %d\n' % (fpath, fsize)) 244 # Copy them to the target 245 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 246 if ret != 0: 247 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 248 'get a complete error message' % args.target) 249 finally: 250 shutil.rmtree(tmpdir) 251 252 # Now run the script 253 ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) 254 if ret != 0: 255 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 256 'error message') 257 258 logger.info('Successfully deployed %s' % recipe_outdir) 259 260 files_list = [] 261 for root, _, files in os.walk(recipe_outdir): 262 for filename in files: 263 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) 264 files_list.append(os.path.join(destdir, filename)) 265 finally: 266 tinfoil.shutdown() 267 268 return 0 269 270def undeploy(args, config, basepath, workspace): 271 """Entry point for the devtool 'undeploy' subcommand""" 272 if args.all and args.recipename: 273 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') 274 elif not args.recipename and not args.all: 275 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') 276 277 extraoptions = '' 278 if args.no_host_check: 279 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 280 if not args.show_status: 281 extraoptions += ' -q' 282 283 scp_sshexec = '' 284 ssh_sshexec = 'ssh' 285 if args.ssh_exec: 286 scp_sshexec = "-S %s" % args.ssh_exec 287 ssh_sshexec = args.ssh_exec 288 scp_port = '' 289 ssh_port = '' 290 if args.port: 291 scp_port = "-P %s" % args.port 292 ssh_port = "-p %s" % args.port 293 294 args.target = args.target.split(':')[0] 295 296 tmpdir = tempfile.mkdtemp(prefix='devtool') 297 try: 298 tmpscript = '/tmp/devtool_undeploy.sh' 299 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) 300 # Write out the script to a file 301 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 302 f.write(shellscript) 303 # Copy it to the target 304 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 305 if ret != 0: 306 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 307 'get a complete error message' % args.target) 308 finally: 309 shutil.rmtree(tmpdir) 310 311 # Now run the script 312 ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) 313 if ret != 0: 314 raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 315 'error message') 316 317 if not args.all and not args.dry_run: 318 logger.info('Successfully undeployed %s' % args.recipename) 319 return 0 320 321 322def register_commands(subparsers, context): 323 """Register devtool subcommands from the deploy plugin""" 324 325 parser_deploy = subparsers.add_parser('deploy-target', 326 help='Deploy recipe output files to live target machine', 327 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', 328 group='testbuild') 329 parser_deploy.add_argument('recipename', help='Recipe to deploy') 330 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') 331 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 332 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 333 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') 334 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') 335 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') 336 parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 337 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 338 parser_deploy.add_argument('-I', '--key', 339 help='Specify ssh private key for connection to the target') 340 341 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) 342 strip_opts.add_argument('-S', '--strip', 343 help='Strip executables prior to deploying (default: %(default)s). ' 344 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', 345 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), 346 action='store_true') 347 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') 348 349 parser_deploy.set_defaults(func=deploy) 350 351 parser_undeploy = subparsers.add_parser('undeploy-target', 352 help='Undeploy recipe output files in live target machine', 353 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', 354 group='testbuild') 355 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') 356 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') 357 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 358 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 359 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') 360 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') 361 parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') 362 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 363 parser_undeploy.add_argument('-I', '--key', 364 help='Specify ssh private key for connection to the target') 365 366 parser_undeploy.set_defaults(func=undeploy) 367