1# Development tool - deploy/undeploy command plugin 2# 3# Copyright (C) 2014-2016 Intel Corporation 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License version 2 as 7# published by the Free Software Foundation. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License along 15# with this program; if not, write to the Free Software Foundation, Inc., 16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17"""Devtool plugin containing the deploy subcommands""" 18 19import logging 20import os 21import shutil 22import subprocess 23import tempfile 24 25import bb.utils 26import argparse_oe 27import oe.types 28 29from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError 30 31logger = logging.getLogger('devtool') 32 33deploylist_path = '/.devtool' 34 35def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): 36 """ 37 Prepare a shell script for running on the target to 38 deploy/undeploy files. We have to be careful what we put in this 39 script - only commands that are likely to be available on the 40 target are suitable (the target might be constrained, e.g. using 41 busybox rather than bash with coreutils). 42 """ 43 lines = [] 44 lines.append('#!/bin/sh') 45 lines.append('set -e') 46 if undeployall: 47 # Yes, I know this is crude - but it does work 48 lines.append('for entry in %s/*.list; do' % deploylist_path) 49 lines.append('[ ! -f $entry ] && exit') 50 lines.append('set `basename $entry | sed "s/.list//"`') 51 if dryrun: 52 if not deploy: 53 lines.append('echo "Previously deployed files for $1:"') 54 lines.append('manifest="%s/$1.list"' % deploylist_path) 55 lines.append('preservedir="%s/$1.preserve"' % deploylist_path) 56 lines.append('if [ -f $manifest ] ; then') 57 # Read manifest in reverse and delete files / remove empty dirs 58 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') 59 lines.append(' do') 60 if dryrun: 61 lines.append(' if [ ! -d $file ] ; then') 62 lines.append(' echo $file') 63 lines.append(' fi') 64 else: 65 lines.append(' if [ -d $file ] ; then') 66 # Avoid deleting a preserved directory in case it has special perms 67 lines.append(' if [ ! -d $preservedir/$file ] ; then') 68 lines.append(' rmdir $file > /dev/null 2>&1 || true') 69 lines.append(' fi') 70 lines.append(' else') 71 lines.append(' rm -f $file') 72 lines.append(' fi') 73 lines.append(' done') 74 if not dryrun: 75 lines.append(' rm $manifest') 76 if not deploy and not dryrun: 77 # May as well remove all traces 78 lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true') 79 lines.append('fi') 80 81 if deploy: 82 if not nocheckspace: 83 # Check for available space 84 # FIXME This doesn't take into account files spread across multiple 85 # partitions, but doing that is non-trivial 86 # Find the part of the destination path that exists 87 lines.append('checkpath="$2"') 88 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]') 89 lines.append('do') 90 lines.append(' checkpath=`dirname "$checkpath"`') 91 lines.append('done') 92 lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') 93 # First line of the file is the total space 94 lines.append('total=`head -n1 $3`') 95 lines.append('if [ $total -gt $freespace ] ; then') 96 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') 97 lines.append(' exit 1') 98 lines.append('fi') 99 if not nopreserve: 100 # Preserve any files that exist. Note that this will add to the 101 # preserved list with successive deployments if the list of files 102 # deployed changes, but because we've deleted any previously 103 # deployed files at this point it will never preserve anything 104 # that was deployed, only files that existed prior to any deploying 105 # (which makes the most sense) 106 lines.append('cat $3 | sed "1d" | while read file fsize') 107 lines.append('do') 108 lines.append(' if [ -e $file ] ; then') 109 lines.append(' dest="$preservedir/$file"') 110 lines.append(' mkdir -p `dirname $dest`') 111 lines.append(' mv $file $dest') 112 lines.append(' fi') 113 lines.append('done') 114 lines.append('rm $3') 115 lines.append('mkdir -p `dirname $manifest`') 116 lines.append('mkdir -p $2') 117 if verbose: 118 lines.append(' tar xv -C $2 -f - | tee $manifest') 119 else: 120 lines.append(' tar xv -C $2 -f - > $manifest') 121 lines.append('sed -i "s!^./!$2!" $manifest') 122 elif not dryrun: 123 # Put any preserved files back 124 lines.append('if [ -d $preservedir ] ; then') 125 lines.append(' cd $preservedir') 126 # find from busybox might not have -exec, so we don't use that 127 lines.append(' find . -type f | while read file') 128 lines.append(' do') 129 lines.append(' mv $file /$file') 130 lines.append(' done') 131 lines.append(' cd /') 132 lines.append(' rm -rf $preservedir') 133 lines.append('fi') 134 135 if undeployall: 136 if not dryrun: 137 lines.append('echo "NOTE: Successfully undeployed $1"') 138 lines.append('done') 139 140 # Delete the script itself 141 lines.append('rm $0') 142 lines.append('') 143 144 return '\n'.join(lines) 145 146 147 148def deploy(args, config, basepath, workspace): 149 """Entry point for the devtool 'deploy' subcommand""" 150 import math 151 import oe.recipeutils 152 import oe.package 153 154 check_workspace_recipe(workspace, args.recipename, checksrc=False) 155 156 try: 157 host, destdir = args.target.split(':') 158 except ValueError: 159 destdir = '/' 160 else: 161 args.target = host 162 if not destdir.endswith('/'): 163 destdir += '/' 164 165 tinfoil = setup_tinfoil(basepath=basepath) 166 try: 167 try: 168 rd = tinfoil.parse_recipe(args.recipename) 169 except Exception as e: 170 raise DevtoolError('Exception parsing recipe %s: %s' % 171 (args.recipename, e)) 172 recipe_outdir = rd.getVar('D') 173 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): 174 raise DevtoolError('No files to deploy - have you built the %s ' 175 'recipe? If so, the install step has not installed ' 176 'any files.' % args.recipename) 177 178 if args.strip and not args.dry_run: 179 # Fakeroot copy to new destination 180 srcdir = recipe_outdir 181 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped') 182 if os.path.isdir(recipe_outdir): 183 bb.utils.remove(recipe_outdir, True) 184 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) 185 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) 186 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), 187 rd.getVar('base_libdir'), rd) 188 189 filelist = [] 190 ftotalsize = 0 191 for root, _, files in os.walk(recipe_outdir): 192 for fn in files: 193 # Get the size in kiB (since we'll be comparing it to the output of du -k) 194 # MUST use lstat() here not stat() or getfilesize() since we don't want to 195 # dereference symlinks 196 fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024)) 197 ftotalsize += fsize 198 # The path as it would appear on the target 199 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) 200 filelist.append((fpath, fsize)) 201 202 if args.dry_run: 203 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) 204 for item, _ in filelist: 205 print(' %s' % item) 206 return 0 207 208 extraoptions = '' 209 if args.no_host_check: 210 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 211 if not args.show_status: 212 extraoptions += ' -q' 213 214 scp_port = '' 215 ssh_port = '' 216 if args.port: 217 scp_port = "-P %s" % args.port 218 ssh_port = "-p %s" % args.port 219 220 # In order to delete previously deployed files and have the manifest file on 221 # the target, we write out a shell script and then copy it to the target 222 # so we can then run it (piping tar output to it). 223 # (We cannot use scp here, because it doesn't preserve symlinks.) 224 tmpdir = tempfile.mkdtemp(prefix='devtool') 225 try: 226 tmpscript = '/tmp/devtool_deploy.sh' 227 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') 228 shellscript = _prepare_remote_script(deploy=True, 229 verbose=args.show_status, 230 nopreserve=args.no_preserve, 231 nocheckspace=args.no_check_space) 232 # Write out the script to a file 233 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 234 f.write(shellscript) 235 # Write out the file list 236 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: 237 f.write('%d\n' % ftotalsize) 238 for fpath, fsize in filelist: 239 f.write('%s %d\n' % (fpath, fsize)) 240 # Copy them to the target 241 ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 242 if ret != 0: 243 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 244 'get a complete error message' % args.target) 245 finally: 246 shutil.rmtree(tmpdir) 247 248 # Now run the script 249 ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s %s \'sh %s %s %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) 250 if ret != 0: 251 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 252 'error message') 253 254 logger.info('Successfully deployed %s' % recipe_outdir) 255 256 files_list = [] 257 for root, _, files in os.walk(recipe_outdir): 258 for filename in files: 259 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) 260 files_list.append(os.path.join(destdir, filename)) 261 finally: 262 tinfoil.shutdown() 263 264 return 0 265 266def undeploy(args, config, basepath, workspace): 267 """Entry point for the devtool 'undeploy' subcommand""" 268 if args.all and args.recipename: 269 raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') 270 elif not args.recipename and not args.all: 271 raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') 272 273 extraoptions = '' 274 if args.no_host_check: 275 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 276 if not args.show_status: 277 extraoptions += ' -q' 278 279 scp_port = '' 280 ssh_port = '' 281 if args.port: 282 scp_port = "-P %s" % args.port 283 ssh_port = "-p %s" % args.port 284 285 args.target = args.target.split(':')[0] 286 287 tmpdir = tempfile.mkdtemp(prefix='devtool') 288 try: 289 tmpscript = '/tmp/devtool_undeploy.sh' 290 shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) 291 # Write out the script to a file 292 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 293 f.write(shellscript) 294 # Copy it to the target 295 ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 296 if ret != 0: 297 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 298 'get a complete error message' % args.target) 299 finally: 300 shutil.rmtree(tmpdir) 301 302 # Now run the script 303 ret = subprocess.call('ssh %s %s %s \'sh %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) 304 if ret != 0: 305 raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 306 'error message') 307 308 if not args.all and not args.dry_run: 309 logger.info('Successfully undeployed %s' % args.recipename) 310 return 0 311 312 313def register_commands(subparsers, context): 314 """Register devtool subcommands from the deploy plugin""" 315 316 parser_deploy = subparsers.add_parser('deploy-target', 317 help='Deploy recipe output files to live target machine', 318 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', 319 group='testbuild') 320 parser_deploy.add_argument('recipename', help='Recipe to deploy') 321 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') 322 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 323 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 324 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') 325 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') 326 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') 327 parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 328 329 strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) 330 strip_opts.add_argument('-S', '--strip', 331 help='Strip executables prior to deploying (default: %(default)s). ' 332 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', 333 default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), 334 action='store_true') 335 strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') 336 337 parser_deploy.set_defaults(func=deploy) 338 339 parser_undeploy = subparsers.add_parser('undeploy-target', 340 help='Undeploy recipe output files in live target machine', 341 description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', 342 group='testbuild') 343 parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') 344 parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') 345 parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 346 parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 347 parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') 348 parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') 349 parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') 350 parser_undeploy.set_defaults(func=undeploy) 351