1# Development tool - deploy/undeploy command plugin
2#
3# Copyright (C) 2014-2016 Intel Corporation
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7"""Devtool plugin containing the deploy subcommands"""
8
9import logging
10import os
11import shutil
12import subprocess
13import tempfile
14
15import bb.utils
16import argparse_oe
17import oe.types
18
19from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
20
21logger = logging.getLogger('devtool')
22
23deploylist_path = '/.devtool'
24
25def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
26    """
27    Prepare a shell script for running on the target to
28    deploy/undeploy files. We have to be careful what we put in this
29    script - only commands that are likely to be available on the
30    target are suitable (the target might be constrained, e.g. using
31    busybox rather than bash with coreutils).
32    """
33    lines = []
34    lines.append('#!/bin/sh')
35    lines.append('set -e')
36    if undeployall:
37        # Yes, I know this is crude - but it does work
38        lines.append('for entry in %s/*.list; do' % deploylist_path)
39        lines.append('[ ! -f $entry ] && exit')
40        lines.append('set `basename $entry | sed "s/.list//"`')
41    if dryrun:
42        if not deploy:
43            lines.append('echo "Previously deployed files for $1:"')
44    lines.append('manifest="%s/$1.list"' % deploylist_path)
45    lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
46    lines.append('if [ -f $manifest ] ; then')
47    # Read manifest in reverse and delete files / remove empty dirs
48    lines.append('    sed \'1!G;h;$!d\' $manifest | while read file')
49    lines.append('    do')
50    if dryrun:
51        lines.append('        if [ ! -d $file ] ; then')
52        lines.append('            echo $file')
53        lines.append('        fi')
54    else:
55        lines.append('        if [ -d $file ] ; then')
56        # Avoid deleting a preserved directory in case it has special perms
57        lines.append('            if [ ! -d $preservedir/$file ] ; then')
58        lines.append('                rmdir $file > /dev/null 2>&1 || true')
59        lines.append('            fi')
60        lines.append('        else')
61        lines.append('            rm -f $file')
62        lines.append('        fi')
63    lines.append('    done')
64    if not dryrun:
65        lines.append('    rm $manifest')
66    if not deploy and not dryrun:
67        # May as well remove all traces
68        lines.append('    rmdir `dirname $manifest` > /dev/null 2>&1 || true')
69    lines.append('fi')
70
71    if deploy:
72        if not nocheckspace:
73            # Check for available space
74            # FIXME This doesn't take into account files spread across multiple
75            # partitions, but doing that is non-trivial
76            # Find the part of the destination path that exists
77            lines.append('checkpath="$2"')
78            lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
79            lines.append('do')
80            lines.append('    checkpath=`dirname "$checkpath"`')
81            lines.append('done')
82            lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
83            # First line of the file is the total space
84            lines.append('total=`head -n1 $3`')
85            lines.append('if [ $total -gt $freespace ] ; then')
86            lines.append('    echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
87            lines.append('    exit 1')
88            lines.append('fi')
89        if not nopreserve:
90            # Preserve any files that exist. Note that this will add to the
91            # preserved list with successive deployments if the list of files
92            # deployed changes, but because we've deleted any previously
93            # deployed files at this point it will never preserve anything
94            # that was deployed, only files that existed prior to any deploying
95            # (which makes the most sense)
96            lines.append('cat $3 | sed "1d" | while read file fsize')
97            lines.append('do')
98            lines.append('    if [ -e $file ] ; then')
99            lines.append('    dest="$preservedir/$file"')
100            lines.append('    mkdir -p `dirname $dest`')
101            lines.append('    mv $file $dest')
102            lines.append('    fi')
103            lines.append('done')
104            lines.append('rm $3')
105        lines.append('mkdir -p `dirname $manifest`')
106        lines.append('mkdir -p $2')
107        if verbose:
108            lines.append('    tar xv -C $2 -f - | tee $manifest')
109        else:
110            lines.append('    tar xv -C $2 -f - > $manifest')
111        lines.append('sed -i "s!^./!$2!" $manifest')
112    elif not dryrun:
113        # Put any preserved files back
114        lines.append('if [ -d $preservedir ] ; then')
115        lines.append('    cd $preservedir')
116        # find from busybox might not have -exec, so we don't use that
117        lines.append('    find . -type f | while read file')
118        lines.append('    do')
119        lines.append('        mv $file /$file')
120        lines.append('    done')
121        lines.append('    cd /')
122        lines.append('    rm -rf $preservedir')
123        lines.append('fi')
124
125    if undeployall:
126        if not dryrun:
127            lines.append('echo "NOTE: Successfully undeployed $1"')
128        lines.append('done')
129
130    # Delete the script itself
131    lines.append('rm $0')
132    lines.append('')
133
134    return '\n'.join(lines)
135
136
137
138def deploy(args, config, basepath, workspace):
139    """Entry point for the devtool 'deploy' subcommand"""
140    import math
141    import oe.recipeutils
142    import oe.package
143    import oe.utils
144
145    check_workspace_recipe(workspace, args.recipename, checksrc=False)
146
147    try:
148        host, destdir = args.target.split(':')
149    except ValueError:
150        destdir = '/'
151    else:
152        args.target = host
153    if not destdir.endswith('/'):
154        destdir += '/'
155
156    tinfoil = setup_tinfoil(basepath=basepath)
157    try:
158        try:
159            rd = tinfoil.parse_recipe(args.recipename)
160        except Exception as e:
161            raise DevtoolError('Exception parsing recipe %s: %s' %
162                            (args.recipename, e))
163        recipe_outdir = rd.getVar('D')
164        if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
165            raise DevtoolError('No files to deploy - have you built the %s '
166                            'recipe? If so, the install step has not installed '
167                            'any files.' % args.recipename)
168
169        if args.strip and not args.dry_run:
170            # Fakeroot copy to new destination
171            srcdir = recipe_outdir
172            recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped')
173            if os.path.isdir(recipe_outdir):
174                exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True)
175            exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
176            os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
177            oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
178                        rd.getVar('base_libdir'), oe.utils.get_bb_number_threads(rd), rd)
179
180        filelist = []
181        inodes = set({})
182        ftotalsize = 0
183        for root, _, files in os.walk(recipe_outdir):
184            for fn in files:
185                fstat = os.lstat(os.path.join(root, fn))
186                # Get the size in kiB (since we'll be comparing it to the output of du -k)
187                # MUST use lstat() here not stat() or getfilesize() since we don't want to
188                # dereference symlinks
189                if fstat.st_ino in inodes:
190                    fsize = 0
191                else:
192                    fsize = int(math.ceil(float(fstat.st_size)/1024))
193                inodes.add(fstat.st_ino)
194                ftotalsize += fsize
195                # The path as it would appear on the target
196                fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
197                filelist.append((fpath, fsize))
198
199        if args.dry_run:
200            print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
201            for item, _ in filelist:
202                print('  %s' % item)
203            return 0
204
205        extraoptions = ''
206        if args.no_host_check:
207            extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
208        if not args.show_status:
209            extraoptions += ' -q'
210
211        scp_sshexec = ''
212        ssh_sshexec = 'ssh'
213        if args.ssh_exec:
214            scp_sshexec = "-S %s" % args.ssh_exec
215            ssh_sshexec = args.ssh_exec
216        scp_port = ''
217        ssh_port = ''
218        if args.port:
219            scp_port = "-P %s" % args.port
220            ssh_port = "-p %s" % args.port
221
222        if args.key:
223            extraoptions += ' -i %s' % args.key
224
225        # In order to delete previously deployed files and have the manifest file on
226        # the target, we write out a shell script and then copy it to the target
227        # so we can then run it (piping tar output to it).
228        # (We cannot use scp here, because it doesn't preserve symlinks.)
229        tmpdir = tempfile.mkdtemp(prefix='devtool')
230        try:
231            tmpscript = '/tmp/devtool_deploy.sh'
232            tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
233            shellscript = _prepare_remote_script(deploy=True,
234                                                verbose=args.show_status,
235                                                nopreserve=args.no_preserve,
236                                                nocheckspace=args.no_check_space)
237            # Write out the script to a file
238            with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
239                f.write(shellscript)
240            # Write out the file list
241            with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
242                f.write('%d\n' % ftotalsize)
243                for fpath, fsize in filelist:
244                    f.write('%s %d\n' % (fpath, fsize))
245            # Copy them to the target
246            ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
247            if ret != 0:
248                raise DevtoolError('Failed to copy script to %s - rerun with -s to '
249                                'get a complete error message' % args.target)
250        finally:
251            shutil.rmtree(tmpdir)
252
253        # Now run the script
254        ret = exec_fakeroot(rd, 'tar cf - . | %s  %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
255        if ret != 0:
256            raise DevtoolError('Deploy failed - rerun with -s to get a complete '
257                            'error message')
258
259        logger.info('Successfully deployed %s' % recipe_outdir)
260
261        files_list = []
262        for root, _, files in os.walk(recipe_outdir):
263            for filename in files:
264                filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
265                files_list.append(os.path.join(destdir, filename))
266    finally:
267        tinfoil.shutdown()
268
269    return 0
270
271def undeploy(args, config, basepath, workspace):
272    """Entry point for the devtool 'undeploy' subcommand"""
273    if args.all and args.recipename:
274        raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
275    elif not args.recipename and not args.all:
276        raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
277
278    extraoptions = ''
279    if args.no_host_check:
280        extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
281    if not args.show_status:
282        extraoptions += ' -q'
283
284    scp_sshexec = ''
285    ssh_sshexec = 'ssh'
286    if args.ssh_exec:
287        scp_sshexec = "-S %s" % args.ssh_exec
288        ssh_sshexec = args.ssh_exec
289    scp_port = ''
290    ssh_port = ''
291    if args.port:
292        scp_port = "-P %s" % args.port
293        ssh_port = "-p %s" % args.port
294
295    args.target = args.target.split(':')[0]
296
297    tmpdir = tempfile.mkdtemp(prefix='devtool')
298    try:
299        tmpscript = '/tmp/devtool_undeploy.sh'
300        shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
301        # Write out the script to a file
302        with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
303            f.write(shellscript)
304        # Copy it to the target
305        ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
306        if ret != 0:
307            raise DevtoolError('Failed to copy script to %s - rerun with -s to '
308                                'get a complete error message' % args.target)
309    finally:
310        shutil.rmtree(tmpdir)
311
312    # Now run the script
313    ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
314    if ret != 0:
315        raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
316                           'error message')
317
318    if not args.all and not args.dry_run:
319        logger.info('Successfully undeployed %s' % args.recipename)
320    return 0
321
322
323def register_commands(subparsers, context):
324    """Register devtool subcommands from the deploy plugin"""
325
326    parser_deploy = subparsers.add_parser('deploy-target',
327                                          help='Deploy recipe output files to live target machine',
328                                          description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
329                                          group='testbuild')
330    parser_deploy.add_argument('recipename', help='Recipe to deploy')
331    parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
332    parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
333    parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
334    parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
335    parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
336    parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
337    parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
338    parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
339    parser_deploy.add_argument('-I', '--key',
340                               help='Specify ssh private key for connection to the target')
341
342    strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
343    strip_opts.add_argument('-S', '--strip',
344                               help='Strip executables prior to deploying (default: %(default)s). '
345                                    'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
346                               default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
347                               action='store_true')
348    strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
349
350    parser_deploy.set_defaults(func=deploy)
351
352    parser_undeploy = subparsers.add_parser('undeploy-target',
353                                            help='Undeploy recipe output files in live target machine',
354                                            description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
355                                            group='testbuild')
356    parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
357    parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
358    parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
359    parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
360    parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
361    parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
362    parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
363    parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
364    parser_undeploy.add_argument('-I', '--key',
365                               help='Specify ssh private key for connection to the target')
366
367    parser_undeploy.set_defaults(func=undeploy)
368