1# Development tool - standard commands plugin
2#
3# Copyright (C) 2014-2017 Intel Corporation
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7"""Devtool standard plugins"""
8
9import os
10import sys
11import re
12import shutil
13import subprocess
14import tempfile
15import logging
16import argparse
17import argparse_oe
18import scriptutils
19import errno
20import glob
21import filecmp
22from collections import OrderedDict
23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
24from devtool import parse_recipe
25
26logger = logging.getLogger('devtool')
27
28override_branch_prefix = 'devtool-override-'
29
30
31def add(args, config, basepath, workspace):
32    """Entry point for the devtool 'add' subcommand"""
33    import bb
34    import oe.recipeutils
35
36    if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
37        raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add')
38
39    # These are positional arguments, but because we're nice, allow
40    # specifying e.g. source tree without name, or fetch URI without name or
41    # source tree (if we can detect that that is what the user meant)
42    if scriptutils.is_src_url(args.recipename):
43        if not args.fetchuri:
44            if args.fetch:
45                raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
46            args.fetchuri = args.recipename
47            args.recipename = ''
48    elif scriptutils.is_src_url(args.srctree):
49        if not args.fetchuri:
50            if args.fetch:
51                raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
52            args.fetchuri = args.srctree
53            args.srctree = ''
54    elif args.recipename and not args.srctree:
55        if os.sep in args.recipename:
56            args.srctree = args.recipename
57            args.recipename = None
58        elif os.path.isdir(args.recipename):
59            logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename)
60
61    if not args.fetchuri:
62        if args.srcrev:
63            raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository')
64        if args.srcbranch:
65            raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository')
66
67    if args.srctree and os.path.isfile(args.srctree):
68        args.fetchuri = 'file://' + os.path.abspath(args.srctree)
69        args.srctree = ''
70
71    if args.fetch:
72        if args.fetchuri:
73            raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
74        else:
75            logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead')
76            args.fetchuri = args.fetch
77
78    if args.recipename:
79        if args.recipename in workspace:
80            raise DevtoolError("recipe %s is already in your workspace" %
81                               args.recipename)
82        reason = oe.recipeutils.validate_pn(args.recipename)
83        if reason:
84            raise DevtoolError(reason)
85
86    if args.srctree:
87        srctree = os.path.abspath(args.srctree)
88        srctreeparent = None
89        tmpsrcdir = None
90    else:
91        srctree = None
92        srctreeparent = get_default_srctree(config)
93        bb.utils.mkdirhier(srctreeparent)
94        tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
95
96    if srctree and os.path.exists(srctree):
97        if args.fetchuri:
98            if not os.path.isdir(srctree):
99                raise DevtoolError("Cannot fetch into source tree path %s as "
100                                   "it exists and is not a directory" %
101                                   srctree)
102            elif os.listdir(srctree):
103                raise DevtoolError("Cannot fetch into source tree path %s as "
104                                   "it already exists and is non-empty" %
105                                   srctree)
106    elif not args.fetchuri:
107        if args.srctree:
108            raise DevtoolError("Specified source tree %s could not be found" %
109                               args.srctree)
110        elif srctree:
111            raise DevtoolError("No source tree exists at default path %s - "
112                               "either create and populate this directory, "
113                               "or specify a path to a source tree, or a "
114                               "URI to fetch source from" % srctree)
115        else:
116            raise DevtoolError("You must either specify a source tree "
117                               "or a URI to fetch source from")
118
119    if args.version:
120        if '_' in args.version or ' ' in args.version:
121            raise DevtoolError('Invalid version string "%s"' % args.version)
122
123    if args.color == 'auto' and sys.stdout.isatty():
124        color = 'always'
125    else:
126        color = args.color
127    extracmdopts = ''
128    if args.fetchuri:
129        source = args.fetchuri
130        if srctree:
131            extracmdopts += ' -x %s' % srctree
132        else:
133            extracmdopts += ' -x %s' % tmpsrcdir
134    else:
135        source = srctree
136    if args.recipename:
137        extracmdopts += ' -N %s' % args.recipename
138    if args.version:
139        extracmdopts += ' -V %s' % args.version
140    if args.binary:
141        extracmdopts += ' -b'
142    if args.also_native:
143        extracmdopts += ' --also-native'
144    if args.src_subdir:
145        extracmdopts += ' --src-subdir "%s"' % args.src_subdir
146    if args.autorev:
147        extracmdopts += ' -a'
148    if args.npm_dev:
149        extracmdopts += ' --npm-dev'
150    if args.no_pypi:
151        extracmdopts += ' --no-pypi'
152    if args.mirrors:
153        extracmdopts += ' --mirrors'
154    if args.srcrev:
155        extracmdopts += ' --srcrev %s' % args.srcrev
156    if args.srcbranch:
157        extracmdopts += ' --srcbranch %s' % args.srcbranch
158    if args.provides:
159        extracmdopts += ' --provides %s' % args.provides
160
161    tempdir = tempfile.mkdtemp(prefix='devtool')
162    try:
163        try:
164            stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True)
165        except bb.process.ExecutionError as e:
166            if e.exitcode == 15:
167                raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line')
168            else:
169                raise DevtoolError('Command \'%s\' failed' % e.command)
170
171        recipes = glob.glob(os.path.join(tempdir, '*.bb'))
172        if recipes:
173            recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
174            if recipename in workspace:
175                raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
176            recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
177            bb.utils.mkdirhier(recipedir)
178            recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
179            appendfile = recipe_to_append(recipefile, config)
180            if os.path.exists(appendfile):
181                # This shouldn't be possible, but just in case
182                raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
183            if os.path.exists(recipefile):
184                raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
185            if tmpsrcdir:
186                srctree = os.path.join(srctreeparent, recipename)
187                if os.path.exists(tmpsrcdir):
188                    if os.path.exists(srctree):
189                        if os.path.isdir(srctree):
190                            try:
191                                os.rmdir(srctree)
192                            except OSError as e:
193                                if e.errno == errno.ENOTEMPTY:
194                                    raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
195                                else:
196                                    raise
197                        else:
198                            raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
199                    logger.info('Using default source tree path %s' % srctree)
200                    shutil.move(tmpsrcdir, srctree)
201                else:
202                    raise DevtoolError('Couldn\'t find source tree created by recipetool')
203            bb.utils.mkdirhier(recipedir)
204            shutil.move(recipes[0], recipefile)
205            # Move any additional files created by recipetool
206            for fn in os.listdir(tempdir):
207                shutil.move(os.path.join(tempdir, fn), recipedir)
208        else:
209            raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
210        attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
211        if os.path.exists(attic_recipe):
212            logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
213    finally:
214        if tmpsrcdir and os.path.exists(tmpsrcdir):
215            shutil.rmtree(tmpsrcdir)
216        shutil.rmtree(tempdir)
217
218    for fn in os.listdir(recipedir):
219        _add_md5(config, recipename, os.path.join(recipedir, fn))
220
221    tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
222    try:
223        try:
224            rd = tinfoil.parse_recipe_file(recipefile, False)
225        except Exception as e:
226            logger.error(str(e))
227            rd = None
228        if not rd:
229            # Parsing failed. We just created this recipe and we shouldn't
230            # leave it in the workdir or it'll prevent bitbake from starting
231            movefn = '%s.parsefailed' % recipefile
232            logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn)
233            shutil.move(recipefile, movefn)
234            return 1
235
236        if args.fetchuri and not args.no_git:
237            setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
238
239        initial_rev = {}
240        if os.path.exists(os.path.join(srctree, '.git')):
241            (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
242            initial_rev["."] = stdout.rstrip()
243            (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree)
244            for line in stdout.splitlines():
245                (rev, submodule) = line.split()
246                initial_rev[os.path.relpath(submodule, srctree)] = rev
247
248        if args.src_subdir:
249            srctree = os.path.join(srctree, args.src_subdir)
250
251        bb.utils.mkdirhier(os.path.dirname(appendfile))
252        with open(appendfile, 'w') as f:
253            f.write('inherit externalsrc\n')
254            f.write('EXTERNALSRC = "%s"\n' % srctree)
255
256            b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
257            if b_is_s:
258                f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
259            if initial_rev:
260                for key, value in initial_rev.items():
261                    f.write('\n# initial_rev %s: %s\n' % (key, value))
262
263            if args.binary:
264                f.write('do_install:append() {\n')
265                f.write('    rm -rf ${D}/.git\n')
266                f.write('    rm -f ${D}/singletask.lock\n')
267                f.write('}\n')
268
269            if bb.data.inherits_class('npm', rd):
270                f.write('python do_configure:append() {\n')
271                f.write('    pkgdir = d.getVar("NPM_PACKAGE")\n')
272                f.write('    lockfile = os.path.join(pkgdir, "singletask.lock")\n')
273                f.write('    bb.utils.remove(lockfile)\n')
274                f.write('}\n')
275
276        # Check if the new layer provides recipes whose priorities have been
277        # overriden by PREFERRED_PROVIDER.
278        recipe_name = rd.getVar('PN')
279        provides = rd.getVar('PROVIDES')
280        # Search every item defined in PROVIDES
281        for recipe_provided in provides.split():
282            preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided
283            current_pprovider = rd.getVar(preferred_provider)
284            if current_pprovider and current_pprovider != recipe_name:
285                if args.fixed_setup:
286                    #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf
287                    layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf")
288                    with open(layerconf_file, 'a') as f:
289                        f.write('%s = "%s"\n' % (preferred_provider, recipe_name))
290                else:
291                    logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider)
292                break
293
294        _add_md5(config, recipename, appendfile)
295
296        check_prerelease_version(rd.getVar('PV'), 'devtool add')
297
298        logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
299
300    finally:
301        tinfoil.shutdown()
302
303    return 0
304
305
306def _check_compatible_recipe(pn, d):
307    """Check if the recipe is supported by devtool"""
308    if pn == 'perf':
309        raise DevtoolError("The perf recipe does not actually check out "
310                           "source and thus cannot be supported by this tool",
311                           4)
312
313    if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'):
314        raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4)
315
316    if bb.data.inherits_class('image', d):
317        raise DevtoolError("The %s recipe is an image, and therefore is not "
318                           "supported by this tool" % pn, 4)
319
320    if bb.data.inherits_class('populate_sdk', d):
321        raise DevtoolError("The %s recipe is an SDK, and therefore is not "
322                           "supported by this tool" % pn, 4)
323
324    if bb.data.inherits_class('packagegroup', d):
325        raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
326                           "not supported by this tool" % pn, 4)
327
328    if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
329        # Not an incompatibility error per se, so we don't pass the error code
330        raise DevtoolError("externalsrc is currently enabled for the %s "
331                           "recipe. This prevents the normal do_patch task "
332                           "from working. You will need to disable this "
333                           "first." % pn)
334
335def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
336    """Common function for copying a file to the dry run output directory"""
337    relpath = os.path.relpath(dst, base_outdir)
338    if relpath.startswith('..'):
339        raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
340    dst = os.path.join(dry_run_outdir, relpath)
341    dst_d = os.path.dirname(dst)
342    if dst_d:
343        bb.utils.mkdirhier(dst_d)
344    # Don't overwrite existing files, otherwise in the case of an upgrade
345    # the dry-run written out recipe will be overwritten with an unmodified
346    # version
347    if not os.path.exists(dst):
348        shutil.copy(src, dst)
349
350def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
351    """Move a file. Creates all the directory components of destination path."""
352    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
353    logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
354    if dry_run_outdir:
355        # We want to copy here, not move
356        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
357    else:
358        dst_d = os.path.dirname(dst)
359        if dst_d:
360            bb.utils.mkdirhier(dst_d)
361        shutil.move(src, dst)
362
363def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None):
364    """Copy a file. Creates all the directory components of destination path."""
365    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
366    logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
367    if dry_run_outdir:
368        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
369    else:
370        dst_d = os.path.dirname(dst)
371        if dst_d:
372            bb.utils.mkdirhier(dst_d)
373        shutil.copy(src, dst)
374
375def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
376    """List contents of a git treeish"""
377    import bb
378    cmd = ['git', 'ls-tree', '-z', treeish]
379    if recursive:
380        cmd.append('-r')
381    out, _ = bb.process.run(cmd, cwd=repodir)
382    ret = {}
383    if out:
384        for line in out.split('\0'):
385            if line:
386                split = line.split(None, 4)
387                ret[split[3]] = split[0:3]
388    return ret
389
390def _git_exclude_path(srctree, path):
391    """Return pathspec (list of paths) that excludes certain path"""
392    # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
393    # we don't catch files that are deleted, for example. A more reliable way
394    # to implement this would be to use "negative pathspecs" which were
395    # introduced in Git v1.9.0. Revisit this when/if the required Git version
396    # becomes greater than that.
397    path = os.path.normpath(path)
398    recurse = True if len(path.split(os.path.sep)) > 1 else False
399    git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys())
400    if path in git_files:
401        git_files.remove(path)
402        return git_files
403    else:
404        return ['.']
405
406def _ls_tree(directory):
407    """Recursive listing of files in a directory"""
408    ret = []
409    for root, dirs, files in os.walk(directory):
410        ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
411                    fname in files])
412    return ret
413
414
415def extract(args, config, basepath, workspace):
416    """Entry point for the devtool 'extract' subcommand"""
417    import bb
418
419    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
420    if not tinfoil:
421        # Error already shown
422        return 1
423    try:
424        rd = parse_recipe(config, tinfoil, args.recipename, True)
425        if not rd:
426            return 1
427
428        srctree = os.path.abspath(args.srctree)
429        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
430        logger.info('Source tree extracted to %s' % srctree)
431
432        if initial_rev:
433            return 0
434        else:
435            return 1
436    finally:
437        tinfoil.shutdown()
438
439def sync(args, config, basepath, workspace):
440    """Entry point for the devtool 'sync' subcommand"""
441    import bb
442
443    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
444    if not tinfoil:
445        # Error already shown
446        return 1
447    try:
448        rd = parse_recipe(config, tinfoil, args.recipename, True)
449        if not rd:
450            return 1
451
452        srctree = os.path.abspath(args.srctree)
453        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True)
454        logger.info('Source tree %s synchronized' % srctree)
455
456        if initial_rev:
457            return 0
458        else:
459            return 1
460    finally:
461        tinfoil.shutdown()
462
463def symlink_oelocal_files_srctree(rd, srctree):
464    import oe.patch
465    if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
466        # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
467        # (otherwise the recipe won't build as expected)
468        local_files_dir = os.path.join(srctree, 'oe-local-files')
469        addfiles = []
470        for root, _, files in os.walk(local_files_dir):
471            relpth = os.path.relpath(root, local_files_dir)
472            if relpth != '.':
473                bb.utils.mkdirhier(os.path.join(srctree, relpth))
474            for fn in files:
475                if fn == '.gitignore':
476                    continue
477                destpth = os.path.join(srctree, relpth, fn)
478                if os.path.exists(destpth):
479                    os.unlink(destpth)
480                if relpth != '.':
481                    back_relpth = os.path.relpath(local_files_dir, root)
482                    os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth)
483                else:
484                    os.symlink('oe-local-files/%s' % fn, destpth)
485                addfiles.append(os.path.join(relpth, fn))
486        if addfiles:
487            oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd)
488
489def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
490    """Extract sources of a recipe"""
491    import oe.recipeutils
492    import oe.patch
493    import oe.path
494
495    pn = d.getVar('PN')
496
497    _check_compatible_recipe(pn, d)
498
499    if sync:
500        if not os.path.exists(srctree):
501                raise DevtoolError("output path %s does not exist" % srctree)
502    else:
503        if os.path.exists(srctree):
504            if not os.path.isdir(srctree):
505                raise DevtoolError("output path %s exists and is not a directory" %
506                                   srctree)
507            elif os.listdir(srctree):
508                raise DevtoolError("output path %s already exists and is "
509                                   "non-empty" % srctree)
510
511        if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
512            raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
513                               "extract source" % pn, 4)
514
515    if not sync:
516        # Prepare for shutil.move later on
517        bb.utils.mkdirhier(srctree)
518        os.rmdir(srctree)
519
520    extra_overrides = []
521    if not no_overrides:
522        history = d.varhistory.variable('SRC_URI')
523        for event in history:
524            if not 'flag' in event:
525                if event['op'].startswith((':append[', ':prepend[')):
526                    override = event['op'].split('[')[1].split(']')[0]
527                    if not override.startswith('pn-'):
528                        extra_overrides.append(override)
529        # We want to remove duplicate overrides. If a recipe had multiple
530        # SRC_URI_override += values it would cause mulitple instances of
531        # overrides. This doesn't play nicely with things like creating a
532        # branch for every instance of DEVTOOL_EXTRA_OVERRIDES.
533        extra_overrides = list(set(extra_overrides))
534        if extra_overrides:
535            logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these')
536
537    initial_rev = None
538
539    recipefile = d.getVar('FILE')
540    appendfile = recipe_to_append(recipefile, config)
541    is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
542
543    # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
544    # directory so that:
545    # (a) we pick up all files that get unpacked to the WORKDIR, and
546    # (b) we don't disturb the existing build
547    # However, with recipe-specific sysroots the sysroots for the recipe
548    # will be prepared under WORKDIR, and if we used the system temporary
549    # directory (i.e. usually /tmp) as used by mkdtemp by default, then
550    # our attempts to hardlink files into the recipe-specific sysroots
551    # will fail on systems where /tmp is a different filesystem, and it
552    # would have to fall back to copying the files which is a waste of
553    # time. Put the temp directory under the WORKDIR to prevent that from
554    # being a problem.
555    tempbasedir = d.getVar('WORKDIR')
556    bb.utils.mkdirhier(tempbasedir)
557    tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
558    try:
559        tinfoil.logger.setLevel(logging.WARNING)
560
561        # FIXME this results in a cache reload under control of tinfoil, which is fine
562        # except we don't get the knotty progress bar
563
564        if os.path.exists(appendfile):
565            appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak')
566            shutil.copyfile(appendfile, appendbackup)
567        else:
568            appendbackup = None
569            bb.utils.mkdirhier(os.path.dirname(appendfile))
570        logger.debug('writing append file %s' % appendfile)
571        with open(appendfile, 'a') as f:
572            f.write('###--- _extract_source\n')
573            f.write('deltask do_recipe_qa\n')
574            f.write('deltask do_recipe_qa_setscene\n')
575            f.write('ERROR_QA:remove = "patch-fuzz"\n')
576            f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
577            f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
578            if not is_kernel_yocto:
579                f.write('PATCHTOOL = "git"\n')
580                f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
581            if extra_overrides:
582                f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides))
583            f.write('inherit devtool-source\n')
584            f.write('###--- _extract_source\n')
585
586        update_unlockedsigs(basepath, workspace, fixed_setup, [pn])
587
588        sstate_manifests = d.getVar('SSTATE_MANIFESTS')
589        bb.utils.mkdirhier(sstate_manifests)
590        preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
591        with open(preservestampfile, 'w') as f:
592            f.write(d.getVar('STAMP'))
593        tinfoil.modified_files()
594        try:
595            if is_kernel_yocto:
596                # We need to generate the kernel config
597                task = 'do_configure'
598            else:
599                task = 'do_patch'
600
601                if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []):
602                    logger.info('The %s recipe has %s disabled. Running only '
603                                       'do_configure task dependencies' % (pn, task))
604
605                    if 'depends' in d.getVarFlags('do_configure', False):
606                        pn = d.getVarFlags('do_configure', False)['depends']
607                        pn = pn.replace('${PV}', d.getVar('PV'))
608                        pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP'))
609                        task = None
610
611            # Run the fetch + unpack tasks
612            res = tinfoil.build_targets(pn,
613                                        task,
614                                        handle_events=True)
615        finally:
616            if os.path.exists(preservestampfile):
617                os.remove(preservestampfile)
618
619        if not res:
620            raise DevtoolError('Extracting source for %s failed' % pn)
621
622        if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])):
623            workshareddir = d.getVar('S')
624            if os.path.islink(srctree):
625                os.unlink(srctree)
626
627            os.symlink(workshareddir, srctree)
628
629            # The initial_rev file is created in devtool_post_unpack function that will not be executed if
630            # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful
631            return True, True
632
633        try:
634            with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
635                initial_rev = f.read()
636
637            with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
638                srcsubdir = f.read()
639        except FileNotFoundError as e:
640            raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
641        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
642
643        # Check if work-shared is empty, if yes
644        # find source and copy to work-shared
645        if is_kernel_yocto:
646            workshareddir = d.getVar('STAGING_KERNEL_DIR')
647            staging_kerVer = get_staging_kver(workshareddir)
648            kernelVersion = d.getVar('LINUX_VERSION')
649
650            # handle dangling symbolic link in work-shared:
651            if os.path.islink(workshareddir):
652                os.unlink(workshareddir)
653
654            if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer):
655                shutil.rmtree(workshareddir)
656                oe.path.copyhardlinktree(srcsubdir, workshareddir)
657            elif not os.path.exists(workshareddir):
658                oe.path.copyhardlinktree(srcsubdir, workshareddir)
659
660        tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
661        srctree_localdir = os.path.join(srctree, 'oe-local-files')
662
663        if sync:
664            bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
665
666            # Move the oe-local-files directory to srctree.
667            # As oe-local-files is not part of the constructed git tree,
668            # removing it directly during the synchronization might surprise
669            # the user.  Instead, we move it to oe-local-files.bak and remind
670            # the user in the log message.
671            if os.path.exists(srctree_localdir + '.bak'):
672                shutil.rmtree(srctree_localdir + '.bak')
673
674            if os.path.exists(srctree_localdir):
675                logger.info('Backing up current local file directory %s' % srctree_localdir)
676                shutil.move(srctree_localdir, srctree_localdir + '.bak')
677
678            if os.path.exists(tempdir_localdir):
679                logger.info('Syncing local source files to srctree...')
680                shutil.copytree(tempdir_localdir, srctree_localdir)
681        else:
682            # Move oe-local-files directory to srctree
683            if os.path.exists(tempdir_localdir):
684                logger.info('Adding local source files to srctree...')
685                shutil.move(tempdir_localdir, srcsubdir)
686
687            shutil.move(srcsubdir, srctree)
688            symlink_oelocal_files_srctree(d, srctree)
689
690        if is_kernel_yocto:
691            logger.info('Copying kernel config to srctree')
692            shutil.copy2(os.path.join(tempdir, '.config'), srctree)
693
694    finally:
695        if appendbackup:
696            shutil.copyfile(appendbackup, appendfile)
697        elif os.path.exists(appendfile):
698            os.remove(appendfile)
699        if keep_temp:
700            logger.info('Preserving temporary directory %s' % tempdir)
701        else:
702            shutil.rmtree(tempdir)
703    return initial_rev, srcsubdir_rel
704
705def _add_md5(config, recipename, filename):
706    """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
707    import bb.utils
708
709    def addfile(fn):
710        md5 = bb.utils.md5_file(fn)
711        with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f:
712            md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)
713            f.seek(0, os.SEEK_SET)
714            if not md5_str in f.read():
715                f.write(md5_str)
716
717    if os.path.isdir(filename):
718        for root, _, files in os.walk(filename):
719            for f in files:
720                addfile(os.path.join(root, f))
721    else:
722        addfile(filename)
723
724def _check_preserve(config, recipename):
725    """Check if a file was manually changed and needs to be saved in 'attic'
726       directory"""
727    import bb.utils
728    origfile = os.path.join(config.workspace_path, '.devtool_md5')
729    newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
730    preservepath = os.path.join(config.workspace_path, 'attic', recipename)
731    with open(origfile, 'r') as f:
732        with open(newfile, 'w') as tf:
733            for line in f.readlines():
734                splitline = line.rstrip().split('|')
735                if splitline[0] == recipename:
736                    removefile = os.path.join(config.workspace_path, splitline[1])
737                    try:
738                        md5 = bb.utils.md5_file(removefile)
739                    except IOError as err:
740                        if err.errno == 2:
741                            # File no longer exists, skip it
742                            continue
743                        else:
744                            raise
745                    if splitline[2] != md5:
746                        bb.utils.mkdirhier(preservepath)
747                        preservefile = os.path.basename(removefile)
748                        logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath))
749                        shutil.move(removefile, os.path.join(preservepath, preservefile))
750                    else:
751                        os.remove(removefile)
752                else:
753                    tf.write(line)
754    bb.utils.rename(newfile, origfile)
755
756def get_staging_kver(srcdir):
757    # Kernel version from work-shared
758    kerver = []
759    staging_kerVer=""
760    if os.path.exists(srcdir) and os.listdir(srcdir):
761        with open(os.path.join(srcdir, "Makefile")) as f:
762            version = [next(f) for x in range(5)][1:4]
763            for word in version:
764                kerver.append(word.split('= ')[1].split('\n')[0])
765            staging_kerVer = ".".join(kerver)
766    return staging_kerVer
767
768def get_staging_kbranch(srcdir):
769    staging_kbranch = ""
770    if os.path.exists(srcdir) and os.listdir(srcdir):
771        (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir)
772        staging_kbranch = "".join(branch.split('\n')[0])
773    return staging_kbranch
774
775def get_real_srctree(srctree, s, workdir):
776    # Check that recipe isn't using a shared workdir
777    s = os.path.abspath(s)
778    workdir = os.path.abspath(workdir)
779    if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
780        # Handle if S is set to a subdirectory of the source
781        srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
782        srctree = os.path.join(srctree, srcsubdir)
783    return srctree
784
785def modify(args, config, basepath, workspace):
786    """Entry point for the devtool 'modify' subcommand"""
787    import bb
788    import oe.recipeutils
789    import oe.patch
790    import oe.path
791
792    if args.recipename in workspace:
793        raise DevtoolError("recipe %s is already in your workspace" %
794                           args.recipename)
795
796    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
797    try:
798        rd = parse_recipe(config, tinfoil, args.recipename, True)
799        if not rd:
800            return 1
801
802        pn = rd.getVar('PN')
803        if pn != args.recipename:
804            logger.info('Mapping %s to %s' % (args.recipename, pn))
805        if pn in workspace:
806            raise DevtoolError("recipe %s is already in your workspace" %
807                            pn)
808
809        if args.srctree:
810            srctree = os.path.abspath(args.srctree)
811        else:
812            srctree = get_default_srctree(config, pn)
813
814        if args.no_extract and not os.path.isdir(srctree):
815            raise DevtoolError("--no-extract specified and source path %s does "
816                            "not exist or is not a directory" %
817                            srctree)
818
819        recipefile = rd.getVar('FILE')
820        appendfile = recipe_to_append(recipefile, config, args.wildcard)
821        if os.path.exists(appendfile):
822            raise DevtoolError("Another variant of recipe %s is already in your "
823                            "workspace (only one variant of a recipe can "
824                            "currently be worked on at once)"
825                            % pn)
826
827        _check_compatible_recipe(pn, rd)
828
829        initial_revs = {}
830        commits = {}
831        check_commits = False
832
833        if bb.data.inherits_class('kernel-yocto', rd):
834            # Current set kernel version
835            kernelVersion = rd.getVar('LINUX_VERSION')
836            srcdir = rd.getVar('STAGING_KERNEL_DIR')
837            kbranch = rd.getVar('KBRANCH')
838
839            staging_kerVer = get_staging_kver(srcdir)
840            staging_kbranch = get_staging_kbranch(srcdir)
841            if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
842                oe.path.copyhardlinktree(srcdir, srctree)
843                workdir = rd.getVar('WORKDIR')
844                srcsubdir = rd.getVar('S')
845                localfilesdir = os.path.join(srctree, 'oe-local-files')
846                # Move local source files into separate subdir
847                recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
848                local_files = oe.recipeutils.get_recipe_local_files(rd)
849
850                for key in local_files.copy():
851                    if key.endswith('scc'):
852                        sccfile = open(local_files[key], 'r')
853                        for l in sccfile:
854                            line = l.split()
855                            if line and line[0] in ('kconf', 'patch'):
856                                cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
857                                if not cfg in local_files.values():
858                                    local_files[line[-1]] = cfg
859                                    shutil.copy2(cfg, workdir)
860                        sccfile.close()
861
862                # Ignore local files with subdir={BP}
863                srcabspath = os.path.abspath(srcsubdir)
864                local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and  (srcabspath == workdir or not  os.path.join(workdir, fname).startswith(srcabspath + os.sep))]
865                if local_files:
866                    for fname in local_files:
867                        _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
868                    with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
869                        f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n')
870
871                symlink_oelocal_files_srctree(rd, srctree)
872
873                task = 'do_configure'
874                res = tinfoil.build_targets(pn, task, handle_events=True)
875
876                # Copy .config to workspace
877                kconfpath = rd.getVar('B')
878                logger.info('Copying kernel config to workspace')
879                shutil.copy2(os.path.join(kconfpath, '.config'), srctree)
880
881                # Set this to true, we still need to get initial_rev
882                # by parsing the git repo
883                args.no_extract = True
884
885        if not args.no_extract:
886            initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
887            if not initial_revs["."]:
888                return 1
889            logger.info('Source tree extracted to %s' % srctree)
890
891            if os.path.exists(os.path.join(srctree, '.git')):
892                # Get list of commits since this revision
893                (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree)
894                commits["."] = stdout.split()
895                check_commits = True
896                (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
897                for line in stdout.splitlines():
898                    (rev, submodule_path) = line.split()
899                    submodule = os.path.relpath(submodule_path, srctree)
900                    initial_revs[submodule] = rev
901                    (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path)
902                    commits[submodule] = stdout.split()
903        else:
904            if os.path.exists(os.path.join(srctree, '.git')):
905                # Check if it's a tree previously extracted by us. This is done
906                # by ensuring that devtool-base and args.branch (devtool) exist.
907                # The check_commits logic will cause an exception if either one
908                # of these doesn't exist
909                try:
910                    (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
911                    bb.process.run('git rev-parse %s' % args.branch, cwd=srctree)
912                except bb.process.ExecutionError:
913                    stdout = ''
914                if stdout:
915                    check_commits = True
916                for line in stdout.splitlines():
917                    if line.startswith('*'):
918                        (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
919                        initial_revs["."] = stdout.rstrip()
920                if "." not in initial_revs:
921                    # Otherwise, just grab the head revision
922                    (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
923                    initial_revs["."] = stdout.rstrip()
924
925        branch_patches = {}
926        if check_commits:
927            # Check if there are override branches
928            (stdout, _) = bb.process.run('git branch', cwd=srctree)
929            branches = []
930            for line in stdout.rstrip().splitlines():
931                branchname = line[2:].rstrip()
932                if branchname.startswith(override_branch_prefix):
933                    branches.append(branchname)
934            if branches:
935                logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix))
936            branches.insert(0, args.branch)
937            seen_patches = []
938            for branch in branches:
939                branch_patches[branch] = []
940                (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree)
941                for sha1 in stdout.splitlines():
942                    notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip())
943                    origpatch = notes.get(oe.patch.GitApplyTree.original_patch)
944                    if origpatch and origpatch not in seen_patches:
945                        seen_patches.append(origpatch)
946                        branch_patches[branch].append(origpatch)
947
948        # Need to grab this here in case the source is within a subdirectory
949        srctreebase = srctree
950        srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
951
952        bb.utils.mkdirhier(os.path.dirname(appendfile))
953        with open(appendfile, 'w') as f:
954            # if not present, add type=git-dependency to the secondary sources
955            # (non local files) so they can be extracted correctly when building a recipe after
956            #  doing a devtool modify on it
957            src_uri = rd.getVar('SRC_URI').split()
958            src_uri_append = []
959            src_uri_remove = []
960
961            # Assume first entry is main source extracted in ${S} so skip it
962            src_uri = src_uri[1::]
963
964            # Add "type=git-dependency" to all non local sources
965            for url in src_uri:
966                if not url.startswith('file://') and not 'type=' in url:
967                    src_uri_remove.append(url)
968                    src_uri_append.append('%s;type=git-dependency' % url)
969
970            if src_uri_remove:
971                f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove))
972                f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append))
973
974            f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n')
975            # Local files can be modified/tracked in separate subdir under srctree
976            # Mostly useful for packages with S != WORKDIR
977            f.write('FILESPATH:prepend := "%s:"\n' %
978                    os.path.join(srctreebase, 'oe-local-files'))
979            f.write('# srctreebase: %s\n' % srctreebase)
980
981            f.write('\ninherit externalsrc\n')
982            f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
983            f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
984
985            b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
986            if b_is_s:
987                f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
988
989            if bb.data.inherits_class('kernel', rd):
990                f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
991                        'do_fetch do_unpack do_kernel_configcheck"\n')
992                f.write('\ndo_patch[noexec] = "1"\n')
993                f.write('\ndo_configure:append() {\n'
994                        '    cp ${B}/.config ${S}/.config.baseline\n'
995                        '    ln -sfT ${B}/.config ${S}/.config.new\n'
996                        '}\n')
997                f.write('\ndo_kernel_configme:prepend() {\n'
998                        '    if [ -e ${S}/.config ]; then\n'
999                        '        mv ${S}/.config ${S}/.config.old\n'
1000                        '    fi\n'
1001                        '}\n')
1002            if rd.getVarFlag('do_menuconfig', 'task'):
1003                f.write('\ndo_configure:append() {\n'
1004                '    if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n'
1005                '        cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n'
1006                '        ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n'
1007                '    fi\n'
1008                '}\n')
1009            if initial_revs:
1010                for name, rev in initial_revs.items():
1011                    f.write('\n# initial_rev %s: %s\n' % (name, rev))
1012                    if name in commits:
1013                        for commit in commits[name]:
1014                            f.write('# commit %s: %s\n' % (name, commit))
1015            if branch_patches:
1016                for branch in branch_patches:
1017                    if branch == args.branch:
1018                        continue
1019                    f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
1020
1021        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
1022
1023        _add_md5(config, pn, appendfile)
1024
1025        logger.info('Recipe %s now set up to build from %s' % (pn, srctree))
1026
1027    finally:
1028        tinfoil.shutdown()
1029
1030    return 0
1031
1032
1033def rename(args, config, basepath, workspace):
1034    """Entry point for the devtool 'rename' subcommand"""
1035    import bb
1036    import oe.recipeutils
1037
1038    check_workspace_recipe(workspace, args.recipename)
1039
1040    if not (args.newname or args.version):
1041        raise DevtoolError('You must specify a new name, a version with -V/--version, or both')
1042
1043    recipefile = workspace[args.recipename]['recipefile']
1044    if not recipefile:
1045        raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)')
1046
1047    if args.newname and args.newname != args.recipename:
1048        reason = oe.recipeutils.validate_pn(args.newname)
1049        if reason:
1050            raise DevtoolError(reason)
1051        newname = args.newname
1052    else:
1053        newname = args.recipename
1054
1055    append = workspace[args.recipename]['bbappend']
1056    appendfn = os.path.splitext(os.path.basename(append))[0]
1057    splitfn = appendfn.split('_')
1058    if len(splitfn) > 1:
1059        origfnver = appendfn.split('_')[1]
1060    else:
1061        origfnver = ''
1062
1063    recipefilemd5 = None
1064    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1065    try:
1066        rd = parse_recipe(config, tinfoil, args.recipename, True)
1067        if not rd:
1068            return 1
1069
1070        bp = rd.getVar('BP')
1071        bpn = rd.getVar('BPN')
1072        if newname != args.recipename:
1073            localdata = rd.createCopy()
1074            localdata.setVar('PN', newname)
1075            newbpn = localdata.getVar('BPN')
1076        else:
1077            newbpn = bpn
1078        s = rd.getVar('S', False)
1079        src_uri = rd.getVar('SRC_URI', False)
1080        pv = rd.getVar('PV')
1081
1082        # Correct variable values that refer to the upstream source - these
1083        # values must stay the same, so if the name/version are changing then
1084        # we need to fix them up
1085        new_s = s
1086        new_src_uri = src_uri
1087        if newbpn != bpn:
1088            # ${PN} here is technically almost always incorrect, but people do use it
1089            new_s = new_s.replace('${BPN}', bpn)
1090            new_s = new_s.replace('${PN}', bpn)
1091            new_s = new_s.replace('${BP}', '%s-${PV}' % bpn)
1092            new_src_uri = new_src_uri.replace('${BPN}', bpn)
1093            new_src_uri = new_src_uri.replace('${PN}', bpn)
1094            new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn)
1095        if args.version and origfnver == pv:
1096            new_s = new_s.replace('${PV}', pv)
1097            new_s = new_s.replace('${BP}', '${BPN}-%s' % pv)
1098            new_src_uri = new_src_uri.replace('${PV}', pv)
1099            new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv)
1100        patchfields = {}
1101        if new_s != s:
1102            patchfields['S'] = new_s
1103        if new_src_uri != src_uri:
1104            patchfields['SRC_URI'] = new_src_uri
1105        if patchfields:
1106            recipefilemd5 = bb.utils.md5_file(recipefile)
1107            oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
1108            newrecipefilemd5 = bb.utils.md5_file(recipefile)
1109    finally:
1110        tinfoil.shutdown()
1111
1112    if args.version:
1113        newver = args.version
1114    else:
1115        newver = origfnver
1116
1117    if newver:
1118        newappend = '%s_%s.bbappend' % (newname, newver)
1119        newfile =  '%s_%s.bb' % (newname, newver)
1120    else:
1121        newappend = '%s.bbappend' % newname
1122        newfile = '%s.bb' % newname
1123
1124    oldrecipedir = os.path.dirname(recipefile)
1125    newrecipedir = os.path.join(config.workspace_path, 'recipes', newname)
1126    if oldrecipedir != newrecipedir:
1127        bb.utils.mkdirhier(newrecipedir)
1128
1129    newappend = os.path.join(os.path.dirname(append), newappend)
1130    newfile = os.path.join(newrecipedir, newfile)
1131
1132    # Rename bbappend
1133    logger.info('Renaming %s to %s' % (append, newappend))
1134    bb.utils.rename(append, newappend)
1135    # Rename recipe file
1136    logger.info('Renaming %s to %s' % (recipefile, newfile))
1137    bb.utils.rename(recipefile, newfile)
1138
1139    # Rename source tree if it's the default path
1140    appendmd5 = None
1141    if not args.no_srctree:
1142        srctree = workspace[args.recipename]['srctree']
1143        if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
1144            newsrctree = os.path.join(config.workspace_path, 'sources', newname)
1145            logger.info('Renaming %s to %s' % (srctree, newsrctree))
1146            shutil.move(srctree, newsrctree)
1147            # Correct any references (basically EXTERNALSRC*) in the .bbappend
1148            appendmd5 = bb.utils.md5_file(newappend)
1149            appendlines = []
1150            with open(newappend, 'r') as f:
1151                for line in f:
1152                    appendlines.append(line)
1153            with open(newappend, 'w') as f:
1154                for line in appendlines:
1155                    if srctree in line:
1156                        line = line.replace(srctree, newsrctree)
1157                    f.write(line)
1158            newappendmd5 = bb.utils.md5_file(newappend)
1159
1160    bpndir = None
1161    newbpndir = None
1162    if newbpn != bpn:
1163        bpndir = os.path.join(oldrecipedir, bpn)
1164        if os.path.exists(bpndir):
1165            newbpndir = os.path.join(newrecipedir, newbpn)
1166            logger.info('Renaming %s to %s' % (bpndir, newbpndir))
1167            shutil.move(bpndir, newbpndir)
1168
1169    bpdir = None
1170    newbpdir = None
1171    if newver != origfnver or newbpn != bpn:
1172        bpdir = os.path.join(oldrecipedir, bp)
1173        if os.path.exists(bpdir):
1174            newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver))
1175            logger.info('Renaming %s to %s' % (bpdir, newbpdir))
1176            shutil.move(bpdir, newbpdir)
1177
1178    if oldrecipedir != newrecipedir:
1179        # Move any stray files and delete the old recipe directory
1180        for entry in os.listdir(oldrecipedir):
1181            oldpath = os.path.join(oldrecipedir, entry)
1182            newpath = os.path.join(newrecipedir, entry)
1183            logger.info('Renaming %s to %s' % (oldpath, newpath))
1184            shutil.move(oldpath, newpath)
1185        os.rmdir(oldrecipedir)
1186
1187    # Now take care of entries in .devtool_md5
1188    md5entries = []
1189    with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f:
1190        for line in f:
1191            md5entries.append(line)
1192
1193    if bpndir and newbpndir:
1194        relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/'
1195    else:
1196        relbpndir = None
1197    if bpdir and newbpdir:
1198        relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/'
1199    else:
1200        relbpdir = None
1201
1202    with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f:
1203        for entry in md5entries:
1204            splitentry = entry.rstrip().split('|')
1205            if len(splitentry) > 2:
1206                if splitentry[0] == args.recipename:
1207                    splitentry[0] = newname
1208                    if splitentry[1] == os.path.relpath(append, config.workspace_path):
1209                        splitentry[1] = os.path.relpath(newappend, config.workspace_path)
1210                        if appendmd5 and splitentry[2] == appendmd5:
1211                            splitentry[2] = newappendmd5
1212                    elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path):
1213                        splitentry[1] = os.path.relpath(newfile, config.workspace_path)
1214                        if recipefilemd5 and splitentry[2] == recipefilemd5:
1215                            splitentry[2] = newrecipefilemd5
1216                    elif relbpndir and splitentry[1].startswith(relbpndir):
1217                        splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path)
1218                    elif relbpdir and splitentry[1].startswith(relbpdir):
1219                        splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path)
1220                    entry = '|'.join(splitentry) + '\n'
1221            f.write(entry)
1222    return 0
1223
1224
1225def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False):
1226    """Get initial and update rev of a recipe. These are the start point of the
1227    whole patchset and start point for the patches to be re-generated/updated.
1228    """
1229    import bb
1230
1231    # Get current branch
1232    stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1233                               cwd=srctree)
1234    branchname = stdout.rstrip()
1235
1236    # Parse initial rev from recipe if not specified
1237    commits = {}
1238    patches = []
1239    initial_revs = {}
1240    with open(recipe_path, 'r') as f:
1241        for line in f:
1242            pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$'
1243            match = re.search(pattern, line)
1244            if match:
1245                name = match.group(1)
1246                rev = match.group(2)
1247                if line.startswith('# initial_rev'):
1248                    if not (name == "." and initial_rev):
1249                        initial_revs[name] = rev
1250                elif line.startswith('# commit') and not force_patch_refresh:
1251                    if name not in commits:
1252                        commits[name] = [rev]
1253                    else:
1254                        commits[name].append(rev)
1255                elif line.startswith('# patches_%s:' % branchname):
1256                    patches = line.split(':')[-1].strip().split(',')
1257
1258    update_revs = dict(initial_revs)
1259    changed_revs = {}
1260    for name, rev in initial_revs.items():
1261        # Find first actually changed revision
1262        stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
1263                                   rev, cwd=os.path.join(srctree, name))
1264        newcommits = stdout.split()
1265        if name in commits:
1266            for i in range(min(len(commits[name]), len(newcommits))):
1267                if newcommits[i] == commits[name][i]:
1268                    update_revs[name] = commits[name][i]
1269
1270        try:
1271            stdout, _ = bb.process.run('git cherry devtool-patched',
1272                                        cwd=os.path.join(srctree, name))
1273        except bb.process.ExecutionError as err:
1274            stdout = None
1275
1276        if stdout is not None and not force_patch_refresh:
1277            for line in stdout.splitlines():
1278                if line.startswith('+ '):
1279                    rev = line.split()[1]
1280                    if rev in newcommits:
1281                        if name not in changed_revs:
1282                            changed_revs[name] = [rev]
1283                        else:
1284                            changed_revs[name].append(rev)
1285
1286    return initial_revs, update_revs, changed_revs, patches
1287
1288def _remove_file_entries(srcuri, filelist):
1289    """Remove file:// entries from SRC_URI"""
1290    remaining = filelist[:]
1291    entries = []
1292    for fname in filelist:
1293        basename = os.path.basename(fname)
1294        for i in range(len(srcuri)):
1295            if (srcuri[i].startswith('file://') and
1296                    os.path.basename(srcuri[i].split(';')[0]) == basename):
1297                entries.append(srcuri[i])
1298                remaining.remove(fname)
1299                srcuri.pop(i)
1300                break
1301    return entries, remaining
1302
1303def _replace_srcuri_entry(srcuri, filename, newentry):
1304    """Replace entry corresponding to specified file with a new entry"""
1305    basename = os.path.basename(filename)
1306    for i in range(len(srcuri)):
1307        if os.path.basename(srcuri[i].split(';')[0]) == basename:
1308            srcuri.pop(i)
1309            srcuri.insert(i, newentry)
1310            break
1311
1312def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
1313    """Unlink existing patch files"""
1314
1315    dry_run_suffix = ' (dry-run)' if dry_run else ''
1316
1317    for path in files:
1318        if append:
1319            if not destpath:
1320                raise Exception('destpath should be set here')
1321            path = os.path.join(destpath, os.path.basename(path))
1322
1323        if os.path.exists(path):
1324            if not no_report_remove:
1325                logger.info('Removing file %s%s' % (path, dry_run_suffix))
1326            if not dry_run:
1327                # FIXME "git rm" here would be nice if the file in question is
1328                #       tracked
1329                # FIXME there's a chance that this file is referred to by
1330                #       another recipe, in which case deleting wouldn't be the
1331                #       right thing to do
1332                os.remove(path)
1333                # Remove directory if empty
1334                try:
1335                    os.rmdir(os.path.dirname(path))
1336                except OSError as ose:
1337                    if ose.errno != errno.ENOTEMPTY:
1338                        raise
1339
1340
1341def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None):
1342    """Export patches from srctree to given location.
1343       Returns three-tuple of dicts:
1344         1. updated - patches that already exist in SRCURI
1345         2. added - new patches that don't exist in SRCURI
1346         3  removed - patches that exist in SRCURI but not in exported patches
1347       In each dict the key is the 'basepath' of the URI and value is:
1348         - for updated and added dicts, a dict with 2 optionnal keys:
1349            - 'path': the absolute path to the existing file in recipe space (if any)
1350            - 'patchdir': the directory in wich the patch should be applied (if any)
1351         - for removed dict, the absolute path to the existing file in recipe space
1352    """
1353    import oe.recipeutils
1354    from oe.patch import GitApplyTree
1355    updated = OrderedDict()
1356    added = OrderedDict()
1357    seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
1358
1359    existing_patches = dict((os.path.basename(path), path) for path in
1360                            oe.recipeutils.get_recipe_patches(rd))
1361    logger.debug('Existing patches: %s' % existing_patches)
1362
1363    # Generate patches from Git, exclude local files directory
1364    patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
1365    GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec)
1366    for dirpath, dirnames, filenames in os.walk(destdir):
1367        new_patches = filenames
1368        reldirpath = os.path.relpath(dirpath, destdir)
1369        for new_patch in new_patches:
1370            # Strip numbering from patch names. If it's a git sequence named patch,
1371            # the numbers might not match up since we are starting from a different
1372            # revision This does assume that people are using unique shortlog
1373            # values, but they ought to be anyway...
1374            new_basename = seqpatch_re.match(new_patch).group(2)
1375            match_name = None
1376            for old_patch in existing_patches:
1377                old_basename = seqpatch_re.match(old_patch).group(2)
1378                old_basename_splitext = os.path.splitext(old_basename)
1379                if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
1380                    old_patch_noext = os.path.splitext(old_patch)[0]
1381                    match_name = old_patch_noext
1382                    break
1383                elif new_basename == old_basename:
1384                    match_name = old_patch
1385                    break
1386            if match_name:
1387                # Rename patch files
1388                if new_patch != match_name:
1389                    bb.utils.rename(os.path.join(destdir, new_patch),
1390                              os.path.join(destdir, match_name))
1391                # Need to pop it off the list now before checking changed_revs
1392                oldpath = existing_patches.pop(old_patch)
1393                if changed_revs is not None and dirpath in changed_revs:
1394                    # Avoid updating patches that have not actually changed
1395                    with open(os.path.join(dirpath, match_name), 'r') as f:
1396                        firstlineitems = f.readline().split()
1397                        # Looking for "From <hash>" line
1398                        if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
1399                            if not firstlineitems[1] in changed_revs[dirpath]:
1400                                continue
1401                # Recompress if necessary
1402                if oldpath.endswith(('.gz', '.Z')):
1403                    bb.process.run(['gzip', match_name], cwd=destdir)
1404                    if oldpath.endswith('.gz'):
1405                        match_name += '.gz'
1406                    else:
1407                        match_name += '.Z'
1408                elif oldpath.endswith('.bz2'):
1409                    bb.process.run(['bzip2', match_name], cwd=destdir)
1410                    match_name += '.bz2'
1411                updated[match_name] = {'path' : oldpath}
1412                if reldirpath != ".":
1413                    updated[match_name]['patchdir'] = reldirpath
1414            else:
1415                added[new_patch] = {}
1416                if reldirpath != ".":
1417                    added[new_patch]['patchdir'] = reldirpath
1418
1419    return (updated, added, existing_patches)
1420
1421
1422def _create_kconfig_diff(srctree, rd, outfile):
1423    """Create a kconfig fragment"""
1424    # Only update config fragment if both config files exist
1425    orig_config = os.path.join(srctree, '.config.baseline')
1426    new_config = os.path.join(srctree, '.config.new')
1427    if os.path.exists(orig_config) and os.path.exists(new_config):
1428        cmd = ['diff', '--new-line-format=%L', '--old-line-format=',
1429               '--unchanged-line-format=', orig_config, new_config]
1430        pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1431                                stderr=subprocess.PIPE)
1432        stdout, stderr = pipe.communicate()
1433        if pipe.returncode == 1:
1434            logger.info("Updating config fragment %s" % outfile)
1435            with open(outfile, 'wb') as fobj:
1436                fobj.write(stdout)
1437        elif pipe.returncode == 0:
1438            logger.info("Would remove config fragment %s" % outfile)
1439            if os.path.exists(outfile):
1440                # Remove fragment file in case of empty diff
1441                logger.info("Removing config fragment %s" % outfile)
1442                os.unlink(outfile)
1443        else:
1444            raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr)
1445        return True
1446    return False
1447
1448
1449def _export_local_files(srctree, rd, destdir, srctreebase):
1450    """Copy local files from srctree to given location.
1451       Returns three-tuple of dicts:
1452         1. updated - files that already exist in SRCURI
1453         2. added - new files files that don't exist in SRCURI
1454         3  removed - files that exist in SRCURI but not in exported files
1455      In each dict the key is the 'basepath' of the URI and value is the
1456      absolute path to the existing file in recipe space (if any).
1457    """
1458    import oe.recipeutils
1459
1460    # Find out local files (SRC_URI files that exist in the "recipe space").
1461    # Local files that reside in srctree are not included in patch generation.
1462    # Instead they are directly copied over the original source files (in
1463    # recipe space).
1464    existing_files = oe.recipeutils.get_recipe_local_files(rd)
1465    new_set = None
1466    updated = OrderedDict()
1467    added = OrderedDict()
1468    removed = OrderedDict()
1469
1470    # Get current branch and return early with empty lists
1471    # if on one of the override branches
1472    # (local files are provided only for the main branch and processing
1473    # them against lists from recipe overrides will result in mismatches
1474    # and broken modifications to recipes).
1475    stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1476                               cwd=srctree)
1477    branchname = stdout.rstrip()
1478    if branchname.startswith(override_branch_prefix):
1479        return (updated, added, removed)
1480
1481    local_files_dir = os.path.join(srctreebase, 'oe-local-files')
1482    git_files = _git_ls_tree(srctree)
1483    if 'oe-local-files' in git_files:
1484        # If tracked by Git, take the files from srctree HEAD. First get
1485        # the tree object of the directory
1486        tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
1487        tree = git_files['oe-local-files'][2]
1488        bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
1489                        env=dict(os.environ, GIT_WORK_TREE=destdir,
1490                                 GIT_INDEX_FILE=tmp_index))
1491        new_set = list(_git_ls_tree(srctree, tree, True).keys())
1492    elif os.path.isdir(local_files_dir):
1493        # If not tracked by Git, just copy from working copy
1494        new_set = _ls_tree(local_files_dir)
1495        bb.process.run(['cp', '-ax',
1496                        os.path.join(local_files_dir, '.'), destdir])
1497    else:
1498        new_set = []
1499
1500    # Special handling for kernel config
1501    if bb.data.inherits_class('kernel-yocto', rd):
1502        fragment_fn = 'devtool-fragment.cfg'
1503        fragment_path = os.path.join(destdir, fragment_fn)
1504        if _create_kconfig_diff(srctree, rd, fragment_path):
1505            if os.path.exists(fragment_path):
1506                if fragment_fn not in new_set:
1507                    new_set.append(fragment_fn)
1508                # Copy fragment to local-files
1509                if os.path.isdir(local_files_dir):
1510                    shutil.copy2(fragment_path, local_files_dir)
1511            else:
1512                if fragment_fn in new_set:
1513                    new_set.remove(fragment_fn)
1514                # Remove fragment from local-files
1515                if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
1516                    os.unlink(os.path.join(local_files_dir, fragment_fn))
1517
1518    # Special handling for cml1, ccmake, etc bbclasses that generated
1519    # configuration fragment files that are consumed as source files
1520    for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]:
1521        if bb.data.inherits_class(frag_class, rd):
1522            srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
1523            if os.path.exists(srcpath):
1524                if frag_name not in new_set:
1525                    new_set.append(frag_name)
1526                # copy fragment into destdir
1527                shutil.copy2(srcpath, destdir)
1528                # copy fragment into local files if exists
1529                if os.path.isdir(local_files_dir):
1530                    shutil.copy2(srcpath, local_files_dir)
1531
1532    if new_set is not None:
1533        for fname in new_set:
1534            if fname in existing_files:
1535                origpath = existing_files.pop(fname)
1536                workpath = os.path.join(local_files_dir, fname)
1537                if not filecmp.cmp(origpath, workpath):
1538                    updated[fname] = origpath
1539            elif fname != '.gitignore':
1540                added[fname] = None
1541
1542        workdir = rd.getVar('WORKDIR')
1543        s = rd.getVar('S')
1544        if not s.endswith(os.sep):
1545            s += os.sep
1546
1547        if workdir != s:
1548            # Handle files where subdir= was specified
1549            for fname in list(existing_files.keys()):
1550                # FIXME handle both subdir starting with BP and not?
1551                fworkpath = os.path.join(workdir, fname)
1552                if fworkpath.startswith(s):
1553                    fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
1554                    if os.path.exists(fpath):
1555                        origpath = existing_files.pop(fname)
1556                        if not filecmp.cmp(origpath, fpath):
1557                            updated[fpath] = origpath
1558
1559        removed = existing_files
1560    return (updated, added, removed)
1561
1562
1563def _determine_files_dir(rd):
1564    """Determine the appropriate files directory for a recipe"""
1565    recipedir = rd.getVar('FILE_DIRNAME')
1566    for entry in rd.getVar('FILESPATH').split(':'):
1567        relpth = os.path.relpath(entry, recipedir)
1568        if not os.sep in relpth:
1569            # One (or zero) levels below only, so we don't put anything in machine-specific directories
1570            if os.path.isdir(entry):
1571                return entry
1572    return os.path.join(recipedir, rd.getVar('BPN'))
1573
1574
1575def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
1576    """Implement the 'srcrev' mode of update-recipe"""
1577    import bb
1578    import oe.recipeutils
1579
1580    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1581
1582    recipefile = rd.getVar('FILE')
1583    recipedir = os.path.basename(recipefile)
1584    logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
1585
1586    # Get original SRCREV
1587    old_srcrev = rd.getVar('SRCREV') or ''
1588    if old_srcrev == "INVALID":
1589            raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository')
1590    old_srcrev = {'.': old_srcrev}
1591
1592    # Get HEAD revision
1593    try:
1594        stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1595    except bb.process.ExecutionError as err:
1596        raise DevtoolError('Failed to get HEAD revision in %s: %s' %
1597                           (srctree, err))
1598    srcrev = stdout.strip()
1599    if len(srcrev) != 40:
1600        raise DevtoolError('Invalid hash returned by git: %s' % stdout)
1601
1602    destpath = None
1603    remove_files = []
1604    patchfields = {}
1605    patchfields['SRCREV'] = srcrev
1606    orig_src_uri = rd.getVar('SRC_URI', False) or ''
1607    srcuri = orig_src_uri.split()
1608    tempdir = tempfile.mkdtemp(prefix='devtool')
1609    update_srcuri = False
1610    appendfile = None
1611    try:
1612        local_files_dir = tempfile.mkdtemp(dir=tempdir)
1613        srctreebase = workspace[recipename]['srctreebase']
1614        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1615        if not no_remove:
1616            # Find list of existing patches in recipe file
1617            patches_dir = tempfile.mkdtemp(dir=tempdir)
1618            upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
1619                                                  patches_dir)
1620            logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
1621
1622            # Remove deleted local files and "overlapping" patches
1623            remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value]
1624            if remove_files:
1625                removedentries = _remove_file_entries(srcuri, remove_files)[0]
1626                update_srcuri = True
1627
1628        if appendlayerdir:
1629            files = dict((os.path.join(local_files_dir, key), val) for
1630                          key, val in list(upd_f.items()) + list(new_f.items()))
1631            removevalues = {}
1632            if update_srcuri:
1633                removevalues  = {'SRC_URI': removedentries}
1634                patchfields['SRC_URI'] = '\\\n    '.join(srcuri)
1635            if dry_run_outdir:
1636                logger.info('Creating bbappend (dry-run)')
1637            appendfile, destpath = oe.recipeutils.bbappend_recipe(
1638                    rd, appendlayerdir, files, wildcardver=wildcard_version,
1639                    extralines=patchfields, removevalues=removevalues,
1640                    redirect_output=dry_run_outdir)
1641        else:
1642            files_dir = _determine_files_dir(rd)
1643            for basepath, path in upd_f.items():
1644                logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
1645                if os.path.isabs(basepath):
1646                    # Original file (probably with subdir pointing inside source tree)
1647                    # so we do not want to move it, just copy
1648                    _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1649                else:
1650                    _move_file(os.path.join(local_files_dir, basepath), path,
1651                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1652                update_srcuri= True
1653            for basepath, path in new_f.items():
1654                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1655                _move_file(os.path.join(local_files_dir, basepath),
1656                           os.path.join(files_dir, basepath),
1657                           dry_run_outdir=dry_run_outdir,
1658                           base_outdir=recipedir)
1659                srcuri.append('file://%s' % basepath)
1660                update_srcuri = True
1661            if update_srcuri:
1662                patchfields['SRC_URI'] = ' '.join(srcuri)
1663            ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
1664    finally:
1665        shutil.rmtree(tempdir)
1666    if not 'git://' in orig_src_uri:
1667        logger.info('You will need to update SRC_URI within the recipe to '
1668                    'point to a git repository where you have pushed your '
1669                    'changes')
1670
1671    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1672    return True, appendfile, remove_files
1673
1674def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False):
1675    """Implement the 'patch' mode of update-recipe"""
1676    import bb
1677    import oe.recipeutils
1678
1679    recipefile = rd.getVar('FILE')
1680    recipedir = os.path.dirname(recipefile)
1681    append = workspace[recipename]['bbappend']
1682    if not os.path.exists(append):
1683        raise DevtoolError('unable to find workspace bbappend for recipe %s' %
1684                           recipename)
1685    srctreebase = workspace[recipename]['srctreebase']
1686    relpatchdir = os.path.relpath(srctreebase, srctree)
1687    if relpatchdir == '.':
1688        patchdir_params = {}
1689    else:
1690        patchdir_params = {'patchdir': relpatchdir}
1691
1692    def srcuri_entry(basepath, patchdir_params):
1693        if patchdir_params:
1694            paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items())
1695        else:
1696            paramstr = ''
1697        return 'file://%s%s' % (basepath, paramstr)
1698
1699    initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
1700    if not initial_revs:
1701        raise DevtoolError('Unable to find initial revision - please specify '
1702                           'it with --initial-rev')
1703
1704    appendfile = None
1705    dl_dir = rd.getVar('DL_DIR')
1706    if not dl_dir.endswith('/'):
1707        dl_dir += '/'
1708
1709    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1710
1711    tempdir = tempfile.mkdtemp(prefix='devtool')
1712    try:
1713        local_files_dir = tempfile.mkdtemp(dir=tempdir)
1714        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1715
1716        # Get updated patches from source tree
1717        patches_dir = tempfile.mkdtemp(dir=tempdir)
1718        upd_p, new_p, _ = _export_patches(srctree, rd, update_revs,
1719                                          patches_dir, changed_revs)
1720        # Get all patches from source tree and check if any should be removed
1721        all_patches_dir = tempfile.mkdtemp(dir=tempdir)
1722        _, _, del_p = _export_patches(srctree, rd, initial_revs,
1723                                      all_patches_dir)
1724        logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
1725        if filter_patches:
1726            new_p = OrderedDict()
1727            upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches)
1728            del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches)
1729        remove_files = []
1730        if not no_remove:
1731            # Remove deleted local files and  patches
1732            remove_files = list(del_f.values()) + list(del_p.values())
1733        updatefiles = False
1734        updaterecipe = False
1735        destpath = None
1736        srcuri = (rd.getVar('SRC_URI', False) or '').split()
1737
1738        if appendlayerdir:
1739            files = OrderedDict((os.path.join(local_files_dir, key), val) for
1740                         key, val in list(upd_f.items()) + list(new_f.items()))
1741            files.update(OrderedDict((os.path.join(patches_dir, key), val) for
1742                              key, val in list(upd_p.items()) + list(new_p.items())))
1743
1744            params = []
1745            for file, param in files.items():
1746                patchdir_param = dict(patchdir_params)
1747                patchdir = param.get('patchdir', ".")
1748                if patchdir != "." :
1749                    if patchdir_param:
1750                       patchdir_param['patchdir'] += patchdir
1751                    else:
1752                        patchdir_param['patchdir'] = patchdir
1753                params.append(patchdir_param)
1754
1755            if files or remove_files:
1756                removevalues = None
1757                if remove_files:
1758                    removedentries, remaining = _remove_file_entries(
1759                                                    srcuri, remove_files)
1760                    if removedentries or remaining:
1761                        remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for
1762                                     item in remaining]
1763                        removevalues = {'SRC_URI': removedentries + remaining}
1764                appendfile, destpath = oe.recipeutils.bbappend_recipe(
1765                                rd, appendlayerdir, files,
1766                                wildcardver=wildcard_version,
1767                                removevalues=removevalues,
1768                                redirect_output=dry_run_outdir,
1769                                params=params)
1770            else:
1771                logger.info('No patches or local source files needed updating')
1772        else:
1773            # Update existing files
1774            files_dir = _determine_files_dir(rd)
1775            for basepath, path in upd_f.items():
1776                logger.info('Updating file %s' % basepath)
1777                if os.path.isabs(basepath):
1778                    # Original file (probably with subdir pointing inside source tree)
1779                    # so we do not want to move it, just copy
1780                    _copy_file(basepath, path,
1781                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1782                else:
1783                    _move_file(os.path.join(local_files_dir, basepath), path,
1784                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1785                updatefiles = True
1786            for basepath, param in upd_p.items():
1787                path = param['path']
1788                patchdir = param.get('patchdir', ".")
1789                if patchdir != "." :
1790                    patchdir_param = dict(patchdir_params)
1791                    if patchdir_param:
1792                       patchdir_param['patchdir'] += patchdir
1793                    else:
1794                        patchdir_param['patchdir'] = patchdir
1795                patchfn = os.path.join(patches_dir, patchdir, basepath)
1796                if os.path.dirname(path) + '/' == dl_dir:
1797                    # This is a a downloaded patch file - we now need to
1798                    # replace the entry in SRC_URI with our local version
1799                    logger.info('Replacing remote patch %s with updated local version' % basepath)
1800                    path = os.path.join(files_dir, basepath)
1801                    _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param))
1802                    updaterecipe = True
1803                else:
1804                    logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
1805                _move_file(patchfn, path,
1806                           dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1807                updatefiles = True
1808            # Add any new files
1809            for basepath, path in new_f.items():
1810                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1811                _move_file(os.path.join(local_files_dir, basepath),
1812                           os.path.join(files_dir, basepath),
1813                           dry_run_outdir=dry_run_outdir,
1814                           base_outdir=recipedir)
1815                srcuri.append(srcuri_entry(basepath, patchdir_params))
1816                updaterecipe = True
1817            for basepath, param in new_p.items():
1818                patchdir = param.get('patchdir', ".")
1819                logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
1820                _move_file(os.path.join(patches_dir, patchdir, basepath),
1821                           os.path.join(files_dir, basepath),
1822                           dry_run_outdir=dry_run_outdir,
1823                           base_outdir=recipedir)
1824                params = dict(patchdir_params)
1825                if patchdir != "." :
1826                    if params:
1827                       params['patchdir'] += patchdir
1828                    else:
1829                        params['patchdir'] = patchdir
1830
1831                srcuri.append(srcuri_entry(basepath, params))
1832                updaterecipe = True
1833            # Update recipe, if needed
1834            if _remove_file_entries(srcuri, remove_files)[0]:
1835                updaterecipe = True
1836            if updaterecipe:
1837                if not dry_run_outdir:
1838                    logger.info('Updating recipe %s' % os.path.basename(recipefile))
1839                ret = oe.recipeutils.patch_recipe(rd, recipefile,
1840                                                  {'SRC_URI': ' '.join(srcuri)},
1841                                                  redirect_output=dry_run_outdir)
1842            elif not updatefiles:
1843                # Neither patches nor recipe were updated
1844                logger.info('No patches or files need updating')
1845                return False, None, []
1846    finally:
1847        shutil.rmtree(tempdir)
1848
1849    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1850    return True, appendfile, remove_files
1851
1852def _guess_recipe_update_mode(srctree, rdata):
1853    """Guess the recipe update mode to use"""
1854    src_uri = (rdata.getVar('SRC_URI') or '').split()
1855    git_uris = [uri for uri in src_uri if uri.startswith('git://')]
1856    if not git_uris:
1857        return 'patch'
1858    # Just use the first URI for now
1859    uri = git_uris[0]
1860    # Check remote branch
1861    params = bb.fetch.decodeurl(uri)[5]
1862    upstr_branch = params['branch'] if 'branch' in params else 'master'
1863    # Check if current branch HEAD is found in upstream branch
1864    stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1865    head_rev = stdout.rstrip()
1866    stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev,
1867                               cwd=srctree)
1868    remote_brs = [branch.strip() for branch in stdout.splitlines()]
1869    if 'origin/' + upstr_branch in remote_brs:
1870        return 'srcrev'
1871
1872    return 'patch'
1873
1874def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False):
1875    srctree = workspace[recipename]['srctree']
1876    if mode == 'auto':
1877        mode = _guess_recipe_update_mode(srctree, rd)
1878
1879    override_branches = []
1880    mainbranch = None
1881    startbranch = None
1882    if not no_overrides:
1883        stdout, _ = bb.process.run('git branch', cwd=srctree)
1884        other_branches = []
1885        for line in stdout.splitlines():
1886            branchname = line[2:]
1887            if line.startswith('* '):
1888                startbranch = branchname
1889            if branchname.startswith(override_branch_prefix):
1890                override_branches.append(branchname)
1891            else:
1892                other_branches.append(branchname)
1893
1894        if override_branches:
1895            logger.debug('_update_recipe: override branches: %s' % override_branches)
1896            logger.debug('_update_recipe: other branches: %s' % other_branches)
1897            if startbranch.startswith(override_branch_prefix):
1898                if len(other_branches) == 1:
1899                    mainbranch = other_branches[1]
1900                else:
1901                    raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
1902            else:
1903                mainbranch = startbranch
1904
1905    checkedout = None
1906    anyupdated = False
1907    appendfile = None
1908    allremoved = []
1909    if override_branches:
1910        logger.info('Handling main branch (%s)...' % mainbranch)
1911        if startbranch != mainbranch:
1912            bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
1913        checkedout = mainbranch
1914    try:
1915        branchlist = [mainbranch] + override_branches
1916        for branch in branchlist:
1917            crd = bb.data.createCopy(rd)
1918            if branch != mainbranch:
1919                logger.info('Handling branch %s...' % branch)
1920                override = branch[len(override_branch_prefix):]
1921                crd.appendVar('OVERRIDES', ':%s' % override)
1922                bb.process.run('git checkout %s' % branch, cwd=srctree)
1923                checkedout = branch
1924
1925            if mode == 'srcrev':
1926                updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
1927            elif mode == 'patch':
1928                updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh)
1929            else:
1930                raise DevtoolError('update_recipe: invalid mode %s' % mode)
1931            if updated:
1932                anyupdated = True
1933            if appendf:
1934                appendfile = appendf
1935            allremoved.extend(removed)
1936    finally:
1937        if startbranch and checkedout != startbranch:
1938            bb.process.run('git checkout %s' % startbranch, cwd=srctree)
1939
1940    return anyupdated, appendfile, allremoved
1941
1942def update_recipe(args, config, basepath, workspace):
1943    """Entry point for the devtool 'update-recipe' subcommand"""
1944    check_workspace_recipe(workspace, args.recipename)
1945
1946    if args.append:
1947        if not os.path.exists(args.append):
1948            raise DevtoolError('bbappend destination layer directory "%s" '
1949                               'does not exist' % args.append)
1950        if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')):
1951            raise DevtoolError('conf/layer.conf not found in bbappend '
1952                               'destination layer "%s"' % args.append)
1953
1954    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1955    try:
1956
1957        rd = parse_recipe(config, tinfoil, args.recipename, True)
1958        if not rd:
1959            return 1
1960
1961        dry_run_output = None
1962        dry_run_outdir = None
1963        if args.dry_run:
1964            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
1965            dry_run_outdir = dry_run_output.name
1966        updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
1967
1968        if updated:
1969            rf = rd.getVar('FILE')
1970            if rf.startswith(config.workspace_path):
1971                logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
1972    finally:
1973        tinfoil.shutdown()
1974
1975    return 0
1976
1977
1978def status(args, config, basepath, workspace):
1979    """Entry point for the devtool 'status' subcommand"""
1980    if workspace:
1981        for recipe, value in sorted(workspace.items()):
1982            recipefile = value['recipefile']
1983            if recipefile:
1984                recipestr = ' (%s)' % recipefile
1985            else:
1986                recipestr = ''
1987            print("%s: %s%s" % (recipe, value['srctree'], recipestr))
1988    else:
1989        logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
1990    return 0
1991
1992
1993def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
1994    """Reset one or more recipes"""
1995    import oe.path
1996
1997    def clean_preferred_provider(pn, layerconf_path):
1998        """Remove PREFERRED_PROVIDER from layer.conf'"""
1999        import re
2000        layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf')
2001        new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf')
2002        pprovider_found = False
2003        with open(layerconf_file, 'r') as f:
2004            lines = f.readlines()
2005            with open(new_layerconf_file, 'a') as nf:
2006                for line in lines:
2007                    pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$'
2008                    if not re.match(pprovider_exp, line):
2009                        nf.write(line)
2010                    else:
2011                        pprovider_found = True
2012        if pprovider_found:
2013            shutil.move(new_layerconf_file, layerconf_file)
2014        else:
2015            os.remove(new_layerconf_file)
2016
2017    if recipes and not no_clean:
2018        if len(recipes) == 1:
2019            logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
2020        else:
2021            logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes))
2022        # If the recipe file itself was created in the workspace, and
2023        # it uses BBCLASSEXTEND, then we need to also clean the other
2024        # variants
2025        targets = []
2026        for recipe in recipes:
2027            targets.append(recipe)
2028            recipefile = workspace[recipe]['recipefile']
2029            if recipefile and os.path.exists(recipefile):
2030                targets.extend(get_bbclassextend_targets(recipefile, recipe))
2031        try:
2032            exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets))
2033        except bb.process.ExecutionError as e:
2034            raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you '
2035                                'wish, you may specify -n/--no-clean to '
2036                                'skip running this command when resetting' %
2037                                (e.command, e.stdout))
2038
2039    for pn in recipes:
2040        _check_preserve(config, pn)
2041
2042        appendfile = workspace[pn]['bbappend']
2043        if os.path.exists(appendfile):
2044            # This shouldn't happen, but is possible if devtool errored out prior to
2045            # writing the md5 file. We need to delete this here or the recipe won't
2046            # actually be reset
2047            os.remove(appendfile)
2048
2049        preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
2050        def preservedir(origdir):
2051            if os.path.exists(origdir):
2052                for root, dirs, files in os.walk(origdir):
2053                    for fn in files:
2054                        logger.warning('Preserving %s in %s' % (fn, preservepath))
2055                        _move_file(os.path.join(origdir, fn),
2056                                   os.path.join(preservepath, fn))
2057                    for dn in dirs:
2058                        preservedir(os.path.join(root, dn))
2059                os.rmdir(origdir)
2060
2061        recipefile = workspace[pn]['recipefile']
2062        if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile):
2063            # This should always be true if recipefile is set, but just in case
2064            preservedir(os.path.dirname(recipefile))
2065        # We don't automatically create this dir next to appends, but the user can
2066        preservedir(os.path.join(config.workspace_path, 'appends', pn))
2067
2068        srctreebase = workspace[pn]['srctreebase']
2069        if os.path.isdir(srctreebase):
2070            if os.listdir(srctreebase):
2071                    if remove_work:
2072                        logger.info('-r argument used on %s, removing source tree.'
2073                                    ' You will lose any unsaved work' %pn)
2074                        shutil.rmtree(srctreebase)
2075                    else:
2076                        # We don't want to risk wiping out any work in progress
2077                        if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')):
2078                            from datetime import datetime
2079                            preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S")))
2080                            logger.info('Preserving source tree in %s\nIf you no '
2081                                        'longer need it then please delete it manually.\n'
2082                                        'It is also possible to reuse it via devtool source tree argument.'
2083                                        % preservesrc)
2084                            bb.utils.mkdirhier(os.path.dirname(preservesrc))
2085                            shutil.move(srctreebase, preservesrc)
2086                        else:
2087                            logger.info('Leaving source tree %s as-is; if you no '
2088                                        'longer need it then please delete it manually'
2089                                        % srctreebase)
2090            else:
2091                # This is unlikely, but if it's empty we can just remove it
2092                os.rmdir(srctreebase)
2093
2094        clean_preferred_provider(pn, config.workspace_path)
2095
2096def reset(args, config, basepath, workspace):
2097    """Entry point for the devtool 'reset' subcommand"""
2098    import bb
2099    import shutil
2100
2101    recipes = ""
2102
2103    if args.recipename:
2104        if args.all:
2105            raise DevtoolError("Recipe cannot be specified if -a/--all is used")
2106        else:
2107            for recipe in args.recipename:
2108                check_workspace_recipe(workspace, recipe, checksrc=False)
2109    elif not args.all:
2110        raise DevtoolError("Recipe must be specified, or specify -a/--all to "
2111                           "reset all recipes")
2112    if args.all:
2113        recipes = list(workspace.keys())
2114    else:
2115        recipes = args.recipename
2116
2117    _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace)
2118
2119    return 0
2120
2121
2122def _get_layer(layername, d):
2123    """Determine the base layer path for the specified layer name/path"""
2124    layerdirs = d.getVar('BBLAYERS').split()
2125    layers = {}    # {basename: layer_paths}
2126    for p in layerdirs:
2127        bn = os.path.basename(p)
2128        if bn not in layers:
2129            layers[bn] = [p]
2130        else:
2131            layers[bn].append(p)
2132    # Provide some shortcuts
2133    if layername.lower() in ['oe-core', 'openembedded-core']:
2134        layername = 'meta'
2135    layer_paths = layers.get(layername, None)
2136    if not layer_paths:
2137        return os.path.abspath(layername)
2138    elif len(layer_paths) == 1:
2139        return os.path.abspath(layer_paths[0])
2140    else:
2141        # multiple layers having the same base name
2142        logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0]))
2143        logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths))
2144        return os.path.abspath(layer_paths[0])
2145
2146
2147def finish(args, config, basepath, workspace):
2148    """Entry point for the devtool 'finish' subcommand"""
2149    import bb
2150    import oe.recipeutils
2151
2152    check_workspace_recipe(workspace, args.recipename)
2153
2154    dry_run_suffix = ' (dry-run)' if args.dry_run else ''
2155
2156    # Grab the equivalent of COREBASE without having to initialise tinfoil
2157    corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
2158
2159    srctree = workspace[args.recipename]['srctree']
2160    check_git_repo_op(srctree, [corebasedir])
2161    dirty = check_git_repo_dirty(srctree)
2162    if dirty:
2163        if args.force:
2164            logger.warning('Source tree is not clean, continuing as requested by -f/--force')
2165        else:
2166            raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty)
2167
2168    no_clean = args.no_clean
2169    remove_work=args.remove_work
2170    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
2171    try:
2172        rd = parse_recipe(config, tinfoil, args.recipename, True)
2173        if not rd:
2174            return 1
2175
2176        destlayerdir = _get_layer(args.destination, tinfoil.config_data)
2177        recipefile = rd.getVar('FILE')
2178        recipedir = os.path.dirname(recipefile)
2179        origlayerdir = oe.recipeutils.find_layerdir(recipefile)
2180
2181        if not os.path.isdir(destlayerdir):
2182            raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
2183
2184        if os.path.abspath(destlayerdir) == config.workspace_path:
2185            raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
2186
2187        # If it's an upgrade, grab the original path
2188        origpath = None
2189        origfilelist = None
2190        append = workspace[args.recipename]['bbappend']
2191        with open(append, 'r') as f:
2192            for line in f:
2193                if line.startswith('# original_path:'):
2194                    origpath = line.split(':')[1].strip()
2195                elif line.startswith('# original_files:'):
2196                    origfilelist = line.split(':')[1].split()
2197
2198        destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir)
2199
2200        if origlayerdir == config.workspace_path:
2201            # Recipe file itself is in workspace, update it there first
2202            appendlayerdir = None
2203            origrelpath = None
2204            if origpath:
2205                origlayerpath = oe.recipeutils.find_layerdir(origpath)
2206                if origlayerpath:
2207                    origrelpath = os.path.relpath(origpath, origlayerpath)
2208            destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
2209            if not destpath:
2210                raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
2211            # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
2212            layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
2213            if not os.path.abspath(destlayerbasedir) in layerdirs:
2214                bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
2215
2216        elif destlayerdir == origlayerdir:
2217            # Same layer, update the original recipe
2218            appendlayerdir = None
2219            destpath = None
2220        else:
2221            # Create/update a bbappend in the specified layer
2222            appendlayerdir = destlayerdir
2223            destpath = None
2224
2225        # Actually update the recipe / bbappend
2226        removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
2227        dry_run_output = None
2228        dry_run_outdir = None
2229        if args.dry_run:
2230            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
2231            dry_run_outdir = dry_run_output.name
2232        updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
2233        removed = [os.path.relpath(pth, recipedir) for pth in removed]
2234
2235        # Remove any old files in the case of an upgrade
2236        if removing_original:
2237            for fn in origfilelist:
2238                fnp = os.path.join(origpath, fn)
2239                if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
2240                    logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
2241                if not args.dry_run:
2242                    try:
2243                        os.remove(fnp)
2244                    except FileNotFoundError:
2245                        pass
2246
2247        if origlayerdir == config.workspace_path and destpath:
2248            # Recipe file itself is in the workspace - need to move it and any
2249            # associated files to the specified layer
2250            no_clean = True
2251            logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
2252            for root, _, files in os.walk(recipedir):
2253                for fn in files:
2254                    srcpath = os.path.join(root, fn)
2255                    relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
2256                    destdir = os.path.abspath(os.path.join(destpath, relpth))
2257                    destfp = os.path.join(destdir, fn)
2258                    _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
2259
2260        if dry_run_outdir:
2261            import difflib
2262            comparelist = []
2263            for root, _, files in os.walk(dry_run_outdir):
2264                for fn in files:
2265                    outf = os.path.join(root, fn)
2266                    relf = os.path.relpath(outf, dry_run_outdir)
2267                    logger.debug('dry-run: output file %s' % relf)
2268                    if fn.endswith('.bb'):
2269                        if origfilelist and origpath and destpath:
2270                            # Need to match this up with the pre-upgrade recipe file
2271                            for origf in origfilelist:
2272                                if origf.endswith('.bb'):
2273                                    comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
2274                                                        outf,
2275                                                        os.path.abspath(os.path.join(destpath, relf))))
2276                                    break
2277                        else:
2278                            # Compare to the existing recipe
2279                            comparelist.append((recipefile, outf, recipefile))
2280                    elif fn.endswith('.bbappend'):
2281                        if appendfile:
2282                            if os.path.exists(appendfile):
2283                                comparelist.append((appendfile, outf, appendfile))
2284                            else:
2285                                comparelist.append((None, outf, appendfile))
2286                    else:
2287                        if destpath:
2288                            recipedest = destpath
2289                        elif appendfile:
2290                            recipedest = os.path.dirname(appendfile)
2291                        else:
2292                            recipedest = os.path.dirname(recipefile)
2293                        destfp = os.path.join(recipedest, relf)
2294                        if os.path.exists(destfp):
2295                            comparelist.append((destfp, outf, destfp))
2296            output = ''
2297            for oldfile, newfile, newfileshow in comparelist:
2298                if oldfile:
2299                    with open(oldfile, 'r') as f:
2300                        oldlines = f.readlines()
2301                else:
2302                    oldfile = '/dev/null'
2303                    oldlines = []
2304                with open(newfile, 'r') as f:
2305                    newlines = f.readlines()
2306                if not newfileshow:
2307                    newfileshow = newfile
2308                diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
2309                difflines = list(diff)
2310                if difflines:
2311                    output += ''.join(difflines)
2312            if output:
2313                logger.info('Diff of changed files:\n%s' % output)
2314    finally:
2315        tinfoil.shutdown()
2316
2317    # Everything else has succeeded, we can now reset
2318    if args.dry_run:
2319        logger.info('Resetting recipe (dry-run)')
2320    else:
2321        _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace)
2322
2323    return 0
2324
2325
2326def get_default_srctree(config, recipename=''):
2327    """Get the default srctree path"""
2328    srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
2329    if recipename:
2330        return os.path.join(srctreeparent, 'sources', recipename)
2331    else:
2332        return os.path.join(srctreeparent, 'sources')
2333
2334def register_commands(subparsers, context):
2335    """Register devtool subcommands from this plugin"""
2336
2337    defsrctree = get_default_srctree(context.config)
2338    parser_add = subparsers.add_parser('add', help='Add a new recipe',
2339                                       description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
2340                                       group='starting', order=100)
2341    parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
2342    parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
2343    parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')
2344    group = parser_add.add_mutually_exclusive_group()
2345    group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2346    group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
2347    parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
2348    parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
2349    parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true")
2350    parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
2351    parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
2352    group = parser_add.add_mutually_exclusive_group()
2353    group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)')
2354    group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
2355    parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)')
2356    parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
2357    parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
2358    parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
2359    parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true")
2360    parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl')
2361    parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup)
2362
2363    parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
2364                                       description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
2365                                       group='starting', order=90)
2366    parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
2367    parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
2368    parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
2369    group = parser_modify.add_mutually_exclusive_group()
2370    group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
2371    group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
2372    group = parser_modify.add_mutually_exclusive_group()
2373    group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2374    group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
2375    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
2376    parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
2377    parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
2378    parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
2379
2380    parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
2381                                       description='Extracts the source for an existing recipe',
2382                                       group='advanced')
2383    parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
2384    parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
2385    parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
2386    parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
2387    parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
2388    parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
2389
2390    parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
2391                                       description='Synchronize the previously extracted source tree for an existing recipe',
2392                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter,
2393                                       group='advanced')
2394    parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
2395    parser_sync.add_argument('srctree', help='Path to the source tree')
2396    parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
2397    parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
2398    parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup)
2399
2400    parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
2401                                       description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',
2402                                       group='working', order=10)
2403    parser_rename.add_argument('recipename', help='Current name of recipe to rename')
2404    parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)')
2405    parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)')
2406    parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path')
2407    parser_rename.set_defaults(func=rename)
2408
2409    parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
2410                                       description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
2411                                       group='working', order=-90)
2412    parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
2413    parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
2414    parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
2415    parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
2416    parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
2417    parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
2418    parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2419    parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2420    parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
2421    parser_update_recipe.set_defaults(func=update_recipe)
2422
2423    parser_status = subparsers.add_parser('status', help='Show workspace status',
2424                                          description='Lists recipes currently in your workspace and the paths to their respective external source trees',
2425                                          group='info', order=100)
2426    parser_status.set_defaults(func=status)
2427
2428    parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
2429                                         description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).',
2430                                         group='working', order=-100)
2431    parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset')
2432    parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
2433    parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
2434    parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append')
2435    parser_reset.set_defaults(func=reset)
2436
2437    parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
2438                                         description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.',
2439                                         group='working', order=-100)
2440    parser_finish.add_argument('recipename', help='Recipe to finish')
2441    parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
2442    parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
2443    parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
2444    parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
2445    parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace')
2446    parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
2447    parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2448    parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2449    parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
2450    parser_finish.set_defaults(func=finish)
2451