1#!/usr/bin/env python3
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9"""Devtool plugins module"""
10
11import os
12import sys
13import subprocess
14import logging
15import re
16import codecs
17
18logger = logging.getLogger('devtool')
19
20class DevtoolError(Exception):
21    """Exception for handling devtool errors"""
22    def __init__(self, message, exitcode=1):
23        super(DevtoolError, self).__init__(message)
24        self.exitcode = exitcode
25
26
27def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28    """Run a program in bitbake build context"""
29    import bb
30    if not 'cwd' in options:
31        options["cwd"] = builddir
32    if init_path:
33        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34        # and can't determine it when running under dash, we need to set
35        # the executable to bash to correctly set things up
36        if not 'executable' in options:
37            options['executable'] = 'bash'
38        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40    else:
41        logger.debug('Executing command "%s"' % cmd)
42        init_prefix = ''
43    if watch:
44        if sys.stdout.isatty():
45            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46            cmd = 'script -e -q -c "%s" /dev/null' % cmd
47        return exec_watch('%s%s' % (init_prefix, cmd), **options)
48    else:
49        return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50
51def exec_watch(cmd, **options):
52    """Run program with stdout shown on sys.stdout"""
53    import bb
54    if isinstance(cmd, str) and not "shell" in options:
55        options["shell"] = True
56
57    process = subprocess.Popen(
58        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59    )
60
61    reader = codecs.getreader('utf-8')(process.stdout)
62    buf = ''
63    while True:
64        out = reader.read(1, 1)
65        if out:
66            sys.stdout.write(out)
67            sys.stdout.flush()
68            buf += out
69        elif out == '' and process.poll() != None:
70            break
71
72    if process.returncode != 0:
73        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74
75    return buf, None
76
77def exec_fakeroot(d, cmd, **kwargs):
78    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79    # Grab the command and check it actually exists
80    fakerootcmd = d.getVar('FAKEROOTCMD')
81    if not os.path.exists(fakerootcmd):
82        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83        return 2
84    # Set up the appropriate environment
85    newenv = dict(os.environ)
86    fakerootenv = d.getVar('FAKEROOTENV')
87    for varvalue in fakerootenv.split():
88        if '=' in varvalue:
89            splitval = varvalue.split('=', 1)
90            newenv[splitval[0]] = splitval[1]
91    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
92
93def setup_tinfoil(config_only=False, basepath=None, tracking=False):
94    """Initialize tinfoil api from bitbake"""
95    import scriptpath
96    orig_cwd = os.path.abspath(os.curdir)
97    try:
98        if basepath:
99            os.chdir(basepath)
100        bitbakepath = scriptpath.add_bitbake_lib_path()
101        if not bitbakepath:
102            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
103            sys.exit(1)
104
105        import bb.tinfoil
106        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
107        try:
108            tinfoil.logger.setLevel(logger.getEffectiveLevel())
109            tinfoil.prepare(config_only)
110        except bb.tinfoil.TinfoilUIException:
111            tinfoil.shutdown()
112            raise DevtoolError('Failed to start bitbake environment')
113        except:
114            tinfoil.shutdown()
115            raise
116    finally:
117        os.chdir(orig_cwd)
118    return tinfoil
119
120def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
121    """Parse the specified recipe"""
122    try:
123        recipefile = tinfoil.get_recipe_file(pn)
124    except bb.providers.NoProvider as e:
125        logger.error(str(e))
126        return None
127    if appends:
128        append_files = tinfoil.get_file_appends(recipefile)
129        if filter_workspace:
130            # Filter out appends from the workspace
131            append_files = [path for path in append_files if
132                            not path.startswith(config.workspace_path)]
133    else:
134        append_files = None
135    try:
136        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
137    except Exception as e:
138        logger.error(str(e))
139        return None
140    return rd
141
142def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
143    """
144    Check that a recipe is in the workspace and (optionally) that source
145    is present.
146    """
147
148    workspacepn = pn
149
150    for recipe, value in workspace.items():
151        if recipe == pn:
152            break
153        if bbclassextend:
154            recipefile = value['recipefile']
155            if recipefile:
156                targets = get_bbclassextend_targets(recipefile, recipe)
157                if pn in targets:
158                    workspacepn = recipe
159                    break
160    else:
161        raise DevtoolError("No recipe named '%s' in your workspace" % pn)
162
163    if checksrc:
164        srctree = workspace[workspacepn]['srctree']
165        if not os.path.exists(srctree):
166            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
167        if not os.listdir(srctree):
168            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
169
170    return workspacepn
171
172def use_external_build(same_dir, no_same_dir, d):
173    """
174    Determine if we should use B!=S (separate build and source directories) or not
175    """
176    b_is_s = True
177    if no_same_dir:
178        logger.info('Using separate build directory since --no-same-dir specified')
179        b_is_s = False
180    elif same_dir:
181        logger.info('Using source tree as build directory since --same-dir specified')
182    elif bb.data.inherits_class('autotools-brokensep', d):
183        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
184    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
185        logger.info('Using source tree as build directory since that would be the default for this recipe')
186    else:
187        b_is_s = False
188    return b_is_s
189
190def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
191    """
192    Set up the git repository for the source tree
193    """
194    import bb.process
195    import oe.patch
196    if not os.path.exists(os.path.join(repodir, '.git')):
197        bb.process.run('git init', cwd=repodir)
198        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
199        bb.process.run('git add .', cwd=repodir)
200        commit_cmd = ['git']
201        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
202        commit_cmd += ['commit', '-q']
203        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
204        if not stdout:
205            commit_cmd.append('--allow-empty')
206            commitmsg = "Initial empty commit with no upstream sources"
207        elif version:
208            commitmsg = "Initial commit from upstream at version %s" % version
209        else:
210            commitmsg = "Initial commit from upstream"
211        commit_cmd += ['-m', commitmsg]
212        bb.process.run(commit_cmd, cwd=repodir)
213
214    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
215    excludes = []
216    excludefile = os.path.join(repodir, '.git', 'info', 'exclude')
217    try:
218        with open(excludefile, 'r') as f:
219            excludes = f.readlines()
220    except FileNotFoundError:
221        pass
222    if 'singletask.lock\n' not in excludes:
223        excludes.append('singletask.lock\n')
224    with open(excludefile, 'w') as f:
225        for line in excludes:
226            f.write(line)
227
228    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
229    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
230
231def recipe_to_append(recipefile, config, wildcard=False):
232    """
233    Convert a recipe file to a bbappend file path within the workspace.
234    NOTE: if the bbappend already exists, you should be using
235    workspace[args.recipename]['bbappend'] instead of calling this
236    function.
237    """
238    appendname = os.path.splitext(os.path.basename(recipefile))[0]
239    if wildcard:
240        appendname = re.sub(r'_.*', '_%', appendname)
241    appendpath = os.path.join(config.workspace_path, 'appends')
242    appendfile = os.path.join(appendpath, appendname + '.bbappend')
243    return appendfile
244
245def get_bbclassextend_targets(recipefile, pn):
246    """
247    Cheap function to get BBCLASSEXTEND and then convert that to the
248    list of targets that would result.
249    """
250    import bb.utils
251
252    values = {}
253    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
254        values[varname] = origvalue
255        return origvalue, None, 0, True
256    with open(recipefile, 'r') as f:
257        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
258
259    targets = []
260    bbclassextend = values.get('BBCLASSEXTEND', '').split()
261    if bbclassextend:
262        for variant in bbclassextend:
263            if variant == 'nativesdk':
264                targets.append('%s-%s' % (variant, pn))
265            elif variant in ['native', 'cross', 'crosssdk']:
266                targets.append('%s-%s' % (pn, variant))
267    return targets
268
269def replace_from_file(path, old, new):
270    """Replace strings on a file"""
271
272    def read_file(path):
273        data = None
274        with open(path) as f:
275            data = f.read()
276        return data
277
278    def write_file(path, data):
279        if data is None:
280            return
281        wdata = data.rstrip() + "\n"
282        with open(path, "w") as f:
283            f.write(wdata)
284
285    # In case old is None, return immediately
286    if old is None:
287        return
288    try:
289        rdata = read_file(path)
290    except IOError as e:
291        # if file does not exit, just quit, otherwise raise an exception
292        if e.errno == errno.ENOENT:
293            return
294        else:
295            raise
296
297    old_contents = rdata.splitlines()
298    new_contents = []
299    for old_content in old_contents:
300        try:
301            new_contents.append(old_content.replace(old, new))
302        except ValueError:
303            pass
304    write_file(path, "\n".join(new_contents))
305
306
307def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
308    """ This function will make unlocked-sigs.inc match the recipes in the
309    workspace plus any extras we want unlocked. """
310
311    if not fixed_setup:
312        # Only need to write this out within the eSDK
313        return
314
315    if not extra:
316        extra = []
317
318    confdir = os.path.join(basepath, 'conf')
319    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
320
321    # Get current unlocked list if any
322    values = {}
323    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
324        values[varname] = origvalue
325        return origvalue, None, 0, True
326    if os.path.exists(unlockedsigs):
327        with open(unlockedsigs, 'r') as f:
328            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
329    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
330
331    # If the new list is different to the current list, write it out
332    newunlocked = sorted(list(workspace.keys()) + extra)
333    if unlocked != newunlocked:
334        bb.utils.mkdirhier(confdir)
335        with open(unlockedsigs, 'w') as f:
336            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
337                    "# This layer was created by the OpenEmbedded devtool" +
338                    " utility in order to\n" +
339                    "# contain recipes that are unlocked.\n")
340
341            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
342            for pn in newunlocked:
343                f.write('    ' + pn)
344            f.write('"')
345
346def check_prerelease_version(ver, operation):
347    if 'pre' in ver or 'rc' in ver:
348        logger.warning('Version "%s" looks like a pre-release version. '
349                       'If that is the case, in order to ensure that the '
350                       'version doesn\'t appear to go backwards when you '
351                       'later upgrade to the final release version, it is '
352                       'recommmended that instead you use '
353                       '<current version>+<pre-release version> e.g. if '
354                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
355                       'If you prefer not to reset and re-try, you can change '
356                       'the version after %s succeeds using "devtool rename" '
357                       'with -V/--version.' % (ver, operation))
358
359def check_git_repo_dirty(repodir):
360    """Check if a git repository is clean or not"""
361    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
362    return stdout
363
364def check_git_repo_op(srctree, ignoredirs=None):
365    """Check if a git repository is in the middle of a rebase"""
366    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
367    topleveldir = stdout.strip()
368    if ignoredirs and topleveldir in ignoredirs:
369        return
370    gitdir = os.path.join(topleveldir, '.git')
371    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
372        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
373    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
374        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
375