1#!/usr/bin/env python3
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9"""Devtool plugins module"""
10
11import os
12import sys
13import subprocess
14import logging
15import re
16import codecs
17
18logger = logging.getLogger('devtool')
19
20class DevtoolError(Exception):
21    """Exception for handling devtool errors"""
22    def __init__(self, message, exitcode=1):
23        super(DevtoolError, self).__init__(message)
24        self.exitcode = exitcode
25
26
27def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28    """Run a program in bitbake build context"""
29    import bb
30    if not 'cwd' in options:
31        options["cwd"] = builddir
32    if init_path:
33        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34        # and can't determine it when running under dash, we need to set
35        # the executable to bash to correctly set things up
36        if not 'executable' in options:
37            options['executable'] = 'bash'
38        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40    else:
41        logger.debug('Executing command "%s"' % cmd)
42        init_prefix = ''
43    if watch:
44        if sys.stdout.isatty():
45            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46            cmd = 'script -e -q -c "%s" /dev/null' % cmd
47        return exec_watch('%s%s' % (init_prefix, cmd), **options)
48    else:
49        return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50
51def exec_watch(cmd, **options):
52    """Run program with stdout shown on sys.stdout"""
53    import bb
54    if isinstance(cmd, str) and not "shell" in options:
55        options["shell"] = True
56
57    process = subprocess.Popen(
58        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59    )
60
61    reader = codecs.getreader('utf-8')(process.stdout)
62    buf = ''
63    while True:
64        out = reader.read(1, 1)
65        if out:
66            sys.stdout.write(out)
67            sys.stdout.flush()
68            buf += out
69        elif out == '' and process.poll() != None:
70            break
71
72    if process.returncode != 0:
73        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74
75    return buf, None
76
77def exec_fakeroot(d, cmd, **kwargs):
78    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79    # Grab the command and check it actually exists
80    fakerootcmd = d.getVar('FAKEROOTCMD')
81    if not os.path.exists(fakerootcmd):
82        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83        return 2
84    # Set up the appropriate environment
85    newenv = dict(os.environ)
86    fakerootenv = d.getVar('FAKEROOTENV')
87    for varvalue in fakerootenv.split():
88        if '=' in varvalue:
89            splitval = varvalue.split('=', 1)
90            newenv[splitval[0]] = splitval[1]
91    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
92
93def setup_tinfoil(config_only=False, basepath=None, tracking=False):
94    """Initialize tinfoil api from bitbake"""
95    import scriptpath
96    orig_cwd = os.path.abspath(os.curdir)
97    try:
98        if basepath:
99            os.chdir(basepath)
100        bitbakepath = scriptpath.add_bitbake_lib_path()
101        if not bitbakepath:
102            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
103            sys.exit(1)
104
105        import bb.tinfoil
106        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
107        try:
108            tinfoil.logger.setLevel(logger.getEffectiveLevel())
109            tinfoil.prepare(config_only)
110        except bb.tinfoil.TinfoilUIException:
111            tinfoil.shutdown()
112            raise DevtoolError('Failed to start bitbake environment')
113        except:
114            tinfoil.shutdown()
115            raise
116    finally:
117        os.chdir(orig_cwd)
118    return tinfoil
119
120def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
121    """Parse the specified recipe"""
122    try:
123        recipefile = tinfoil.get_recipe_file(pn)
124    except bb.providers.NoProvider as e:
125        logger.error(str(e))
126        return None
127    if appends:
128        append_files = tinfoil.get_file_appends(recipefile)
129        if filter_workspace:
130            # Filter out appends from the workspace
131            append_files = [path for path in append_files if
132                            not path.startswith(config.workspace_path)]
133    else:
134        append_files = None
135    try:
136        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
137    except Exception as e:
138        logger.error(str(e))
139        return None
140    return rd
141
142def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
143    """
144    Check that a recipe is in the workspace and (optionally) that source
145    is present.
146    """
147
148    workspacepn = pn
149
150    for recipe, value in workspace.items():
151        if recipe == pn:
152            break
153        if bbclassextend:
154            recipefile = value['recipefile']
155            if recipefile:
156                targets = get_bbclassextend_targets(recipefile, recipe)
157                if pn in targets:
158                    workspacepn = recipe
159                    break
160    else:
161        raise DevtoolError("No recipe named '%s' in your workspace" % pn)
162
163    if checksrc:
164        srctree = workspace[workspacepn]['srctree']
165        if not os.path.exists(srctree):
166            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
167        if not os.listdir(srctree):
168            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
169
170    return workspacepn
171
172def use_external_build(same_dir, no_same_dir, d):
173    """
174    Determine if we should use B!=S (separate build and source directories) or not
175    """
176    b_is_s = True
177    if no_same_dir:
178        logger.info('Using separate build directory since --no-same-dir specified')
179        b_is_s = False
180    elif same_dir:
181        logger.info('Using source tree as build directory since --same-dir specified')
182    elif bb.data.inherits_class('autotools-brokensep', d):
183        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
184    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
185        logger.info('Using source tree as build directory since that would be the default for this recipe')
186    else:
187        b_is_s = False
188    return b_is_s
189
190def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
191    """
192    Set up the git repository for the source tree
193    """
194    import bb.process
195    import oe.patch
196    if not os.path.exists(os.path.join(repodir, '.git')):
197        bb.process.run('git init', cwd=repodir)
198        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
199        bb.process.run('git add -f -A .', cwd=repodir)
200        commit_cmd = ['git']
201        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
202        commit_cmd += ['commit', '-q']
203        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
204        if not stdout:
205            commit_cmd.append('--allow-empty')
206            commitmsg = "Initial empty commit with no upstream sources"
207        elif version:
208            commitmsg = "Initial commit from upstream at version %s" % version
209        else:
210            commitmsg = "Initial commit from upstream"
211        commit_cmd += ['-m', commitmsg]
212        bb.process.run(commit_cmd, cwd=repodir)
213
214    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
215    gitinfodir = os.path.join(repodir, '.git', 'info')
216    try:
217        os.mkdir(gitinfodir)
218    except FileExistsError:
219        pass
220    excludes = []
221    excludefile = os.path.join(gitinfodir, 'exclude')
222    try:
223        with open(excludefile, 'r') as f:
224            excludes = f.readlines()
225    except FileNotFoundError:
226        pass
227    if 'singletask.lock\n' not in excludes:
228        excludes.append('singletask.lock\n')
229    with open(excludefile, 'w') as f:
230        for line in excludes:
231            f.write(line)
232
233    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
234    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
235
236def recipe_to_append(recipefile, config, wildcard=False):
237    """
238    Convert a recipe file to a bbappend file path within the workspace.
239    NOTE: if the bbappend already exists, you should be using
240    workspace[args.recipename]['bbappend'] instead of calling this
241    function.
242    """
243    appendname = os.path.splitext(os.path.basename(recipefile))[0]
244    if wildcard:
245        appendname = re.sub(r'_.*', '_%', appendname)
246    appendpath = os.path.join(config.workspace_path, 'appends')
247    appendfile = os.path.join(appendpath, appendname + '.bbappend')
248    return appendfile
249
250def get_bbclassextend_targets(recipefile, pn):
251    """
252    Cheap function to get BBCLASSEXTEND and then convert that to the
253    list of targets that would result.
254    """
255    import bb.utils
256
257    values = {}
258    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
259        values[varname] = origvalue
260        return origvalue, None, 0, True
261    with open(recipefile, 'r') as f:
262        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
263
264    targets = []
265    bbclassextend = values.get('BBCLASSEXTEND', '').split()
266    if bbclassextend:
267        for variant in bbclassextend:
268            if variant == 'nativesdk':
269                targets.append('%s-%s' % (variant, pn))
270            elif variant in ['native', 'cross', 'crosssdk']:
271                targets.append('%s-%s' % (pn, variant))
272    return targets
273
274def replace_from_file(path, old, new):
275    """Replace strings on a file"""
276
277    def read_file(path):
278        data = None
279        with open(path) as f:
280            data = f.read()
281        return data
282
283    def write_file(path, data):
284        if data is None:
285            return
286        wdata = data.rstrip() + "\n"
287        with open(path, "w") as f:
288            f.write(wdata)
289
290    # In case old is None, return immediately
291    if old is None:
292        return
293    try:
294        rdata = read_file(path)
295    except IOError as e:
296        # if file does not exit, just quit, otherwise raise an exception
297        if e.errno == errno.ENOENT:
298            return
299        else:
300            raise
301
302    old_contents = rdata.splitlines()
303    new_contents = []
304    for old_content in old_contents:
305        try:
306            new_contents.append(old_content.replace(old, new))
307        except ValueError:
308            pass
309    write_file(path, "\n".join(new_contents))
310
311
312def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
313    """ This function will make unlocked-sigs.inc match the recipes in the
314    workspace plus any extras we want unlocked. """
315
316    if not fixed_setup:
317        # Only need to write this out within the eSDK
318        return
319
320    if not extra:
321        extra = []
322
323    confdir = os.path.join(basepath, 'conf')
324    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
325
326    # Get current unlocked list if any
327    values = {}
328    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
329        values[varname] = origvalue
330        return origvalue, None, 0, True
331    if os.path.exists(unlockedsigs):
332        with open(unlockedsigs, 'r') as f:
333            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
334    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
335
336    # If the new list is different to the current list, write it out
337    newunlocked = sorted(list(workspace.keys()) + extra)
338    if unlocked != newunlocked:
339        bb.utils.mkdirhier(confdir)
340        with open(unlockedsigs, 'w') as f:
341            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
342                    "# This layer was created by the OpenEmbedded devtool" +
343                    " utility in order to\n" +
344                    "# contain recipes that are unlocked.\n")
345
346            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
347            for pn in newunlocked:
348                f.write('    ' + pn)
349            f.write('"')
350
351def check_prerelease_version(ver, operation):
352    if 'pre' in ver or 'rc' in ver:
353        logger.warning('Version "%s" looks like a pre-release version. '
354                       'If that is the case, in order to ensure that the '
355                       'version doesn\'t appear to go backwards when you '
356                       'later upgrade to the final release version, it is '
357                       'recommmended that instead you use '
358                       '<current version>+<pre-release version> e.g. if '
359                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
360                       'If you prefer not to reset and re-try, you can change '
361                       'the version after %s succeeds using "devtool rename" '
362                       'with -V/--version.' % (ver, operation))
363
364def check_git_repo_dirty(repodir):
365    """Check if a git repository is clean or not"""
366    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
367    return stdout
368
369def check_git_repo_op(srctree, ignoredirs=None):
370    """Check if a git repository is in the middle of a rebase"""
371    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
372    topleveldir = stdout.strip()
373    if ignoredirs and topleveldir in ignoredirs:
374        return
375    gitdir = os.path.join(topleveldir, '.git')
376    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
377        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
378    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
379        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
380