1#!/usr/bin/env python3
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9"""Devtool plugins module"""
10
11import os
12import sys
13import subprocess
14import logging
15import re
16import codecs
17
18logger = logging.getLogger('devtool')
19
20class DevtoolError(Exception):
21    """Exception for handling devtool errors"""
22    def __init__(self, message, exitcode=1):
23        super(DevtoolError, self).__init__(message)
24        self.exitcode = exitcode
25
26
27def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28    """Run a program in bitbake build context"""
29    import bb
30    if not 'cwd' in options:
31        options["cwd"] = builddir
32    if init_path:
33        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34        # and can't determine it when running under dash, we need to set
35        # the executable to bash to correctly set things up
36        if not 'executable' in options:
37            options['executable'] = 'bash'
38        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40    else:
41        logger.debug('Executing command "%s"' % cmd)
42        init_prefix = ''
43    if watch:
44        if sys.stdout.isatty():
45            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46            cmd = 'script -e -q -c "%s" /dev/null' % cmd
47        return exec_watch('%s%s' % (init_prefix, cmd), **options)
48    else:
49        return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50
51def exec_watch(cmd, **options):
52    """Run program with stdout shown on sys.stdout"""
53    import bb
54    if isinstance(cmd, str) and not "shell" in options:
55        options["shell"] = True
56
57    process = subprocess.Popen(
58        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59    )
60
61    reader = codecs.getreader('utf-8')(process.stdout)
62    buf = ''
63    while True:
64        out = reader.read(1, 1)
65        if out:
66            sys.stdout.write(out)
67            sys.stdout.flush()
68            buf += out
69        elif out == '' and process.poll() != None:
70            break
71
72    if process.returncode != 0:
73        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74
75    return buf, None
76
77def exec_fakeroot(d, cmd, **kwargs):
78    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79    # Grab the command and check it actually exists
80    fakerootcmd = d.getVar('FAKEROOTCMD')
81    if not os.path.exists(fakerootcmd):
82        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83        return 2
84    # Set up the appropriate environment
85    newenv = dict(os.environ)
86    fakerootenv = d.getVar('FAKEROOTENV')
87    for varvalue in fakerootenv.split():
88        if '=' in varvalue:
89            splitval = varvalue.split('=', 1)
90            newenv[splitval[0]] = splitval[1]
91    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
92
93def setup_tinfoil(config_only=False, basepath=None, tracking=False):
94    """Initialize tinfoil api from bitbake"""
95    import scriptpath
96    orig_cwd = os.path.abspath(os.curdir)
97    try:
98        if basepath:
99            os.chdir(basepath)
100        bitbakepath = scriptpath.add_bitbake_lib_path()
101        if not bitbakepath:
102            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
103            sys.exit(1)
104
105        import bb.tinfoil
106        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
107        try:
108            tinfoil.logger.setLevel(logger.getEffectiveLevel())
109            tinfoil.prepare(config_only)
110        except bb.tinfoil.TinfoilUIException:
111            tinfoil.shutdown()
112            raise DevtoolError('Failed to start bitbake environment')
113        except:
114            tinfoil.shutdown()
115            raise
116    finally:
117        os.chdir(orig_cwd)
118    return tinfoil
119
120def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
121    """Parse the specified recipe"""
122    try:
123        recipefile = tinfoil.get_recipe_file(pn)
124    except bb.providers.NoProvider as e:
125        logger.error(str(e))
126        return None
127    if appends:
128        append_files = tinfoil.get_file_appends(recipefile)
129        if filter_workspace:
130            # Filter out appends from the workspace
131            append_files = [path for path in append_files if
132                            not path.startswith(config.workspace_path)]
133    else:
134        append_files = None
135    try:
136        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
137    except Exception as e:
138        logger.error(str(e))
139        return None
140    return rd
141
142def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
143    """
144    Check that a recipe is in the workspace and (optionally) that source
145    is present.
146    """
147
148    workspacepn = pn
149
150    for recipe, value in workspace.items():
151        if recipe == pn:
152            break
153        if bbclassextend:
154            recipefile = value['recipefile']
155            if recipefile:
156                targets = get_bbclassextend_targets(recipefile, recipe)
157                if pn in targets:
158                    workspacepn = recipe
159                    break
160    else:
161        raise DevtoolError("No recipe named '%s' in your workspace" % pn)
162
163    if checksrc:
164        srctree = workspace[workspacepn]['srctree']
165        if not os.path.exists(srctree):
166            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
167        if not os.listdir(srctree):
168            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
169
170    return workspacepn
171
172def use_external_build(same_dir, no_same_dir, d):
173    """
174    Determine if we should use B!=S (separate build and source directories) or not
175    """
176    b_is_s = True
177    if no_same_dir:
178        logger.info('Using separate build directory since --no-same-dir specified')
179        b_is_s = False
180    elif same_dir:
181        logger.info('Using source tree as build directory since --same-dir specified')
182    elif bb.data.inherits_class('autotools-brokensep', d):
183        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
184    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
185        logger.info('Using source tree as build directory since that would be the default for this recipe')
186    else:
187        b_is_s = False
188    return b_is_s
189
190def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
191    """
192    Set up the git repository for the source tree
193    """
194    import bb.process
195    import oe.patch
196    if not os.path.exists(os.path.join(repodir, '.git')):
197        bb.process.run('git init', cwd=repodir)
198        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
199        bb.process.run('git add -f -A .', cwd=repodir)
200        commit_cmd = ['git']
201        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
202        commit_cmd += ['commit', '-q']
203        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
204        if not stdout:
205            commit_cmd.append('--allow-empty')
206            commitmsg = "Initial empty commit with no upstream sources"
207        elif version:
208            commitmsg = "Initial commit from upstream at version %s" % version
209        else:
210            commitmsg = "Initial commit from upstream"
211        commit_cmd += ['-m', commitmsg]
212        bb.process.run(commit_cmd, cwd=repodir)
213
214    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
215    gitinfodir = os.path.join(repodir, '.git', 'info')
216    try:
217        os.mkdir(gitinfodir)
218    except FileExistsError:
219        pass
220    excludes = []
221    excludefile = os.path.join(gitinfodir, 'exclude')
222    try:
223        with open(excludefile, 'r') as f:
224            excludes = f.readlines()
225    except FileNotFoundError:
226        pass
227    if 'singletask.lock\n' not in excludes:
228        excludes.append('singletask.lock\n')
229    with open(excludefile, 'w') as f:
230        for line in excludes:
231            f.write(line)
232
233    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
234    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
235
236    # if recipe unpacks another git repo inside S, we need to declare it as a regular git submodule now,
237    # so we will be able to tag branches on it and extract patches when doing finish/update on the recipe
238    stdout, _ = bb.process.run("git status --porcelain", cwd=repodir)
239    found = False
240    for line in stdout.splitlines():
241        if line.endswith("/"):
242            new_dir = line.split()[1]
243            for root, dirs, files in os.walk(os.path.join(repodir, new_dir)):
244                if ".git" in dirs + files:
245                    (stdout, _) = bb.process.run('git remote', cwd=root)
246                    remote = stdout.splitlines()[0]
247                    (stdout, _) = bb.process.run('git remote get-url %s' % remote, cwd=root)
248                    remote_url = stdout.splitlines()[0]
249                    logger.error(os.path.relpath(os.path.join(root, ".."), root))
250                    bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, ".."))
251                    found = True
252                if found:
253                    useroptions = []
254                    oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
255                    bb.process.run('git %s commit -m "Adding additionnal submodule from SRC_URI\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=os.path.join(root, ".."))
256                    found = False
257    if os.path.exists(os.path.join(repodir, '.gitmodules')):
258        bb.process.run('git submodule foreach --recursive  "git tag -f %s"' % basetag, cwd=repodir)
259
260def recipe_to_append(recipefile, config, wildcard=False):
261    """
262    Convert a recipe file to a bbappend file path within the workspace.
263    NOTE: if the bbappend already exists, you should be using
264    workspace[args.recipename]['bbappend'] instead of calling this
265    function.
266    """
267    appendname = os.path.splitext(os.path.basename(recipefile))[0]
268    if wildcard:
269        appendname = re.sub(r'_.*', '_%', appendname)
270    appendpath = os.path.join(config.workspace_path, 'appends')
271    appendfile = os.path.join(appendpath, appendname + '.bbappend')
272    return appendfile
273
274def get_bbclassextend_targets(recipefile, pn):
275    """
276    Cheap function to get BBCLASSEXTEND and then convert that to the
277    list of targets that would result.
278    """
279    import bb.utils
280
281    values = {}
282    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
283        values[varname] = origvalue
284        return origvalue, None, 0, True
285    with open(recipefile, 'r') as f:
286        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
287
288    targets = []
289    bbclassextend = values.get('BBCLASSEXTEND', '').split()
290    if bbclassextend:
291        for variant in bbclassextend:
292            if variant == 'nativesdk':
293                targets.append('%s-%s' % (variant, pn))
294            elif variant in ['native', 'cross', 'crosssdk']:
295                targets.append('%s-%s' % (pn, variant))
296    return targets
297
298def replace_from_file(path, old, new):
299    """Replace strings on a file"""
300
301    def read_file(path):
302        data = None
303        with open(path) as f:
304            data = f.read()
305        return data
306
307    def write_file(path, data):
308        if data is None:
309            return
310        wdata = data.rstrip() + "\n"
311        with open(path, "w") as f:
312            f.write(wdata)
313
314    # In case old is None, return immediately
315    if old is None:
316        return
317    try:
318        rdata = read_file(path)
319    except IOError as e:
320        # if file does not exit, just quit, otherwise raise an exception
321        if e.errno == errno.ENOENT:
322            return
323        else:
324            raise
325
326    old_contents = rdata.splitlines()
327    new_contents = []
328    for old_content in old_contents:
329        try:
330            new_contents.append(old_content.replace(old, new))
331        except ValueError:
332            pass
333    write_file(path, "\n".join(new_contents))
334
335
336def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
337    """ This function will make unlocked-sigs.inc match the recipes in the
338    workspace plus any extras we want unlocked. """
339
340    if not fixed_setup:
341        # Only need to write this out within the eSDK
342        return
343
344    if not extra:
345        extra = []
346
347    confdir = os.path.join(basepath, 'conf')
348    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
349
350    # Get current unlocked list if any
351    values = {}
352    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
353        values[varname] = origvalue
354        return origvalue, None, 0, True
355    if os.path.exists(unlockedsigs):
356        with open(unlockedsigs, 'r') as f:
357            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
358    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
359
360    # If the new list is different to the current list, write it out
361    newunlocked = sorted(list(workspace.keys()) + extra)
362    if unlocked != newunlocked:
363        bb.utils.mkdirhier(confdir)
364        with open(unlockedsigs, 'w') as f:
365            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
366                    "# This layer was created by the OpenEmbedded devtool" +
367                    " utility in order to\n" +
368                    "# contain recipes that are unlocked.\n")
369
370            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
371            for pn in newunlocked:
372                f.write('    ' + pn)
373            f.write('"')
374
375def check_prerelease_version(ver, operation):
376    if 'pre' in ver or 'rc' in ver:
377        logger.warning('Version "%s" looks like a pre-release version. '
378                       'If that is the case, in order to ensure that the '
379                       'version doesn\'t appear to go backwards when you '
380                       'later upgrade to the final release version, it is '
381                       'recommmended that instead you use '
382                       '<current version>+<pre-release version> e.g. if '
383                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
384                       'If you prefer not to reset and re-try, you can change '
385                       'the version after %s succeeds using "devtool rename" '
386                       'with -V/--version.' % (ver, operation))
387
388def check_git_repo_dirty(repodir):
389    """Check if a git repository is clean or not"""
390    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
391    return stdout
392
393def check_git_repo_op(srctree, ignoredirs=None):
394    """Check if a git repository is in the middle of a rebase"""
395    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
396    topleveldir = stdout.strip()
397    if ignoredirs and topleveldir in ignoredirs:
398        return
399    gitdir = os.path.join(topleveldir, '.git')
400    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
401        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
402    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
403        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
404