1#!/usr/bin/env python3
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9"""Devtool plugins module"""
10
11import os
12import sys
13import subprocess
14import logging
15import re
16import codecs
17
18logger = logging.getLogger('devtool')
19
20class DevtoolError(Exception):
21    """Exception for handling devtool errors"""
22    def __init__(self, message, exitcode=1):
23        super(DevtoolError, self).__init__(message)
24        self.exitcode = exitcode
25
26
27def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28    """Run a program in bitbake build context"""
29    import bb
30    if not 'cwd' in options:
31        options["cwd"] = builddir
32    if init_path:
33        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34        # and can't determine it when running under dash, we need to set
35        # the executable to bash to correctly set things up
36        if not 'executable' in options:
37            options['executable'] = 'bash'
38        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40    else:
41        logger.debug('Executing command "%s"' % cmd)
42        init_prefix = ''
43    if watch:
44        if sys.stdout.isatty():
45            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46            cmd = 'script -e -q -c "%s" /dev/null' % cmd
47        return exec_watch('%s%s' % (init_prefix, cmd), **options)
48    else:
49        return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50
51def exec_watch(cmd, **options):
52    """Run program with stdout shown on sys.stdout"""
53    import bb
54    if isinstance(cmd, str) and not "shell" in options:
55        options["shell"] = True
56
57    process = subprocess.Popen(
58        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59    )
60
61    reader = codecs.getreader('utf-8')(process.stdout)
62    buf = ''
63    while True:
64        out = reader.read(1, 1)
65        if out:
66            sys.stdout.write(out)
67            sys.stdout.flush()
68            buf += out
69        elif out == '' and process.poll() != None:
70            break
71
72    if process.returncode != 0:
73        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74
75    return buf, None
76
77def exec_fakeroot(d, cmd, **kwargs):
78    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79    # Grab the command and check it actually exists
80    fakerootcmd = d.getVar('FAKEROOTCMD')
81    fakerootenv = d.getVar('FAKEROOTENV')
82    exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, kwargs)
83
84def exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, **kwargs):
85    if not os.path.exists(fakerootcmd):
86        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
87        return 2
88    # Set up the appropriate environment
89    newenv = dict(os.environ)
90    for varvalue in fakerootenv.split():
91        if '=' in varvalue:
92            splitval = varvalue.split('=', 1)
93            newenv[splitval[0]] = splitval[1]
94    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
95
96def setup_tinfoil(config_only=False, basepath=None, tracking=False):
97    """Initialize tinfoil api from bitbake"""
98    import scriptpath
99    orig_cwd = os.path.abspath(os.curdir)
100    try:
101        if basepath:
102            os.chdir(basepath)
103        bitbakepath = scriptpath.add_bitbake_lib_path()
104        if not bitbakepath:
105            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
106            sys.exit(1)
107
108        import bb.tinfoil
109        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
110        try:
111            tinfoil.logger.setLevel(logger.getEffectiveLevel())
112            tinfoil.prepare(config_only)
113        except bb.tinfoil.TinfoilUIException:
114            tinfoil.shutdown()
115            raise DevtoolError('Failed to start bitbake environment')
116        except:
117            tinfoil.shutdown()
118            raise
119    finally:
120        os.chdir(orig_cwd)
121    return tinfoil
122
123def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
124    """Parse the specified recipe"""
125    try:
126        recipefile = tinfoil.get_recipe_file(pn)
127    except bb.providers.NoProvider as e:
128        logger.error(str(e))
129        return None
130    if appends:
131        append_files = tinfoil.get_file_appends(recipefile)
132        if filter_workspace:
133            # Filter out appends from the workspace
134            append_files = [path for path in append_files if
135                            not path.startswith(config.workspace_path)]
136    else:
137        append_files = None
138    try:
139        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
140    except Exception as e:
141        logger.error(str(e))
142        return None
143    return rd
144
145def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
146    """
147    Check that a recipe is in the workspace and (optionally) that source
148    is present.
149    """
150
151    workspacepn = pn
152
153    for recipe, value in workspace.items():
154        if recipe == pn:
155            break
156        if bbclassextend:
157            recipefile = value['recipefile']
158            if recipefile:
159                targets = get_bbclassextend_targets(recipefile, recipe)
160                if pn in targets:
161                    workspacepn = recipe
162                    break
163    else:
164        raise DevtoolError("No recipe named '%s' in your workspace" % pn)
165
166    if checksrc:
167        srctree = workspace[workspacepn]['srctree']
168        if not os.path.exists(srctree):
169            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
170        if not os.listdir(srctree):
171            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
172
173    return workspacepn
174
175def use_external_build(same_dir, no_same_dir, d):
176    """
177    Determine if we should use B!=S (separate build and source directories) or not
178    """
179    b_is_s = True
180    if no_same_dir:
181        logger.info('Using separate build directory since --no-same-dir specified')
182        b_is_s = False
183    elif same_dir:
184        logger.info('Using source tree as build directory since --same-dir specified')
185    elif bb.data.inherits_class('autotools-brokensep', d):
186        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
187    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
188        logger.info('Using source tree as build directory since that would be the default for this recipe')
189    else:
190        b_is_s = False
191    return b_is_s
192
193def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
194    """
195    Set up the git repository for the source tree
196    """
197    import bb.process
198    import oe.patch
199    if not os.path.exists(os.path.join(repodir, '.git')):
200        bb.process.run('git init', cwd=repodir)
201        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
202        bb.process.run('git add -f -A .', cwd=repodir)
203        commit_cmd = ['git']
204        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
205        commit_cmd += ['commit', '-q']
206        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
207        if not stdout:
208            commit_cmd.append('--allow-empty')
209            commitmsg = "Initial empty commit with no upstream sources"
210        elif version:
211            commitmsg = "Initial commit from upstream at version %s" % version
212        else:
213            commitmsg = "Initial commit from upstream"
214        commit_cmd += ['-m', commitmsg]
215        bb.process.run(commit_cmd, cwd=repodir)
216
217    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
218    gitinfodir = os.path.join(repodir, '.git', 'info')
219    try:
220        os.mkdir(gitinfodir)
221    except FileExistsError:
222        pass
223    excludes = []
224    excludefile = os.path.join(gitinfodir, 'exclude')
225    try:
226        with open(excludefile, 'r') as f:
227            excludes = f.readlines()
228    except FileNotFoundError:
229        pass
230    if 'singletask.lock\n' not in excludes:
231        excludes.append('singletask.lock\n')
232    with open(excludefile, 'w') as f:
233        for line in excludes:
234            f.write(line)
235
236    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
237    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
238
239    # if recipe unpacks another git repo inside S, we need to declare it as a regular git submodule now,
240    # so we will be able to tag branches on it and extract patches when doing finish/update on the recipe
241    stdout, _ = bb.process.run("git status --porcelain", cwd=repodir)
242    found = False
243    for line in stdout.splitlines():
244        if line.endswith("/"):
245            new_dir = line.split()[1]
246            for root, dirs, files in os.walk(os.path.join(repodir, new_dir)):
247                if ".git" in dirs + files:
248                    (stdout, _) = bb.process.run('git remote', cwd=root)
249                    remote = stdout.splitlines()[0]
250                    (stdout, _) = bb.process.run('git remote get-url %s' % remote, cwd=root)
251                    remote_url = stdout.splitlines()[0]
252                    logger.error(os.path.relpath(os.path.join(root, ".."), root))
253                    bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, ".."))
254                    found = True
255                if found:
256                    oe.patch.GitApplyTree.commitIgnored("Add additional submodule from SRC_URI", dir=os.path.join(root, ".."), d=d)
257                    found = False
258    if os.path.exists(os.path.join(repodir, '.gitmodules')):
259        bb.process.run('git submodule foreach --recursive  "git tag -f %s"' % basetag, cwd=repodir)
260
261def recipe_to_append(recipefile, config, wildcard=False):
262    """
263    Convert a recipe file to a bbappend file path within the workspace.
264    NOTE: if the bbappend already exists, you should be using
265    workspace[args.recipename]['bbappend'] instead of calling this
266    function.
267    """
268    appendname = os.path.splitext(os.path.basename(recipefile))[0]
269    if wildcard:
270        appendname = re.sub(r'_.*', '_%', appendname)
271    appendpath = os.path.join(config.workspace_path, 'appends')
272    appendfile = os.path.join(appendpath, appendname + '.bbappend')
273    return appendfile
274
275def get_bbclassextend_targets(recipefile, pn):
276    """
277    Cheap function to get BBCLASSEXTEND and then convert that to the
278    list of targets that would result.
279    """
280    import bb.utils
281
282    values = {}
283    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
284        values[varname] = origvalue
285        return origvalue, None, 0, True
286    with open(recipefile, 'r') as f:
287        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
288
289    targets = []
290    bbclassextend = values.get('BBCLASSEXTEND', '').split()
291    if bbclassextend:
292        for variant in bbclassextend:
293            if variant == 'nativesdk':
294                targets.append('%s-%s' % (variant, pn))
295            elif variant in ['native', 'cross', 'crosssdk']:
296                targets.append('%s-%s' % (pn, variant))
297    return targets
298
299def replace_from_file(path, old, new):
300    """Replace strings on a file"""
301
302    def read_file(path):
303        data = None
304        with open(path) as f:
305            data = f.read()
306        return data
307
308    def write_file(path, data):
309        if data is None:
310            return
311        wdata = data.rstrip() + "\n"
312        with open(path, "w") as f:
313            f.write(wdata)
314
315    # In case old is None, return immediately
316    if old is None:
317        return
318    try:
319        rdata = read_file(path)
320    except IOError as e:
321        # if file does not exit, just quit, otherwise raise an exception
322        if e.errno == errno.ENOENT:
323            return
324        else:
325            raise
326
327    old_contents = rdata.splitlines()
328    new_contents = []
329    for old_content in old_contents:
330        try:
331            new_contents.append(old_content.replace(old, new))
332        except ValueError:
333            pass
334    write_file(path, "\n".join(new_contents))
335
336
337def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
338    """ This function will make unlocked-sigs.inc match the recipes in the
339    workspace plus any extras we want unlocked. """
340
341    if not fixed_setup:
342        # Only need to write this out within the eSDK
343        return
344
345    if not extra:
346        extra = []
347
348    confdir = os.path.join(basepath, 'conf')
349    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
350
351    # Get current unlocked list if any
352    values = {}
353    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
354        values[varname] = origvalue
355        return origvalue, None, 0, True
356    if os.path.exists(unlockedsigs):
357        with open(unlockedsigs, 'r') as f:
358            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
359    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
360
361    # If the new list is different to the current list, write it out
362    newunlocked = sorted(list(workspace.keys()) + extra)
363    if unlocked != newunlocked:
364        bb.utils.mkdirhier(confdir)
365        with open(unlockedsigs, 'w') as f:
366            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
367                    "# This layer was created by the OpenEmbedded devtool" +
368                    " utility in order to\n" +
369                    "# contain recipes that are unlocked.\n")
370
371            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
372            for pn in newunlocked:
373                f.write('    ' + pn)
374            f.write('"')
375
376def check_prerelease_version(ver, operation):
377    if 'pre' in ver or 'rc' in ver:
378        logger.warning('Version "%s" looks like a pre-release version. '
379                       'If that is the case, in order to ensure that the '
380                       'version doesn\'t appear to go backwards when you '
381                       'later upgrade to the final release version, it is '
382                       'recommmended that instead you use '
383                       '<current version>+<pre-release version> e.g. if '
384                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
385                       'If you prefer not to reset and re-try, you can change '
386                       'the version after %s succeeds using "devtool rename" '
387                       'with -V/--version.' % (ver, operation))
388
389def check_git_repo_dirty(repodir):
390    """Check if a git repository is clean or not"""
391    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
392    return stdout
393
394def check_git_repo_op(srctree, ignoredirs=None):
395    """Check if a git repository is in the middle of a rebase"""
396    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
397    topleveldir = stdout.strip()
398    if ignoredirs and topleveldir in ignoredirs:
399        return
400    gitdir = os.path.join(topleveldir, '.git')
401    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
402        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
403    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
404        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
405