1#!/usr/bin/env python3
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19"""Devtool plugins module"""
20
21import os
22import sys
23import subprocess
24import logging
25import re
26import codecs
27
28logger = logging.getLogger('devtool')
29
30class DevtoolError(Exception):
31    """Exception for handling devtool errors"""
32    def __init__(self, message, exitcode=1):
33        super(DevtoolError, self).__init__(message)
34        self.exitcode = exitcode
35
36
37def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
38    """Run a program in bitbake build context"""
39    import bb
40    if not 'cwd' in options:
41        options["cwd"] = builddir
42    if init_path:
43        # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
44        # and can't determine it when running under dash, we need to set
45        # the executable to bash to correctly set things up
46        if not 'executable' in options:
47            options['executable'] = 'bash'
48        logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
49        init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
50    else:
51        logger.debug('Executing command "%s"' % cmd)
52        init_prefix = ''
53    if watch:
54        if sys.stdout.isatty():
55            # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
56            cmd = 'script -e -q -c "%s" /dev/null' % cmd
57        return exec_watch('%s%s' % (init_prefix, cmd), **options)
58    else:
59        return bb.process.run('%s%s' % (init_prefix, cmd), **options)
60
61def exec_watch(cmd, **options):
62    """Run program with stdout shown on sys.stdout"""
63    import bb
64    if isinstance(cmd, str) and not "shell" in options:
65        options["shell"] = True
66
67    process = subprocess.Popen(
68        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
69    )
70
71    reader = codecs.getreader('utf-8')(process.stdout)
72    buf = ''
73    while True:
74        out = reader.read(1, 1)
75        if out:
76            sys.stdout.write(out)
77            sys.stdout.flush()
78            buf += out
79        elif out == '' and process.poll() != None:
80            break
81
82    if process.returncode != 0:
83        raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
84
85    return buf, None
86
87def exec_fakeroot(d, cmd, **kwargs):
88    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
89    # Grab the command and check it actually exists
90    fakerootcmd = d.getVar('FAKEROOTCMD')
91    if not os.path.exists(fakerootcmd):
92        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
93        return 2
94    # Set up the appropriate environment
95    newenv = dict(os.environ)
96    fakerootenv = d.getVar('FAKEROOTENV')
97    for varvalue in fakerootenv.split():
98        if '=' in varvalue:
99            splitval = varvalue.split('=', 1)
100            newenv[splitval[0]] = splitval[1]
101    return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
102
103def setup_tinfoil(config_only=False, basepath=None, tracking=False):
104    """Initialize tinfoil api from bitbake"""
105    import scriptpath
106    orig_cwd = os.path.abspath(os.curdir)
107    try:
108        if basepath:
109            os.chdir(basepath)
110        bitbakepath = scriptpath.add_bitbake_lib_path()
111        if not bitbakepath:
112            logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
113            sys.exit(1)
114
115        import bb.tinfoil
116        tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
117        try:
118            tinfoil.logger.setLevel(logger.getEffectiveLevel())
119            tinfoil.prepare(config_only)
120        except bb.tinfoil.TinfoilUIException:
121            tinfoil.shutdown()
122            raise DevtoolError('Failed to start bitbake environment')
123        except:
124            tinfoil.shutdown()
125            raise
126    finally:
127        os.chdir(orig_cwd)
128    return tinfoil
129
130def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
131    """Parse the specified recipe"""
132    try:
133        recipefile = tinfoil.get_recipe_file(pn)
134    except bb.providers.NoProvider as e:
135        logger.error(str(e))
136        return None
137    if appends:
138        append_files = tinfoil.get_file_appends(recipefile)
139        if filter_workspace:
140            # Filter out appends from the workspace
141            append_files = [path for path in append_files if
142                            not path.startswith(config.workspace_path)]
143    else:
144        append_files = None
145    try:
146        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
147    except Exception as e:
148        logger.error(str(e))
149        return None
150    return rd
151
152def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
153    """
154    Check that a recipe is in the workspace and (optionally) that source
155    is present.
156    """
157
158    workspacepn = pn
159
160    for recipe, value in workspace.items():
161        if recipe == pn:
162            break
163        if bbclassextend:
164            recipefile = value['recipefile']
165            if recipefile:
166                targets = get_bbclassextend_targets(recipefile, recipe)
167                if pn in targets:
168                    workspacepn = recipe
169                    break
170    else:
171        raise DevtoolError("No recipe named '%s' in your workspace" % pn)
172
173    if checksrc:
174        srctree = workspace[workspacepn]['srctree']
175        if not os.path.exists(srctree):
176            raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
177        if not os.listdir(srctree):
178            raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
179
180    return workspacepn
181
182def use_external_build(same_dir, no_same_dir, d):
183    """
184    Determine if we should use B!=S (separate build and source directories) or not
185    """
186    b_is_s = True
187    if no_same_dir:
188        logger.info('Using separate build directory since --no-same-dir specified')
189        b_is_s = False
190    elif same_dir:
191        logger.info('Using source tree as build directory since --same-dir specified')
192    elif bb.data.inherits_class('autotools-brokensep', d):
193        logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
194    elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
195        logger.info('Using source tree as build directory since that would be the default for this recipe')
196    else:
197        b_is_s = False
198    return b_is_s
199
200def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
201    """
202    Set up the git repository for the source tree
203    """
204    import bb.process
205    import oe.patch
206    if not os.path.exists(os.path.join(repodir, '.git')):
207        bb.process.run('git init', cwd=repodir)
208        bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
209        bb.process.run('git add .', cwd=repodir)
210        commit_cmd = ['git']
211        oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
212        commit_cmd += ['commit', '-q']
213        stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
214        if not stdout:
215            commit_cmd.append('--allow-empty')
216            commitmsg = "Initial empty commit with no upstream sources"
217        elif version:
218            commitmsg = "Initial commit from upstream at version %s" % version
219        else:
220            commitmsg = "Initial commit from upstream"
221        commit_cmd += ['-m', commitmsg]
222        bb.process.run(commit_cmd, cwd=repodir)
223
224    # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
225    excludes = []
226    excludefile = os.path.join(repodir, '.git', 'info', 'exclude')
227    try:
228        with open(excludefile, 'r') as f:
229            excludes = f.readlines()
230    except FileNotFoundError:
231        pass
232    if 'singletask.lock\n' not in excludes:
233        excludes.append('singletask.lock\n')
234    with open(excludefile, 'w') as f:
235        for line in excludes:
236            f.write(line)
237
238    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
239    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
240
241def recipe_to_append(recipefile, config, wildcard=False):
242    """
243    Convert a recipe file to a bbappend file path within the workspace.
244    NOTE: if the bbappend already exists, you should be using
245    workspace[args.recipename]['bbappend'] instead of calling this
246    function.
247    """
248    appendname = os.path.splitext(os.path.basename(recipefile))[0]
249    if wildcard:
250        appendname = re.sub(r'_.*', '_%', appendname)
251    appendpath = os.path.join(config.workspace_path, 'appends')
252    appendfile = os.path.join(appendpath, appendname + '.bbappend')
253    return appendfile
254
255def get_bbclassextend_targets(recipefile, pn):
256    """
257    Cheap function to get BBCLASSEXTEND and then convert that to the
258    list of targets that would result.
259    """
260    import bb.utils
261
262    values = {}
263    def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
264        values[varname] = origvalue
265        return origvalue, None, 0, True
266    with open(recipefile, 'r') as f:
267        bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
268
269    targets = []
270    bbclassextend = values.get('BBCLASSEXTEND', '').split()
271    if bbclassextend:
272        for variant in bbclassextend:
273            if variant == 'nativesdk':
274                targets.append('%s-%s' % (variant, pn))
275            elif variant in ['native', 'cross', 'crosssdk']:
276                targets.append('%s-%s' % (pn, variant))
277    return targets
278
279def replace_from_file(path, old, new):
280    """Replace strings on a file"""
281
282    def read_file(path):
283        data = None
284        with open(path) as f:
285            data = f.read()
286        return data
287
288    def write_file(path, data):
289        if data is None:
290            return
291        wdata = data.rstrip() + "\n"
292        with open(path, "w") as f:
293            f.write(wdata)
294
295    # In case old is None, return immediately
296    if old is None:
297        return
298    try:
299        rdata = read_file(path)
300    except IOError as e:
301        # if file does not exit, just quit, otherwise raise an exception
302        if e.errno == errno.ENOENT:
303            return
304        else:
305            raise
306
307    old_contents = rdata.splitlines()
308    new_contents = []
309    for old_content in old_contents:
310        try:
311            new_contents.append(old_content.replace(old, new))
312        except ValueError:
313            pass
314    write_file(path, "\n".join(new_contents))
315
316
317def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
318    """ This function will make unlocked-sigs.inc match the recipes in the
319    workspace plus any extras we want unlocked. """
320
321    if not fixed_setup:
322        # Only need to write this out within the eSDK
323        return
324
325    if not extra:
326        extra = []
327
328    confdir = os.path.join(basepath, 'conf')
329    unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
330
331    # Get current unlocked list if any
332    values = {}
333    def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
334        values[varname] = origvalue
335        return origvalue, None, 0, True
336    if os.path.exists(unlockedsigs):
337        with open(unlockedsigs, 'r') as f:
338            bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
339    unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
340
341    # If the new list is different to the current list, write it out
342    newunlocked = sorted(list(workspace.keys()) + extra)
343    if unlocked != newunlocked:
344        bb.utils.mkdirhier(confdir)
345        with open(unlockedsigs, 'w') as f:
346            f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
347                    "# This layer was created by the OpenEmbedded devtool" +
348                    " utility in order to\n" +
349                    "# contain recipes that are unlocked.\n")
350
351            f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
352            for pn in newunlocked:
353                f.write('    ' + pn)
354            f.write('"')
355
356def check_prerelease_version(ver, operation):
357    if 'pre' in ver or 'rc' in ver:
358        logger.warning('Version "%s" looks like a pre-release version. '
359                       'If that is the case, in order to ensure that the '
360                       'version doesn\'t appear to go backwards when you '
361                       'later upgrade to the final release version, it is '
362                       'recommmended that instead you use '
363                       '<current version>+<pre-release version> e.g. if '
364                       'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
365                       'If you prefer not to reset and re-try, you can change '
366                       'the version after %s succeeds using "devtool rename" '
367                       'with -V/--version.' % (ver, operation))
368
369def check_git_repo_dirty(repodir):
370    """Check if a git repository is clean or not"""
371    stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
372    return stdout
373
374def check_git_repo_op(srctree, ignoredirs=None):
375    """Check if a git repository is in the middle of a rebase"""
376    stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
377    topleveldir = stdout.strip()
378    if ignoredirs and topleveldir in ignoredirs:
379        return
380    gitdir = os.path.join(topleveldir, '.git')
381    if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
382        raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
383    if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
384        raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
385