1 #!/usr/bin/env python3
2 
3 # Development tool - utility functions for plugins
4 #
5 # Copyright (C) 2014 Intel Corporation
6 #
7 # SPDX-License-Identifier: GPL-2.0-only
8 #
9 """Devtool plugins module"""
10 
11 import os
12 import sys
13 import subprocess
14 import logging
15 import re
16 import codecs
17 
18 logger = logging.getLogger('devtool')
19 
20 class DevtoolError(Exception):
21     """Exception for handling devtool errors"""
22     def __init__(self, message, exitcode=1):
23         super(DevtoolError, self).__init__(message)
24         self.exitcode = exitcode
25 
26 
27 def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
28     """Run a program in bitbake build context"""
29     import bb
30     if not 'cwd' in options:
31         options["cwd"] = builddir
32     if init_path:
33         # As the OE init script makes use of BASH_SOURCE to determine OEROOT,
34         # and can't determine it when running under dash, we need to set
35         # the executable to bash to correctly set things up
36         if not 'executable' in options:
37             options['executable'] = 'bash'
38         logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
39         init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
40     else:
41         logger.debug('Executing command "%s"' % cmd)
42         init_prefix = ''
43     if watch:
44         if sys.stdout.isatty():
45             # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
46             cmd = 'script -e -q -c "%s" /dev/null' % cmd
47         return exec_watch('%s%s' % (init_prefix, cmd), **options)
48     else:
49         return bb.process.run('%s%s' % (init_prefix, cmd), **options)
50 
51 def exec_watch(cmd, **options):
52     """Run program with stdout shown on sys.stdout"""
53     import bb
54     if isinstance(cmd, str) and not "shell" in options:
55         options["shell"] = True
56 
57     process = subprocess.Popen(
58         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
59     )
60 
61     reader = codecs.getreader('utf-8')(process.stdout)
62     buf = ''
63     while True:
64         out = reader.read(1, 1)
65         if out:
66             sys.stdout.write(out)
67             sys.stdout.flush()
68             buf += out
69         elif out == '' and process.poll() != None:
70             break
71 
72     if process.returncode != 0:
73         raise bb.process.ExecutionError(cmd, process.returncode, buf, None)
74 
75     return buf, None
76 
77 def exec_fakeroot(d, cmd, **kwargs):
78     """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79     # Grab the command and check it actually exists
80     fakerootcmd = d.getVar('FAKEROOTCMD')
81     if not os.path.exists(fakerootcmd):
82         logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83         return 2
84     # Set up the appropriate environment
85     newenv = dict(os.environ)
86     fakerootenv = d.getVar('FAKEROOTENV')
87     for varvalue in fakerootenv.split():
88         if '=' in varvalue:
89             splitval = varvalue.split('=', 1)
90             newenv[splitval[0]] = splitval[1]
91     return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
92 
93 def setup_tinfoil(config_only=False, basepath=None, tracking=False):
94     """Initialize tinfoil api from bitbake"""
95     import scriptpath
96     orig_cwd = os.path.abspath(os.curdir)
97     try:
98         if basepath:
99             os.chdir(basepath)
100         bitbakepath = scriptpath.add_bitbake_lib_path()
101         if not bitbakepath:
102             logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
103             sys.exit(1)
104 
105         import bb.tinfoil
106         tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
107         try:
108             tinfoil.logger.setLevel(logger.getEffectiveLevel())
109             tinfoil.prepare(config_only)
110         except bb.tinfoil.TinfoilUIException:
111             tinfoil.shutdown()
112             raise DevtoolError('Failed to start bitbake environment')
113         except:
114             tinfoil.shutdown()
115             raise
116     finally:
117         os.chdir(orig_cwd)
118     return tinfoil
119 
120 def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
121     """Parse the specified recipe"""
122     try:
123         recipefile = tinfoil.get_recipe_file(pn)
124     except bb.providers.NoProvider as e:
125         logger.error(str(e))
126         return None
127     if appends:
128         append_files = tinfoil.get_file_appends(recipefile)
129         if filter_workspace:
130             # Filter out appends from the workspace
131             append_files = [path for path in append_files if
132                             not path.startswith(config.workspace_path)]
133     else:
134         append_files = None
135     try:
136         rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
137     except Exception as e:
138         logger.error(str(e))
139         return None
140     return rd
141 
142 def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
143     """
144     Check that a recipe is in the workspace and (optionally) that source
145     is present.
146     """
147 
148     workspacepn = pn
149 
150     for recipe, value in workspace.items():
151         if recipe == pn:
152             break
153         if bbclassextend:
154             recipefile = value['recipefile']
155             if recipefile:
156                 targets = get_bbclassextend_targets(recipefile, recipe)
157                 if pn in targets:
158                     workspacepn = recipe
159                     break
160     else:
161         raise DevtoolError("No recipe named '%s' in your workspace" % pn)
162 
163     if checksrc:
164         srctree = workspace[workspacepn]['srctree']
165         if not os.path.exists(srctree):
166             raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
167         if not os.listdir(srctree):
168             raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
169 
170     return workspacepn
171 
172 def use_external_build(same_dir, no_same_dir, d):
173     """
174     Determine if we should use B!=S (separate build and source directories) or not
175     """
176     b_is_s = True
177     if no_same_dir:
178         logger.info('Using separate build directory since --no-same-dir specified')
179         b_is_s = False
180     elif same_dir:
181         logger.info('Using source tree as build directory since --same-dir specified')
182     elif bb.data.inherits_class('autotools-brokensep', d):
183         logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
184     elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')):
185         logger.info('Using source tree as build directory since that would be the default for this recipe')
186     else:
187         b_is_s = False
188     return b_is_s
189 
190 def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
191     """
192     Set up the git repository for the source tree
193     """
194     import bb.process
195     import oe.patch
196     if not os.path.exists(os.path.join(repodir, '.git')):
197         bb.process.run('git init', cwd=repodir)
198         bb.process.run('git config --local gc.autodetach 0', cwd=repodir)
199         bb.process.run('git add .', cwd=repodir)
200         commit_cmd = ['git']
201         oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
202         commit_cmd += ['commit', '-q']
203         stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
204         if not stdout:
205             commit_cmd.append('--allow-empty')
206             commitmsg = "Initial empty commit with no upstream sources"
207         elif version:
208             commitmsg = "Initial commit from upstream at version %s" % version
209         else:
210             commitmsg = "Initial commit from upstream"
211         commit_cmd += ['-m', commitmsg]
212         bb.process.run(commit_cmd, cwd=repodir)
213 
214     # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git
215     excludes = []
216     excludefile = os.path.join(repodir, '.git', 'info', 'exclude')
217     try:
218         with open(excludefile, 'r') as f:
219             excludes = f.readlines()
220     except FileNotFoundError:
221         pass
222     if 'singletask.lock\n' not in excludes:
223         excludes.append('singletask.lock\n')
224     with open(excludefile, 'w') as f:
225         for line in excludes:
226             f.write(line)
227 
228     bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
229     bb.process.run('git tag -f %s' % basetag, cwd=repodir)
230 
231 def recipe_to_append(recipefile, config, wildcard=False):
232     """
233     Convert a recipe file to a bbappend file path within the workspace.
234     NOTE: if the bbappend already exists, you should be using
235     workspace[args.recipename]['bbappend'] instead of calling this
236     function.
237     """
238     appendname = os.path.splitext(os.path.basename(recipefile))[0]
239     if wildcard:
240         appendname = re.sub(r'_.*', '_%', appendname)
241     appendpath = os.path.join(config.workspace_path, 'appends')
242     appendfile = os.path.join(appendpath, appendname + '.bbappend')
243     return appendfile
244 
245 def get_bbclassextend_targets(recipefile, pn):
246     """
247     Cheap function to get BBCLASSEXTEND and then convert that to the
248     list of targets that would result.
249     """
250     import bb.utils
251 
252     values = {}
253     def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
254         values[varname] = origvalue
255         return origvalue, None, 0, True
256     with open(recipefile, 'r') as f:
257         bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
258 
259     targets = []
260     bbclassextend = values.get('BBCLASSEXTEND', '').split()
261     if bbclassextend:
262         for variant in bbclassextend:
263             if variant == 'nativesdk':
264                 targets.append('%s-%s' % (variant, pn))
265             elif variant in ['native', 'cross', 'crosssdk']:
266                 targets.append('%s-%s' % (pn, variant))
267     return targets
268 
269 def replace_from_file(path, old, new):
270     """Replace strings on a file"""
271 
272     def read_file(path):
273         data = None
274         with open(path) as f:
275             data = f.read()
276         return data
277 
278     def write_file(path, data):
279         if data is None:
280             return
281         wdata = data.rstrip() + "\n"
282         with open(path, "w") as f:
283             f.write(wdata)
284 
285     # In case old is None, return immediately
286     if old is None:
287         return
288     try:
289         rdata = read_file(path)
290     except IOError as e:
291         # if file does not exit, just quit, otherwise raise an exception
292         if e.errno == errno.ENOENT:
293             return
294         else:
295             raise
296 
297     old_contents = rdata.splitlines()
298     new_contents = []
299     for old_content in old_contents:
300         try:
301             new_contents.append(old_content.replace(old, new))
302         except ValueError:
303             pass
304     write_file(path, "\n".join(new_contents))
305 
306 
307 def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None):
308     """ This function will make unlocked-sigs.inc match the recipes in the
309     workspace plus any extras we want unlocked. """
310 
311     if not fixed_setup:
312         # Only need to write this out within the eSDK
313         return
314 
315     if not extra:
316         extra = []
317 
318     confdir = os.path.join(basepath, 'conf')
319     unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc')
320 
321     # Get current unlocked list if any
322     values = {}
323     def get_unlockedsigs_varfunc(varname, origvalue, op, newlines):
324         values[varname] = origvalue
325         return origvalue, None, 0, True
326     if os.path.exists(unlockedsigs):
327         with open(unlockedsigs, 'r') as f:
328             bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc)
329     unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', []))
330 
331     # If the new list is different to the current list, write it out
332     newunlocked = sorted(list(workspace.keys()) + extra)
333     if unlocked != newunlocked:
334         bb.utils.mkdirhier(confdir)
335         with open(unlockedsigs, 'w') as f:
336             f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" +
337                     "# This layer was created by the OpenEmbedded devtool" +
338                     " utility in order to\n" +
339                     "# contain recipes that are unlocked.\n")
340 
341             f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n')
342             for pn in newunlocked:
343                 f.write('    ' + pn)
344             f.write('"')
345 
346 def check_prerelease_version(ver, operation):
347     if 'pre' in ver or 'rc' in ver:
348         logger.warning('Version "%s" looks like a pre-release version. '
349                        'If that is the case, in order to ensure that the '
350                        'version doesn\'t appear to go backwards when you '
351                        'later upgrade to the final release version, it is '
352                        'recommmended that instead you use '
353                        '<current version>+<pre-release version> e.g. if '
354                        'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". '
355                        'If you prefer not to reset and re-try, you can change '
356                        'the version after %s succeeds using "devtool rename" '
357                        'with -V/--version.' % (ver, operation))
358 
359 def check_git_repo_dirty(repodir):
360     """Check if a git repository is clean or not"""
361     stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
362     return stdout
363 
364 def check_git_repo_op(srctree, ignoredirs=None):
365     """Check if a git repository is in the middle of a rebase"""
366     stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree)
367     topleveldir = stdout.strip()
368     if ignoredirs and topleveldir in ignoredirs:
369         return
370     gitdir = os.path.join(topleveldir, '.git')
371     if os.path.exists(os.path.join(gitdir, 'rebase-merge')):
372         raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree)
373     if os.path.exists(os.path.join(gitdir, 'rebase-apply')):
374         raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree)
375