1 #!/usr/bin/env python3 2 3 # Development tool - utility functions for plugins 4 # 5 # Copyright (C) 2014 Intel Corporation 6 # 7 # SPDX-License-Identifier: GPL-2.0-only 8 # 9 """Devtool plugins module""" 10 11 import os 12 import sys 13 import subprocess 14 import logging 15 import re 16 import codecs 17 18 logger = logging.getLogger('devtool') 19 20 class DevtoolError(Exception): 21 """Exception for handling devtool errors""" 22 def __init__(self, message, exitcode=1): 23 super(DevtoolError, self).__init__(message) 24 self.exitcode = exitcode 25 26 27 def exec_build_env_command(init_path, builddir, cmd, watch=False, **options): 28 """Run a program in bitbake build context""" 29 import bb 30 if not 'cwd' in options: 31 options["cwd"] = builddir 32 if init_path: 33 # As the OE init script makes use of BASH_SOURCE to determine OEROOT, 34 # and can't determine it when running under dash, we need to set 35 # the executable to bash to correctly set things up 36 if not 'executable' in options: 37 options['executable'] = 'bash' 38 logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path)) 39 init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir) 40 else: 41 logger.debug('Executing command "%s"' % cmd) 42 init_prefix = '' 43 if watch: 44 if sys.stdout.isatty(): 45 # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly) 46 cmd = 'script -e -q -c "%s" /dev/null' % cmd 47 return exec_watch('%s%s' % (init_prefix, cmd), **options) 48 else: 49 return bb.process.run('%s%s' % (init_prefix, cmd), **options) 50 51 def exec_watch(cmd, **options): 52 """Run program with stdout shown on sys.stdout""" 53 import bb 54 if isinstance(cmd, str) and not "shell" in options: 55 options["shell"] = True 56 57 process = subprocess.Popen( 58 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options 59 ) 60 61 reader = codecs.getreader('utf-8')(process.stdout) 62 buf = '' 63 while True: 64 out = reader.read(1, 1) 65 if out: 66 sys.stdout.write(out) 67 sys.stdout.flush() 68 buf += out 69 elif out == '' and process.poll() != None: 70 break 71 72 if process.returncode != 0: 73 raise bb.process.ExecutionError(cmd, process.returncode, buf, None) 74 75 return buf, None 76 77 def exec_fakeroot(d, cmd, **kwargs): 78 """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" 79 # Grab the command and check it actually exists 80 fakerootcmd = d.getVar('FAKEROOTCMD') 81 if not os.path.exists(fakerootcmd): 82 logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') 83 return 2 84 # Set up the appropriate environment 85 newenv = dict(os.environ) 86 fakerootenv = d.getVar('FAKEROOTENV') 87 for varvalue in fakerootenv.split(): 88 if '=' in varvalue: 89 splitval = varvalue.split('=', 1) 90 newenv[splitval[0]] = splitval[1] 91 return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs) 92 93 def setup_tinfoil(config_only=False, basepath=None, tracking=False): 94 """Initialize tinfoil api from bitbake""" 95 import scriptpath 96 orig_cwd = os.path.abspath(os.curdir) 97 try: 98 if basepath: 99 os.chdir(basepath) 100 bitbakepath = scriptpath.add_bitbake_lib_path() 101 if not bitbakepath: 102 logger.error("Unable to find bitbake by searching parent directory of this script or PATH") 103 sys.exit(1) 104 105 import bb.tinfoil 106 tinfoil = bb.tinfoil.Tinfoil(tracking=tracking) 107 try: 108 tinfoil.logger.setLevel(logger.getEffectiveLevel()) 109 tinfoil.prepare(config_only) 110 except bb.tinfoil.TinfoilUIException: 111 tinfoil.shutdown() 112 raise DevtoolError('Failed to start bitbake environment') 113 except: 114 tinfoil.shutdown() 115 raise 116 finally: 117 os.chdir(orig_cwd) 118 return tinfoil 119 120 def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True): 121 """Parse the specified recipe""" 122 try: 123 recipefile = tinfoil.get_recipe_file(pn) 124 except bb.providers.NoProvider as e: 125 logger.error(str(e)) 126 return None 127 if appends: 128 append_files = tinfoil.get_file_appends(recipefile) 129 if filter_workspace: 130 # Filter out appends from the workspace 131 append_files = [path for path in append_files if 132 not path.startswith(config.workspace_path)] 133 else: 134 append_files = None 135 try: 136 rd = tinfoil.parse_recipe_file(recipefile, appends, append_files) 137 except Exception as e: 138 logger.error(str(e)) 139 return None 140 return rd 141 142 def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False): 143 """ 144 Check that a recipe is in the workspace and (optionally) that source 145 is present. 146 """ 147 148 workspacepn = pn 149 150 for recipe, value in workspace.items(): 151 if recipe == pn: 152 break 153 if bbclassextend: 154 recipefile = value['recipefile'] 155 if recipefile: 156 targets = get_bbclassextend_targets(recipefile, recipe) 157 if pn in targets: 158 workspacepn = recipe 159 break 160 else: 161 raise DevtoolError("No recipe named '%s' in your workspace" % pn) 162 163 if checksrc: 164 srctree = workspace[workspacepn]['srctree'] 165 if not os.path.exists(srctree): 166 raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn)) 167 if not os.listdir(srctree): 168 raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn)) 169 170 return workspacepn 171 172 def use_external_build(same_dir, no_same_dir, d): 173 """ 174 Determine if we should use B!=S (separate build and source directories) or not 175 """ 176 b_is_s = True 177 if no_same_dir: 178 logger.info('Using separate build directory since --no-same-dir specified') 179 b_is_s = False 180 elif same_dir: 181 logger.info('Using source tree as build directory since --same-dir specified') 182 elif bb.data.inherits_class('autotools-brokensep', d): 183 logger.info('Using source tree as build directory since recipe inherits autotools-brokensep') 184 elif os.path.abspath(d.getVar('B')) == os.path.abspath(d.getVar('S')): 185 logger.info('Using source tree as build directory since that would be the default for this recipe') 186 else: 187 b_is_s = False 188 return b_is_s 189 190 def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None): 191 """ 192 Set up the git repository for the source tree 193 """ 194 import bb.process 195 import oe.patch 196 if not os.path.exists(os.path.join(repodir, '.git')): 197 bb.process.run('git init', cwd=repodir) 198 bb.process.run('git config --local gc.autodetach 0', cwd=repodir) 199 bb.process.run('git add .', cwd=repodir) 200 commit_cmd = ['git'] 201 oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d) 202 commit_cmd += ['commit', '-q'] 203 stdout, _ = bb.process.run('git status --porcelain', cwd=repodir) 204 if not stdout: 205 commit_cmd.append('--allow-empty') 206 commitmsg = "Initial empty commit with no upstream sources" 207 elif version: 208 commitmsg = "Initial commit from upstream at version %s" % version 209 else: 210 commitmsg = "Initial commit from upstream" 211 commit_cmd += ['-m', commitmsg] 212 bb.process.run(commit_cmd, cwd=repodir) 213 214 # Ensure singletask.lock (as used by externalsrc.bbclass) is ignored by git 215 excludes = [] 216 excludefile = os.path.join(repodir, '.git', 'info', 'exclude') 217 try: 218 with open(excludefile, 'r') as f: 219 excludes = f.readlines() 220 except FileNotFoundError: 221 pass 222 if 'singletask.lock\n' not in excludes: 223 excludes.append('singletask.lock\n') 224 with open(excludefile, 'w') as f: 225 for line in excludes: 226 f.write(line) 227 228 bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) 229 bb.process.run('git tag -f %s' % basetag, cwd=repodir) 230 231 def recipe_to_append(recipefile, config, wildcard=False): 232 """ 233 Convert a recipe file to a bbappend file path within the workspace. 234 NOTE: if the bbappend already exists, you should be using 235 workspace[args.recipename]['bbappend'] instead of calling this 236 function. 237 """ 238 appendname = os.path.splitext(os.path.basename(recipefile))[0] 239 if wildcard: 240 appendname = re.sub(r'_.*', '_%', appendname) 241 appendpath = os.path.join(config.workspace_path, 'appends') 242 appendfile = os.path.join(appendpath, appendname + '.bbappend') 243 return appendfile 244 245 def get_bbclassextend_targets(recipefile, pn): 246 """ 247 Cheap function to get BBCLASSEXTEND and then convert that to the 248 list of targets that would result. 249 """ 250 import bb.utils 251 252 values = {} 253 def get_bbclassextend_varfunc(varname, origvalue, op, newlines): 254 values[varname] = origvalue 255 return origvalue, None, 0, True 256 with open(recipefile, 'r') as f: 257 bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc) 258 259 targets = [] 260 bbclassextend = values.get('BBCLASSEXTEND', '').split() 261 if bbclassextend: 262 for variant in bbclassextend: 263 if variant == 'nativesdk': 264 targets.append('%s-%s' % (variant, pn)) 265 elif variant in ['native', 'cross', 'crosssdk']: 266 targets.append('%s-%s' % (pn, variant)) 267 return targets 268 269 def replace_from_file(path, old, new): 270 """Replace strings on a file""" 271 272 def read_file(path): 273 data = None 274 with open(path) as f: 275 data = f.read() 276 return data 277 278 def write_file(path, data): 279 if data is None: 280 return 281 wdata = data.rstrip() + "\n" 282 with open(path, "w") as f: 283 f.write(wdata) 284 285 # In case old is None, return immediately 286 if old is None: 287 return 288 try: 289 rdata = read_file(path) 290 except IOError as e: 291 # if file does not exit, just quit, otherwise raise an exception 292 if e.errno == errno.ENOENT: 293 return 294 else: 295 raise 296 297 old_contents = rdata.splitlines() 298 new_contents = [] 299 for old_content in old_contents: 300 try: 301 new_contents.append(old_content.replace(old, new)) 302 except ValueError: 303 pass 304 write_file(path, "\n".join(new_contents)) 305 306 307 def update_unlockedsigs(basepath, workspace, fixed_setup, extra=None): 308 """ This function will make unlocked-sigs.inc match the recipes in the 309 workspace plus any extras we want unlocked. """ 310 311 if not fixed_setup: 312 # Only need to write this out within the eSDK 313 return 314 315 if not extra: 316 extra = [] 317 318 confdir = os.path.join(basepath, 'conf') 319 unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc') 320 321 # Get current unlocked list if any 322 values = {} 323 def get_unlockedsigs_varfunc(varname, origvalue, op, newlines): 324 values[varname] = origvalue 325 return origvalue, None, 0, True 326 if os.path.exists(unlockedsigs): 327 with open(unlockedsigs, 'r') as f: 328 bb.utils.edit_metadata(f, ['SIGGEN_UNLOCKED_RECIPES'], get_unlockedsigs_varfunc) 329 unlocked = sorted(values.get('SIGGEN_UNLOCKED_RECIPES', [])) 330 331 # If the new list is different to the current list, write it out 332 newunlocked = sorted(list(workspace.keys()) + extra) 333 if unlocked != newunlocked: 334 bb.utils.mkdirhier(confdir) 335 with open(unlockedsigs, 'w') as f: 336 f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" + 337 "# This layer was created by the OpenEmbedded devtool" + 338 " utility in order to\n" + 339 "# contain recipes that are unlocked.\n") 340 341 f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n') 342 for pn in newunlocked: 343 f.write(' ' + pn) 344 f.write('"') 345 346 def check_prerelease_version(ver, operation): 347 if 'pre' in ver or 'rc' in ver: 348 logger.warning('Version "%s" looks like a pre-release version. ' 349 'If that is the case, in order to ensure that the ' 350 'version doesn\'t appear to go backwards when you ' 351 'later upgrade to the final release version, it is ' 352 'recommmended that instead you use ' 353 '<current version>+<pre-release version> e.g. if ' 354 'upgrading from 1.9 to 2.0-rc2 use "1.9+2.0-rc2". ' 355 'If you prefer not to reset and re-try, you can change ' 356 'the version after %s succeeds using "devtool rename" ' 357 'with -V/--version.' % (ver, operation)) 358 359 def check_git_repo_dirty(repodir): 360 """Check if a git repository is clean or not""" 361 stdout, _ = bb.process.run('git status --porcelain', cwd=repodir) 362 return stdout 363 364 def check_git_repo_op(srctree, ignoredirs=None): 365 """Check if a git repository is in the middle of a rebase""" 366 stdout, _ = bb.process.run('git rev-parse --show-toplevel', cwd=srctree) 367 topleveldir = stdout.strip() 368 if ignoredirs and topleveldir in ignoredirs: 369 return 370 gitdir = os.path.join(topleveldir, '.git') 371 if os.path.exists(os.path.join(gitdir, 'rebase-merge')): 372 raise DevtoolError("Source tree %s appears to be in the middle of a rebase - please resolve this first" % srctree) 373 if os.path.exists(os.path.join(gitdir, 'rebase-apply')): 374 raise DevtoolError("Source tree %s appears to be in the middle of 'git am' or 'git apply' - please resolve this first" % srctree) 375