1# Utility functions for reading and modifying recipes 2# 3# Some code borrowed from the OE layer index 4# 5# Copyright (C) 2013-2017 Intel Corporation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import sys 11import os 12import os.path 13import tempfile 14import textwrap 15import difflib 16from . import utils 17import shutil 18import re 19import fnmatch 20import glob 21import bb.tinfoil 22 23from collections import OrderedDict, defaultdict 24from bb.utils import vercmp_string 25 26# Help us to find places to insert values 27recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND'] 28# Variables that sometimes are a bit long but shouldn't be wrapped 29nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha[0-9]+sum\]'] 30list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] 31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] 32 33 34def simplify_history(history, d): 35 """ 36 Eliminate any irrelevant events from a variable history 37 """ 38 ret_history = [] 39 has_set = False 40 # Go backwards through the history and remove any immediate operations 41 # before the most recent set 42 for event in reversed(history): 43 if 'flag' in event or not 'file' in event: 44 continue 45 if event['op'] == 'set': 46 if has_set: 47 continue 48 has_set = True 49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'): 50 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT :append / :prepend 51 if has_set: 52 continue 53 ret_history.insert(0, event) 54 return ret_history 55 56 57def get_var_files(fn, varlist, d): 58 """Find the file in which each of a list of variables is set. 59 Note: requires variable history to be enabled when parsing. 60 """ 61 varfiles = {} 62 for v in varlist: 63 files = [] 64 if '[' in v: 65 varsplit = v.split('[') 66 varflag = varsplit[1].split(']')[0] 67 history = d.varhistory.variable(varsplit[0]) 68 for event in history: 69 if 'file' in event and event.get('flag', '') == varflag: 70 files.append(event['file']) 71 else: 72 history = d.varhistory.variable(v) 73 for event in history: 74 if 'file' in event and not 'flag' in event: 75 files.append(event['file']) 76 if files: 77 actualfile = files[-1] 78 else: 79 actualfile = None 80 varfiles[v] = actualfile 81 82 return varfiles 83 84 85def split_var_value(value, assignment=True): 86 """ 87 Split a space-separated variable's value into a list of items, 88 taking into account that some of the items might be made up of 89 expressions containing spaces that should not be split. 90 Parameters: 91 value: 92 The string value to split 93 assignment: 94 True to assume that the value represents an assignment 95 statement, False otherwise. If True, and an assignment 96 statement is passed in the first item in 97 the returned list will be the part of the assignment 98 statement up to and including the opening quote character, 99 and the last item will be the closing quote. 100 """ 101 inexpr = 0 102 lastchar = None 103 out = [] 104 buf = '' 105 for char in value: 106 if char == '{': 107 if lastchar == '$': 108 inexpr += 1 109 elif char == '}': 110 inexpr -= 1 111 elif assignment and char in '"\'' and inexpr == 0: 112 if buf: 113 out.append(buf) 114 out.append(char) 115 char = '' 116 buf = '' 117 elif char.isspace() and inexpr == 0: 118 char = '' 119 if buf: 120 out.append(buf) 121 buf = '' 122 buf += char 123 lastchar = char 124 if buf: 125 out.append(buf) 126 127 # Join together assignment statement and opening quote 128 outlist = out 129 if assignment: 130 assigfound = False 131 for idx, item in enumerate(out): 132 if '=' in item: 133 assigfound = True 134 if assigfound: 135 if '"' in item or "'" in item: 136 outlist = [' '.join(out[:idx+1])] 137 outlist.extend(out[idx+1:]) 138 break 139 return outlist 140 141 142def patch_recipe_lines(fromlines, values, trailing_newline=True): 143 """Update or insert variable values into lines from a recipe. 144 Note that some manual inspection/intervention may be required 145 since this cannot handle all situations. 146 """ 147 148 import bb.utils 149 150 if trailing_newline: 151 newline = '\n' 152 else: 153 newline = '' 154 155 nowrap_vars_res = [] 156 for item in nowrap_vars: 157 nowrap_vars_res.append(re.compile('^%s$' % item)) 158 159 recipe_progression_res = [] 160 recipe_progression_restrs = [] 161 for item in recipe_progression: 162 if item.endswith('()'): 163 key = item[:-2] 164 else: 165 key = item 166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key 167 if item.endswith('()'): 168 recipe_progression_restrs.append(restr + '()') 169 else: 170 recipe_progression_restrs.append(restr) 171 recipe_progression_res.append(re.compile('^%s$' % restr)) 172 173 def get_recipe_pos(variable): 174 for i, p in enumerate(recipe_progression_res): 175 if p.match(variable): 176 return i 177 return -1 178 179 remainingnames = {} 180 for k in values.keys(): 181 remainingnames[k] = get_recipe_pos(k) 182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) 183 184 modifying = False 185 186 def outputvalue(name, lines, rewindcomments=False): 187 if values[name] is None: 188 return 189 if isinstance(values[name], tuple): 190 op, value = values[name] 191 if op == '+=' and value.strip() == '': 192 return 193 else: 194 value = values[name] 195 op = '=' 196 rawtext = '%s %s "%s"%s' % (name, op, value, newline) 197 addlines = [] 198 nowrap = False 199 for nowrap_re in nowrap_vars_res: 200 if nowrap_re.match(name): 201 nowrap = True 202 break 203 if nowrap: 204 addlines.append(rawtext) 205 elif name in list_vars: 206 splitvalue = split_var_value(value, assignment=False) 207 if len(splitvalue) > 1: 208 linesplit = ' \\\n' + (' ' * (len(name) + 4)) 209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline)) 210 else: 211 addlines.append(rawtext) 212 else: 213 wrapped = textwrap.wrap(rawtext) 214 for wrapline in wrapped[:-1]: 215 addlines.append('%s \\%s' % (wrapline, newline)) 216 addlines.append('%s%s' % (wrapped[-1], newline)) 217 218 # Split on newlines - this isn't strictly necessary if you are only 219 # going to write the output to disk, but if you want to compare it 220 # (as patch_recipe_file() will do if patch=True) then it's important. 221 addlines = [line for l in addlines for line in l.splitlines(True)] 222 if rewindcomments: 223 # Ensure we insert the lines before any leading comments 224 # (that we'd want to ensure remain leading the next value) 225 for i, ln in reversed(list(enumerate(lines))): 226 if not ln.startswith('#'): 227 lines[i+1:i+1] = addlines 228 break 229 else: 230 lines.extend(addlines) 231 else: 232 lines.extend(addlines) 233 234 existingnames = [] 235 def patch_recipe_varfunc(varname, origvalue, op, newlines): 236 if modifying: 237 # Insert anything that should come before this variable 238 pos = get_recipe_pos(varname) 239 for k in list(remainingnames): 240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: 241 outputvalue(k, newlines, rewindcomments=True) 242 del remainingnames[k] 243 # Now change this variable, if it needs to be changed 244 if varname in existingnames and op in ['+=', '=', '=+']: 245 if varname in remainingnames: 246 outputvalue(varname, newlines) 247 del remainingnames[varname] 248 return None, None, 0, True 249 else: 250 if varname in values: 251 existingnames.append(varname) 252 return origvalue, None, 0, True 253 254 # First run - establish which values we want to set are already in the file 255 varlist = [re.escape(item) for item in values.keys()] 256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) 257 # Second run - actually set everything 258 modifying = True 259 varlist.extend(recipe_progression_restrs) 260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) 261 262 if remainingnames: 263 if tolines and tolines[-1].strip() != '': 264 tolines.append('\n') 265 for k in remainingnames.keys(): 266 outputvalue(k, tolines) 267 268 return changed, tolines 269 270 271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None): 272 """Update or insert variable values into a recipe file (assuming you 273 have already identified the exact file you want to update.) 274 Note that some manual inspection/intervention may be required 275 since this cannot handle all situations. 276 """ 277 278 with open(fn, 'r') as f: 279 fromlines = f.readlines() 280 281 _, tolines = patch_recipe_lines(fromlines, values) 282 283 if redirect_output: 284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f: 285 f.writelines(tolines) 286 return None 287 elif patch: 288 relfn = os.path.relpath(fn, relpath) 289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) 290 return diff 291 else: 292 with open(fn, 'w') as f: 293 f.writelines(tolines) 294 return None 295 296 297def localise_file_vars(fn, varfiles, varlist): 298 """Given a list of variables and variable history (fetched with get_var_files()) 299 find where each variable should be set/changed. This handles for example where a 300 recipe includes an inc file where variables might be changed - in most cases 301 we want to update the inc file when changing the variable value rather than adding 302 it to the recipe itself. 303 """ 304 fndir = os.path.dirname(fn) + os.sep 305 306 first_meta_file = None 307 for v in meta_vars: 308 f = varfiles.get(v, None) 309 if f: 310 actualdir = os.path.dirname(f) + os.sep 311 if actualdir.startswith(fndir): 312 first_meta_file = f 313 break 314 315 filevars = defaultdict(list) 316 for v in varlist: 317 f = varfiles[v] 318 # Only return files that are in the same directory as the recipe or in some directory below there 319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable 320 # in if we were going to set a value specific to this recipe) 321 if f: 322 actualfile = f 323 else: 324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it 325 if first_meta_file: 326 actualfile = first_meta_file 327 else: 328 actualfile = fn 329 330 actualdir = os.path.dirname(actualfile) + os.sep 331 if not actualdir.startswith(fndir): 332 actualfile = fn 333 filevars[actualfile].append(v) 334 335 return filevars 336 337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None): 338 """Modify a list of variable values in the specified recipe. Handles inc files if 339 used by the recipe. 340 """ 341 overrides = d.getVar('OVERRIDES').split(':') 342 def override_applicable(hevent): 343 op = hevent['op'] 344 if '[' in op: 345 opoverrides = op.split('[')[1].split(']')[0].split(':') 346 for opoverride in opoverrides: 347 if not opoverride in overrides: 348 return False 349 return True 350 351 varlist = varvalues.keys() 352 fn = os.path.abspath(fn) 353 varfiles = get_var_files(fn, varlist, d) 354 locs = localise_file_vars(fn, varfiles, varlist) 355 patches = [] 356 for f,v in locs.items(): 357 vals = {k: varvalues[k] for k in v} 358 f = os.path.abspath(f) 359 if f == fn: 360 extravals = {} 361 for var, value in vals.items(): 362 if var in list_vars: 363 history = simplify_history(d.varhistory.variable(var), d) 364 recipe_set = False 365 for event in history: 366 if os.path.abspath(event['file']) == fn: 367 if event['op'] == 'set': 368 recipe_set = True 369 if not recipe_set: 370 for event in history: 371 if event['op'].startswith(':remove'): 372 continue 373 if not override_applicable(event): 374 continue 375 newvalue = value.replace(event['detail'], '') 376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith(':'): 377 op = event['op'].replace('[', ':').replace(']', '') 378 extravals[var + op] = None 379 value = newvalue 380 vals[var] = ('+=', value) 381 vals.update(extravals) 382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output) 383 if patch: 384 patches.append(patchdata) 385 386 if patch: 387 return patches 388 else: 389 return None 390 391 392 393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False): 394 """Copy (local) recipe files, including both files included via include/require, 395 and files referred to in the SRC_URI variable.""" 396 import bb.fetch2 397 import oe.path 398 399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references 400 401 uri_values = [] 402 localpaths = [] 403 def fetch_urls(rdata): 404 # Collect the local paths from SRC_URI 405 srcuri = rdata.getVar('SRC_URI') or "" 406 if srcuri not in uri_values: 407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata) 408 if download: 409 fetch.download() 410 for pth in fetch.localpaths(): 411 if pth not in localpaths: 412 localpaths.append(os.path.abspath(pth)) 413 uri_values.append(srcuri) 414 415 fetch_urls(d) 416 if all_variants: 417 # Get files for other variants e.g. in the case of a SRC_URI:append 418 localdata = bb.data.createCopy(d) 419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split() 420 if variants: 421 # Ensure we handle class-target if we're dealing with one of the variants 422 variants.append('target') 423 for variant in variants: 424 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant) 425 fetch_urls(localdata) 426 427 # Copy local files to target directory and gather any remote files 428 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep 429 remotes = [] 430 copied = [] 431 # Need to do this in two steps since we want to check against the absolute path 432 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)] 433 # We also check this below, but we don't want any items in this list being considered remotes 434 includes = [path for path in includes if path.startswith(bb_dir)] 435 for path in localpaths + includes: 436 # Only import files that are under the meta directory 437 if path.startswith(bb_dir): 438 if not whole_dir: 439 relpath = os.path.relpath(path, bb_dir) 440 subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) 441 if not os.path.exists(subdir): 442 os.makedirs(subdir) 443 shutil.copy2(path, os.path.join(tgt_dir, relpath)) 444 copied.append(relpath) 445 else: 446 remotes.append(path) 447 # Simply copy whole meta dir, if requested 448 if whole_dir: 449 shutil.copytree(bb_dir, tgt_dir) 450 451 return copied, remotes 452 453 454def get_recipe_local_files(d, patches=False, archives=False): 455 """Get a list of local files in SRC_URI within a recipe.""" 456 import oe.patch 457 uris = (d.getVar('SRC_URI') or "").split() 458 fetch = bb.fetch2.Fetch(uris, d) 459 # FIXME this list should be factored out somewhere else (such as the 460 # fetcher) though note that this only encompasses actual container formats 461 # i.e. that can contain multiple files as opposed to those that only 462 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz) 463 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z'] 464 ret = {} 465 for uri in uris: 466 if fetch.ud[uri].type == 'file': 467 if (not patches and 468 oe.patch.patch_path(uri, fetch, '', expand=False)): 469 continue 470 # Skip files that are referenced by absolute path 471 fname = fetch.ud[uri].basepath 472 if os.path.isabs(fname): 473 continue 474 # Handle subdir= 475 subdir = fetch.ud[uri].parm.get('subdir', '') 476 if subdir: 477 if os.path.isabs(subdir): 478 continue 479 fname = os.path.join(subdir, fname) 480 localpath = fetch.localpath(uri) 481 if not archives: 482 # Ignore archives that will be unpacked 483 if localpath.endswith(tuple(archive_exts)): 484 unpack = fetch.ud[uri].parm.get('unpack', True) 485 if unpack: 486 continue 487 if os.path.isdir(localpath): 488 for root, dirs, files in os.walk(localpath): 489 for fname in files: 490 fileabspath = os.path.join(root,fname) 491 srcdir = os.path.dirname(localpath) 492 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath 493 else: 494 ret[fname] = localpath 495 return ret 496 497 498def get_recipe_patches(d): 499 """Get a list of the patches included in SRC_URI within a recipe.""" 500 import oe.patch 501 patches = oe.patch.src_patches(d, expand=False) 502 patchfiles = [] 503 for patch in patches: 504 _, _, local, _, _, parm = bb.fetch.decodeurl(patch) 505 patchfiles.append(local) 506 return patchfiles 507 508 509def get_recipe_patched_files(d): 510 """ 511 Get the list of patches for a recipe along with the files each patch modifies. 512 Params: 513 d: the datastore for the recipe 514 Returns: 515 a dict mapping patch file path to a list of tuples of changed files and 516 change mode ('A' for add, 'D' for delete or 'M' for modify) 517 """ 518 import oe.patch 519 patches = oe.patch.src_patches(d, expand=False) 520 patchedfiles = {} 521 for patch in patches: 522 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch) 523 striplevel = int(parm['striplevel']) 524 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', ''))) 525 return patchedfiles 526 527 528def validate_pn(pn): 529 """Perform validation on a recipe name (PN) for a new recipe.""" 530 reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] 531 if not re.match('^[0-9a-z-.+]+$', pn): 532 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn 533 elif pn in reserved_names: 534 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn 535 elif pn.startswith('pn-'): 536 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn 537 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): 538 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn 539 return '' 540 541 542def get_bbfile_path(d, destdir, extrapathhint=None): 543 """ 544 Determine the correct path for a recipe within a layer 545 Parameters: 546 d: Recipe-specific datastore 547 destdir: destination directory. Can be the path to the base of the layer or a 548 partial path somewhere within the layer. 549 extrapathhint: a path relative to the base of the layer to try 550 """ 551 import bb.cookerdata 552 553 destdir = os.path.abspath(destdir) 554 destlayerdir = find_layerdir(destdir) 555 556 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf 557 confdata = d.createCopy() 558 confdata.setVar('BBFILES', '') 559 confdata.setVar('LAYERDIR', destlayerdir) 560 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") 561 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) 562 pn = d.getVar('PN') 563 564 # Parse BBFILES_DYNAMIC and append to BBFILES 565 bbfiles_dynamic = (confdata.getVar('BBFILES_DYNAMIC') or "").split() 566 collections = (confdata.getVar('BBFILE_COLLECTIONS') or "").split() 567 invalid = [] 568 for entry in bbfiles_dynamic: 569 parts = entry.split(":", 1) 570 if len(parts) != 2: 571 invalid.append(entry) 572 continue 573 l, f = parts 574 invert = l[0] == "!" 575 if invert: 576 l = l[1:] 577 if (l in collections and not invert) or (l not in collections and invert): 578 confdata.appendVar("BBFILES", " " + f) 579 if invalid: 580 return None 581 bbfilespecs = (confdata.getVar('BBFILES') or '').split() 582 if destdir == destlayerdir: 583 for bbfilespec in bbfilespecs: 584 if not bbfilespec.endswith('.bbappend'): 585 for match in glob.glob(bbfilespec): 586 splitext = os.path.splitext(os.path.basename(match)) 587 if splitext[1] == '.bb': 588 mpn = splitext[0].split('_')[0] 589 if mpn == pn: 590 return os.path.dirname(match) 591 592 # Try to make up a path that matches BBFILES 593 # this is a little crude, but better than nothing 594 bpn = d.getVar('BPN') 595 recipefn = os.path.basename(d.getVar('FILE')) 596 pathoptions = [destdir] 597 if extrapathhint: 598 pathoptions.append(os.path.join(destdir, extrapathhint)) 599 if destdir == destlayerdir: 600 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) 601 pathoptions.append(os.path.join(destdir, 'recipes', bpn)) 602 pathoptions.append(os.path.join(destdir, bpn)) 603 elif not destdir.endswith(('/' + pn, '/' + bpn)): 604 pathoptions.append(os.path.join(destdir, bpn)) 605 closepath = '' 606 for pathoption in pathoptions: 607 bbfilepath = os.path.join(pathoption, 'test.bb') 608 for bbfilespec in bbfilespecs: 609 if fnmatch.fnmatchcase(bbfilepath, bbfilespec): 610 return pathoption 611 return None 612 613def get_bbappend_path(d, destlayerdir, wildcardver=False): 614 """Determine how a bbappend for a recipe should be named and located within another layer""" 615 616 import bb.cookerdata 617 618 destlayerdir = os.path.abspath(destlayerdir) 619 recipefile = d.getVar('FILE') 620 recipefn = os.path.splitext(os.path.basename(recipefile))[0] 621 if wildcardver and '_' in recipefn: 622 recipefn = recipefn.split('_', 1)[0] + '_%' 623 appendfn = recipefn + '.bbappend' 624 625 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf 626 confdata = d.createCopy() 627 confdata.setVar('BBFILES', '') 628 confdata.setVar('LAYERDIR', destlayerdir) 629 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") 630 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) 631 632 origlayerdir = find_layerdir(recipefile) 633 if not origlayerdir: 634 return (None, False) 635 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES 636 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn) 637 closepath = '' 638 pathok = True 639 for bbfilespec in confdata.getVar('BBFILES').split(): 640 if fnmatch.fnmatchcase(appendpath, bbfilespec): 641 # Our append path works, we're done 642 break 643 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)): 644 # Try to find the longest matching path 645 if len(bbfilespec) > len(closepath): 646 closepath = bbfilespec 647 else: 648 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure 649 if closepath: 650 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files 651 # Now we just need to substitute out any wildcards 652 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir) 653 if 'recipes-*' in appendsubdir: 654 # Try to copy this part from the original recipe path 655 res = re.search('/recipes-[^/]+/', recipefile) 656 if res: 657 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0)) 658 # This is crude, but we have to do something 659 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0]) 660 appendsubdir = appendsubdir.replace('?', 'a') 661 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn) 662 else: 663 pathok = False 664 return (appendpath, pathok) 665 666 667def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None, update_original_recipe=False): 668 """ 669 Writes a bbappend file for a recipe 670 Parameters: 671 rd: data dictionary for the recipe 672 destlayerdir: base directory of the layer to place the bbappend in 673 (subdirectory path from there will be determined automatically) 674 srcfiles: dict of source files to add to SRC_URI, where the key 675 is the full path to the file to be added, and the value is a 676 dict with following optional keys: 677 path: the original filename as it would appear in SRC_URI 678 or None if it isn't already present. 679 patchdir: the patchdir parameter 680 newname: the name to give to the new added file. None to use 681 the default value: basename(path) 682 You may pass None for this parameter if you simply want to specify 683 your own content via the extralines parameter. 684 install: dict mapping entries in srcfiles to a tuple of two elements: 685 install path (*without* ${D} prefix) and permission value (as a 686 string, e.g. '0644'). 687 wildcardver: True to use a % wildcard in the bbappend filename, or 688 False to make the bbappend specific to the recipe version. 689 machine: 690 If specified, make the changes in the bbappend specific to this 691 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}" 692 to be added to the bbappend. 693 extralines: 694 Extra lines to add to the bbappend. This may be a dict of name 695 value pairs, or simply a list of the lines. 696 removevalues: 697 Variable values to remove - a dict of names/values. 698 redirect_output: 699 If specified, redirects writing the output file to the 700 specified directory (for dry-run purposes) 701 params: 702 Parameters to use when adding entries to SRC_URI. If specified, 703 should be a list of dicts with the same length as srcfiles. 704 update_original_recipe: 705 Force to update the original recipe instead of creating/updating 706 a bbapend. destlayerdir must contain the original recipe 707 """ 708 709 if not removevalues: 710 removevalues = {} 711 712 recipefile = rd.getVar('FILE') 713 if update_original_recipe: 714 if destlayerdir not in recipefile: 715 bb.error("destlayerdir %s doesn't contain the original recipe (%s), cannot update it" % (destlayerdir, recipefile)) 716 return (None, None) 717 718 appendpath = recipefile 719 else: 720 # Determine how the bbappend should be named 721 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) 722 if not appendpath: 723 bb.error('Unable to determine layer directory containing %s' % recipefile) 724 return (None, None) 725 if not pathok: 726 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) 727 728 appenddir = os.path.dirname(appendpath) 729 if not redirect_output: 730 bb.utils.mkdirhier(appenddir) 731 732 # FIXME check if the bbappend doesn't get overridden by a higher priority layer? 733 734 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 735 if not os.path.abspath(destlayerdir) in layerdirs: 736 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') 737 738 bbappendlines = [] 739 if extralines: 740 if isinstance(extralines, dict): 741 for name, value in extralines.items(): 742 bbappendlines.append((name, '=', value)) 743 else: 744 # Do our best to split it 745 for line in extralines: 746 if line[-1] == '\n': 747 line = line[:-1] 748 splitline = line.split(None, 2) 749 if len(splitline) == 3: 750 bbappendlines.append(tuple(splitline)) 751 else: 752 raise Exception('Invalid extralines value passed') 753 754 def popline(varname): 755 for i in range(0, len(bbappendlines)): 756 if bbappendlines[i][0] == varname: 757 line = bbappendlines.pop(i) 758 return line 759 return None 760 761 def appendline(varname, op, value): 762 for i in range(0, len(bbappendlines)): 763 item = bbappendlines[i] 764 if item[0] == varname: 765 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) 766 break 767 else: 768 bbappendlines.append((varname, op, value)) 769 770 destsubdir = rd.getVar('PN') 771 if not update_original_recipe and srcfiles: 772 bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:')) 773 774 appendoverride = '' 775 if machine: 776 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}')) 777 appendoverride = ':%s' % machine 778 copyfiles = {} 779 if srcfiles: 780 instfunclines = [] 781 for i, (newfile, param) in enumerate(srcfiles.items()): 782 srcurientry = None 783 if not 'path' in param or not param['path']: 784 if 'newname' in param and param['newname']: 785 srcfile = param['newname'] 786 else: 787 srcfile = os.path.basename(newfile) 788 srcurientry = 'file://%s' % srcfile 789 oldentry = None 790 for uri in rd.getVar('SRC_URI').split(): 791 if srcurientry in uri: 792 oldentry = uri 793 if params and params[i]: 794 srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items())) 795 # Double-check it's not there already 796 # FIXME do we care if the entry is added by another bbappend that might go away? 797 if not srcurientry in rd.getVar('SRC_URI').split(): 798 if machine: 799 if oldentry: 800 appendline('SRC_URI:remove%s' % appendoverride, '=', ' ' + oldentry) 801 appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) 802 else: 803 if oldentry: 804 if update_original_recipe: 805 removevalues['SRC_URI'] = oldentry 806 else: 807 appendline('SRC_URI:remove', '=', oldentry) 808 appendline('SRC_URI', '+=', srcurientry) 809 param['path'] = srcfile 810 else: 811 srcfile = param['path'] 812 copyfiles[newfile] = param 813 if install: 814 institem = install.pop(newfile, None) 815 if institem: 816 (destpath, perms) = institem 817 instdestpath = replace_dir_vars(destpath, rd) 818 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath) 819 if not instdirline in instfunclines: 820 instfunclines.append(instdirline) 821 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath)) 822 if instfunclines: 823 bbappendlines.append(('do_install:append%s()' % appendoverride, '', instfunclines)) 824 825 if redirect_output: 826 bb.note('Writing append file %s (dry-run)' % appendpath) 827 outfile = os.path.join(redirect_output, os.path.basename(appendpath)) 828 # Only take a copy if the file isn't already there (this function may be called 829 # multiple times per operation when we're handling overrides) 830 if os.path.exists(appendpath) and not os.path.exists(outfile): 831 shutil.copy2(appendpath, outfile) 832 elif update_original_recipe: 833 outfile = recipefile 834 else: 835 bb.note('Writing append file %s' % appendpath) 836 outfile = appendpath 837 838 if os.path.exists(outfile): 839 # Work around lack of nonlocal in python 2 840 extvars = {'destsubdir': destsubdir} 841 842 def appendfile_varfunc(varname, origvalue, op, newlines): 843 if varname == 'FILESEXTRAPATHS:prepend': 844 if origvalue.startswith('${THISDIR}/'): 845 popline('FILESEXTRAPATHS:prepend') 846 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':')) 847 elif varname == 'PACKAGE_ARCH': 848 if machine: 849 popline('PACKAGE_ARCH') 850 return (machine, None, 4, False) 851 elif varname.startswith('do_install:append'): 852 func = popline(varname) 853 if func: 854 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()] 855 for line in func[2]: 856 if not line in instfunclines: 857 instfunclines.append(line) 858 return (instfunclines, None, 4, False) 859 else: 860 splitval = split_var_value(origvalue, assignment=False) 861 changed = False 862 removevar = varname 863 if varname in ['SRC_URI', 'SRC_URI:append%s' % appendoverride]: 864 removevar = 'SRC_URI' 865 line = popline(varname) 866 if line: 867 if line[2] not in splitval: 868 splitval.append(line[2]) 869 changed = True 870 else: 871 line = popline(varname) 872 if line: 873 splitval = [line[2]] 874 changed = True 875 876 if removevar in removevalues: 877 remove = removevalues[removevar] 878 if isinstance(remove, str): 879 if remove in splitval: 880 splitval.remove(remove) 881 changed = True 882 else: 883 for removeitem in remove: 884 if removeitem in splitval: 885 splitval.remove(removeitem) 886 changed = True 887 888 if changed: 889 newvalue = splitval 890 if len(newvalue) == 1: 891 # Ensure it's written out as one line 892 if ':append' in varname: 893 newvalue = ' ' + newvalue[0] 894 else: 895 newvalue = newvalue[0] 896 if not newvalue and (op in ['+=', '.='] or ':append' in varname): 897 # There's no point appending nothing 898 newvalue = None 899 if varname.endswith('()'): 900 indent = 4 901 else: 902 indent = -1 903 return (newvalue, None, indent, True) 904 return (origvalue, None, 4, False) 905 906 varnames = [item[0] for item in bbappendlines] 907 if removevalues: 908 varnames.extend(list(removevalues.keys())) 909 910 with open(outfile, 'r') as f: 911 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) 912 913 destsubdir = extvars['destsubdir'] 914 else: 915 updated = False 916 newlines = [] 917 918 if bbappendlines: 919 for line in bbappendlines: 920 if line[0].endswith('()'): 921 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2]))) 922 else: 923 newlines.append('%s %s "%s"\n\n' % line) 924 updated = True 925 926 if updated: 927 with open(outfile, 'w') as f: 928 f.writelines(newlines) 929 930 if copyfiles: 931 if machine: 932 destsubdir = os.path.join(destsubdir, machine) 933 if redirect_output: 934 outdir = redirect_output 935 else: 936 outdir = appenddir 937 for newfile, param in copyfiles.items(): 938 srcfile = param['path'] 939 patchdir = param.get('patchdir', ".") 940 941 if patchdir != ".": 942 newfile = os.path.join(os.path.split(newfile)[0], patchdir, os.path.split(newfile)[1]) 943 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile)) 944 if os.path.abspath(newfile) != os.path.abspath(filedest): 945 if newfile.startswith(tempfile.gettempdir()): 946 newfiledisp = os.path.basename(newfile) 947 else: 948 newfiledisp = newfile 949 if redirect_output: 950 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile)))) 951 else: 952 bb.note('Copying %s to %s' % (newfiledisp, filedest)) 953 bb.utils.mkdirhier(os.path.dirname(filedest)) 954 shutil.copyfile(newfile, filedest) 955 956 return (appendpath, os.path.join(appenddir, destsubdir)) 957 958 959def find_layerdir(fn): 960 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" 961 pth = os.path.abspath(fn) 962 layerdir = '' 963 while pth: 964 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): 965 layerdir = pth 966 break 967 pth = os.path.dirname(pth) 968 if pth == '/': 969 return None 970 return layerdir 971 972 973def replace_dir_vars(path, d): 974 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" 975 dirvars = {} 976 # Sort by length so we get the variables we're interested in first 977 for var in sorted(list(d.keys()), key=len): 978 if var.endswith('dir') and var.lower() == var: 979 value = d.getVar(var) 980 if value.startswith('/') and not '\n' in value and value not in dirvars: 981 dirvars[value] = var 982 for dirpath in sorted(list(dirvars.keys()), reverse=True): 983 path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) 984 return path 985 986def get_recipe_pv_with_pfx_sfx(pv, uri_type): 987 """ 988 Get PV separating prefix and suffix components. 989 990 Returns tuple with pv, prefix and suffix. 991 """ 992 pfx = '' 993 sfx = '' 994 995 if uri_type == 'git': 996 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+)?)(?P<rev>.*)") 997 m = git_regex.match(pv) 998 999 if m: 1000 pv = m.group('ver') 1001 pfx = m.group('pfx') 1002 sfx = m.group('sfx') 1003 else: 1004 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)") 1005 m = regex.match(pv) 1006 if m: 1007 pv = m.group('ver') 1008 pfx = m.group('pfx') 1009 1010 return (pv, pfx, sfx) 1011 1012def get_recipe_upstream_version(rd): 1013 """ 1014 Get upstream version of recipe using bb.fetch2 methods with support for 1015 http, https, ftp and git. 1016 1017 bb.fetch2 exceptions can be raised, 1018 FetchError when don't have network access or upstream site don't response. 1019 NoMethodError when uri latest_versionstring method isn't implemented. 1020 1021 Returns a dictonary with version, repository revision, current_version, type and datetime. 1022 Type can be A for Automatic, M for Manual and U for Unknown. 1023 """ 1024 from bb.fetch2 import decodeurl 1025 from datetime import datetime 1026 1027 ru = {} 1028 ru['current_version'] = rd.getVar('PV') 1029 ru['version'] = '' 1030 ru['type'] = 'U' 1031 ru['datetime'] = '' 1032 ru['revision'] = '' 1033 1034 # XXX: If don't have SRC_URI means that don't have upstream sources so 1035 # returns the current recipe version, so that upstream version check 1036 # declares a match. 1037 src_uris = rd.getVar('SRC_URI') 1038 if not src_uris: 1039 ru['version'] = ru['current_version'] 1040 ru['type'] = 'M' 1041 ru['datetime'] = datetime.now() 1042 return ru 1043 1044 # XXX: we suppose that the first entry points to the upstream sources 1045 src_uri = src_uris.split()[0] 1046 uri_type, _, _, _, _, _ = decodeurl(src_uri) 1047 1048 (pv, pfx, sfx) = get_recipe_pv_with_pfx_sfx(rd.getVar('PV'), uri_type) 1049 ru['current_version'] = pv 1050 1051 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") 1052 if manual_upstream_version: 1053 # manual tracking of upstream version. 1054 ru['version'] = manual_upstream_version 1055 ru['type'] = 'M' 1056 1057 manual_upstream_date = rd.getVar("CHECK_DATE") 1058 if manual_upstream_date: 1059 date = datetime.strptime(manual_upstream_date, "%b %d, %Y") 1060 else: 1061 date = datetime.now() 1062 ru['datetime'] = date 1063 1064 elif uri_type == "file": 1065 # files are always up-to-date 1066 ru['version'] = pv 1067 ru['type'] = 'A' 1068 ru['datetime'] = datetime.now() 1069 else: 1070 ud = bb.fetch2.FetchData(src_uri, rd) 1071 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1": 1072 bb.fetch2.get_srcrev(rd) 1073 revision = ud.method.latest_revision(ud, rd, 'default') 1074 upversion = pv 1075 if revision != rd.getVar("SRCREV"): 1076 upversion = upversion + "-new-commits-available" 1077 else: 1078 pupver = ud.method.latest_versionstring(ud, rd) 1079 (upversion, revision) = pupver 1080 1081 if upversion: 1082 ru['version'] = upversion 1083 ru['type'] = 'A' 1084 1085 if revision: 1086 ru['revision'] = revision 1087 1088 ru['datetime'] = datetime.now() 1089 1090 return ru 1091 1092def _get_recipe_upgrade_status(data): 1093 uv = get_recipe_upstream_version(data) 1094 1095 pn = data.getVar('PN') 1096 cur_ver = uv['current_version'] 1097 1098 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') 1099 if not uv['version']: 1100 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" 1101 else: 1102 cmp = vercmp_string(uv['current_version'], uv['version']) 1103 if cmp == -1: 1104 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" 1105 elif cmp == 0: 1106 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" 1107 else: 1108 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" 1109 1110 next_ver = uv['version'] if uv['version'] else "N/A" 1111 revision = uv['revision'] if uv['revision'] else "N/A" 1112 maintainer = data.getVar('RECIPE_MAINTAINER') 1113 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') 1114 1115 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) 1116 1117def get_recipe_upgrade_status(recipes=None): 1118 pkgs_list = [] 1119 data_copy_list = [] 1120 copy_vars = ('SRC_URI', 1121 'PV', 1122 'DL_DIR', 1123 'PN', 1124 'CACHE', 1125 'PERSISTENT_DIR', 1126 'BB_URI_HEADREVS', 1127 'UPSTREAM_CHECK_COMMITS', 1128 'UPSTREAM_CHECK_GITTAGREGEX', 1129 'UPSTREAM_CHECK_REGEX', 1130 'UPSTREAM_CHECK_URI', 1131 'UPSTREAM_VERSION_UNKNOWN', 1132 'RECIPE_MAINTAINER', 1133 'RECIPE_NO_UPDATE_REASON', 1134 'RECIPE_UPSTREAM_VERSION', 1135 'RECIPE_UPSTREAM_DATE', 1136 'CHECK_DATE', 1137 'FETCHCMD_bzr', 1138 'FETCHCMD_ccrc', 1139 'FETCHCMD_cvs', 1140 'FETCHCMD_git', 1141 'FETCHCMD_hg', 1142 'FETCHCMD_npm', 1143 'FETCHCMD_osc', 1144 'FETCHCMD_p4', 1145 'FETCHCMD_repo', 1146 'FETCHCMD_s3', 1147 'FETCHCMD_svn', 1148 'FETCHCMD_wget', 1149 ) 1150 1151 with bb.tinfoil.Tinfoil() as tinfoil: 1152 tinfoil.prepare(config_only=False) 1153 1154 if not recipes: 1155 recipes = tinfoil.all_recipe_files(variants=False) 1156 1157 for fn in recipes: 1158 try: 1159 if fn.startswith("/"): 1160 data = tinfoil.parse_recipe_file(fn) 1161 else: 1162 data = tinfoil.parse_recipe(fn) 1163 except bb.providers.NoProvider: 1164 bb.note(" No provider for %s" % fn) 1165 continue 1166 1167 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE') 1168 if unreliable == "1": 1169 bb.note(" Skip package %s as upstream check unreliable" % pn) 1170 continue 1171 1172 data_copy = bb.data.init() 1173 for var in copy_vars: 1174 data_copy.setVar(var, data.getVar(var)) 1175 for k in data: 1176 if k.startswith('SRCREV'): 1177 data_copy.setVar(k, data.getVar(k)) 1178 1179 data_copy_list.append(data_copy) 1180 1181 from concurrent.futures import ProcessPoolExecutor 1182 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: 1183 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) 1184 1185 return pkgs_list 1186