1 """ 2 BitBake 'Data' implementations 3 4 Functions for interacting with the data structure used by the 5 BitBake build tools. 6 7 expandKeys and datastore iteration are the most expensive 8 operations. Updating overrides is now "on the fly" but still based 9 on the idea of the cookie monster introduced by zecke: 10 "At night the cookie monster came by and 11 suggested 'give me cookies on setting the variables and 12 things will work out'. Taking this suggestion into account 13 applying the skills from the not yet passed 'Entwurf und 14 Analyse von Algorithmen' lecture and the cookie 15 monster seems to be right. We will track setVar more carefully 16 to have faster datastore operations." 17 18 This is a trade-off between speed and memory again but 19 the speed is more critical here. 20 """ 21 22 # Copyright (C) 2003, 2004 Chris Larson 23 # Copyright (C) 2005 Holger Hans Peter Freyther 24 # 25 # SPDX-License-Identifier: GPL-2.0-only 26 # 27 # Based on functions from the base bb module, Copyright 2003 Holger Schurig 28 29 import sys, os, re 30 import hashlib 31 from itertools import groupby 32 33 from bb import data_smart 34 from bb import codeparser 35 import bb 36 37 logger = data_smart.logger 38 _dict_type = data_smart.DataSmart 39 40 def init(): 41 """Return a new object representing the Bitbake data""" 42 return _dict_type() 43 44 def init_db(parent = None): 45 """Return a new object representing the Bitbake data, 46 optionally based on an existing object""" 47 if parent is not None: 48 return parent.createCopy() 49 else: 50 return _dict_type() 51 52 def createCopy(source): 53 """Link the source set to the destination 54 If one does not find the value in the destination set, 55 search will go on to the source set to get the value. 56 Value from source are copy-on-write. i.e. any try to 57 modify one of them will end up putting the modified value 58 in the destination set. 59 """ 60 return source.createCopy() 61 62 def initVar(var, d): 63 """Non-destructive var init for data structure""" 64 d.initVar(var) 65 66 def keys(d): 67 """Return a list of keys in d""" 68 return d.keys() 69 70 def expand(s, d, varname = None): 71 """Variable expansion using the data store""" 72 return d.expand(s, varname) 73 74 def expandKeys(alterdata, readdata = None): 75 if readdata is None: 76 readdata = alterdata 77 78 todolist = {} 79 for key in alterdata: 80 if not '${' in key: 81 continue 82 83 ekey = expand(key, readdata) 84 if key == ekey: 85 continue 86 todolist[key] = ekey 87 88 # These two for loops are split for performance to maximise the 89 # usefulness of the expand cache 90 for key in sorted(todolist): 91 ekey = todolist[key] 92 newval = alterdata.getVar(ekey, False) 93 if newval is not None: 94 val = alterdata.getVar(key, False) 95 if val is not None: 96 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval)) 97 alterdata.renameVar(key, ekey) 98 99 def inheritFromOS(d, savedenv, permitted): 100 """Inherit variables from the initial environment.""" 101 exportlist = bb.utils.preserved_envvars_exported() 102 for s in savedenv.keys(): 103 if s in permitted: 104 try: 105 d.setVar(s, savedenv.getVar(s), op = 'from env') 106 if s in exportlist: 107 d.setVarFlag(s, "export", True, op = 'auto env export') 108 except TypeError: 109 pass 110 111 def emit_var(var, o=sys.__stdout__, d = init(), all=False): 112 """Emit a variable to be sourced by a shell.""" 113 func = d.getVarFlag(var, "func", False) 114 if d.getVarFlag(var, 'python', False) and func: 115 return False 116 117 export = bb.utils.to_boolean(d.getVarFlag(var, "export")) 118 unexport = bb.utils.to_boolean(d.getVarFlag(var, "unexport")) 119 if not all and not export and not unexport and not func: 120 return False 121 122 try: 123 if all: 124 oval = d.getVar(var, False) 125 val = d.getVar(var) 126 except (KeyboardInterrupt): 127 raise 128 except Exception as exc: 129 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc))) 130 return False 131 132 if all: 133 d.varhistory.emit(var, oval, val, o, d) 134 135 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: 136 return False 137 138 varExpanded = d.expand(var) 139 140 if unexport: 141 o.write('unset %s\n' % varExpanded) 142 return False 143 144 if val is None: 145 return False 146 147 val = str(val) 148 149 if varExpanded.startswith("BASH_FUNC_"): 150 varExpanded = varExpanded[10:-2] 151 val = val[3:] # Strip off "() " 152 o.write("%s() %s\n" % (varExpanded, val)) 153 o.write("export -f %s\n" % (varExpanded)) 154 return True 155 156 if func: 157 # Write a comment indicating where the shell function came from (line number and filename) to make it easier 158 # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata 159 # location of shell functions. 160 o.write("# line: {0}, file: {1}\n".format( 161 d.getVarFlag(var, "lineno", False), 162 d.getVarFlag(var, "filename", False))) 163 # NOTE: should probably check for unbalanced {} within the var 164 val = val.rstrip('\n') 165 o.write("%s() {\n%s\n}\n" % (varExpanded, val)) 166 return 1 167 168 if export: 169 o.write('export ') 170 171 # if we're going to output this within doublequotes, 172 # to a shell, we need to escape the quotes in the var 173 alter = re.sub('"', '\\"', val) 174 alter = re.sub('\n', ' \\\n', alter) 175 alter = re.sub('\\$', '\\\\$', alter) 176 o.write('%s="%s"\n' % (varExpanded, alter)) 177 return False 178 179 def emit_env(o=sys.__stdout__, d = init(), all=False): 180 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 181 182 isfunc = lambda key: bool(d.getVarFlag(key, "func", False)) 183 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc) 184 grouped = groupby(keys, isfunc) 185 for isfunc, keys in grouped: 186 for key in sorted(keys): 187 emit_var(key, o, d, all and not isfunc) and o.write('\n') 188 189 def exported_keys(d): 190 return (key for key in d.keys() if not key.startswith('__') and 191 bb.utils.to_boolean(d.getVarFlag(key, 'export')) and 192 not bb.utils.to_boolean(d.getVarFlag(key, 'unexport'))) 193 194 def exported_vars(d): 195 k = list(exported_keys(d)) 196 for key in k: 197 try: 198 value = d.getVar(key) 199 except Exception as err: 200 bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err)) 201 continue 202 203 if value is not None: 204 yield key, str(value) 205 206 def emit_func(func, o=sys.__stdout__, d = init()): 207 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 208 209 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False)) 210 for key in sorted(keys): 211 emit_var(key, o, d, False) 212 213 o.write('\n') 214 emit_var(func, o, d, False) and o.write('\n') 215 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func)) 216 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split()) 217 seen = set() 218 while newdeps: 219 deps = newdeps 220 seen |= deps 221 newdeps = set() 222 for dep in sorted(deps): 223 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False): 224 emit_var(dep, o, d, False) and o.write('\n') 225 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep)) 226 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) 227 newdeps -= seen 228 229 _functionfmt = """ 230 def {function}(d): 231 {body}""" 232 233 def emit_func_python(func, o=sys.__stdout__, d = init()): 234 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 235 236 def write_func(func, o, call = False): 237 body = d.getVar(func, False) 238 if not body.startswith("def"): 239 body = _functionfmt.format(function=func, body=body) 240 241 o.write(body.strip() + "\n\n") 242 if call: 243 o.write(func + "(d)" + "\n\n") 244 245 write_func(func, o, True) 246 pp = bb.codeparser.PythonParser(func, logger) 247 pp.parse_python(d.getVar(func, False)) 248 newdeps = pp.execs 249 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split()) 250 seen = set() 251 while newdeps: 252 deps = newdeps 253 seen |= deps 254 newdeps = set() 255 for dep in deps: 256 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False): 257 write_func(dep, o) 258 pp = bb.codeparser.PythonParser(dep, logger) 259 pp.parse_python(d.getVar(dep, False)) 260 newdeps |= pp.execs 261 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) 262 newdeps -= seen 263 264 def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparsedata): 265 def handle_contains(value, contains, exclusions, d): 266 newvalue = [] 267 if value: 268 newvalue.append(str(value)) 269 for k in sorted(contains): 270 if k in exclusions or k in ignored_vars: 271 continue 272 l = (d.getVar(k) or "").split() 273 for item in sorted(contains[k]): 274 for word in item.split(): 275 if not word in l: 276 newvalue.append("\n%s{%s} = Unset" % (k, item)) 277 break 278 else: 279 newvalue.append("\n%s{%s} = Set" % (k, item)) 280 return "".join(newvalue) 281 282 def handle_remove(value, deps, removes, d): 283 for r in sorted(removes): 284 r2 = d.expandWithRefs(r, None) 285 value += "\n_remove of %s" % r 286 deps |= r2.references 287 deps = deps | (keys & r2.execs) 288 value = handle_contains(value, r2.contains, exclusions, d) 289 return value 290 291 deps = set() 292 try: 293 if key in mod_funcs: 294 exclusions = set() 295 moddep = bb.codeparser.modulecode_deps[key] 296 value = handle_contains(moddep[4], moddep[3], exclusions, d) 297 return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value 298 299 if key[-1] == ']': 300 vf = key[:-1].split('[') 301 if vf[1] == "vardepvalueexclude": 302 return deps, "" 303 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True) 304 deps |= parser.references 305 deps = deps | (keys & parser.execs) 306 deps -= ignored_vars 307 return frozenset(deps), value 308 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {} 309 vardeps = varflags.get("vardeps") 310 exclusions = varflags.get("vardepsexclude", "").split() 311 312 if "vardepvalue" in varflags: 313 value = varflags.get("vardepvalue") 314 elif varflags.get("func"): 315 if varflags.get("python"): 316 value = codeparsedata.getVarFlag(key, "_content", False) 317 parser = bb.codeparser.PythonParser(key, logger) 318 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno")) 319 deps = deps | parser.references 320 deps = deps | (keys & parser.execs) 321 value = handle_contains(value, parser.contains, exclusions, d) 322 else: 323 value, parsedvar = codeparsedata.getVarFlag(key, "_content", False, retparser=True) 324 parser = bb.codeparser.ShellParser(key, logger) 325 parser.parse_shell(parsedvar.value) 326 deps = deps | shelldeps 327 deps = deps | parsedvar.references 328 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs) 329 value = handle_contains(value, parsedvar.contains, exclusions, d) 330 if hasattr(parsedvar, "removes"): 331 value = handle_remove(value, deps, parsedvar.removes, d) 332 if vardeps is None: 333 parser.log.flush() 334 if "prefuncs" in varflags: 335 deps = deps | set(varflags["prefuncs"].split()) 336 if "postfuncs" in varflags: 337 deps = deps | set(varflags["postfuncs"].split()) 338 if "exports" in varflags: 339 deps = deps | set(varflags["exports"].split()) 340 else: 341 value, parser = d.getVarFlag(key, "_content", False, retparser=True) 342 deps |= parser.references 343 deps = deps | (keys & parser.execs) 344 value = handle_contains(value, parser.contains, exclusions, d) 345 if hasattr(parser, "removes"): 346 value = handle_remove(value, deps, parser.removes, d) 347 348 if "vardepvalueexclude" in varflags: 349 exclude = varflags.get("vardepvalueexclude") 350 for excl in exclude.split('|'): 351 if excl: 352 value = value.replace(excl, '') 353 354 # Add varflags, assuming an exclusion list is set 355 if varflagsexcl: 356 varfdeps = [] 357 for f in varflags: 358 if f not in varflagsexcl: 359 varfdeps.append('%s[%s]' % (key, f)) 360 if varfdeps: 361 deps |= set(varfdeps) 362 363 deps |= set((vardeps or "").split()) 364 deps -= set(exclusions) 365 deps -= ignored_vars 366 except bb.parse.SkipRecipe: 367 raise 368 except Exception as e: 369 bb.warn("Exception during build_dependencies for %s" % key) 370 raise 371 return frozenset(deps), value 372 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) 373 #d.setVarFlag(key, "vardeps", deps) 374 375 def generate_dependencies(d, ignored_vars): 376 377 mod_funcs = set(bb.codeparser.modulecode_deps.keys()) 378 keys = set(key for key in d if not key.startswith("__")) | mod_funcs 379 shelldeps = set(key for key in d.getVar("__exportlist", False) if bb.utils.to_boolean(d.getVarFlag(key, "export")) and not bb.utils.to_boolean(d.getVarFlag(key, "unexport"))) 380 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') 381 382 codeparserd = d.createCopy() 383 for forced in (d.getVar('BB_HASH_CODEPARSER_VALS') or "").split(): 384 key, value = forced.split("=", 1) 385 codeparserd.setVar(key, value) 386 387 deps = {} 388 values = {} 389 390 tasklist = d.getVar('__BBTASKS', False) or [] 391 for task in tasklist: 392 deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd) 393 newdeps = deps[task] 394 seen = set() 395 while newdeps: 396 nextdeps = newdeps 397 seen |= nextdeps 398 newdeps = set() 399 for dep in nextdeps: 400 if dep not in deps: 401 deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd) 402 newdeps |= deps[dep] 403 newdeps -= seen 404 #print "For %s: %s" % (task, str(deps[task])) 405 return tasklist, deps, values 406 407 def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): 408 taskdeps = {} 409 basehash = {} 410 411 for task in tasklist: 412 data = lookupcache[task] 413 414 if data is None: 415 bb.error("Task %s from %s seems to be empty?!" % (task, fn)) 416 data = [] 417 else: 418 data = [data] 419 420 newdeps = gendeps[task] 421 seen = set() 422 while newdeps: 423 nextdeps = newdeps 424 seen |= nextdeps 425 newdeps = set() 426 for dep in nextdeps: 427 newdeps |= gendeps[dep] 428 newdeps -= seen 429 430 alldeps = sorted(seen) 431 for dep in alldeps: 432 data.append(dep) 433 var = lookupcache[dep] 434 if var is not None: 435 data.append(str(var)) 436 k = fn + ":" + task 437 basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest() 438 taskdeps[task] = frozenset(seen) 439 440 return taskdeps, basehash 441 442 def inherits_class(klass, d): 443 val = d.getVar('__inherit_cache', False) or [] 444 needle = '/%s.bbclass' % klass 445 for v in val: 446 if v.endswith(needle): 447 return True 448 return False 449