1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import hashlib 6import logging 7import os 8import re 9import tempfile 10import pickle 11import bb.data 12import difflib 13import simplediff 14from bb.checksum import FileChecksumCache 15 16logger = logging.getLogger('BitBake.SigGen') 17 18def init(d): 19 siggens = [obj for obj in globals().values() 20 if type(obj) is type and issubclass(obj, SignatureGenerator)] 21 22 desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop" 23 for sg in siggens: 24 if desired == sg.name: 25 return sg(d) 26 break 27 else: 28 logger.error("Invalid signature generator '%s', using default 'noop'\n" 29 "Available generators: %s", desired, 30 ', '.join(obj.name for obj in siggens)) 31 return SignatureGenerator(d) 32 33class SignatureGenerator(object): 34 """ 35 """ 36 name = "noop" 37 38 def __init__(self, data): 39 self.basehash = {} 40 self.taskhash = {} 41 self.runtaskdeps = {} 42 self.file_checksum_values = {} 43 self.taints = {} 44 45 def finalise(self, fn, d, varient): 46 return 47 48 def get_unihash(self, task): 49 return self.taskhash[task] 50 51 def get_taskhash(self, fn, task, deps, dataCache): 52 return "0" 53 54 def writeout_file_checksum_cache(self): 55 """Write/update the file checksum cache onto disk""" 56 return 57 58 def stampfile(self, stampbase, file_name, taskname, extrainfo): 59 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 60 61 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo): 62 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 63 64 def dump_sigtask(self, fn, task, stampbase, runtime): 65 return 66 67 def invalidate_task(self, task, d, fn): 68 bb.build.del_stamp(task, d, fn) 69 70 def dump_sigs(self, dataCache, options): 71 return 72 73 def get_taskdata(self): 74 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) 75 76 def set_taskdata(self, data): 77 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data 78 79 def reset(self, data): 80 self.__init__(data) 81 82 83class SignatureGeneratorBasic(SignatureGenerator): 84 """ 85 """ 86 name = "basic" 87 88 def __init__(self, data): 89 self.basehash = {} 90 self.taskhash = {} 91 self.taskdeps = {} 92 self.runtaskdeps = {} 93 self.file_checksum_values = {} 94 self.taints = {} 95 self.gendeps = {} 96 self.lookupcache = {} 97 self.pkgnameextract = re.compile(r"(?P<fn>.*)\..*") 98 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) 99 self.taskwhitelist = None 100 self.init_rundepcheck(data) 101 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") 102 if checksum_cache_file: 103 self.checksum_cache = FileChecksumCache() 104 self.checksum_cache.init_cache(data, checksum_cache_file) 105 else: 106 self.checksum_cache = None 107 108 def init_rundepcheck(self, data): 109 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None 110 if self.taskwhitelist: 111 self.twl = re.compile(self.taskwhitelist) 112 else: 113 self.twl = None 114 115 def _build_data(self, fn, d): 116 117 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') 118 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) 119 120 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) 121 122 for task in tasklist: 123 k = fn + "." + task 124 if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]: 125 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k])) 126 bb.error("The following commands may help:") 127 cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task) 128 # Make sure sigdata is dumped before run printdiff 129 bb.error("%s -Snone" % cmd) 130 bb.error("Then:") 131 bb.error("%s -Sprintdiff\n" % cmd) 132 self.basehash[k] = basehash[k] 133 134 self.taskdeps[fn] = taskdeps 135 self.gendeps[fn] = gendeps 136 self.lookupcache[fn] = lookupcache 137 138 return taskdeps 139 140 def finalise(self, fn, d, variant): 141 142 mc = d.getVar("__BBMULTICONFIG", False) or "" 143 if variant or mc: 144 fn = bb.cache.realfn2virtual(fn, variant, mc) 145 146 try: 147 taskdeps = self._build_data(fn, d) 148 except bb.parse.SkipRecipe: 149 raise 150 except: 151 bb.warn("Error during finalise of %s" % fn) 152 raise 153 154 #Slow but can be useful for debugging mismatched basehashes 155 #for task in self.taskdeps[fn]: 156 # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) 157 158 for task in taskdeps: 159 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) 160 161 def rundep_check(self, fn, recipename, task, dep, depname, dataCache): 162 # Return True if we should keep the dependency, False to drop it 163 # We only manipulate the dependencies for packages not in the whitelist 164 if self.twl and not self.twl.search(recipename): 165 # then process the actual dependencies 166 if self.twl.search(depname): 167 return False 168 return True 169 170 def read_taint(self, fn, task, stampbase): 171 taint = None 172 try: 173 with open(stampbase + '.' + task + '.taint', 'r') as taintf: 174 taint = taintf.read() 175 except IOError: 176 pass 177 return taint 178 179 def get_taskhash(self, fn, task, deps, dataCache): 180 181 mc = '' 182 if fn.startswith('mc:'): 183 mc = fn.split(':')[1] 184 k = fn + "." + task 185 186 data = dataCache.basetaskhash[k] 187 self.basehash[k] = data 188 self.runtaskdeps[k] = [] 189 self.file_checksum_values[k] = [] 190 recipename = dataCache.pkg_fn[fn] 191 for dep in sorted(deps, key=clean_basepath): 192 pkgname = self.pkgnameextract.search(dep).group('fn') 193 if mc: 194 depmc = pkgname.split(':')[1] 195 if mc != depmc: 196 continue 197 if dep.startswith("mc:") and not mc: 198 continue 199 depname = dataCache.pkg_fn[pkgname] 200 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): 201 continue 202 if dep not in self.taskhash: 203 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) 204 data = data + self.get_unihash(dep) 205 self.runtaskdeps[k].append(dep) 206 207 if task in dataCache.file_checksums[fn]: 208 if self.checksum_cache: 209 checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) 210 else: 211 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) 212 for (f,cs) in checksums: 213 self.file_checksum_values[k].append((f,cs)) 214 if cs: 215 data = data + cs 216 217 taskdep = dataCache.task_deps[fn] 218 if 'nostamp' in taskdep and task in taskdep['nostamp']: 219 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run 220 import uuid 221 taint = str(uuid.uuid4()) 222 data = data + taint 223 self.taints[k] = "nostamp:" + taint 224 225 taint = self.read_taint(fn, task, dataCache.stamp[fn]) 226 if taint: 227 data = data + taint 228 self.taints[k] = taint 229 logger.warning("%s is tainted from a forced run" % k) 230 231 h = hashlib.sha256(data.encode("utf-8")).hexdigest() 232 self.taskhash[k] = h 233 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 234 return h 235 236 def writeout_file_checksum_cache(self): 237 """Write/update the file checksum cache onto disk""" 238 if self.checksum_cache: 239 self.checksum_cache.save_extras() 240 self.checksum_cache.save_merge() 241 else: 242 bb.fetch2.fetcher_parse_save() 243 bb.fetch2.fetcher_parse_done() 244 245 def dump_sigtask(self, fn, task, stampbase, runtime): 246 247 k = fn + "." + task 248 referencestamp = stampbase 249 if isinstance(runtime, str) and runtime.startswith("customfile"): 250 sigfile = stampbase 251 referencestamp = runtime[11:] 252 elif runtime and k in self.taskhash: 253 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] 254 else: 255 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] 256 257 bb.utils.mkdirhier(os.path.dirname(sigfile)) 258 259 data = {} 260 data['task'] = task 261 data['basewhitelist'] = self.basewhitelist 262 data['taskwhitelist'] = self.taskwhitelist 263 data['taskdeps'] = self.taskdeps[fn][task] 264 data['basehash'] = self.basehash[k] 265 data['gendeps'] = {} 266 data['varvals'] = {} 267 data['varvals'][task] = self.lookupcache[fn][task] 268 for dep in self.taskdeps[fn][task]: 269 if dep in self.basewhitelist: 270 continue 271 data['gendeps'][dep] = self.gendeps[fn][dep] 272 data['varvals'][dep] = self.lookupcache[fn][dep] 273 274 if runtime and k in self.taskhash: 275 data['runtaskdeps'] = self.runtaskdeps[k] 276 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] 277 data['runtaskhashes'] = {} 278 for dep in data['runtaskdeps']: 279 data['runtaskhashes'][dep] = self.get_unihash(dep) 280 data['taskhash'] = self.taskhash[k] 281 282 taint = self.read_taint(fn, task, referencestamp) 283 if taint: 284 data['taint'] = taint 285 286 if runtime and k in self.taints: 287 if 'nostamp:' in self.taints[k]: 288 data['taint'] = self.taints[k] 289 290 computed_basehash = calc_basehash(data) 291 if computed_basehash != self.basehash[k]: 292 bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) 293 if runtime and k in self.taskhash: 294 computed_taskhash = calc_taskhash(data) 295 if computed_taskhash != self.taskhash[k]: 296 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) 297 sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) 298 299 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") 300 try: 301 with os.fdopen(fd, "wb") as stream: 302 p = pickle.dump(data, stream, -1) 303 stream.flush() 304 os.chmod(tmpfile, 0o664) 305 os.rename(tmpfile, sigfile) 306 except (OSError, IOError) as err: 307 try: 308 os.unlink(tmpfile) 309 except OSError: 310 pass 311 raise err 312 313 def dump_sigfn(self, fn, dataCaches, options): 314 if fn in self.taskdeps: 315 for task in self.taskdeps[fn]: 316 tid = fn + ":" + task 317 (mc, _, _) = bb.runqueue.split_tid(tid) 318 k = fn + "." + task 319 if k not in self.taskhash: 320 continue 321 if dataCaches[mc].basetaskhash[k] != self.basehash[k]: 322 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) 323 bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) 324 self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) 325 326class SignatureGeneratorBasicHash(SignatureGeneratorBasic): 327 name = "basichash" 328 329 def get_stampfile_hash(self, task): 330 if task in self.taskhash: 331 return self.taskhash[task] 332 333 # If task is not in basehash, then error 334 return self.basehash[task] 335 336 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): 337 if taskname != "do_setscene" and taskname.endswith("_setscene"): 338 k = fn + "." + taskname[:-9] 339 else: 340 k = fn + "." + taskname 341 if clean: 342 h = "*" 343 else: 344 h = self.get_stampfile_hash(k) 345 346 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') 347 348 def stampcleanmask(self, stampbase, fn, taskname, extrainfo): 349 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True) 350 351 def invalidate_task(self, task, d, fn): 352 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) 353 bb.build.write_taint(task, d, fn) 354 355def dump_this_task(outfile, d): 356 import bb.parse 357 fn = d.getVar("BB_FILENAME") 358 task = "do_" + d.getVar("BB_CURRENTTASK") 359 referencestamp = bb.build.stamp_internal(task, d, None, True) 360 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) 361 362def init_colors(enable_color): 363 """Initialise colour dict for passing to compare_sigfiles()""" 364 # First set up the colours 365 colors = {'color_title': '\033[1m', 366 'color_default': '\033[0m', 367 'color_add': '\033[0;32m', 368 'color_remove': '\033[0;31m', 369 } 370 # Leave all keys present but clear the values 371 if not enable_color: 372 for k in colors.keys(): 373 colors[k] = '' 374 return colors 375 376def worddiff_str(oldstr, newstr, colors=None): 377 if not colors: 378 colors = init_colors(False) 379 diff = simplediff.diff(oldstr.split(' '), newstr.split(' ')) 380 ret = [] 381 for change, value in diff: 382 value = ' '.join(value) 383 if change == '=': 384 ret.append(value) 385 elif change == '+': 386 item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors) 387 ret.append(item) 388 elif change == '-': 389 item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors) 390 ret.append(item) 391 whitespace_note = '' 392 if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()): 393 whitespace_note = ' (whitespace changed)' 394 return '"%s"%s' % (' '.join(ret), whitespace_note) 395 396def list_inline_diff(oldlist, newlist, colors=None): 397 if not colors: 398 colors = init_colors(False) 399 diff = simplediff.diff(oldlist, newlist) 400 ret = [] 401 for change, value in diff: 402 value = ' '.join(value) 403 if change == '=': 404 ret.append("'%s'" % value) 405 elif change == '+': 406 item = '{color_add}+{value}{color_default}'.format(value=value, **colors) 407 ret.append(item) 408 elif change == '-': 409 item = '{color_remove}-{value}{color_default}'.format(value=value, **colors) 410 ret.append(item) 411 return '[%s]' % (', '.join(ret)) 412 413def clean_basepath(a): 414 mc = None 415 if a.startswith("mc:"): 416 _, mc, a = a.split(":", 2) 417 b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] 418 if a.startswith("virtual:"): 419 b = b + ":" + a.rsplit(":", 1)[0] 420 if mc: 421 b = b + ":mc:" + mc 422 return b 423 424def clean_basepaths(a): 425 b = {} 426 for x in a: 427 b[clean_basepath(x)] = a[x] 428 return b 429 430def clean_basepaths_list(a): 431 b = [] 432 for x in a: 433 b.append(clean_basepath(x)) 434 return b 435 436def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): 437 output = [] 438 439 colors = init_colors(color) 440 def color_format(formatstr, **values): 441 """ 442 Return colour formatted string. 443 NOTE: call with the format string, not an already formatted string 444 containing values (otherwise you could have trouble with { and } 445 characters) 446 """ 447 if not formatstr.endswith('{color_default}'): 448 formatstr += '{color_default}' 449 # In newer python 3 versions you can pass both of these directly, 450 # but we only require 3.4 at the moment 451 formatparams = {} 452 formatparams.update(colors) 453 formatparams.update(values) 454 return formatstr.format(**formatparams) 455 456 with open(a, 'rb') as f: 457 p1 = pickle.Unpickler(f) 458 a_data = p1.load() 459 with open(b, 'rb') as f: 460 p2 = pickle.Unpickler(f) 461 b_data = p2.load() 462 463 def dict_diff(a, b, whitelist=set()): 464 sa = set(a.keys()) 465 sb = set(b.keys()) 466 common = sa & sb 467 changed = set() 468 for i in common: 469 if a[i] != b[i] and i not in whitelist: 470 changed.add(i) 471 added = sb - sa 472 removed = sa - sb 473 return changed, added, removed 474 475 def file_checksums_diff(a, b): 476 from collections import Counter 477 # Handle old siginfo format 478 if isinstance(a, dict): 479 a = [(os.path.basename(f), cs) for f, cs in a.items()] 480 if isinstance(b, dict): 481 b = [(os.path.basename(f), cs) for f, cs in b.items()] 482 # Compare lists, ensuring we can handle duplicate filenames if they exist 483 removedcount = Counter(a) 484 removedcount.subtract(b) 485 addedcount = Counter(b) 486 addedcount.subtract(a) 487 added = [] 488 for x in b: 489 if addedcount[x] > 0: 490 addedcount[x] -= 1 491 added.append(x) 492 removed = [] 493 changed = [] 494 for x in a: 495 if removedcount[x] > 0: 496 removedcount[x] -= 1 497 for y in added: 498 if y[0] == x[0]: 499 changed.append((x[0], x[1], y[1])) 500 added.remove(y) 501 break 502 else: 503 removed.append(x) 504 added = [x[0] for x in added] 505 removed = [x[0] for x in removed] 506 return changed, added, removed 507 508 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: 509 output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) 510 if a_data['basewhitelist'] and b_data['basewhitelist']: 511 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) 512 513 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: 514 output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) 515 if a_data['taskwhitelist'] and b_data['taskwhitelist']: 516 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) 517 518 if a_data['taskdeps'] != b_data['taskdeps']: 519 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) 520 521 if a_data['basehash'] != b_data['basehash'] and not collapsed: 522 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) 523 524 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) 525 if changed: 526 for dep in changed: 527 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) 528 if a_data['gendeps'][dep] and b_data['gendeps'][dep]: 529 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) 530 if added: 531 for dep in added: 532 output.append(color_format("{color_title}Dependency on variable %s was added") % (dep)) 533 if removed: 534 for dep in removed: 535 output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep)) 536 537 538 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) 539 if changed: 540 for dep in changed: 541 oldval = a_data['varvals'][dep] 542 newval = b_data['varvals'][dep] 543 if newval and oldval and ('\n' in oldval or '\n' in newval): 544 diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='') 545 # Cut off the first two lines, since we aren't interested in 546 # the old/new filename (they are blank anyway in this case) 547 difflines = list(diff)[2:] 548 if color: 549 # Add colour to diff output 550 for i, line in enumerate(difflines): 551 if line.startswith('+'): 552 line = color_format('{color_add}{line}', line=line) 553 difflines[i] = line 554 elif line.startswith('-'): 555 line = color_format('{color_remove}{line}', line=line) 556 difflines[i] = line 557 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines))) 558 elif newval and oldval and (' ' in oldval or ' ' in newval): 559 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors))) 560 else: 561 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval)) 562 563 if not 'file_checksum_values' in a_data: 564 a_data['file_checksum_values'] = {} 565 if not 'file_checksum_values' in b_data: 566 b_data['file_checksum_values'] = {} 567 568 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) 569 if changed: 570 for f, old, new in changed: 571 output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new)) 572 if added: 573 for f in added: 574 output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f)) 575 if removed: 576 for f in removed: 577 output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f)) 578 579 if not 'runtaskdeps' in a_data: 580 a_data['runtaskdeps'] = {} 581 if not 'runtaskdeps' in b_data: 582 b_data['runtaskdeps'] = {} 583 584 if not collapsed: 585 if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): 586 changed = ["Number of task dependencies changed"] 587 else: 588 changed = [] 589 for idx, task in enumerate(a_data['runtaskdeps']): 590 a = a_data['runtaskdeps'][idx] 591 b = b_data['runtaskdeps'][idx] 592 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed: 593 changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b])) 594 595 if changed: 596 clean_a = clean_basepaths_list(a_data['runtaskdeps']) 597 clean_b = clean_basepaths_list(b_data['runtaskdeps']) 598 if clean_a != clean_b: 599 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors)) 600 else: 601 output.append(color_format("{color_title}runtaskdeps changed:")) 602 output.append("\n".join(changed)) 603 604 605 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: 606 a = a_data['runtaskhashes'] 607 b = b_data['runtaskhashes'] 608 changed, added, removed = dict_diff(a, b) 609 if added: 610 for dep in added: 611 bdep_found = False 612 if removed: 613 for bdep in removed: 614 if b[dep] == a[bdep]: 615 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) 616 bdep_found = True 617 if not bdep_found: 618 output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep])) 619 if removed: 620 for dep in removed: 621 adep_found = False 622 if added: 623 for adep in added: 624 if b[adep] == a[dep]: 625 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) 626 adep_found = True 627 if not adep_found: 628 output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep])) 629 if changed: 630 for dep in changed: 631 if not collapsed: 632 output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep])) 633 if callable(recursecb): 634 recout = recursecb(dep, a[dep], b[dep]) 635 if recout: 636 if collapsed: 637 output.extend(recout) 638 else: 639 # If a dependent hash changed, might as well print the line above and then defer to the changes in 640 # that hash since in all likelyhood, they're the same changes this task also saw. 641 output = [output[-1]] + recout 642 643 a_taint = a_data.get('taint', None) 644 b_taint = b_data.get('taint', None) 645 if a_taint != b_taint: 646 if a_taint.startswith('nostamp:'): 647 a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):') 648 if b_taint.startswith('nostamp:'): 649 b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):') 650 output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint)) 651 652 return output 653 654 655def calc_basehash(sigdata): 656 task = sigdata['task'] 657 basedata = sigdata['varvals'][task] 658 659 if basedata is None: 660 basedata = '' 661 662 alldeps = sigdata['taskdeps'] 663 for dep in alldeps: 664 basedata = basedata + dep 665 val = sigdata['varvals'][dep] 666 if val is not None: 667 basedata = basedata + str(val) 668 669 return hashlib.sha256(basedata.encode("utf-8")).hexdigest() 670 671def calc_taskhash(sigdata): 672 data = sigdata['basehash'] 673 674 for dep in sigdata['runtaskdeps']: 675 data = data + sigdata['runtaskhashes'][dep] 676 677 for c in sigdata['file_checksum_values']: 678 if c[1]: 679 data = data + c[1] 680 681 if 'taint' in sigdata: 682 if 'nostamp:' in sigdata['taint']: 683 data = data + sigdata['taint'][8:] 684 else: 685 data = data + sigdata['taint'] 686 687 return hashlib.sha256(data.encode("utf-8")).hexdigest() 688 689 690def dump_sigfile(a): 691 output = [] 692 693 with open(a, 'rb') as f: 694 p1 = pickle.Unpickler(f) 695 a_data = p1.load() 696 697 output.append("basewhitelist: %s" % (a_data['basewhitelist'])) 698 699 output.append("taskwhitelist: %s" % (a_data['taskwhitelist'])) 700 701 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) 702 703 output.append("basehash: %s" % (a_data['basehash'])) 704 705 for dep in a_data['gendeps']: 706 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])) 707 708 for dep in a_data['varvals']: 709 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep])) 710 711 if 'runtaskdeps' in a_data: 712 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps'])) 713 714 if 'file_checksum_values' in a_data: 715 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])) 716 717 if 'runtaskhashes' in a_data: 718 for dep in a_data['runtaskhashes']: 719 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])) 720 721 if 'taint' in a_data: 722 if a_data['taint'].startswith('nostamp:'): 723 msg = a_data['taint'].replace('nostamp:', 'nostamp(uuid4):') 724 else: 725 msg = a_data['taint'] 726 output.append("Tainted (by forced/invalidated task): %s" % msg) 727 728 if 'task' in a_data: 729 computed_basehash = calc_basehash(a_data) 730 output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash'])) 731 else: 732 output.append("Unable to compute base hash") 733 734 computed_taskhash = calc_taskhash(a_data) 735 output.append("Computed task hash is %s" % computed_taskhash) 736 737 return output 738