1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import hashlib 6import logging 7import os 8import re 9import tempfile 10import pickle 11import bb.data 12import difflib 13import simplediff 14from bb.checksum import FileChecksumCache 15 16logger = logging.getLogger('BitBake.SigGen') 17 18def init(d): 19 siggens = [obj for obj in globals().values() 20 if type(obj) is type and issubclass(obj, SignatureGenerator)] 21 22 desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop" 23 for sg in siggens: 24 if desired == sg.name: 25 return sg(d) 26 break 27 else: 28 logger.error("Invalid signature generator '%s', using default 'noop'\n" 29 "Available generators: %s", desired, 30 ', '.join(obj.name for obj in siggens)) 31 return SignatureGenerator(d) 32 33class SignatureGenerator(object): 34 """ 35 """ 36 name = "noop" 37 38 def __init__(self, data): 39 self.basehash = {} 40 self.taskhash = {} 41 self.runtaskdeps = {} 42 self.file_checksum_values = {} 43 self.taints = {} 44 45 def finalise(self, fn, d, varient): 46 return 47 48 def get_unihash(self, task): 49 return self.taskhash[task] 50 51 def get_taskhash(self, fn, task, deps, dataCache): 52 k = fn + "." + task 53 self.taskhash[k] = hashlib.sha256(k.encode("utf-8")).hexdigest() 54 return self.taskhash[k] 55 56 def writeout_file_checksum_cache(self): 57 """Write/update the file checksum cache onto disk""" 58 return 59 60 def stampfile(self, stampbase, file_name, taskname, extrainfo): 61 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 62 63 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo): 64 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 65 66 def dump_sigtask(self, fn, task, stampbase, runtime): 67 return 68 69 def invalidate_task(self, task, d, fn): 70 bb.build.del_stamp(task, d, fn) 71 72 def dump_sigs(self, dataCache, options): 73 return 74 75 def get_taskdata(self): 76 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) 77 78 def set_taskdata(self, data): 79 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data 80 81 def reset(self, data): 82 self.__init__(data) 83 84 85class SignatureGeneratorBasic(SignatureGenerator): 86 """ 87 """ 88 name = "basic" 89 90 def __init__(self, data): 91 self.basehash = {} 92 self.taskhash = {} 93 self.taskdeps = {} 94 self.runtaskdeps = {} 95 self.file_checksum_values = {} 96 self.taints = {} 97 self.gendeps = {} 98 self.lookupcache = {} 99 self.pkgnameextract = re.compile(r"(?P<fn>.*)\..*") 100 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) 101 self.taskwhitelist = None 102 self.init_rundepcheck(data) 103 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") 104 if checksum_cache_file: 105 self.checksum_cache = FileChecksumCache() 106 self.checksum_cache.init_cache(data, checksum_cache_file) 107 else: 108 self.checksum_cache = None 109 110 def init_rundepcheck(self, data): 111 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None 112 if self.taskwhitelist: 113 self.twl = re.compile(self.taskwhitelist) 114 else: 115 self.twl = None 116 117 def _build_data(self, fn, d): 118 119 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') 120 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) 121 122 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) 123 124 for task in tasklist: 125 k = fn + "." + task 126 if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]: 127 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k])) 128 bb.error("The following commands may help:") 129 cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task) 130 # Make sure sigdata is dumped before run printdiff 131 bb.error("%s -Snone" % cmd) 132 bb.error("Then:") 133 bb.error("%s -Sprintdiff\n" % cmd) 134 self.basehash[k] = basehash[k] 135 136 self.taskdeps[fn] = taskdeps 137 self.gendeps[fn] = gendeps 138 self.lookupcache[fn] = lookupcache 139 140 return taskdeps 141 142 def finalise(self, fn, d, variant): 143 144 mc = d.getVar("__BBMULTICONFIG", False) or "" 145 if variant or mc: 146 fn = bb.cache.realfn2virtual(fn, variant, mc) 147 148 try: 149 taskdeps = self._build_data(fn, d) 150 except bb.parse.SkipRecipe: 151 raise 152 except: 153 bb.warn("Error during finalise of %s" % fn) 154 raise 155 156 #Slow but can be useful for debugging mismatched basehashes 157 #for task in self.taskdeps[fn]: 158 # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) 159 160 for task in taskdeps: 161 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) 162 163 def rundep_check(self, fn, recipename, task, dep, depname, dataCache): 164 # Return True if we should keep the dependency, False to drop it 165 # We only manipulate the dependencies for packages not in the whitelist 166 if self.twl and not self.twl.search(recipename): 167 # then process the actual dependencies 168 if self.twl.search(depname): 169 return False 170 return True 171 172 def read_taint(self, fn, task, stampbase): 173 taint = None 174 try: 175 with open(stampbase + '.' + task + '.taint', 'r') as taintf: 176 taint = taintf.read() 177 except IOError: 178 pass 179 return taint 180 181 def get_taskhash(self, fn, task, deps, dataCache): 182 183 mc = '' 184 if fn.startswith('mc:'): 185 mc = fn.split(':')[1] 186 k = fn + "." + task 187 188 data = dataCache.basetaskhash[k] 189 self.basehash[k] = data 190 self.runtaskdeps[k] = [] 191 self.file_checksum_values[k] = [] 192 recipename = dataCache.pkg_fn[fn] 193 for dep in sorted(deps, key=clean_basepath): 194 pkgname = self.pkgnameextract.search(dep).group('fn') 195 if mc: 196 depmc = pkgname.split(':')[1] 197 if mc != depmc: 198 continue 199 if dep.startswith("mc:") and not mc: 200 continue 201 depname = dataCache.pkg_fn[pkgname] 202 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): 203 continue 204 if dep not in self.taskhash: 205 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) 206 data = data + self.get_unihash(dep) 207 self.runtaskdeps[k].append(dep) 208 209 if task in dataCache.file_checksums[fn]: 210 if self.checksum_cache: 211 checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) 212 else: 213 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) 214 for (f,cs) in checksums: 215 self.file_checksum_values[k].append((f,cs)) 216 if cs: 217 data = data + cs 218 219 taskdep = dataCache.task_deps[fn] 220 if 'nostamp' in taskdep and task in taskdep['nostamp']: 221 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run 222 import uuid 223 taint = str(uuid.uuid4()) 224 data = data + taint 225 self.taints[k] = "nostamp:" + taint 226 227 taint = self.read_taint(fn, task, dataCache.stamp[fn]) 228 if taint: 229 data = data + taint 230 self.taints[k] = taint 231 logger.warning("%s is tainted from a forced run" % k) 232 233 h = hashlib.sha256(data.encode("utf-8")).hexdigest() 234 self.taskhash[k] = h 235 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 236 return h 237 238 def writeout_file_checksum_cache(self): 239 """Write/update the file checksum cache onto disk""" 240 if self.checksum_cache: 241 self.checksum_cache.save_extras() 242 self.checksum_cache.save_merge() 243 else: 244 bb.fetch2.fetcher_parse_save() 245 bb.fetch2.fetcher_parse_done() 246 247 def dump_sigtask(self, fn, task, stampbase, runtime): 248 249 k = fn + "." + task 250 referencestamp = stampbase 251 if isinstance(runtime, str) and runtime.startswith("customfile"): 252 sigfile = stampbase 253 referencestamp = runtime[11:] 254 elif runtime and k in self.taskhash: 255 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] 256 else: 257 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] 258 259 bb.utils.mkdirhier(os.path.dirname(sigfile)) 260 261 data = {} 262 data['task'] = task 263 data['basewhitelist'] = self.basewhitelist 264 data['taskwhitelist'] = self.taskwhitelist 265 data['taskdeps'] = self.taskdeps[fn][task] 266 data['basehash'] = self.basehash[k] 267 data['gendeps'] = {} 268 data['varvals'] = {} 269 data['varvals'][task] = self.lookupcache[fn][task] 270 for dep in self.taskdeps[fn][task]: 271 if dep in self.basewhitelist: 272 continue 273 data['gendeps'][dep] = self.gendeps[fn][dep] 274 data['varvals'][dep] = self.lookupcache[fn][dep] 275 276 if runtime and k in self.taskhash: 277 data['runtaskdeps'] = self.runtaskdeps[k] 278 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] 279 data['runtaskhashes'] = {} 280 for dep in data['runtaskdeps']: 281 data['runtaskhashes'][dep] = self.get_unihash(dep) 282 data['taskhash'] = self.taskhash[k] 283 284 taint = self.read_taint(fn, task, referencestamp) 285 if taint: 286 data['taint'] = taint 287 288 if runtime and k in self.taints: 289 if 'nostamp:' in self.taints[k]: 290 data['taint'] = self.taints[k] 291 292 computed_basehash = calc_basehash(data) 293 if computed_basehash != self.basehash[k]: 294 bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) 295 if runtime and k in self.taskhash: 296 computed_taskhash = calc_taskhash(data) 297 if computed_taskhash != self.taskhash[k]: 298 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) 299 sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) 300 301 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") 302 try: 303 with os.fdopen(fd, "wb") as stream: 304 p = pickle.dump(data, stream, -1) 305 stream.flush() 306 os.chmod(tmpfile, 0o664) 307 os.rename(tmpfile, sigfile) 308 except (OSError, IOError) as err: 309 try: 310 os.unlink(tmpfile) 311 except OSError: 312 pass 313 raise err 314 315 def dump_sigfn(self, fn, dataCaches, options): 316 if fn in self.taskdeps: 317 for task in self.taskdeps[fn]: 318 tid = fn + ":" + task 319 (mc, _, _) = bb.runqueue.split_tid(tid) 320 k = fn + "." + task 321 if k not in self.taskhash: 322 continue 323 if dataCaches[mc].basetaskhash[k] != self.basehash[k]: 324 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) 325 bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) 326 self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) 327 328class SignatureGeneratorBasicHash(SignatureGeneratorBasic): 329 name = "basichash" 330 331 def get_stampfile_hash(self, task): 332 if task in self.taskhash: 333 return self.taskhash[task] 334 335 # If task is not in basehash, then error 336 return self.basehash[task] 337 338 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): 339 if taskname != "do_setscene" and taskname.endswith("_setscene"): 340 k = fn + "." + taskname[:-9] 341 else: 342 k = fn + "." + taskname 343 if clean: 344 h = "*" 345 else: 346 h = self.get_stampfile_hash(k) 347 348 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') 349 350 def stampcleanmask(self, stampbase, fn, taskname, extrainfo): 351 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True) 352 353 def invalidate_task(self, task, d, fn): 354 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) 355 bb.build.write_taint(task, d, fn) 356 357def dump_this_task(outfile, d): 358 import bb.parse 359 fn = d.getVar("BB_FILENAME") 360 task = "do_" + d.getVar("BB_CURRENTTASK") 361 referencestamp = bb.build.stamp_internal(task, d, None, True) 362 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) 363 364def init_colors(enable_color): 365 """Initialise colour dict for passing to compare_sigfiles()""" 366 # First set up the colours 367 colors = {'color_title': '\033[1m', 368 'color_default': '\033[0m', 369 'color_add': '\033[0;32m', 370 'color_remove': '\033[0;31m', 371 } 372 # Leave all keys present but clear the values 373 if not enable_color: 374 for k in colors.keys(): 375 colors[k] = '' 376 return colors 377 378def worddiff_str(oldstr, newstr, colors=None): 379 if not colors: 380 colors = init_colors(False) 381 diff = simplediff.diff(oldstr.split(' '), newstr.split(' ')) 382 ret = [] 383 for change, value in diff: 384 value = ' '.join(value) 385 if change == '=': 386 ret.append(value) 387 elif change == '+': 388 item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors) 389 ret.append(item) 390 elif change == '-': 391 item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors) 392 ret.append(item) 393 whitespace_note = '' 394 if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()): 395 whitespace_note = ' (whitespace changed)' 396 return '"%s"%s' % (' '.join(ret), whitespace_note) 397 398def list_inline_diff(oldlist, newlist, colors=None): 399 if not colors: 400 colors = init_colors(False) 401 diff = simplediff.diff(oldlist, newlist) 402 ret = [] 403 for change, value in diff: 404 value = ' '.join(value) 405 if change == '=': 406 ret.append("'%s'" % value) 407 elif change == '+': 408 item = '{color_add}+{value}{color_default}'.format(value=value, **colors) 409 ret.append(item) 410 elif change == '-': 411 item = '{color_remove}-{value}{color_default}'.format(value=value, **colors) 412 ret.append(item) 413 return '[%s]' % (', '.join(ret)) 414 415def clean_basepath(a): 416 mc = None 417 if a.startswith("mc:"): 418 _, mc, a = a.split(":", 2) 419 b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] 420 if a.startswith("virtual:"): 421 b = b + ":" + a.rsplit(":", 1)[0] 422 if mc: 423 b = b + ":mc:" + mc 424 return b 425 426def clean_basepaths(a): 427 b = {} 428 for x in a: 429 b[clean_basepath(x)] = a[x] 430 return b 431 432def clean_basepaths_list(a): 433 b = [] 434 for x in a: 435 b.append(clean_basepath(x)) 436 return b 437 438def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): 439 output = [] 440 441 colors = init_colors(color) 442 def color_format(formatstr, **values): 443 """ 444 Return colour formatted string. 445 NOTE: call with the format string, not an already formatted string 446 containing values (otherwise you could have trouble with { and } 447 characters) 448 """ 449 if not formatstr.endswith('{color_default}'): 450 formatstr += '{color_default}' 451 # In newer python 3 versions you can pass both of these directly, 452 # but we only require 3.4 at the moment 453 formatparams = {} 454 formatparams.update(colors) 455 formatparams.update(values) 456 return formatstr.format(**formatparams) 457 458 with open(a, 'rb') as f: 459 p1 = pickle.Unpickler(f) 460 a_data = p1.load() 461 with open(b, 'rb') as f: 462 p2 = pickle.Unpickler(f) 463 b_data = p2.load() 464 465 def dict_diff(a, b, whitelist=set()): 466 sa = set(a.keys()) 467 sb = set(b.keys()) 468 common = sa & sb 469 changed = set() 470 for i in common: 471 if a[i] != b[i] and i not in whitelist: 472 changed.add(i) 473 added = sb - sa 474 removed = sa - sb 475 return changed, added, removed 476 477 def file_checksums_diff(a, b): 478 from collections import Counter 479 # Handle old siginfo format 480 if isinstance(a, dict): 481 a = [(os.path.basename(f), cs) for f, cs in a.items()] 482 if isinstance(b, dict): 483 b = [(os.path.basename(f), cs) for f, cs in b.items()] 484 # Compare lists, ensuring we can handle duplicate filenames if they exist 485 removedcount = Counter(a) 486 removedcount.subtract(b) 487 addedcount = Counter(b) 488 addedcount.subtract(a) 489 added = [] 490 for x in b: 491 if addedcount[x] > 0: 492 addedcount[x] -= 1 493 added.append(x) 494 removed = [] 495 changed = [] 496 for x in a: 497 if removedcount[x] > 0: 498 removedcount[x] -= 1 499 for y in added: 500 if y[0] == x[0]: 501 changed.append((x[0], x[1], y[1])) 502 added.remove(y) 503 break 504 else: 505 removed.append(x) 506 added = [x[0] for x in added] 507 removed = [x[0] for x in removed] 508 return changed, added, removed 509 510 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: 511 output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) 512 if a_data['basewhitelist'] and b_data['basewhitelist']: 513 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) 514 515 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: 516 output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) 517 if a_data['taskwhitelist'] and b_data['taskwhitelist']: 518 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) 519 520 if a_data['taskdeps'] != b_data['taskdeps']: 521 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) 522 523 if a_data['basehash'] != b_data['basehash'] and not collapsed: 524 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) 525 526 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) 527 if changed: 528 for dep in changed: 529 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) 530 if a_data['gendeps'][dep] and b_data['gendeps'][dep]: 531 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) 532 if added: 533 for dep in added: 534 output.append(color_format("{color_title}Dependency on variable %s was added") % (dep)) 535 if removed: 536 for dep in removed: 537 output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep)) 538 539 540 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) 541 if changed: 542 for dep in changed: 543 oldval = a_data['varvals'][dep] 544 newval = b_data['varvals'][dep] 545 if newval and oldval and ('\n' in oldval or '\n' in newval): 546 diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='') 547 # Cut off the first two lines, since we aren't interested in 548 # the old/new filename (they are blank anyway in this case) 549 difflines = list(diff)[2:] 550 if color: 551 # Add colour to diff output 552 for i, line in enumerate(difflines): 553 if line.startswith('+'): 554 line = color_format('{color_add}{line}', line=line) 555 difflines[i] = line 556 elif line.startswith('-'): 557 line = color_format('{color_remove}{line}', line=line) 558 difflines[i] = line 559 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines))) 560 elif newval and oldval and (' ' in oldval or ' ' in newval): 561 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors))) 562 else: 563 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval)) 564 565 if not 'file_checksum_values' in a_data: 566 a_data['file_checksum_values'] = {} 567 if not 'file_checksum_values' in b_data: 568 b_data['file_checksum_values'] = {} 569 570 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) 571 if changed: 572 for f, old, new in changed: 573 output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new)) 574 if added: 575 for f in added: 576 output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f)) 577 if removed: 578 for f in removed: 579 output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f)) 580 581 if not 'runtaskdeps' in a_data: 582 a_data['runtaskdeps'] = {} 583 if not 'runtaskdeps' in b_data: 584 b_data['runtaskdeps'] = {} 585 586 if not collapsed: 587 if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): 588 changed = ["Number of task dependencies changed"] 589 else: 590 changed = [] 591 for idx, task in enumerate(a_data['runtaskdeps']): 592 a = a_data['runtaskdeps'][idx] 593 b = b_data['runtaskdeps'][idx] 594 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed: 595 changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b])) 596 597 if changed: 598 clean_a = clean_basepaths_list(a_data['runtaskdeps']) 599 clean_b = clean_basepaths_list(b_data['runtaskdeps']) 600 if clean_a != clean_b: 601 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors)) 602 else: 603 output.append(color_format("{color_title}runtaskdeps changed:")) 604 output.append("\n".join(changed)) 605 606 607 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: 608 a = a_data['runtaskhashes'] 609 b = b_data['runtaskhashes'] 610 changed, added, removed = dict_diff(a, b) 611 if added: 612 for dep in added: 613 bdep_found = False 614 if removed: 615 for bdep in removed: 616 if b[dep] == a[bdep]: 617 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) 618 bdep_found = True 619 if not bdep_found: 620 output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep])) 621 if removed: 622 for dep in removed: 623 adep_found = False 624 if added: 625 for adep in added: 626 if b[adep] == a[dep]: 627 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) 628 adep_found = True 629 if not adep_found: 630 output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep])) 631 if changed: 632 for dep in changed: 633 if not collapsed: 634 output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep])) 635 if callable(recursecb): 636 recout = recursecb(dep, a[dep], b[dep]) 637 if recout: 638 if collapsed: 639 output.extend(recout) 640 else: 641 # If a dependent hash changed, might as well print the line above and then defer to the changes in 642 # that hash since in all likelyhood, they're the same changes this task also saw. 643 output = [output[-1]] + recout 644 645 a_taint = a_data.get('taint', None) 646 b_taint = b_data.get('taint', None) 647 if a_taint != b_taint: 648 if a_taint and a_taint.startswith('nostamp:'): 649 a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):') 650 if b_taint and b_taint.startswith('nostamp:'): 651 b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):') 652 output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint)) 653 654 return output 655 656 657def calc_basehash(sigdata): 658 task = sigdata['task'] 659 basedata = sigdata['varvals'][task] 660 661 if basedata is None: 662 basedata = '' 663 664 alldeps = sigdata['taskdeps'] 665 for dep in alldeps: 666 basedata = basedata + dep 667 val = sigdata['varvals'][dep] 668 if val is not None: 669 basedata = basedata + str(val) 670 671 return hashlib.sha256(basedata.encode("utf-8")).hexdigest() 672 673def calc_taskhash(sigdata): 674 data = sigdata['basehash'] 675 676 for dep in sigdata['runtaskdeps']: 677 data = data + sigdata['runtaskhashes'][dep] 678 679 for c in sigdata['file_checksum_values']: 680 if c[1]: 681 data = data + c[1] 682 683 if 'taint' in sigdata: 684 if 'nostamp:' in sigdata['taint']: 685 data = data + sigdata['taint'][8:] 686 else: 687 data = data + sigdata['taint'] 688 689 return hashlib.sha256(data.encode("utf-8")).hexdigest() 690 691 692def dump_sigfile(a): 693 output = [] 694 695 with open(a, 'rb') as f: 696 p1 = pickle.Unpickler(f) 697 a_data = p1.load() 698 699 output.append("basewhitelist: %s" % (a_data['basewhitelist'])) 700 701 output.append("taskwhitelist: %s" % (a_data['taskwhitelist'])) 702 703 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) 704 705 output.append("basehash: %s" % (a_data['basehash'])) 706 707 for dep in a_data['gendeps']: 708 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])) 709 710 for dep in a_data['varvals']: 711 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep])) 712 713 if 'runtaskdeps' in a_data: 714 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps'])) 715 716 if 'file_checksum_values' in a_data: 717 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])) 718 719 if 'runtaskhashes' in a_data: 720 for dep in a_data['runtaskhashes']: 721 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])) 722 723 if 'taint' in a_data: 724 if a_data['taint'].startswith('nostamp:'): 725 msg = a_data['taint'].replace('nostamp:', 'nostamp(uuid4):') 726 else: 727 msg = a_data['taint'] 728 output.append("Tainted (by forced/invalidated task): %s" % msg) 729 730 if 'task' in a_data: 731 computed_basehash = calc_basehash(a_data) 732 output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash'])) 733 else: 734 output.append("Unable to compute base hash") 735 736 computed_taskhash = calc_taskhash(a_data) 737 output.append("Computed task hash is %s" % computed_taskhash) 738 739 return output 740