1import hashlib 2import logging 3import os 4import re 5import tempfile 6import pickle 7import bb.data 8import difflib 9import simplediff 10from bb.checksum import FileChecksumCache 11 12logger = logging.getLogger('BitBake.SigGen') 13 14def init(d): 15 siggens = [obj for obj in globals().values() 16 if type(obj) is type and issubclass(obj, SignatureGenerator)] 17 18 desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop" 19 for sg in siggens: 20 if desired == sg.name: 21 return sg(d) 22 break 23 else: 24 logger.error("Invalid signature generator '%s', using default 'noop'\n" 25 "Available generators: %s", desired, 26 ', '.join(obj.name for obj in siggens)) 27 return SignatureGenerator(d) 28 29class SignatureGenerator(object): 30 """ 31 """ 32 name = "noop" 33 34 def __init__(self, data): 35 self.basehash = {} 36 self.taskhash = {} 37 self.runtaskdeps = {} 38 self.file_checksum_values = {} 39 self.taints = {} 40 41 def finalise(self, fn, d, varient): 42 return 43 44 def get_taskhash(self, fn, task, deps, dataCache): 45 return "0" 46 47 def writeout_file_checksum_cache(self): 48 """Write/update the file checksum cache onto disk""" 49 return 50 51 def stampfile(self, stampbase, file_name, taskname, extrainfo): 52 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 53 54 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo): 55 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') 56 57 def dump_sigtask(self, fn, task, stampbase, runtime): 58 return 59 60 def invalidate_task(self, task, d, fn): 61 bb.build.del_stamp(task, d, fn) 62 63 def dump_sigs(self, dataCache, options): 64 return 65 66 def get_taskdata(self): 67 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) 68 69 def set_taskdata(self, data): 70 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data 71 72 def reset(self, data): 73 self.__init__(data) 74 75 76class SignatureGeneratorBasic(SignatureGenerator): 77 """ 78 """ 79 name = "basic" 80 81 def __init__(self, data): 82 self.basehash = {} 83 self.taskhash = {} 84 self.taskdeps = {} 85 self.runtaskdeps = {} 86 self.file_checksum_values = {} 87 self.taints = {} 88 self.gendeps = {} 89 self.lookupcache = {} 90 self.pkgnameextract = re.compile("(?P<fn>.*)\..*") 91 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split()) 92 self.taskwhitelist = None 93 self.init_rundepcheck(data) 94 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE") 95 if checksum_cache_file: 96 self.checksum_cache = FileChecksumCache() 97 self.checksum_cache.init_cache(data, checksum_cache_file) 98 else: 99 self.checksum_cache = None 100 101 def init_rundepcheck(self, data): 102 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None 103 if self.taskwhitelist: 104 self.twl = re.compile(self.taskwhitelist) 105 else: 106 self.twl = None 107 108 def _build_data(self, fn, d): 109 110 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') 111 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) 112 113 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn) 114 115 for task in tasklist: 116 k = fn + "." + task 117 if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]: 118 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k])) 119 self.basehash[k] = basehash[k] 120 121 self.taskdeps[fn] = taskdeps 122 self.gendeps[fn] = gendeps 123 self.lookupcache[fn] = lookupcache 124 125 return taskdeps 126 127 def finalise(self, fn, d, variant): 128 129 mc = d.getVar("__BBMULTICONFIG", False) or "" 130 if variant or mc: 131 fn = bb.cache.realfn2virtual(fn, variant, mc) 132 133 try: 134 taskdeps = self._build_data(fn, d) 135 except bb.parse.SkipRecipe: 136 raise 137 except: 138 bb.warn("Error during finalise of %s" % fn) 139 raise 140 141 #Slow but can be useful for debugging mismatched basehashes 142 #for task in self.taskdeps[fn]: 143 # self.dump_sigtask(fn, task, d.getVar("STAMP"), False) 144 145 for task in taskdeps: 146 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) 147 148 def rundep_check(self, fn, recipename, task, dep, depname, dataCache): 149 # Return True if we should keep the dependency, False to drop it 150 # We only manipulate the dependencies for packages not in the whitelist 151 if self.twl and not self.twl.search(recipename): 152 # then process the actual dependencies 153 if self.twl.search(depname): 154 return False 155 return True 156 157 def read_taint(self, fn, task, stampbase): 158 taint = None 159 try: 160 with open(stampbase + '.' + task + '.taint', 'r') as taintf: 161 taint = taintf.read() 162 except IOError: 163 pass 164 return taint 165 166 def get_taskhash(self, fn, task, deps, dataCache): 167 168 mc = '' 169 if fn.startswith('multiconfig:'): 170 mc = fn.split(':')[1] 171 k = fn + "." + task 172 173 data = dataCache.basetaskhash[k] 174 self.basehash[k] = data 175 self.runtaskdeps[k] = [] 176 self.file_checksum_values[k] = [] 177 recipename = dataCache.pkg_fn[fn] 178 for dep in sorted(deps, key=clean_basepath): 179 pkgname = self.pkgnameextract.search(dep).group('fn') 180 if mc: 181 depmc = pkgname.split(':')[1] 182 if mc != depmc: 183 continue 184 depname = dataCache.pkg_fn[pkgname] 185 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): 186 continue 187 if dep not in self.taskhash: 188 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) 189 data = data + self.taskhash[dep] 190 self.runtaskdeps[k].append(dep) 191 192 if task in dataCache.file_checksums[fn]: 193 if self.checksum_cache: 194 checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename) 195 else: 196 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) 197 for (f,cs) in checksums: 198 self.file_checksum_values[k].append((f,cs)) 199 if cs: 200 data = data + cs 201 202 taskdep = dataCache.task_deps[fn] 203 if 'nostamp' in taskdep and task in taskdep['nostamp']: 204 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run 205 import uuid 206 taint = str(uuid.uuid4()) 207 data = data + taint 208 self.taints[k] = "nostamp:" + taint 209 210 taint = self.read_taint(fn, task, dataCache.stamp[fn]) 211 if taint: 212 data = data + taint 213 self.taints[k] = taint 214 logger.warning("%s is tainted from a forced run" % k) 215 216 h = hashlib.md5(data.encode("utf-8")).hexdigest() 217 self.taskhash[k] = h 218 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 219 return h 220 221 def writeout_file_checksum_cache(self): 222 """Write/update the file checksum cache onto disk""" 223 if self.checksum_cache: 224 self.checksum_cache.save_extras() 225 self.checksum_cache.save_merge() 226 else: 227 bb.fetch2.fetcher_parse_save() 228 bb.fetch2.fetcher_parse_done() 229 230 def dump_sigtask(self, fn, task, stampbase, runtime): 231 232 k = fn + "." + task 233 referencestamp = stampbase 234 if isinstance(runtime, str) and runtime.startswith("customfile"): 235 sigfile = stampbase 236 referencestamp = runtime[11:] 237 elif runtime and k in self.taskhash: 238 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] 239 else: 240 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] 241 242 bb.utils.mkdirhier(os.path.dirname(sigfile)) 243 244 data = {} 245 data['task'] = task 246 data['basewhitelist'] = self.basewhitelist 247 data['taskwhitelist'] = self.taskwhitelist 248 data['taskdeps'] = self.taskdeps[fn][task] 249 data['basehash'] = self.basehash[k] 250 data['gendeps'] = {} 251 data['varvals'] = {} 252 data['varvals'][task] = self.lookupcache[fn][task] 253 for dep in self.taskdeps[fn][task]: 254 if dep in self.basewhitelist: 255 continue 256 data['gendeps'][dep] = self.gendeps[fn][dep] 257 data['varvals'][dep] = self.lookupcache[fn][dep] 258 259 if runtime and k in self.taskhash: 260 data['runtaskdeps'] = self.runtaskdeps[k] 261 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k]] 262 data['runtaskhashes'] = {} 263 for dep in data['runtaskdeps']: 264 data['runtaskhashes'][dep] = self.taskhash[dep] 265 data['taskhash'] = self.taskhash[k] 266 267 taint = self.read_taint(fn, task, referencestamp) 268 if taint: 269 data['taint'] = taint 270 271 if runtime and k in self.taints: 272 if 'nostamp:' in self.taints[k]: 273 data['taint'] = self.taints[k] 274 275 computed_basehash = calc_basehash(data) 276 if computed_basehash != self.basehash[k]: 277 bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k)) 278 if runtime and k in self.taskhash: 279 computed_taskhash = calc_taskhash(data) 280 if computed_taskhash != self.taskhash[k]: 281 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k)) 282 sigfile = sigfile.replace(self.taskhash[k], computed_taskhash) 283 284 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") 285 try: 286 with os.fdopen(fd, "wb") as stream: 287 p = pickle.dump(data, stream, -1) 288 stream.flush() 289 os.chmod(tmpfile, 0o664) 290 os.rename(tmpfile, sigfile) 291 except (OSError, IOError) as err: 292 try: 293 os.unlink(tmpfile) 294 except OSError: 295 pass 296 raise err 297 298 def dump_sigfn(self, fn, dataCaches, options): 299 if fn in self.taskdeps: 300 for task in self.taskdeps[fn]: 301 tid = fn + ":" + task 302 (mc, _, _) = bb.runqueue.split_tid(tid) 303 k = fn + "." + task 304 if k not in self.taskhash: 305 continue 306 if dataCaches[mc].basetaskhash[k] != self.basehash[k]: 307 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) 308 bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k])) 309 self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) 310 311class SignatureGeneratorBasicHash(SignatureGeneratorBasic): 312 name = "basichash" 313 314 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): 315 if taskname != "do_setscene" and taskname.endswith("_setscene"): 316 k = fn + "." + taskname[:-9] 317 else: 318 k = fn + "." + taskname 319 if clean: 320 h = "*" 321 elif k in self.taskhash: 322 h = self.taskhash[k] 323 else: 324 # If k is not in basehash, then error 325 h = self.basehash[k] 326 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') 327 328 def stampcleanmask(self, stampbase, fn, taskname, extrainfo): 329 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True) 330 331 def invalidate_task(self, task, d, fn): 332 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) 333 bb.build.write_taint(task, d, fn) 334 335def dump_this_task(outfile, d): 336 import bb.parse 337 fn = d.getVar("BB_FILENAME") 338 task = "do_" + d.getVar("BB_CURRENTTASK") 339 referencestamp = bb.build.stamp_internal(task, d, None, True) 340 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp) 341 342def init_colors(enable_color): 343 """Initialise colour dict for passing to compare_sigfiles()""" 344 # First set up the colours 345 colors = {'color_title': '\033[1m', 346 'color_default': '\033[0m', 347 'color_add': '\033[0;32m', 348 'color_remove': '\033[0;31m', 349 } 350 # Leave all keys present but clear the values 351 if not enable_color: 352 for k in colors.keys(): 353 colors[k] = '' 354 return colors 355 356def worddiff_str(oldstr, newstr, colors=None): 357 if not colors: 358 colors = init_colors(False) 359 diff = simplediff.diff(oldstr.split(' '), newstr.split(' ')) 360 ret = [] 361 for change, value in diff: 362 value = ' '.join(value) 363 if change == '=': 364 ret.append(value) 365 elif change == '+': 366 item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors) 367 ret.append(item) 368 elif change == '-': 369 item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors) 370 ret.append(item) 371 whitespace_note = '' 372 if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()): 373 whitespace_note = ' (whitespace changed)' 374 return '"%s"%s' % (' '.join(ret), whitespace_note) 375 376def list_inline_diff(oldlist, newlist, colors=None): 377 if not colors: 378 colors = init_colors(False) 379 diff = simplediff.diff(oldlist, newlist) 380 ret = [] 381 for change, value in diff: 382 value = ' '.join(value) 383 if change == '=': 384 ret.append("'%s'" % value) 385 elif change == '+': 386 item = '{color_add}+{value}{color_default}'.format(value=value, **colors) 387 ret.append(item) 388 elif change == '-': 389 item = '{color_remove}-{value}{color_default}'.format(value=value, **colors) 390 ret.append(item) 391 return '[%s]' % (', '.join(ret)) 392 393def clean_basepath(a): 394 mc = None 395 if a.startswith("multiconfig:"): 396 _, mc, a = a.split(":", 2) 397 b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2] 398 if a.startswith("virtual:"): 399 b = b + ":" + a.rsplit(":", 1)[0] 400 if mc: 401 b = b + ":multiconfig:" + mc 402 return b 403 404def clean_basepaths(a): 405 b = {} 406 for x in a: 407 b[clean_basepath(x)] = a[x] 408 return b 409 410def clean_basepaths_list(a): 411 b = [] 412 for x in a: 413 b.append(clean_basepath(x)) 414 return b 415 416def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): 417 output = [] 418 419 colors = init_colors(color) 420 def color_format(formatstr, **values): 421 """ 422 Return colour formatted string. 423 NOTE: call with the format string, not an already formatted string 424 containing values (otherwise you could have trouble with { and } 425 characters) 426 """ 427 if not formatstr.endswith('{color_default}'): 428 formatstr += '{color_default}' 429 # In newer python 3 versions you can pass both of these directly, 430 # but we only require 3.4 at the moment 431 formatparams = {} 432 formatparams.update(colors) 433 formatparams.update(values) 434 return formatstr.format(**formatparams) 435 436 with open(a, 'rb') as f: 437 p1 = pickle.Unpickler(f) 438 a_data = p1.load() 439 with open(b, 'rb') as f: 440 p2 = pickle.Unpickler(f) 441 b_data = p2.load() 442 443 def dict_diff(a, b, whitelist=set()): 444 sa = set(a.keys()) 445 sb = set(b.keys()) 446 common = sa & sb 447 changed = set() 448 for i in common: 449 if a[i] != b[i] and i not in whitelist: 450 changed.add(i) 451 added = sb - sa 452 removed = sa - sb 453 return changed, added, removed 454 455 def file_checksums_diff(a, b): 456 from collections import Counter 457 # Handle old siginfo format 458 if isinstance(a, dict): 459 a = [(os.path.basename(f), cs) for f, cs in a.items()] 460 if isinstance(b, dict): 461 b = [(os.path.basename(f), cs) for f, cs in b.items()] 462 # Compare lists, ensuring we can handle duplicate filenames if they exist 463 removedcount = Counter(a) 464 removedcount.subtract(b) 465 addedcount = Counter(b) 466 addedcount.subtract(a) 467 added = [] 468 for x in b: 469 if addedcount[x] > 0: 470 addedcount[x] -= 1 471 added.append(x) 472 removed = [] 473 changed = [] 474 for x in a: 475 if removedcount[x] > 0: 476 removedcount[x] -= 1 477 for y in added: 478 if y[0] == x[0]: 479 changed.append((x[0], x[1], y[1])) 480 added.remove(y) 481 break 482 else: 483 removed.append(x) 484 added = [x[0] for x in added] 485 removed = [x[0] for x in removed] 486 return changed, added, removed 487 488 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: 489 output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist'])) 490 if a_data['basewhitelist'] and b_data['basewhitelist']: 491 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) 492 493 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: 494 output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist'])) 495 if a_data['taskwhitelist'] and b_data['taskwhitelist']: 496 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) 497 498 if a_data['taskdeps'] != b_data['taskdeps']: 499 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) 500 501 if a_data['basehash'] != b_data['basehash'] and not collapsed: 502 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash'])) 503 504 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) 505 if changed: 506 for dep in changed: 507 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) 508 if a_data['gendeps'][dep] and b_data['gendeps'][dep]: 509 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) 510 if added: 511 for dep in added: 512 output.append(color_format("{color_title}Dependency on variable %s was added") % (dep)) 513 if removed: 514 for dep in removed: 515 output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep)) 516 517 518 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) 519 if changed: 520 for dep in changed: 521 oldval = a_data['varvals'][dep] 522 newval = b_data['varvals'][dep] 523 if newval and oldval and ('\n' in oldval or '\n' in newval): 524 diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='') 525 # Cut off the first two lines, since we aren't interested in 526 # the old/new filename (they are blank anyway in this case) 527 difflines = list(diff)[2:] 528 if color: 529 # Add colour to diff output 530 for i, line in enumerate(difflines): 531 if line.startswith('+'): 532 line = color_format('{color_add}{line}', line=line) 533 difflines[i] = line 534 elif line.startswith('-'): 535 line = color_format('{color_remove}{line}', line=line) 536 difflines[i] = line 537 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines))) 538 elif newval and oldval and (' ' in oldval or ' ' in newval): 539 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors))) 540 else: 541 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval)) 542 543 if not 'file_checksum_values' in a_data: 544 a_data['file_checksum_values'] = {} 545 if not 'file_checksum_values' in b_data: 546 b_data['file_checksum_values'] = {} 547 548 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) 549 if changed: 550 for f, old, new in changed: 551 output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new)) 552 if added: 553 for f in added: 554 output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f)) 555 if removed: 556 for f in removed: 557 output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f)) 558 559 if not 'runtaskdeps' in a_data: 560 a_data['runtaskdeps'] = {} 561 if not 'runtaskdeps' in b_data: 562 b_data['runtaskdeps'] = {} 563 564 if not collapsed: 565 if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']): 566 changed = ["Number of task dependencies changed"] 567 else: 568 changed = [] 569 for idx, task in enumerate(a_data['runtaskdeps']): 570 a = a_data['runtaskdeps'][idx] 571 b = b_data['runtaskdeps'][idx] 572 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed: 573 changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b])) 574 575 if changed: 576 clean_a = clean_basepaths_list(a_data['runtaskdeps']) 577 clean_b = clean_basepaths_list(b_data['runtaskdeps']) 578 if clean_a != clean_b: 579 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors)) 580 else: 581 output.append(color_format("{color_title}runtaskdeps changed:")) 582 output.append("\n".join(changed)) 583 584 585 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: 586 a = a_data['runtaskhashes'] 587 b = b_data['runtaskhashes'] 588 changed, added, removed = dict_diff(a, b) 589 if added: 590 for dep in added: 591 bdep_found = False 592 if removed: 593 for bdep in removed: 594 if b[dep] == a[bdep]: 595 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) 596 bdep_found = True 597 if not bdep_found: 598 output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep])) 599 if removed: 600 for dep in removed: 601 adep_found = False 602 if added: 603 for adep in added: 604 if b[adep] == a[dep]: 605 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) 606 adep_found = True 607 if not adep_found: 608 output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep])) 609 if changed: 610 for dep in changed: 611 if not collapsed: 612 output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep])) 613 if callable(recursecb): 614 recout = recursecb(dep, a[dep], b[dep]) 615 if recout: 616 if collapsed: 617 output.extend(recout) 618 else: 619 # If a dependent hash changed, might as well print the line above and then defer to the changes in 620 # that hash since in all likelyhood, they're the same changes this task also saw. 621 output = [output[-1]] + recout 622 623 a_taint = a_data.get('taint', None) 624 b_taint = b_data.get('taint', None) 625 if a_taint != b_taint: 626 output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint)) 627 628 return output 629 630 631def calc_basehash(sigdata): 632 task = sigdata['task'] 633 basedata = sigdata['varvals'][task] 634 635 if basedata is None: 636 basedata = '' 637 638 alldeps = sigdata['taskdeps'] 639 for dep in alldeps: 640 basedata = basedata + dep 641 val = sigdata['varvals'][dep] 642 if val is not None: 643 basedata = basedata + str(val) 644 645 return hashlib.md5(basedata.encode("utf-8")).hexdigest() 646 647def calc_taskhash(sigdata): 648 data = sigdata['basehash'] 649 650 for dep in sigdata['runtaskdeps']: 651 data = data + sigdata['runtaskhashes'][dep] 652 653 for c in sigdata['file_checksum_values']: 654 if c[1]: 655 data = data + c[1] 656 657 if 'taint' in sigdata: 658 if 'nostamp:' in sigdata['taint']: 659 data = data + sigdata['taint'][8:] 660 else: 661 data = data + sigdata['taint'] 662 663 return hashlib.md5(data.encode("utf-8")).hexdigest() 664 665 666def dump_sigfile(a): 667 output = [] 668 669 with open(a, 'rb') as f: 670 p1 = pickle.Unpickler(f) 671 a_data = p1.load() 672 673 output.append("basewhitelist: %s" % (a_data['basewhitelist'])) 674 675 output.append("taskwhitelist: %s" % (a_data['taskwhitelist'])) 676 677 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) 678 679 output.append("basehash: %s" % (a_data['basehash'])) 680 681 for dep in a_data['gendeps']: 682 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])) 683 684 for dep in a_data['varvals']: 685 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep])) 686 687 if 'runtaskdeps' in a_data: 688 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps'])) 689 690 if 'file_checksum_values' in a_data: 691 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])) 692 693 if 'runtaskhashes' in a_data: 694 for dep in a_data['runtaskhashes']: 695 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])) 696 697 if 'taint' in a_data: 698 output.append("Tainted (by forced/invalidated task): %s" % a_data['taint']) 699 700 if 'task' in a_data: 701 computed_basehash = calc_basehash(a_data) 702 output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash'])) 703 else: 704 output.append("Unable to compute base hash") 705 706 computed_taskhash = calc_taskhash(a_data) 707 output.append("Computed task hash is %s" % computed_taskhash) 708 709 return output 710