1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6import bb.siggen 7import bb.runqueue 8import oe 9 10def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches): 11 # Return True if we should keep the dependency, False to drop it 12 def isNative(x): 13 return x.endswith("-native") 14 def isCross(x): 15 return "-cross-" in x 16 def isNativeSDK(x): 17 return x.startswith("nativesdk-") 18 def isKernel(mc, fn): 19 inherits = " ".join(dataCaches[mc].inherits[fn]) 20 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1 21 def isPackageGroup(mc, fn): 22 inherits = " ".join(dataCaches[mc].inherits[fn]) 23 return "/packagegroup.bbclass" in inherits 24 def isAllArch(mc, fn): 25 inherits = " ".join(dataCaches[mc].inherits[fn]) 26 return "/allarch.bbclass" in inherits 27 def isImage(mc, fn): 28 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn]) 29 30 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep) 31 mc, _ = bb.runqueue.split_mc(fn) 32 33 # We can skip the rm_work task signature to avoid running the task 34 # when we remove some tasks from the dependencie chain 35 # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work 36 if task == "do_rm_work": 37 return False 38 39 # (Almost) always include our own inter-task dependencies (unless it comes 40 # from a mcdepends). The exception is the special 41 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass. 42 if recipename == depname and depmc == mc: 43 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch": 44 return False 45 return True 46 47 # Exclude well defined recipe->dependency 48 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps: 49 return False 50 51 # Check for special wildcard 52 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname: 53 return False 54 55 # Don't change native/cross/nativesdk recipe dependencies any further 56 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename): 57 return True 58 59 # Only target packages beyond here 60 61 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes 62 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname): 63 return False 64 65 # Exclude well defined machine specific configurations which don't change ABI 66 if depname in siggen.abisaferecipes and not isImage(mc, fn): 67 return False 68 69 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum 70 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum 71 # is machine specific. 72 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes) 73 # and we reccomend a kernel-module, we exclude the dependency. 74 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn): 75 for pkg in dataCaches[mc].runrecs[fn]: 76 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1: 77 return False 78 79 # Default to keep dependencies 80 return True 81 82def sstate_lockedsigs(d): 83 sigs = {} 84 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() 85 for t in types: 86 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t 87 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() 88 for ls in lockedsigs: 89 pn, task, h = ls.split(":", 2) 90 if pn not in sigs: 91 sigs[pn] = {} 92 sigs[pn][task] = [h, siggen_lockedsigs_var] 93 return sigs 94 95class SignatureGeneratorOEBasicHashMixIn(object): 96 supports_multiconfig_datacaches = True 97 98 def init_rundepcheck(self, data): 99 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() 100 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() 101 self.lockedsigs = sstate_lockedsigs(data) 102 self.lockedhashes = {} 103 self.lockedpnmap = {} 104 self.lockedhashfn = {} 105 self.machine = data.getVar("MACHINE") 106 self.mismatch_msgs = [] 107 self.mismatch_number = 0 108 self.lockedsigs_msgs = "" 109 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or 110 "").split() 111 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } 112 self._internal = False 113 pass 114 115 def tasks_resolved(self, virtmap, virtpnmap, dataCache): 116 # Translate virtual/xxx entries to PN values 117 newabisafe = [] 118 for a in self.abisaferecipes: 119 if a in virtpnmap: 120 newabisafe.append(virtpnmap[a]) 121 else: 122 newabisafe.append(a) 123 self.abisaferecipes = newabisafe 124 newsafedeps = [] 125 for a in self.saferecipedeps: 126 a1, a2 = a.split("->") 127 if a1 in virtpnmap: 128 a1 = virtpnmap[a1] 129 if a2 in virtpnmap: 130 a2 = virtpnmap[a2] 131 newsafedeps.append(a1 + "->" + a2) 132 self.saferecipedeps = newsafedeps 133 134 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None): 135 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches) 136 137 def get_taskdata(self): 138 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata() 139 140 def set_taskdata(self, data): 141 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3] 142 super().set_taskdata(data[3:]) 143 144 def dump_sigs(self, dataCache, options): 145 if 'lockedsigs' in options: 146 sigfile = os.getcwd() + "/locked-sigs.inc" 147 bb.plain("Writing locked sigs to %s" % sigfile) 148 self.dump_lockedsigs(sigfile) 149 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) 150 151 152 def get_taskhash(self, tid, deps, dataCaches): 153 if tid in self.lockedhashes: 154 if self.lockedhashes[tid]: 155 return self.lockedhashes[tid] 156 else: 157 return super().get_taskhash(tid, deps, dataCaches) 158 159 h = super().get_taskhash(tid, deps, dataCaches) 160 161 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 162 163 recipename = dataCaches[mc].pkg_fn[fn] 164 self.lockedpnmap[fn] = recipename 165 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn] 166 167 unlocked = False 168 if recipename in self.unlockedrecipes: 169 unlocked = True 170 else: 171 def recipename_from_dep(dep): 172 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep) 173 return dataCaches[depmc].pkg_fn[depfn] 174 175 # If any unlocked recipe is in the direct dependencies then the 176 # current recipe should be unlocked as well. 177 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)] 178 if any(x in y for y in depnames for x in self.unlockedrecipes): 179 self.unlockedrecipes[recipename] = '' 180 unlocked = True 181 182 if not unlocked and recipename in self.lockedsigs: 183 if task in self.lockedsigs[recipename]: 184 h_locked = self.lockedsigs[recipename][task][0] 185 var = self.lockedsigs[recipename][task][1] 186 self.lockedhashes[tid] = h_locked 187 self._internal = True 188 unihash = self.get_unihash(tid) 189 self._internal = False 190 #bb.warn("Using %s %s %s" % (recipename, task, h)) 191 192 if h != h_locked and h_locked != unihash: 193 self.mismatch_number += 1 194 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' 195 % (recipename, task, h, h_locked, var)) 196 197 return h_locked 198 199 self.lockedhashes[tid] = False 200 #bb.warn("%s %s %s" % (recipename, task, h)) 201 return h 202 203 def get_stampfile_hash(self, tid): 204 if tid in self.lockedhashes and self.lockedhashes[tid]: 205 return self.lockedhashes[tid] 206 return super().get_stampfile_hash(tid) 207 208 def get_unihash(self, tid): 209 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal: 210 return self.lockedhashes[tid] 211 return super().get_unihash(tid) 212 213 def dump_sigtask(self, fn, task, stampbase, runtime): 214 tid = fn + ":" + task 215 if tid in self.lockedhashes and self.lockedhashes[tid]: 216 return 217 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) 218 219 def dump_lockedsigs(self, sigfile, taskfilter=None): 220 types = {} 221 for tid in self.runtaskdeps: 222 # Bitbake changed this to a tuple in newer versions 223 if isinstance(tid, tuple): 224 tid = tid[1] 225 if taskfilter: 226 if not tid in taskfilter: 227 continue 228 fn = bb.runqueue.fn_from_tid(tid) 229 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] 230 t = 't-' + t.replace('_', '-') 231 if t not in types: 232 types[t] = [] 233 types[t].append(tid) 234 235 with open(sigfile, "w") as f: 236 l = sorted(types) 237 for t in l: 238 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) 239 types[t].sort() 240 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) 241 for tid in sortedtid: 242 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 243 if tid not in self.taskhash: 244 continue 245 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n") 246 f.write(' "\n') 247 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"' % (self.machine, " ".join(l))) 248 249 def dump_siglist(self, sigfile, path_prefix_strip=None): 250 def strip_fn(fn): 251 nonlocal path_prefix_strip 252 if not path_prefix_strip: 253 return fn 254 255 fn_exp = fn.split(":") 256 if fn_exp[-1].startswith(path_prefix_strip): 257 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):] 258 259 return ":".join(fn_exp) 260 261 with open(sigfile, "w") as f: 262 tasks = [] 263 for taskitem in self.taskhash: 264 (fn, task) = taskitem.rsplit(":", 1) 265 pn = self.lockedpnmap[fn] 266 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem])) 267 for (pn, task, fn, taskhash) in sorted(tasks): 268 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash)) 269 270 def checkhashes(self, sq_data, missed, found, d): 271 warn_msgs = [] 272 error_msgs = [] 273 sstate_missing_msgs = [] 274 info_msgs = None 275 276 if self.lockedsigs: 277 if len(self.lockedsigs) > 10: 278 self.lockedsigs_msgs = "There are %s recipes with locked tasks (%s task(s) have non matching signature)" % (len(self.lockedsigs), self.mismatch_number) 279 else: 280 self.lockedsigs_msgs = "The following recipes have locked tasks:" 281 for pn in self.lockedsigs: 282 self.lockedsigs_msgs += " %s" % (pn) 283 284 for tid in sq_data['hash']: 285 if tid not in found: 286 for pn in self.lockedsigs: 287 taskname = bb.runqueue.taskname_from_tid(tid) 288 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()): 289 if taskname == 'do_shared_workdir': 290 continue 291 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" 292 % (pn, taskname, sq_data['hash'][tid])) 293 294 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") 295 if checklevel == 'info': 296 info_msgs = self.lockedsigs_msgs 297 if checklevel == 'warn' or checklevel == 'info': 298 warn_msgs += self.mismatch_msgs 299 elif checklevel == 'error': 300 error_msgs += self.mismatch_msgs 301 302 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") 303 if checklevel == 'warn': 304 warn_msgs += sstate_missing_msgs 305 elif checklevel == 'error': 306 error_msgs += sstate_missing_msgs 307 308 if info_msgs: 309 bb.note(info_msgs) 310 if warn_msgs: 311 bb.warn("\n".join(warn_msgs)) 312 if error_msgs: 313 bb.fatal("\n".join(error_msgs)) 314 315class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 316 name = "OEBasicHash" 317 318class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 319 name = "OEEquivHash" 320 321 def init_rundepcheck(self, data): 322 super().init_rundepcheck(data) 323 self.server = data.getVar('BB_HASHSERVE') 324 if not self.server: 325 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set") 326 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 327 if not self.method: 328 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set") 329 330# Insert these classes into siggen's namespace so it can see and select them 331bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash 332bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash 333 334 335def find_siginfo(pn, taskname, taskhashlist, d): 336 """ Find signature data files for comparison purposes """ 337 338 import fnmatch 339 import glob 340 341 if not taskname: 342 # We have to derive pn and taskname 343 key = pn 344 if key.startswith("mc:"): 345 # mc:<mc>:<pn>:<task> 346 _, _, pn, taskname = key.split(':', 3) 347 else: 348 # <pn>:<task> 349 pn, taskname = key.split(':', 1) 350 351 hashfiles = {} 352 353 def get_hashval(siginfo): 354 if siginfo.endswith('.siginfo'): 355 return siginfo.rpartition(':')[2].partition('_')[0] 356 else: 357 return siginfo.rpartition('.')[2] 358 359 def get_time(fullpath): 360 return os.stat(fullpath).st_mtime 361 362 # First search in stamps dir 363 localdata = d.createCopy() 364 localdata.setVar('MULTIMACH_TARGET_SYS', '*') 365 localdata.setVar('PN', pn) 366 localdata.setVar('PV', '*') 367 localdata.setVar('PR', '*') 368 localdata.setVar('EXTENDPE', '') 369 stamp = localdata.getVar('STAMP') 370 if pn.startswith("gcc-source"): 371 # gcc-source shared workdir is a special case :( 372 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") 373 374 filespec = '%s.%s.sigdata.*' % (stamp, taskname) 375 foundall = False 376 import glob 377 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 378 for fullpath in glob.glob(filespec): 379 match = False 380 if taskhashlist: 381 for taskhash in taskhashlist: 382 if fullpath.endswith('.%s' % taskhash): 383 hashfiles[taskhash] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)} 384 if len(hashfiles) == len(taskhashlist): 385 foundall = True 386 break 387 else: 388 hashval = get_hashval(fullpath) 389 hashfiles[hashval] = {'path':fullpath, 'sstate':False, 'time':get_time(fullpath)} 390 391 if not taskhashlist or (len(hashfiles) < 2 and not foundall): 392 # That didn't work, look in sstate-cache 393 hashes = taskhashlist or ['?' * 64] 394 localdata = bb.data.createCopy(d) 395 for hashval in hashes: 396 localdata.setVar('PACKAGE_ARCH', '*') 397 localdata.setVar('TARGET_VENDOR', '*') 398 localdata.setVar('TARGET_OS', '*') 399 localdata.setVar('PN', pn) 400 # gcc-source is a special case, same as with local stamps above 401 if pn.startswith("gcc-source"): 402 localdata.setVar('PN', "gcc") 403 localdata.setVar('PV', '*') 404 localdata.setVar('PR', '*') 405 localdata.setVar('BB_TASKHASH', hashval) 406 localdata.setVar('SSTATE_CURRTASK', taskname[3:]) 407 swspec = localdata.getVar('SSTATE_SWSPEC') 408 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: 409 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') 410 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: 411 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") 412 filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG') 413 414 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 415 matchedfiles = glob.glob(filespec) 416 for fullpath in matchedfiles: 417 actual_hashval = get_hashval(fullpath) 418 if actual_hashval in hashfiles: 419 continue 420 hashfiles[actual_hashval] = {'path':fullpath, 'sstate':True, 'time':get_time(fullpath)} 421 422 return hashfiles 423 424bb.siggen.find_siginfo = find_siginfo 425bb.siggen.find_siginfo_version = 2 426 427 428def sstate_get_manifest_filename(task, d): 429 """ 430 Return the sstate manifest file path for a particular task. 431 Also returns the datastore that can be used to query related variables. 432 """ 433 d2 = d.createCopy() 434 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') 435 if extrainf: 436 d2.setVar("SSTATE_MANMACH", extrainf) 437 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) 438 439def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): 440 d2 = d 441 variant = '' 442 curr_variant = '' 443 if d.getVar("BBEXTENDCURR") == "multilib": 444 curr_variant = d.getVar("BBEXTENDVARIANT") 445 if "virtclass-multilib" not in d.getVar("OVERRIDES"): 446 curr_variant = "invalid" 447 if taskdata2.startswith("virtual:multilib"): 448 variant = taskdata2.split(":")[2] 449 if curr_variant != variant: 450 if variant not in multilibcache: 451 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d) 452 d2 = multilibcache[variant] 453 454 if taskdata.endswith("-native"): 455 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"] 456 elif taskdata.startswith("nativesdk-"): 457 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"] 458 elif "-cross-canadian" in taskdata: 459 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"] 460 elif "-cross-" in taskdata: 461 pkgarchs = ["${BUILD_ARCH}"] 462 elif "-crosssdk" in taskdata: 463 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"] 464 else: 465 pkgarchs = ['${MACHINE_ARCH}'] 466 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split())) 467 pkgarchs.append('allarch') 468 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}') 469 470 searched_manifests = [] 471 472 for pkgarch in pkgarchs: 473 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname)) 474 if os.path.exists(manifest): 475 return manifest, d2 476 searched_manifests.append(manifest) 477 bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s" 478 % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests))) 479 return None, d2 480 481def OEOuthashBasic(path, sigfile, task, d): 482 """ 483 Basic output hash function 484 485 Calculates the output hash of a task by hashing all output file metadata, 486 and file contents. 487 """ 488 import hashlib 489 import stat 490 import pwd 491 import grp 492 import re 493 import fnmatch 494 495 def update_hash(s): 496 s = s.encode('utf-8') 497 h.update(s) 498 if sigfile: 499 sigfile.write(s) 500 501 h = hashlib.sha256() 502 prev_dir = os.getcwd() 503 corebase = d.getVar("COREBASE") 504 tmpdir = d.getVar("TMPDIR") 505 include_owners = os.environ.get('PSEUDO_DISABLED') == '0' 506 if "package_write_" in task or task == "package_qa": 507 include_owners = False 508 include_timestamps = False 509 include_root = True 510 if task == "package": 511 include_timestamps = True 512 include_root = False 513 hash_version = d.getVar('HASHEQUIV_HASH_VERSION') 514 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA") 515 516 filemaps = {} 517 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split(): 518 entry = m.split(":") 519 if len(entry) != 3 or entry[0] != task: 520 continue 521 filemaps.setdefault(entry[1], []) 522 filemaps[entry[1]].append(entry[2]) 523 524 try: 525 os.chdir(path) 526 basepath = os.path.normpath(path) 527 528 update_hash("OEOuthashBasic\n") 529 if hash_version: 530 update_hash(hash_version + "\n") 531 532 if extra_sigdata: 533 update_hash(extra_sigdata + "\n") 534 535 # It is only currently useful to get equivalent hashes for things that 536 # can be restored from sstate. Since the sstate object is named using 537 # SSTATE_PKGSPEC and the task name, those should be included in the 538 # output hash calculation. 539 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) 540 update_hash("task=%s\n" % task) 541 542 for root, dirs, files in os.walk('.', topdown=True): 543 # Sort directories to ensure consistent ordering when recursing 544 dirs.sort() 545 files.sort() 546 547 def process(path): 548 s = os.lstat(path) 549 550 if stat.S_ISDIR(s.st_mode): 551 update_hash('d') 552 elif stat.S_ISCHR(s.st_mode): 553 update_hash('c') 554 elif stat.S_ISBLK(s.st_mode): 555 update_hash('b') 556 elif stat.S_ISSOCK(s.st_mode): 557 update_hash('s') 558 elif stat.S_ISLNK(s.st_mode): 559 update_hash('l') 560 elif stat.S_ISFIFO(s.st_mode): 561 update_hash('p') 562 else: 563 update_hash('-') 564 565 def add_perm(mask, on, off='-'): 566 if mask & s.st_mode: 567 update_hash(on) 568 else: 569 update_hash(off) 570 571 add_perm(stat.S_IRUSR, 'r') 572 add_perm(stat.S_IWUSR, 'w') 573 if stat.S_ISUID & s.st_mode: 574 add_perm(stat.S_IXUSR, 's', 'S') 575 else: 576 add_perm(stat.S_IXUSR, 'x') 577 578 if include_owners: 579 # Group/other permissions are only relevant in pseudo context 580 add_perm(stat.S_IRGRP, 'r') 581 add_perm(stat.S_IWGRP, 'w') 582 if stat.S_ISGID & s.st_mode: 583 add_perm(stat.S_IXGRP, 's', 'S') 584 else: 585 add_perm(stat.S_IXGRP, 'x') 586 587 add_perm(stat.S_IROTH, 'r') 588 add_perm(stat.S_IWOTH, 'w') 589 if stat.S_ISVTX & s.st_mode: 590 update_hash('t') 591 else: 592 add_perm(stat.S_IXOTH, 'x') 593 594 try: 595 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) 596 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) 597 except KeyError as e: 598 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match " 599 "any user/group on target. This may be due to host contamination." % 600 (e, os.path.abspath(path), s.st_uid, s.st_gid)) 601 raise Exception(msg).with_traceback(e.__traceback__) 602 603 if include_timestamps: 604 update_hash(" %10d" % s.st_mtime) 605 606 update_hash(" ") 607 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): 608 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) 609 else: 610 update_hash(" " * 9) 611 612 filterfile = False 613 for entry in filemaps: 614 if fnmatch.fnmatch(path, entry): 615 filterfile = True 616 617 update_hash(" ") 618 if stat.S_ISREG(s.st_mode) and not filterfile: 619 update_hash("%10d" % s.st_size) 620 else: 621 update_hash(" " * 10) 622 623 update_hash(" ") 624 fh = hashlib.sha256() 625 if stat.S_ISREG(s.st_mode): 626 # Hash file contents 627 if filterfile: 628 # Need to ignore paths in crossscripts and postinst-useradd files. 629 with open(path, 'rb') as d: 630 chunk = d.read() 631 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'') 632 for entry in filemaps: 633 if not fnmatch.fnmatch(path, entry): 634 continue 635 for r in filemaps[entry]: 636 if r.startswith("regex-"): 637 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk) 638 else: 639 chunk = chunk.replace(bytes(r, encoding='utf8'), b'') 640 fh.update(chunk) 641 else: 642 with open(path, 'rb') as d: 643 for chunk in iter(lambda: d.read(4096), b""): 644 fh.update(chunk) 645 update_hash(fh.hexdigest()) 646 else: 647 update_hash(" " * len(fh.hexdigest())) 648 649 update_hash(" %s" % path) 650 651 if stat.S_ISLNK(s.st_mode): 652 update_hash(" -> %s" % os.readlink(path)) 653 654 update_hash("\n") 655 656 # Process this directory and all its child files 657 if include_root or root != ".": 658 process(root) 659 for f in files: 660 if f == 'fixmepath': 661 continue 662 process(os.path.join(root, f)) 663 664 for dir in dirs: 665 if os.path.islink(os.path.join(root, dir)): 666 process(os.path.join(root, dir)) 667 finally: 668 os.chdir(prev_dir) 669 670 return h.hexdigest() 671 672 673