1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6import bb.siggen 7import bb.runqueue 8import oe 9import netrc 10 11def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches): 12 # Return True if we should keep the dependency, False to drop it 13 def isNative(x): 14 return x.endswith("-native") 15 def isCross(x): 16 return "-cross-" in x 17 def isNativeSDK(x): 18 return x.startswith("nativesdk-") 19 def isKernel(mc, fn): 20 inherits = " ".join(dataCaches[mc].inherits[fn]) 21 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1 22 def isPackageGroup(mc, fn): 23 inherits = " ".join(dataCaches[mc].inherits[fn]) 24 return "/packagegroup.bbclass" in inherits 25 def isAllArch(mc, fn): 26 inherits = " ".join(dataCaches[mc].inherits[fn]) 27 return "/allarch.bbclass" in inherits 28 def isImage(mc, fn): 29 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn]) 30 31 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep) 32 mc, _ = bb.runqueue.split_mc(fn) 33 34 # We can skip the rm_work task signature to avoid running the task 35 # when we remove some tasks from the dependencie chain 36 # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work 37 if task == "do_rm_work": 38 return False 39 40 # (Almost) always include our own inter-task dependencies (unless it comes 41 # from a mcdepends). The exception is the special 42 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass. 43 if recipename == depname and depmc == mc: 44 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch": 45 return False 46 return True 47 48 # Exclude well defined recipe->dependency 49 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps: 50 return False 51 52 # Check for special wildcard 53 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname: 54 return False 55 56 # Don't change native/cross/nativesdk recipe dependencies any further 57 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename): 58 return True 59 60 # Only target packages beyond here 61 62 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes 63 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname): 64 return False 65 66 # Exclude well defined machine specific configurations which don't change ABI 67 if depname in siggen.abisaferecipes and not isImage(mc, fn): 68 return False 69 70 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum 71 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum 72 # is machine specific. 73 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes) 74 # and we reccomend a kernel-module, we exclude the dependency. 75 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn): 76 for pkg in dataCaches[mc].runrecs[fn]: 77 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1: 78 return False 79 80 # Default to keep dependencies 81 return True 82 83def sstate_lockedsigs(d): 84 sigs = {} 85 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() 86 for t in types: 87 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t 88 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() 89 for ls in lockedsigs: 90 pn, task, h = ls.split(":", 2) 91 if pn not in sigs: 92 sigs[pn] = {} 93 sigs[pn][task] = [h, siggen_lockedsigs_var] 94 return sigs 95 96def lockedsigs_unihashmap(d): 97 unihashmap = {} 98 data = (d.getVar("SIGGEN_UNIHASHMAP") or "").split() 99 for entry in data: 100 pn, task, taskhash, unihash = entry.split(":") 101 unihashmap[(pn, task)] = (taskhash, unihash) 102 return unihashmap 103 104class SignatureGeneratorOEBasicHashMixIn(object): 105 supports_multiconfig_datacaches = True 106 107 def init_rundepcheck(self, data): 108 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() 109 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() 110 self.lockedsigs = sstate_lockedsigs(data) 111 self.unihashmap = lockedsigs_unihashmap(data) 112 self.lockedhashes = {} 113 self.lockedpnmap = {} 114 self.lockedhashfn = {} 115 self.machine = data.getVar("MACHINE") 116 self.mismatch_msgs = [] 117 self.mismatch_number = 0 118 self.lockedsigs_msgs = "" 119 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or 120 "").split() 121 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } 122 self._internal = False 123 pass 124 125 def tasks_resolved(self, virtmap, virtpnmap, dataCache): 126 # Translate virtual/xxx entries to PN values 127 newabisafe = [] 128 for a in self.abisaferecipes: 129 if a in virtpnmap: 130 newabisafe.append(virtpnmap[a]) 131 else: 132 newabisafe.append(a) 133 self.abisaferecipes = newabisafe 134 newsafedeps = [] 135 for a in self.saferecipedeps: 136 a1, a2 = a.split("->") 137 if a1 in virtpnmap: 138 a1 = virtpnmap[a1] 139 if a2 in virtpnmap: 140 a2 = virtpnmap[a2] 141 newsafedeps.append(a1 + "->" + a2) 142 self.saferecipedeps = newsafedeps 143 144 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None): 145 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches) 146 147 def get_taskdata(self): 148 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata() 149 150 def set_taskdata(self, data): 151 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3] 152 super().set_taskdata(data[3:]) 153 154 def dump_sigs(self, dataCache, options): 155 if 'lockedsigs' in options: 156 sigfile = os.getcwd() + "/locked-sigs.inc" 157 bb.plain("Writing locked sigs to %s" % sigfile) 158 self.dump_lockedsigs(sigfile) 159 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) 160 161 162 def get_taskhash(self, tid, deps, dataCaches): 163 if tid in self.lockedhashes: 164 if self.lockedhashes[tid]: 165 return self.lockedhashes[tid] 166 else: 167 return super().get_taskhash(tid, deps, dataCaches) 168 169 h = super().get_taskhash(tid, deps, dataCaches) 170 171 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 172 173 recipename = dataCaches[mc].pkg_fn[fn] 174 self.lockedpnmap[fn] = recipename 175 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn] 176 177 unlocked = False 178 if recipename in self.unlockedrecipes: 179 unlocked = True 180 else: 181 def recipename_from_dep(dep): 182 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep) 183 return dataCaches[depmc].pkg_fn[depfn] 184 185 # If any unlocked recipe is in the direct dependencies then the 186 # current recipe should be unlocked as well. 187 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)] 188 if any(x in y for y in depnames for x in self.unlockedrecipes): 189 self.unlockedrecipes[recipename] = '' 190 unlocked = True 191 192 if not unlocked and recipename in self.lockedsigs: 193 if task in self.lockedsigs[recipename]: 194 h_locked = self.lockedsigs[recipename][task][0] 195 var = self.lockedsigs[recipename][task][1] 196 self.lockedhashes[tid] = h_locked 197 self._internal = True 198 unihash = self.get_unihash(tid) 199 self._internal = False 200 #bb.warn("Using %s %s %s" % (recipename, task, h)) 201 202 if h != h_locked and h_locked != unihash: 203 self.mismatch_number += 1 204 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' 205 % (recipename, task, h, h_locked, var)) 206 207 return h_locked 208 209 self.lockedhashes[tid] = False 210 #bb.warn("%s %s %s" % (recipename, task, h)) 211 return h 212 213 def get_stampfile_hash(self, tid): 214 if tid in self.lockedhashes and self.lockedhashes[tid]: 215 return self.lockedhashes[tid] 216 return super().get_stampfile_hash(tid) 217 218 def get_cached_unihash(self, tid): 219 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal: 220 return self.lockedhashes[tid] 221 222 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 223 recipename = self.lockedpnmap[fn] 224 225 if (recipename, task) in self.unihashmap: 226 taskhash, unihash = self.unihashmap[(recipename, task)] 227 if taskhash == self.taskhash[tid]: 228 return unihash 229 230 return super().get_cached_unihash(tid) 231 232 def dump_sigtask(self, fn, task, stampbase, runtime): 233 tid = fn + ":" + task 234 if tid in self.lockedhashes and self.lockedhashes[tid]: 235 return 236 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) 237 238 def dump_lockedsigs(self, sigfile, taskfilter=None): 239 types = {} 240 unihashmap = {} 241 for tid in self.runtaskdeps: 242 # Bitbake changed this to a tuple in newer versions 243 if isinstance(tid, tuple): 244 tid = tid[1] 245 if taskfilter: 246 if not tid in taskfilter: 247 continue 248 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 249 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] 250 t = 't-' + t.replace('_', '-') 251 if t not in types: 252 types[t] = [] 253 types[t].append(tid) 254 255 taskhash = self.taskhash[tid] 256 unihash = self.get_unihash(tid) 257 if taskhash != unihash: 258 unihashmap[tid] = " " + self.lockedpnmap[fn] + ":" + task + ":" + taskhash + ":" + unihash 259 260 with open(sigfile, "w") as f: 261 l = sorted(types) 262 for t in l: 263 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) 264 types[t].sort() 265 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) 266 for tid in sortedtid: 267 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 268 if tid not in self.taskhash: 269 continue 270 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n") 271 f.write(' "\n') 272 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"\n' % (self.machine, " ".join(l))) 273 f.write('SIGGEN_UNIHASHMAP += "\\\n') 274 sortedtid = sorted(unihashmap, key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) 275 for tid in sortedtid: 276 f.write(unihashmap[tid] + " \\\n") 277 f.write(' "\n') 278 279 def dump_siglist(self, sigfile, path_prefix_strip=None): 280 def strip_fn(fn): 281 nonlocal path_prefix_strip 282 if not path_prefix_strip: 283 return fn 284 285 fn_exp = fn.split(":") 286 if fn_exp[-1].startswith(path_prefix_strip): 287 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):] 288 289 return ":".join(fn_exp) 290 291 with open(sigfile, "w") as f: 292 tasks = [] 293 for taskitem in self.taskhash: 294 (fn, task) = taskitem.rsplit(":", 1) 295 pn = self.lockedpnmap[fn] 296 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem])) 297 for (pn, task, fn, taskhash) in sorted(tasks): 298 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash)) 299 300 def checkhashes(self, sq_data, missed, found, d): 301 warn_msgs = [] 302 error_msgs = [] 303 sstate_missing_msgs = [] 304 info_msgs = None 305 306 if self.lockedsigs: 307 if len(self.lockedsigs) > 10: 308 self.lockedsigs_msgs = "There are %s recipes with locked tasks (%s task(s) have non matching signature)" % (len(self.lockedsigs), self.mismatch_number) 309 else: 310 self.lockedsigs_msgs = "The following recipes have locked tasks:" 311 for pn in self.lockedsigs: 312 self.lockedsigs_msgs += " %s" % (pn) 313 314 for tid in sq_data['hash']: 315 if tid not in found: 316 for pn in self.lockedsigs: 317 taskname = bb.runqueue.taskname_from_tid(tid) 318 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()): 319 if taskname == 'do_shared_workdir': 320 continue 321 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" 322 % (pn, taskname, sq_data['hash'][tid])) 323 324 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") 325 if checklevel == 'info': 326 info_msgs = self.lockedsigs_msgs 327 if checklevel == 'warn' or checklevel == 'info': 328 warn_msgs += self.mismatch_msgs 329 elif checklevel == 'error': 330 error_msgs += self.mismatch_msgs 331 332 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") 333 if checklevel == 'warn': 334 warn_msgs += sstate_missing_msgs 335 elif checklevel == 'error': 336 error_msgs += sstate_missing_msgs 337 338 if info_msgs: 339 bb.note(info_msgs) 340 if warn_msgs: 341 bb.warn("\n".join(warn_msgs)) 342 if error_msgs: 343 bb.fatal("\n".join(error_msgs)) 344 345class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 346 name = "OEBasicHash" 347 348class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 349 name = "OEEquivHash" 350 351 def init_rundepcheck(self, data): 352 super().init_rundepcheck(data) 353 self.server = data.getVar('BB_HASHSERVE') 354 if not self.server: 355 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set") 356 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 357 if not self.method: 358 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set") 359 self.username = data.getVar("BB_HASHSERVE_USERNAME") 360 self.password = data.getVar("BB_HASHSERVE_PASSWORD") 361 if not self.username or not self.password: 362 try: 363 n = netrc.netrc() 364 auth = n.authenticators(self.server) 365 if auth is not None: 366 self.username, _, self.password = auth 367 except FileNotFoundError: 368 pass 369 except netrc.NetrcParseError as e: 370 bb.warn("Error parsing %s:%s: %s" % (e.filename, str(e.lineno), e.msg)) 371 372# Insert these classes into siggen's namespace so it can see and select them 373bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash 374bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash 375 376 377def find_siginfo(pn, taskname, taskhashlist, d): 378 """ Find signature data files for comparison purposes """ 379 380 import fnmatch 381 import glob 382 383 if not taskname: 384 # We have to derive pn and taskname 385 key = pn 386 if key.startswith("mc:"): 387 # mc:<mc>:<pn>:<task> 388 _, _, pn, taskname = key.split(':', 3) 389 else: 390 # <pn>:<task> 391 pn, taskname = key.split(':', 1) 392 393 hashfiles = {} 394 395 def get_hashval(siginfo): 396 if siginfo.endswith('.siginfo'): 397 return siginfo.rpartition(':')[2].partition('_')[0] 398 else: 399 return siginfo.rpartition('.')[2] 400 401 def get_time(fullpath): 402 # NFS can end up in a weird state where the file exists but has no stat info. 403 # If that happens, we assume it doesn't acutally exist and show a warning 404 try: 405 return os.stat(fullpath).st_mtime 406 except FileNotFoundError: 407 bb.warn("Could not obtain mtime for {}".format(fullpath)) 408 return None 409 410 # First search in stamps dir 411 localdata = d.createCopy() 412 localdata.setVar('MULTIMACH_TARGET_SYS', '*') 413 localdata.setVar('PN', pn) 414 localdata.setVar('PV', '*') 415 localdata.setVar('PR', '*') 416 localdata.setVar('EXTENDPE', '') 417 stamp = localdata.getVar('STAMP') 418 if pn.startswith("gcc-source"): 419 # gcc-source shared workdir is a special case :( 420 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") 421 422 filespec = '%s.%s.sigdata.*' % (stamp, taskname) 423 foundall = False 424 import glob 425 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 426 for fullpath in glob.glob(filespec): 427 match = False 428 if taskhashlist: 429 for taskhash in taskhashlist: 430 if fullpath.endswith('.%s' % taskhash): 431 mtime = get_time(fullpath) 432 if mtime: 433 hashfiles[taskhash] = {'path':fullpath, 'sstate':False, 'time':mtime} 434 if len(hashfiles) == len(taskhashlist): 435 foundall = True 436 break 437 else: 438 hashval = get_hashval(fullpath) 439 mtime = get_time(fullpath) 440 if mtime: 441 hashfiles[hashval] = {'path':fullpath, 'sstate':False, 'time':mtime} 442 443 if not taskhashlist or (len(hashfiles) < 2 and not foundall): 444 # That didn't work, look in sstate-cache 445 hashes = taskhashlist or ['?' * 64] 446 localdata = bb.data.createCopy(d) 447 for hashval in hashes: 448 localdata.setVar('PACKAGE_ARCH', '*') 449 localdata.setVar('TARGET_VENDOR', '*') 450 localdata.setVar('TARGET_OS', '*') 451 localdata.setVar('PN', pn) 452 # gcc-source is a special case, same as with local stamps above 453 if pn.startswith("gcc-source"): 454 localdata.setVar('PN', "gcc") 455 localdata.setVar('PV', '*') 456 localdata.setVar('PR', '*') 457 localdata.setVar('BB_TASKHASH', hashval) 458 localdata.setVar('SSTATE_CURRTASK', taskname[3:]) 459 swspec = localdata.getVar('SSTATE_SWSPEC') 460 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: 461 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') 462 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: 463 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") 464 filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG') 465 466 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 467 matchedfiles = glob.glob(filespec) 468 for fullpath in matchedfiles: 469 actual_hashval = get_hashval(fullpath) 470 if actual_hashval in hashfiles: 471 continue 472 mtime = get_time(fullpath) 473 if mtime: 474 hashfiles[actual_hashval] = {'path':fullpath, 'sstate':True, 'time':mtime} 475 476 return hashfiles 477 478bb.siggen.find_siginfo = find_siginfo 479bb.siggen.find_siginfo_version = 2 480 481 482def sstate_get_manifest_filename(task, d): 483 """ 484 Return the sstate manifest file path for a particular task. 485 Also returns the datastore that can be used to query related variables. 486 """ 487 d2 = d.createCopy() 488 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') 489 if extrainf: 490 d2.setVar("SSTATE_MANMACH", extrainf) 491 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) 492 493def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): 494 d2 = d 495 variant = '' 496 curr_variant = '' 497 if d.getVar("BBEXTENDCURR") == "multilib": 498 curr_variant = d.getVar("BBEXTENDVARIANT") 499 if "virtclass-multilib" not in d.getVar("OVERRIDES"): 500 curr_variant = "invalid" 501 if taskdata2.startswith("virtual:multilib"): 502 variant = taskdata2.split(":")[2] 503 if curr_variant != variant: 504 if variant not in multilibcache: 505 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d) 506 d2 = multilibcache[variant] 507 508 if taskdata.endswith("-native"): 509 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"] 510 elif taskdata.startswith("nativesdk-"): 511 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"] 512 elif "-cross-canadian" in taskdata: 513 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"] 514 elif "-cross-" in taskdata: 515 pkgarchs = ["${BUILD_ARCH}"] 516 elif "-crosssdk" in taskdata: 517 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"] 518 else: 519 pkgarchs = ['${MACHINE_ARCH}'] 520 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split())) 521 pkgarchs.append('allarch') 522 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}') 523 524 searched_manifests = [] 525 526 for pkgarch in pkgarchs: 527 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname)) 528 if os.path.exists(manifest): 529 return manifest, d2 530 searched_manifests.append(manifest) 531 bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s" 532 % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests))) 533 return None, d2 534 535def OEOuthashBasic(path, sigfile, task, d): 536 """ 537 Basic output hash function 538 539 Calculates the output hash of a task by hashing all output file metadata, 540 and file contents. 541 """ 542 import hashlib 543 import stat 544 import pwd 545 import grp 546 import re 547 import fnmatch 548 549 def update_hash(s): 550 s = s.encode('utf-8') 551 h.update(s) 552 if sigfile: 553 sigfile.write(s) 554 555 h = hashlib.sha256() 556 prev_dir = os.getcwd() 557 corebase = d.getVar("COREBASE") 558 tmpdir = d.getVar("TMPDIR") 559 include_owners = os.environ.get('PSEUDO_DISABLED') == '0' 560 if "package_write_" in task or task == "package_qa": 561 include_owners = False 562 include_timestamps = False 563 include_root = True 564 if task == "package": 565 include_timestamps = True 566 include_root = False 567 source_date_epoch = float(d.getVar("SOURCE_DATE_EPOCH")) 568 hash_version = d.getVar('HASHEQUIV_HASH_VERSION') 569 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA") 570 571 filemaps = {} 572 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split(): 573 entry = m.split(":") 574 if len(entry) != 3 or entry[0] != task: 575 continue 576 filemaps.setdefault(entry[1], []) 577 filemaps[entry[1]].append(entry[2]) 578 579 try: 580 os.chdir(path) 581 basepath = os.path.normpath(path) 582 583 update_hash("OEOuthashBasic\n") 584 if hash_version: 585 update_hash(hash_version + "\n") 586 587 if extra_sigdata: 588 update_hash(extra_sigdata + "\n") 589 590 # It is only currently useful to get equivalent hashes for things that 591 # can be restored from sstate. Since the sstate object is named using 592 # SSTATE_PKGSPEC and the task name, those should be included in the 593 # output hash calculation. 594 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) 595 update_hash("task=%s\n" % task) 596 597 for root, dirs, files in os.walk('.', topdown=True): 598 # Sort directories to ensure consistent ordering when recursing 599 dirs.sort() 600 files.sort() 601 602 def process(path): 603 s = os.lstat(path) 604 605 if stat.S_ISDIR(s.st_mode): 606 update_hash('d') 607 elif stat.S_ISCHR(s.st_mode): 608 update_hash('c') 609 elif stat.S_ISBLK(s.st_mode): 610 update_hash('b') 611 elif stat.S_ISSOCK(s.st_mode): 612 update_hash('s') 613 elif stat.S_ISLNK(s.st_mode): 614 update_hash('l') 615 elif stat.S_ISFIFO(s.st_mode): 616 update_hash('p') 617 else: 618 update_hash('-') 619 620 def add_perm(mask, on, off='-'): 621 if mask & s.st_mode: 622 update_hash(on) 623 else: 624 update_hash(off) 625 626 add_perm(stat.S_IRUSR, 'r') 627 add_perm(stat.S_IWUSR, 'w') 628 if stat.S_ISUID & s.st_mode: 629 add_perm(stat.S_IXUSR, 's', 'S') 630 else: 631 add_perm(stat.S_IXUSR, 'x') 632 633 if include_owners: 634 # Group/other permissions are only relevant in pseudo context 635 add_perm(stat.S_IRGRP, 'r') 636 add_perm(stat.S_IWGRP, 'w') 637 if stat.S_ISGID & s.st_mode: 638 add_perm(stat.S_IXGRP, 's', 'S') 639 else: 640 add_perm(stat.S_IXGRP, 'x') 641 642 add_perm(stat.S_IROTH, 'r') 643 add_perm(stat.S_IWOTH, 'w') 644 if stat.S_ISVTX & s.st_mode: 645 update_hash('t') 646 else: 647 add_perm(stat.S_IXOTH, 'x') 648 649 try: 650 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) 651 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) 652 except KeyError as e: 653 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match " 654 "any user/group on target. This may be due to host contamination." % 655 (e, os.path.abspath(path), s.st_uid, s.st_gid)) 656 raise Exception(msg).with_traceback(e.__traceback__) 657 658 if include_timestamps: 659 # Need to clamp to SOURCE_DATE_EPOCH 660 if s.st_mtime > source_date_epoch: 661 update_hash(" %10d" % source_date_epoch) 662 else: 663 update_hash(" %10d" % s.st_mtime) 664 665 update_hash(" ") 666 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): 667 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) 668 else: 669 update_hash(" " * 9) 670 671 filterfile = False 672 for entry in filemaps: 673 if fnmatch.fnmatch(path, entry): 674 filterfile = True 675 676 update_hash(" ") 677 if stat.S_ISREG(s.st_mode) and not filterfile: 678 update_hash("%10d" % s.st_size) 679 else: 680 update_hash(" " * 10) 681 682 update_hash(" ") 683 fh = hashlib.sha256() 684 if stat.S_ISREG(s.st_mode): 685 # Hash file contents 686 if filterfile: 687 # Need to ignore paths in crossscripts and postinst-useradd files. 688 with open(path, 'rb') as d: 689 chunk = d.read() 690 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'') 691 for entry in filemaps: 692 if not fnmatch.fnmatch(path, entry): 693 continue 694 for r in filemaps[entry]: 695 if r.startswith("regex-"): 696 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk) 697 else: 698 chunk = chunk.replace(bytes(r, encoding='utf8'), b'') 699 fh.update(chunk) 700 else: 701 with open(path, 'rb') as d: 702 for chunk in iter(lambda: d.read(4096), b""): 703 fh.update(chunk) 704 update_hash(fh.hexdigest()) 705 else: 706 update_hash(" " * len(fh.hexdigest())) 707 708 update_hash(" %s" % path) 709 710 if stat.S_ISLNK(s.st_mode): 711 update_hash(" -> %s" % os.readlink(path)) 712 713 update_hash("\n") 714 715 # Process this directory and all its child files 716 if include_root or root != ".": 717 process(root) 718 for f in files: 719 if f == 'fixmepath': 720 continue 721 process(os.path.join(root, f)) 722 723 for dir in dirs: 724 if os.path.islink(os.path.join(root, dir)): 725 process(os.path.join(root, dir)) 726 finally: 727 os.chdir(prev_dir) 728 729 return h.hexdigest() 730 731 732