1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6import bb.siggen 7import bb.runqueue 8import oe 9import netrc 10 11def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches): 12 # Return True if we should keep the dependency, False to drop it 13 def isNative(x): 14 return x.endswith("-native") 15 def isCross(x): 16 return "-cross-" in x 17 def isNativeSDK(x): 18 return x.startswith("nativesdk-") 19 def isKernel(mc, fn): 20 inherits = " ".join(dataCaches[mc].inherits[fn]) 21 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1 22 def isPackageGroup(mc, fn): 23 inherits = " ".join(dataCaches[mc].inherits[fn]) 24 return "/packagegroup.bbclass" in inherits 25 def isAllArch(mc, fn): 26 inherits = " ".join(dataCaches[mc].inherits[fn]) 27 return "/allarch.bbclass" in inherits 28 def isImage(mc, fn): 29 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn]) 30 31 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep) 32 mc, _ = bb.runqueue.split_mc(fn) 33 34 # We can skip the rm_work task signature to avoid running the task 35 # when we remove some tasks from the dependencie chain 36 # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work 37 if task == "do_rm_work": 38 return False 39 40 # (Almost) always include our own inter-task dependencies (unless it comes 41 # from a mcdepends). The exception is the special 42 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass. 43 if recipename == depname and depmc == mc: 44 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch": 45 return False 46 return True 47 48 # Exclude well defined recipe->dependency 49 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps: 50 return False 51 52 # Check for special wildcard 53 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname: 54 return False 55 56 # Don't change native/cross/nativesdk recipe dependencies any further 57 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename): 58 return True 59 60 # Only target packages beyond here 61 62 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes 63 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname): 64 return False 65 66 # Exclude well defined machine specific configurations which don't change ABI 67 if depname in siggen.abisaferecipes and not isImage(mc, fn): 68 return False 69 70 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum 71 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum 72 # is machine specific. 73 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes) 74 # and we reccomend a kernel-module, we exclude the dependency. 75 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn): 76 for pkg in dataCaches[mc].runrecs[fn]: 77 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1: 78 return False 79 80 # Default to keep dependencies 81 return True 82 83def sstate_lockedsigs(d): 84 sigs = {} 85 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split() 86 for t in types: 87 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t 88 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split() 89 for ls in lockedsigs: 90 pn, task, h = ls.split(":", 2) 91 if pn not in sigs: 92 sigs[pn] = {} 93 sigs[pn][task] = [h, siggen_lockedsigs_var] 94 return sigs 95 96def lockedsigs_unihashmap(d): 97 unihashmap = {} 98 data = (d.getVar("SIGGEN_UNIHASHMAP") or "").split() 99 for entry in data: 100 pn, task, taskhash, unihash = entry.split(":") 101 unihashmap[(pn, task)] = (taskhash, unihash) 102 return unihashmap 103 104class SignatureGeneratorOEBasicHashMixIn(object): 105 supports_multiconfig_datacaches = True 106 107 def init_rundepcheck(self, data): 108 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split() 109 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split() 110 self.lockedsigs = sstate_lockedsigs(data) 111 self.unihashmap = lockedsigs_unihashmap(data) 112 self.lockedhashes = {} 113 self.lockedpnmap = {} 114 self.lockedhashfn = {} 115 self.machine = data.getVar("MACHINE") 116 self.mismatch_msgs = [] 117 self.mismatch_number = 0 118 self.lockedsigs_msgs = "" 119 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or 120 "").split() 121 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } 122 self._internal = False 123 pass 124 125 def tasks_resolved(self, virtmap, virtpnmap, dataCache): 126 # Translate virtual/xxx entries to PN values 127 newabisafe = [] 128 for a in self.abisaferecipes: 129 if a in virtpnmap: 130 newabisafe.append(virtpnmap[a]) 131 else: 132 newabisafe.append(a) 133 self.abisaferecipes = newabisafe 134 newsafedeps = [] 135 for a in self.saferecipedeps: 136 a1, a2 = a.split("->") 137 if a1 in virtpnmap: 138 a1 = virtpnmap[a1] 139 if a2 in virtpnmap: 140 a2 = virtpnmap[a2] 141 newsafedeps.append(a1 + "->" + a2) 142 self.saferecipedeps = newsafedeps 143 144 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None): 145 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches) 146 147 def get_taskdata(self): 148 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata() 149 150 def set_taskdata(self, data): 151 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3] 152 super().set_taskdata(data[3:]) 153 154 def dump_sigs(self, dataCache, options): 155 if 'lockedsigs' in options: 156 sigfile = os.getcwd() + "/locked-sigs.inc" 157 bb.plain("Writing locked sigs to %s" % sigfile) 158 self.dump_lockedsigs(sigfile) 159 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options) 160 161 162 def get_taskhash(self, tid, deps, dataCaches): 163 if tid in self.lockedhashes: 164 if self.lockedhashes[tid]: 165 return self.lockedhashes[tid] 166 else: 167 return super().get_taskhash(tid, deps, dataCaches) 168 169 h = super().get_taskhash(tid, deps, dataCaches) 170 171 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 172 173 recipename = dataCaches[mc].pkg_fn[fn] 174 self.lockedpnmap[fn] = recipename 175 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn] 176 177 unlocked = False 178 if recipename in self.unlockedrecipes: 179 unlocked = True 180 else: 181 def recipename_from_dep(dep): 182 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep) 183 return dataCaches[depmc].pkg_fn[depfn] 184 185 # If any unlocked recipe is in the direct dependencies then the 186 # current recipe should be unlocked as well. 187 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)] 188 if any(x in y for y in depnames for x in self.unlockedrecipes): 189 self.unlockedrecipes[recipename] = '' 190 unlocked = True 191 192 if not unlocked and recipename in self.lockedsigs: 193 if task in self.lockedsigs[recipename]: 194 h_locked = self.lockedsigs[recipename][task][0] 195 var = self.lockedsigs[recipename][task][1] 196 self.lockedhashes[tid] = h_locked 197 self._internal = True 198 unihash = self.get_unihash(tid) 199 self._internal = False 200 #bb.warn("Using %s %s %s" % (recipename, task, h)) 201 202 if h != h_locked and h_locked != unihash: 203 self.mismatch_number += 1 204 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' 205 % (recipename, task, h, h_locked, var)) 206 207 return h_locked 208 209 self.lockedhashes[tid] = False 210 #bb.warn("%s %s %s" % (recipename, task, h)) 211 return h 212 213 def get_stampfile_hash(self, tid): 214 if tid in self.lockedhashes and self.lockedhashes[tid]: 215 return self.lockedhashes[tid] 216 return super().get_stampfile_hash(tid) 217 218 def get_cached_unihash(self, tid): 219 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal: 220 return self.lockedhashes[tid] 221 222 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 223 recipename = self.lockedpnmap[fn] 224 225 if (recipename, task) in self.unihashmap: 226 taskhash, unihash = self.unihashmap[(recipename, task)] 227 if taskhash == self.taskhash[tid]: 228 return unihash 229 230 return super().get_cached_unihash(tid) 231 232 def dump_sigtask(self, fn, task, stampbase, runtime): 233 tid = fn + ":" + task 234 if tid in self.lockedhashes and self.lockedhashes[tid]: 235 return 236 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime) 237 238 def dump_lockedsigs(self, sigfile, taskfilter=None): 239 types = {} 240 unihashmap = {} 241 for tid in self.runtaskdeps: 242 # Bitbake changed this to a tuple in newer versions 243 if isinstance(tid, tuple): 244 tid = tid[1] 245 if taskfilter: 246 if not tid in taskfilter: 247 continue 248 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 249 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5] 250 t = 't-' + t.replace('_', '-') 251 if t not in types: 252 types[t] = [] 253 types[t].append(tid) 254 255 taskhash = self.taskhash[tid] 256 unihash = self.get_unihash(tid) 257 if taskhash != unihash: 258 unihashmap[tid] = " " + self.lockedpnmap[fn] + ":" + task + ":" + taskhash + ":" + unihash 259 260 with open(sigfile, "w") as f: 261 l = sorted(types) 262 for t in l: 263 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t) 264 types[t].sort() 265 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) 266 for tid in sortedtid: 267 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 268 if tid not in self.taskhash: 269 continue 270 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n") 271 f.write(' "\n') 272 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"\n' % (self.machine, " ".join(l))) 273 f.write('SIGGEN_UNIHASHMAP += "\\\n') 274 sortedtid = sorted(unihashmap, key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)]) 275 for tid in sortedtid: 276 f.write(unihashmap[tid] + " \\\n") 277 f.write(' "\n') 278 279 def dump_siglist(self, sigfile, path_prefix_strip=None): 280 def strip_fn(fn): 281 nonlocal path_prefix_strip 282 if not path_prefix_strip: 283 return fn 284 285 fn_exp = fn.split(":") 286 if fn_exp[-1].startswith(path_prefix_strip): 287 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):] 288 289 return ":".join(fn_exp) 290 291 with open(sigfile, "w") as f: 292 tasks = [] 293 for taskitem in self.taskhash: 294 (fn, task) = taskitem.rsplit(":", 1) 295 pn = self.lockedpnmap[fn] 296 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem])) 297 for (pn, task, fn, taskhash) in sorted(tasks): 298 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash)) 299 300 def checkhashes(self, sq_data, missed, found, d): 301 warn_msgs = [] 302 error_msgs = [] 303 sstate_missing_msgs = [] 304 info_msgs = None 305 306 if self.lockedsigs: 307 if len(self.lockedsigs) > 10: 308 self.lockedsigs_msgs = "There are %s recipes with locked tasks (%s task(s) have non matching signature)" % (len(self.lockedsigs), self.mismatch_number) 309 else: 310 self.lockedsigs_msgs = "The following recipes have locked tasks:" 311 for pn in self.lockedsigs: 312 self.lockedsigs_msgs += " %s" % (pn) 313 314 for tid in sq_data['hash']: 315 if tid not in found: 316 for pn in self.lockedsigs: 317 taskname = bb.runqueue.taskname_from_tid(tid) 318 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()): 319 if taskname == 'do_shared_workdir': 320 continue 321 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" 322 % (pn, taskname, sq_data['hash'][tid])) 323 324 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") 325 if checklevel == 'info': 326 info_msgs = self.lockedsigs_msgs 327 if checklevel == 'warn' or checklevel == 'info': 328 warn_msgs += self.mismatch_msgs 329 elif checklevel == 'error': 330 error_msgs += self.mismatch_msgs 331 332 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK") 333 if checklevel == 'warn': 334 warn_msgs += sstate_missing_msgs 335 elif checklevel == 'error': 336 error_msgs += sstate_missing_msgs 337 338 if info_msgs: 339 bb.note(info_msgs) 340 if warn_msgs: 341 bb.warn("\n".join(warn_msgs)) 342 if error_msgs: 343 bb.fatal("\n".join(error_msgs)) 344 345class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 346 name = "OEBasicHash" 347 348class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash): 349 name = "OEEquivHash" 350 351 def init_rundepcheck(self, data): 352 super().init_rundepcheck(data) 353 self.server = data.getVar('BB_HASHSERVE') 354 if not self.server: 355 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set") 356 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 357 if not self.method: 358 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set") 359 self.username = data.getVar("BB_HASHSERVE_USERNAME") 360 self.password = data.getVar("BB_HASHSERVE_PASSWORD") 361 if not self.username or not self.password: 362 try: 363 n = netrc.netrc() 364 auth = n.authenticators(self.server) 365 if auth is not None: 366 self.username, _, self.password = auth 367 except FileNotFoundError: 368 pass 369 except netrc.NetrcParseError as e: 370 bb.warn("Error parsing %s:%s: %s" % (e.filename, str(e.lineno), e.msg)) 371 372# Insert these classes into siggen's namespace so it can see and select them 373bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash 374bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash 375 376 377def find_siginfo(pn, taskname, taskhashlist, d): 378 """ Find signature data files for comparison purposes """ 379 380 import fnmatch 381 import glob 382 383 if not taskname: 384 # We have to derive pn and taskname 385 key = pn 386 if key.startswith("mc:"): 387 # mc:<mc>:<pn>:<task> 388 _, _, pn, taskname = key.split(':', 3) 389 else: 390 # <pn>:<task> 391 pn, taskname = key.split(':', 1) 392 393 hashfiles = {} 394 395 def get_hashval(siginfo): 396 if siginfo.endswith('.siginfo'): 397 return siginfo.rpartition(':')[2].partition('_')[0] 398 else: 399 return siginfo.rpartition('.')[2] 400 401 def get_time(fullpath): 402 # NFS can end up in a weird state where the file exists but has no stat info. 403 # If that happens, we assume it doesn't acutally exist and show a warning 404 try: 405 return os.stat(fullpath).st_mtime 406 except FileNotFoundError: 407 bb.warn("Could not obtain mtime for {}".format(fullpath)) 408 return None 409 410 # First search in stamps dir 411 localdata = d.createCopy() 412 localdata.setVar('MULTIMACH_TARGET_SYS', '*') 413 localdata.setVar('PN', pn) 414 localdata.setVar('PV', '*') 415 localdata.setVar('PR', '*') 416 localdata.setVar('EXTENDPE', '') 417 stamp = localdata.getVar('STAMP') 418 if pn.startswith("gcc-source"): 419 # gcc-source shared workdir is a special case :( 420 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}") 421 elif pn.startswith("llvm-project-source"): 422 # llvm-project-source shared workdir is also a special case :*( 423 stamp = localdata.expand("${STAMPS_DIR}/work-shared/llvm-project-source-${PV}-${PR}") 424 425 filespec = '%s.%s.sigdata.*' % (stamp, taskname) 426 foundall = False 427 import glob 428 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 429 for fullpath in glob.glob(filespec): 430 match = False 431 if taskhashlist: 432 for taskhash in taskhashlist: 433 if fullpath.endswith('.%s' % taskhash): 434 mtime = get_time(fullpath) 435 if mtime: 436 hashfiles[taskhash] = {'path':fullpath, 'sstate':False, 'time':mtime} 437 if len(hashfiles) == len(taskhashlist): 438 foundall = True 439 break 440 else: 441 hashval = get_hashval(fullpath) 442 mtime = get_time(fullpath) 443 if mtime: 444 hashfiles[hashval] = {'path':fullpath, 'sstate':False, 'time':mtime} 445 446 if not taskhashlist or (len(hashfiles) < 2 and not foundall): 447 # That didn't work, look in sstate-cache 448 hashes = taskhashlist or ['?' * 64] 449 localdata = bb.data.createCopy(d) 450 for hashval in hashes: 451 localdata.setVar('PACKAGE_ARCH', '*') 452 localdata.setVar('TARGET_VENDOR', '*') 453 localdata.setVar('TARGET_OS', '*') 454 localdata.setVar('PN', pn) 455 # gcc-source is a special case, same as with local stamps above 456 if pn.startswith("gcc-source"): 457 localdata.setVar('PN', "gcc") 458 localdata.setVar('PV', '*') 459 localdata.setVar('PR', '*') 460 localdata.setVar('BB_TASKHASH', hashval) 461 localdata.setVar('SSTATE_CURRTASK', taskname[3:]) 462 swspec = localdata.getVar('SSTATE_SWSPEC') 463 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec: 464 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}') 465 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn: 466 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") 467 filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG') 468 469 bb.debug(1, "Calling glob.glob on {}".format(filespec)) 470 matchedfiles = glob.glob(filespec) 471 for fullpath in matchedfiles: 472 actual_hashval = get_hashval(fullpath) 473 if actual_hashval in hashfiles: 474 continue 475 mtime = get_time(fullpath) 476 if mtime: 477 hashfiles[actual_hashval] = {'path':fullpath, 'sstate':True, 'time':mtime} 478 479 return hashfiles 480 481bb.siggen.find_siginfo = find_siginfo 482bb.siggen.find_siginfo_version = 2 483 484 485def sstate_get_manifest_filename(task, d): 486 """ 487 Return the sstate manifest file path for a particular task. 488 Also returns the datastore that can be used to query related variables. 489 """ 490 d2 = d.createCopy() 491 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info') 492 if extrainf: 493 d2.setVar("SSTATE_MANMACH", extrainf) 494 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2) 495 496def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): 497 d2 = d 498 variant = '' 499 curr_variant = '' 500 if d.getVar("BBEXTENDCURR") == "multilib": 501 curr_variant = d.getVar("BBEXTENDVARIANT") 502 if "virtclass-multilib" not in d.getVar("OVERRIDES"): 503 curr_variant = "invalid" 504 if taskdata2.startswith("virtual:multilib"): 505 variant = taskdata2.split(":")[2] 506 if curr_variant != variant: 507 if variant not in multilibcache: 508 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d) 509 d2 = multilibcache[variant] 510 511 if taskdata.endswith("-native"): 512 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"] 513 elif taskdata.startswith("nativesdk-"): 514 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"] 515 elif "-cross-canadian" in taskdata: 516 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"] 517 elif "-cross-" in taskdata: 518 pkgarchs = ["${BUILD_ARCH}"] 519 elif "-crosssdk" in taskdata: 520 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"] 521 else: 522 pkgarchs = ['${MACHINE_ARCH}'] 523 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split())) 524 pkgarchs.append('allarch') 525 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}') 526 527 searched_manifests = [] 528 529 for pkgarch in pkgarchs: 530 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname)) 531 if os.path.exists(manifest): 532 return manifest, d2 533 searched_manifests.append(manifest) 534 bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s" 535 % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests))) 536 return None, d2 537 538def OEOuthashBasic(path, sigfile, task, d): 539 """ 540 Basic output hash function 541 542 Calculates the output hash of a task by hashing all output file metadata, 543 and file contents. 544 """ 545 import hashlib 546 import stat 547 import pwd 548 import grp 549 import re 550 import fnmatch 551 552 def update_hash(s): 553 s = s.encode('utf-8') 554 h.update(s) 555 if sigfile: 556 sigfile.write(s) 557 558 h = hashlib.sha256() 559 prev_dir = os.getcwd() 560 corebase = d.getVar("COREBASE") 561 tmpdir = d.getVar("TMPDIR") 562 include_owners = os.environ.get('PSEUDO_DISABLED') == '0' 563 if "package_write_" in task or task == "package_qa": 564 include_owners = False 565 include_timestamps = False 566 include_root = True 567 if task == "package": 568 include_timestamps = True 569 include_root = False 570 source_date_epoch = float(d.getVar("SOURCE_DATE_EPOCH")) 571 hash_version = d.getVar('HASHEQUIV_HASH_VERSION') 572 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA") 573 574 filemaps = {} 575 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split(): 576 entry = m.split(":") 577 if len(entry) != 3 or entry[0] != task: 578 continue 579 filemaps.setdefault(entry[1], []) 580 filemaps[entry[1]].append(entry[2]) 581 582 try: 583 os.chdir(path) 584 basepath = os.path.normpath(path) 585 586 update_hash("OEOuthashBasic\n") 587 if hash_version: 588 update_hash(hash_version + "\n") 589 590 if extra_sigdata: 591 update_hash(extra_sigdata + "\n") 592 593 # It is only currently useful to get equivalent hashes for things that 594 # can be restored from sstate. Since the sstate object is named using 595 # SSTATE_PKGSPEC and the task name, those should be included in the 596 # output hash calculation. 597 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) 598 update_hash("task=%s\n" % task) 599 600 for root, dirs, files in os.walk('.', topdown=True): 601 # Sort directories to ensure consistent ordering when recursing 602 dirs.sort() 603 files.sort() 604 605 def process(path): 606 s = os.lstat(path) 607 608 if stat.S_ISDIR(s.st_mode): 609 update_hash('d') 610 elif stat.S_ISCHR(s.st_mode): 611 update_hash('c') 612 elif stat.S_ISBLK(s.st_mode): 613 update_hash('b') 614 elif stat.S_ISSOCK(s.st_mode): 615 update_hash('s') 616 elif stat.S_ISLNK(s.st_mode): 617 update_hash('l') 618 elif stat.S_ISFIFO(s.st_mode): 619 update_hash('p') 620 else: 621 update_hash('-') 622 623 def add_perm(mask, on, off='-'): 624 if mask & s.st_mode: 625 update_hash(on) 626 else: 627 update_hash(off) 628 629 add_perm(stat.S_IRUSR, 'r') 630 add_perm(stat.S_IWUSR, 'w') 631 if stat.S_ISUID & s.st_mode: 632 add_perm(stat.S_IXUSR, 's', 'S') 633 else: 634 add_perm(stat.S_IXUSR, 'x') 635 636 if include_owners: 637 # Group/other permissions are only relevant in pseudo context 638 add_perm(stat.S_IRGRP, 'r') 639 add_perm(stat.S_IWGRP, 'w') 640 if stat.S_ISGID & s.st_mode: 641 add_perm(stat.S_IXGRP, 's', 'S') 642 else: 643 add_perm(stat.S_IXGRP, 'x') 644 645 add_perm(stat.S_IROTH, 'r') 646 add_perm(stat.S_IWOTH, 'w') 647 if stat.S_ISVTX & s.st_mode: 648 update_hash('t') 649 else: 650 add_perm(stat.S_IXOTH, 'x') 651 652 try: 653 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) 654 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) 655 except KeyError as e: 656 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match " 657 "any user/group on target. This may be due to host contamination." % 658 (e, os.path.abspath(path), s.st_uid, s.st_gid)) 659 raise Exception(msg).with_traceback(e.__traceback__) 660 661 if include_timestamps: 662 # Need to clamp to SOURCE_DATE_EPOCH 663 if s.st_mtime > source_date_epoch: 664 update_hash(" %10d" % source_date_epoch) 665 else: 666 update_hash(" %10d" % s.st_mtime) 667 668 update_hash(" ") 669 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): 670 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) 671 else: 672 update_hash(" " * 9) 673 674 filterfile = False 675 for entry in filemaps: 676 if fnmatch.fnmatch(path, entry): 677 filterfile = True 678 679 update_hash(" ") 680 if stat.S_ISREG(s.st_mode) and not filterfile: 681 update_hash("%10d" % s.st_size) 682 else: 683 update_hash(" " * 10) 684 685 update_hash(" ") 686 fh = hashlib.sha256() 687 if stat.S_ISREG(s.st_mode): 688 # Hash file contents 689 if filterfile: 690 # Need to ignore paths in crossscripts and postinst-useradd files. 691 with open(path, 'rb') as d: 692 chunk = d.read() 693 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'') 694 for entry in filemaps: 695 if not fnmatch.fnmatch(path, entry): 696 continue 697 for r in filemaps[entry]: 698 if r.startswith("regex-"): 699 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk) 700 else: 701 chunk = chunk.replace(bytes(r, encoding='utf8'), b'') 702 fh.update(chunk) 703 else: 704 with open(path, 'rb') as d: 705 for chunk in iter(lambda: d.read(4096), b""): 706 fh.update(chunk) 707 update_hash(fh.hexdigest()) 708 else: 709 update_hash(" " * len(fh.hexdigest())) 710 711 update_hash(" %s" % path) 712 713 if stat.S_ISLNK(s.st_mode): 714 update_hash(" -> %s" % os.readlink(path)) 715 716 update_hash("\n") 717 718 # Process this directory and all its child files 719 if include_root or root != ".": 720 process(root) 721 for f in files: 722 if f == 'fixmepath': 723 continue 724 process(os.path.join(root, f)) 725 726 for dir in dirs: 727 if os.path.islink(os.path.join(root, dir)): 728 process(os.path.join(root, dir)) 729 finally: 730 os.chdir(prev_dir) 731 732 return h.hexdigest() 733 734 735