1# 2# BitBake Cache implementation 3# 4# Caching of bitbake variables before task execution 5 6# Copyright (C) 2006 Richard Purdie 7# Copyright (C) 2012 Intel Corporation 8 9# but small sections based on code from bin/bitbake: 10# Copyright (C) 2003, 2004 Chris Larson 11# Copyright (C) 2003, 2004 Phil Blundell 12# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 13# Copyright (C) 2005 Holger Hans Peter Freyther 14# Copyright (C) 2005 ROAD GmbH 15# 16# SPDX-License-Identifier: GPL-2.0-only 17# 18 19import os 20import logging 21import pickle 22from collections import defaultdict 23from collections.abc import Mapping 24import bb.utils 25from bb import PrefixLoggerAdapter 26import re 27import shutil 28 29logger = logging.getLogger("BitBake.Cache") 30 31__cache_version__ = "155" 32 33def getCacheFile(path, filename, mc, data_hash): 34 mcspec = '' 35 if mc: 36 mcspec = ".%s" % mc 37 return os.path.join(path, filename + mcspec + "." + data_hash) 38 39# RecipeInfoCommon defines common data retrieving methods 40# from meta data for caches. CoreRecipeInfo as well as other 41# Extra RecipeInfo needs to inherit this class 42class RecipeInfoCommon(object): 43 44 @classmethod 45 def listvar(cls, var, metadata): 46 return cls.getvar(var, metadata).split() 47 48 @classmethod 49 def intvar(cls, var, metadata): 50 return int(cls.getvar(var, metadata) or 0) 51 52 @classmethod 53 def depvar(cls, var, metadata): 54 return bb.utils.explode_deps(cls.getvar(var, metadata)) 55 56 @classmethod 57 def pkgvar(cls, var, packages, metadata): 58 return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata)) 59 for pkg in packages) 60 61 @classmethod 62 def taskvar(cls, var, tasks, metadata): 63 return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata)) 64 for task in tasks) 65 66 @classmethod 67 def flaglist(cls, flag, varlist, metadata, squash=False): 68 out_dict = dict((var, metadata.getVarFlag(var, flag)) 69 for var in varlist) 70 if squash: 71 return dict((k,v) for (k,v) in out_dict.items() if v) 72 else: 73 return out_dict 74 75 @classmethod 76 def getvar(cls, var, metadata, expand = True): 77 return metadata.getVar(var, expand) or '' 78 79 80class CoreRecipeInfo(RecipeInfoCommon): 81 __slots__ = () 82 83 cachefile = "bb_cache.dat" 84 85 def __init__(self, filename, metadata): 86 self.file_depends = metadata.getVar('__depends', False) 87 self.timestamp = bb.parse.cached_mtime(filename) 88 self.variants = self.listvar('__VARIANTS', metadata) + [''] 89 self.appends = self.listvar('__BBAPPEND', metadata) 90 self.nocache = self.getvar('BB_DONT_CACHE', metadata) 91 92 self.provides = self.depvar('PROVIDES', metadata) 93 self.rprovides = self.depvar('RPROVIDES', metadata) 94 self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0] 95 self.packages = self.listvar('PACKAGES', metadata) 96 if not self.packages: 97 self.packages.append(self.pn) 98 self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) 99 self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata) 100 101 self.skipreason = self.getvar('__SKIPPED', metadata) 102 if self.skipreason: 103 self.skipped = True 104 return 105 106 self.tasks = metadata.getVar('__BBTASKS', False) 107 108 self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {} 109 self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) 110 111 self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} 112 113 self.skipped = False 114 self.pe = self.getvar('PE', metadata) 115 self.pv = self.getvar('PV', metadata) 116 self.pr = self.getvar('PR', metadata) 117 self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata) 118 self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata) 119 self.stamp = self.getvar('STAMP', metadata) 120 self.stampclean = self.getvar('STAMPCLEAN', metadata) 121 self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) 122 self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True) 123 self.depends = self.depvar('DEPENDS', metadata) 124 self.rdepends = self.depvar('RDEPENDS', metadata) 125 self.rrecommends = self.depvar('RRECOMMENDS', metadata) 126 self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata) 127 self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata) 128 self.inherits = self.getvar('__inherit_cache', metadata, expand=False) 129 self.fakerootenv = self.getvar('FAKEROOTENV', metadata) 130 self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) 131 self.fakerootlogs = self.getvar('FAKEROOTLOGS', metadata) 132 self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata) 133 self.extradepsfunc = self.getvar('calculate_extra_depends', metadata) 134 135 @classmethod 136 def init_cacheData(cls, cachedata): 137 # CacheData in Core RecipeInfo Class 138 cachedata.task_deps = {} 139 cachedata.pkg_fn = {} 140 cachedata.pkg_pn = defaultdict(list) 141 cachedata.pkg_pepvpr = {} 142 cachedata.pkg_dp = {} 143 144 cachedata.stamp = {} 145 cachedata.stampclean = {} 146 cachedata.stamp_extrainfo = {} 147 cachedata.file_checksums = {} 148 cachedata.fn_provides = {} 149 cachedata.pn_provides = defaultdict(list) 150 cachedata.all_depends = [] 151 152 cachedata.deps = defaultdict(list) 153 cachedata.packages = defaultdict(list) 154 cachedata.providers = defaultdict(list) 155 cachedata.rproviders = defaultdict(list) 156 cachedata.packages_dynamic = defaultdict(list) 157 158 cachedata.rundeps = defaultdict(lambda: defaultdict(list)) 159 cachedata.runrecs = defaultdict(lambda: defaultdict(list)) 160 cachedata.possible_world = [] 161 cachedata.universe_target = [] 162 cachedata.hashfn = {} 163 164 cachedata.basetaskhash = {} 165 cachedata.inherits = {} 166 cachedata.fakerootenv = {} 167 cachedata.fakerootnoenv = {} 168 cachedata.fakerootdirs = {} 169 cachedata.fakerootlogs = {} 170 cachedata.extradepsfunc = {} 171 172 def add_cacheData(self, cachedata, fn): 173 cachedata.task_deps[fn] = self.task_deps 174 cachedata.pkg_fn[fn] = self.pn 175 cachedata.pkg_pn[self.pn].append(fn) 176 cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr) 177 cachedata.pkg_dp[fn] = self.defaultpref 178 cachedata.stamp[fn] = self.stamp 179 cachedata.stampclean[fn] = self.stampclean 180 cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo 181 cachedata.file_checksums[fn] = self.file_checksums 182 183 provides = [self.pn] 184 for provide in self.provides: 185 if provide not in provides: 186 provides.append(provide) 187 cachedata.fn_provides[fn] = provides 188 189 for provide in provides: 190 cachedata.providers[provide].append(fn) 191 if provide not in cachedata.pn_provides[self.pn]: 192 cachedata.pn_provides[self.pn].append(provide) 193 194 for dep in self.depends: 195 if dep not in cachedata.deps[fn]: 196 cachedata.deps[fn].append(dep) 197 if dep not in cachedata.all_depends: 198 cachedata.all_depends.append(dep) 199 200 rprovides = self.rprovides 201 for package in self.packages: 202 cachedata.packages[package].append(fn) 203 rprovides += self.rprovides_pkg[package] 204 205 for rprovide in rprovides: 206 if fn not in cachedata.rproviders[rprovide]: 207 cachedata.rproviders[rprovide].append(fn) 208 209 for package in self.packages_dynamic: 210 cachedata.packages_dynamic[package].append(fn) 211 212 # Build hash of runtime depends and recommends 213 for package in self.packages: 214 cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package] 215 cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package] 216 217 # Collect files we may need for possible world-dep 218 # calculations 219 if not bb.utils.to_boolean(self.not_world): 220 cachedata.possible_world.append(fn) 221 #else: 222 # logger.debug2("EXCLUDE FROM WORLD: %s", fn) 223 224 # create a collection of all targets for sanity checking 225 # tasks, such as upstream versions, license, and tools for 226 # task and image creation. 227 cachedata.universe_target.append(self.pn) 228 229 cachedata.hashfn[fn] = self.hashfilename 230 for task, taskhash in self.basetaskhashes.items(): 231 identifier = '%s:%s' % (fn, task) 232 cachedata.basetaskhash[identifier] = taskhash 233 234 cachedata.inherits[fn] = self.inherits 235 cachedata.fakerootenv[fn] = self.fakerootenv 236 cachedata.fakerootnoenv[fn] = self.fakerootnoenv 237 cachedata.fakerootdirs[fn] = self.fakerootdirs 238 cachedata.fakerootlogs[fn] = self.fakerootlogs 239 cachedata.extradepsfunc[fn] = self.extradepsfunc 240 241 242class SiggenRecipeInfo(RecipeInfoCommon): 243 __slots__ = () 244 245 classname = "SiggenRecipeInfo" 246 cachefile = "bb_cache_" + classname +".dat" 247 # we don't want to show this information in graph files so don't set cachefields 248 #cachefields = [] 249 250 def __init__(self, filename, metadata): 251 self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False) 252 self.siggen_varvals = metadata.getVar("__siggen_varvals", False) 253 self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False) 254 255 @classmethod 256 def init_cacheData(cls, cachedata): 257 cachedata.siggen_taskdeps = {} 258 cachedata.siggen_gendeps = {} 259 cachedata.siggen_varvals = {} 260 261 def add_cacheData(self, cachedata, fn): 262 cachedata.siggen_gendeps[fn] = self.siggen_gendeps 263 cachedata.siggen_varvals[fn] = self.siggen_varvals 264 cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps 265 266 # The siggen variable data is large and impacts: 267 # - bitbake's overall memory usage 268 # - the amount of data sent over IPC between parsing processes and the server 269 # - the size of the cache files on disk 270 # - the size of "sigdata" hash information files on disk 271 # The data consists of strings (some large) or frozenset lists of variables 272 # As such, we a) deplicate the data here and b) pass references to the object at second 273 # access (e.g. over IPC or saving into pickle). 274 275 store = {} 276 save_map = {} 277 save_count = 1 278 restore_map = {} 279 restore_count = {} 280 281 @classmethod 282 def reset(cls): 283 # Needs to be called before starting new streamed data in a given process 284 # (e.g. writing out the cache again) 285 cls.save_map = {} 286 cls.save_count = 1 287 cls.restore_map = {} 288 289 @classmethod 290 def _save(cls, deps): 291 ret = [] 292 if not deps: 293 return deps 294 for dep in deps: 295 fs = deps[dep] 296 if fs is None: 297 ret.append((dep, None, None)) 298 elif fs in cls.save_map: 299 ret.append((dep, None, cls.save_map[fs])) 300 else: 301 cls.save_map[fs] = cls.save_count 302 ret.append((dep, fs, cls.save_count)) 303 cls.save_count = cls.save_count + 1 304 return ret 305 306 @classmethod 307 def _restore(cls, deps, pid): 308 ret = {} 309 if not deps: 310 return deps 311 if pid not in cls.restore_map: 312 cls.restore_map[pid] = {} 313 map = cls.restore_map[pid] 314 for dep, fs, mapnum in deps: 315 if fs is None and mapnum is None: 316 ret[dep] = None 317 elif fs is None: 318 ret[dep] = map[mapnum] 319 else: 320 try: 321 fs = cls.store[fs] 322 except KeyError: 323 cls.store[fs] = fs 324 map[mapnum] = fs 325 ret[dep] = fs 326 return ret 327 328 def __getstate__(self): 329 ret = {} 330 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: 331 ret[key] = self._save(self.__dict__[key]) 332 ret['pid'] = os.getpid() 333 return ret 334 335 def __setstate__(self, state): 336 pid = state['pid'] 337 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: 338 setattr(self, key, self._restore(state[key], pid)) 339 340 341def virtualfn2realfn(virtualfn): 342 """ 343 Convert a virtual file name to a real one + the associated subclass keyword 344 """ 345 mc = "" 346 if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2: 347 (_, mc, virtualfn) = virtualfn.split(':', 2) 348 349 fn = virtualfn 350 cls = "" 351 if virtualfn.startswith('virtual:'): 352 elems = virtualfn.split(':') 353 cls = ":".join(elems[1:-1]) 354 fn = elems[-1] 355 356 return (fn, cls, mc) 357 358def realfn2virtual(realfn, cls, mc): 359 """ 360 Convert a real filename + the associated subclass keyword to a virtual filename 361 """ 362 if cls: 363 realfn = "virtual:" + cls + ":" + realfn 364 if mc: 365 realfn = "mc:" + mc + ":" + realfn 366 return realfn 367 368def variant2virtual(realfn, variant): 369 """ 370 Convert a real filename + a variant to a virtual filename 371 """ 372 if variant == "": 373 return realfn 374 if variant.startswith("mc:") and variant.count(':') >= 2: 375 elems = variant.split(":") 376 if elems[2]: 377 return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn 378 return "mc:" + elems[1] + ":" + realfn 379 return "virtual:" + variant + ":" + realfn 380 381# 382# Cooker calls cacheValid on its recipe list, then either calls loadCached 383# from it's main thread or parse from separate processes to generate an up to 384# date cache 385# 386class Cache(object): 387 """ 388 BitBake Cache implementation 389 """ 390 def __init__(self, databuilder, mc, data_hash, caches_array): 391 self.databuilder = databuilder 392 self.data = databuilder.data 393 394 # Pass caches_array information into Cache Constructor 395 # It will be used later for deciding whether we 396 # need extra cache file dump/load support 397 self.mc = mc 398 self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) 399 self.caches_array = caches_array 400 self.cachedir = self.data.getVar("CACHE") 401 self.clean = set() 402 self.checked = set() 403 self.depends_cache = {} 404 self.data_fn = None 405 self.cacheclean = True 406 self.data_hash = data_hash 407 self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') 408 409 if self.cachedir in [None, '']: 410 bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use") 411 412 def getCacheFile(self, cachefile): 413 return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) 414 415 def prepare_cache(self, progress): 416 loaded = 0 417 418 self.cachefile = self.getCacheFile("bb_cache.dat") 419 420 self.logger.debug("Cache dir: %s", self.cachedir) 421 bb.utils.mkdirhier(self.cachedir) 422 423 cache_ok = True 424 if self.caches_array: 425 for cache_class in self.caches_array: 426 cachefile = self.getCacheFile(cache_class.cachefile) 427 cache_exists = os.path.exists(cachefile) 428 self.logger.debug2("Checking if %s exists: %r", cachefile, cache_exists) 429 cache_ok = cache_ok and cache_exists 430 cache_class.init_cacheData(self) 431 if cache_ok: 432 loaded = self.load_cachefile(progress) 433 elif os.path.isfile(self.cachefile): 434 self.logger.info("Out of date cache found, rebuilding...") 435 else: 436 self.logger.debug("Cache file %s not found, building..." % self.cachefile) 437 438 # We don't use the symlink, its just for debugging convinience 439 if self.mc: 440 symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) 441 else: 442 symlink = os.path.join(self.cachedir, "bb_cache.dat") 443 444 if os.path.exists(symlink): 445 bb.utils.remove(symlink) 446 try: 447 os.symlink(os.path.basename(self.cachefile), symlink) 448 except OSError: 449 pass 450 451 return loaded 452 453 def cachesize(self): 454 cachesize = 0 455 for cache_class in self.caches_array: 456 cachefile = self.getCacheFile(cache_class.cachefile) 457 try: 458 with open(cachefile, "rb") as cachefile: 459 cachesize += os.fstat(cachefile.fileno()).st_size 460 except FileNotFoundError: 461 pass 462 463 return cachesize 464 465 def load_cachefile(self, progress): 466 previous_progress = 0 467 468 for cache_class in self.caches_array: 469 cachefile = self.getCacheFile(cache_class.cachefile) 470 self.logger.debug('Loading cache file: %s' % cachefile) 471 with open(cachefile, "rb") as cachefile: 472 pickled = pickle.Unpickler(cachefile) 473 # Check cache version information 474 try: 475 cache_ver = pickled.load() 476 bitbake_ver = pickled.load() 477 except Exception: 478 self.logger.info('Invalid cache, rebuilding...') 479 return 0 480 481 if cache_ver != __cache_version__: 482 self.logger.info('Cache version mismatch, rebuilding...') 483 return 0 484 elif bitbake_ver != bb.__version__: 485 self.logger.info('Bitbake version mismatch, rebuilding...') 486 return 0 487 488 # Load the rest of the cache file 489 current_progress = 0 490 while cachefile: 491 try: 492 key = pickled.load() 493 value = pickled.load() 494 except Exception: 495 break 496 if not isinstance(key, str): 497 bb.warn("%s from extras cache is not a string?" % key) 498 break 499 if not isinstance(value, RecipeInfoCommon): 500 bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value) 501 break 502 503 if key in self.depends_cache: 504 self.depends_cache[key].append(value) 505 else: 506 self.depends_cache[key] = [value] 507 # only fire events on even percentage boundaries 508 current_progress = cachefile.tell() + previous_progress 509 progress(cachefile.tell() + previous_progress) 510 511 previous_progress += current_progress 512 513 return len(self.depends_cache) 514 515 def parse(self, filename, appends, layername): 516 """Parse the specified filename, returning the recipe information""" 517 self.logger.debug("Parsing %s", filename) 518 infos = [] 519 datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername) 520 depends = [] 521 variants = [] 522 # Process the "real" fn last so we can store variants list 523 for variant, data in sorted(datastores.items(), 524 key=lambda i: i[0], 525 reverse=True): 526 virtualfn = variant2virtual(filename, variant) 527 variants.append(variant) 528 depends = depends + (data.getVar("__depends", False) or []) 529 if depends and not variant: 530 data.setVar("__depends", depends) 531 if virtualfn == filename: 532 data.setVar("__VARIANTS", " ".join(variants)) 533 info_array = [] 534 for cache_class in self.caches_array: 535 info = cache_class(filename, data) 536 info_array.append(info) 537 infos.append((virtualfn, info_array)) 538 539 return infos 540 541 def loadCached(self, filename, appends): 542 """Obtain the recipe information for the specified filename, 543 using cached values. 544 """ 545 546 infos = [] 547 # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] 548 info_array = self.depends_cache[filename] 549 for variant in info_array[0].variants: 550 virtualfn = variant2virtual(filename, variant) 551 infos.append((virtualfn, self.depends_cache[virtualfn])) 552 553 return infos 554 555 def cacheValid(self, fn, appends): 556 """ 557 Is the cache valid for fn? 558 Fast version, no timestamps checked. 559 """ 560 if fn not in self.checked: 561 self.cacheValidUpdate(fn, appends) 562 if fn in self.clean: 563 return True 564 return False 565 566 def cacheValidUpdate(self, fn, appends): 567 """ 568 Is the cache valid for fn? 569 Make thorough (slower) checks including timestamps. 570 """ 571 self.checked.add(fn) 572 573 # File isn't in depends_cache 574 if not fn in self.depends_cache: 575 self.logger.debug2("%s is not cached", fn) 576 return False 577 578 mtime = bb.parse.cached_mtime_noerror(fn) 579 580 # Check file still exists 581 if mtime == 0: 582 self.logger.debug2("%s no longer exists", fn) 583 self.remove(fn) 584 return False 585 586 info_array = self.depends_cache[fn] 587 # Check the file's timestamp 588 if mtime != info_array[0].timestamp: 589 self.logger.debug2("%s changed", fn) 590 self.remove(fn) 591 return False 592 593 # Check dependencies are still valid 594 depends = info_array[0].file_depends 595 if depends: 596 for f, old_mtime in depends: 597 fmtime = bb.parse.cached_mtime_noerror(f) 598 # Check if file still exists 599 if old_mtime != 0 and fmtime == 0: 600 self.logger.debug2("%s's dependency %s was removed", 601 fn, f) 602 self.remove(fn) 603 return False 604 605 if (fmtime != old_mtime): 606 self.logger.debug2("%s's dependency %s changed", 607 fn, f) 608 self.remove(fn) 609 return False 610 611 if hasattr(info_array[0], 'file_checksums'): 612 for _, fl in info_array[0].file_checksums.items(): 613 fl = fl.strip() 614 if not fl: 615 continue 616 # Have to be careful about spaces and colons in filenames 617 flist = self.filelist_regex.split(fl) 618 for f in flist: 619 if not f: 620 continue 621 f, exist = f.rsplit(":", 1) 622 if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): 623 self.logger.debug2("%s's file checksum list file %s changed", 624 fn, f) 625 self.remove(fn) 626 return False 627 628 if tuple(appends) != tuple(info_array[0].appends): 629 self.logger.debug2("appends for %s changed", fn) 630 self.logger.debug2("%s to %s" % (str(appends), str(info_array[0].appends))) 631 self.remove(fn) 632 return False 633 634 invalid = False 635 for cls in info_array[0].variants: 636 virtualfn = variant2virtual(fn, cls) 637 self.clean.add(virtualfn) 638 if virtualfn not in self.depends_cache: 639 self.logger.debug2("%s is not cached", virtualfn) 640 invalid = True 641 elif len(self.depends_cache[virtualfn]) != len(self.caches_array): 642 self.logger.debug2("Extra caches missing for %s?" % virtualfn) 643 invalid = True 644 645 # If any one of the variants is not present, mark as invalid for all 646 if invalid: 647 for cls in info_array[0].variants: 648 virtualfn = variant2virtual(fn, cls) 649 if virtualfn in self.clean: 650 self.logger.debug2("Removing %s from cache", virtualfn) 651 self.clean.remove(virtualfn) 652 if fn in self.clean: 653 self.logger.debug2("Marking %s as not clean", fn) 654 self.clean.remove(fn) 655 return False 656 657 self.clean.add(fn) 658 return True 659 660 def remove(self, fn): 661 """ 662 Remove a fn from the cache 663 Called from the parser in error cases 664 """ 665 if fn in self.depends_cache: 666 self.logger.debug("Removing %s from cache", fn) 667 del self.depends_cache[fn] 668 if fn in self.clean: 669 self.logger.debug("Marking %s as unclean", fn) 670 self.clean.remove(fn) 671 672 def sync(self): 673 """ 674 Save the cache 675 Called from the parser when complete (or exiting) 676 """ 677 if self.cacheclean: 678 self.logger.debug2("Cache is clean, not saving.") 679 return 680 681 for cache_class in self.caches_array: 682 cache_class_name = cache_class.__name__ 683 cachefile = self.getCacheFile(cache_class.cachefile) 684 self.logger.debug2("Writing %s", cachefile) 685 with open(cachefile, "wb") as f: 686 p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) 687 p.dump(__cache_version__) 688 p.dump(bb.__version__) 689 690 for key, info_array in self.depends_cache.items(): 691 for info in info_array: 692 if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name: 693 p.dump(key) 694 p.dump(info) 695 696 del self.depends_cache 697 SiggenRecipeInfo.reset() 698 699 @staticmethod 700 def mtime(cachefile): 701 return bb.parse.cached_mtime_noerror(cachefile) 702 703 def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): 704 if self.mc is not None: 705 (fn, cls, mc) = virtualfn2realfn(filename) 706 if mc: 707 self.logger.error("Unexpected multiconfig %s", filename) 708 return 709 710 vfn = realfn2virtual(fn, cls, self.mc) 711 else: 712 vfn = filename 713 714 if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): 715 cacheData.add_from_recipeinfo(vfn, info_array) 716 717 if watcher: 718 watcher(info_array[0].file_depends) 719 720 if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: 721 if parsed: 722 self.cacheclean = False 723 self.depends_cache[filename] = info_array 724 725class MulticonfigCache(Mapping): 726 def __init__(self, databuilder, data_hash, caches_array): 727 def progress(p): 728 nonlocal current_progress 729 nonlocal previous_progress 730 nonlocal previous_percent 731 nonlocal cachesize 732 733 current_progress = previous_progress + p 734 735 if current_progress > cachesize: 736 # we might have calculated incorrect total size because a file 737 # might've been written out just after we checked its size 738 cachesize = current_progress 739 current_percent = 100 * current_progress / cachesize 740 if current_percent > previous_percent: 741 previous_percent = current_percent 742 bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), 743 databuilder.data) 744 745 746 cachesize = 0 747 current_progress = 0 748 previous_progress = 0 749 previous_percent = 0 750 self.__caches = {} 751 752 for mc, mcdata in databuilder.mcdata.items(): 753 self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) 754 755 cachesize += self.__caches[mc].cachesize() 756 757 bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) 758 loaded = 0 759 760 for c in self.__caches.values(): 761 SiggenRecipeInfo.reset() 762 loaded += c.prepare_cache(progress) 763 previous_progress = current_progress 764 765 # Note: depends cache number is corresponding to the parsing file numbers. 766 # The same file has several caches, still regarded as one item in the cache 767 bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) 768 769 def __len__(self): 770 return len(self.__caches) 771 772 def __getitem__(self, key): 773 return self.__caches[key] 774 775 def __contains__(self, key): 776 return key in self.__caches 777 778 def __iter__(self): 779 for k in self.__caches: 780 yield k 781 782def init(cooker): 783 """ 784 The Objective: Cache the minimum amount of data possible yet get to the 785 stage of building packages (i.e. tryBuild) without reparsing any .bb files. 786 787 To do this, we intercept getVar calls and only cache the variables we see 788 being accessed. We rely on the cache getVar calls being made for all 789 variables bitbake might need to use to reach this stage. For each cached 790 file we need to track: 791 792 * Its mtime 793 * The mtimes of all its dependencies 794 * Whether it caused a parse.SkipRecipe exception 795 796 Files causing parsing errors are evicted from the cache. 797 798 """ 799 return Cache(cooker.configuration.data, cooker.configuration.data_hash) 800 801 802class CacheData(object): 803 """ 804 The data structures we compile from the cached data 805 """ 806 807 def __init__(self, caches_array): 808 self.caches_array = caches_array 809 for cache_class in self.caches_array: 810 if not issubclass(cache_class, RecipeInfoCommon): 811 bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class) 812 cache_class.init_cacheData(self) 813 814 # Direct cache variables 815 self.task_queues = {} 816 self.preferred = {} 817 self.tasks = {} 818 # Indirect Cache variables (set elsewhere) 819 self.ignored_dependencies = [] 820 self.world_target = set() 821 self.bbfile_priority = {} 822 823 def add_from_recipeinfo(self, fn, info_array): 824 for info in info_array: 825 info.add_cacheData(self, fn) 826 827class MultiProcessCache(object): 828 """ 829 BitBake multi-process cache implementation 830 831 Used by the codeparser & file checksum caches 832 """ 833 834 def __init__(self): 835 self.cachefile = None 836 self.cachedata = self.create_cachedata() 837 self.cachedata_extras = self.create_cachedata() 838 839 def init_cache(self, cachedir, cache_file_name=None): 840 if not cachedir: 841 return 842 843 bb.utils.mkdirhier(cachedir) 844 self.cachefile = os.path.join(cachedir, 845 cache_file_name or self.__class__.cache_file_name) 846 logger.debug("Using cache in '%s'", self.cachefile) 847 848 glf = bb.utils.lockfile(self.cachefile + ".lock") 849 850 try: 851 with open(self.cachefile, "rb") as f: 852 p = pickle.Unpickler(f) 853 data, version = p.load() 854 except: 855 bb.utils.unlockfile(glf) 856 return 857 858 bb.utils.unlockfile(glf) 859 860 if version != self.__class__.CACHE_VERSION: 861 return 862 863 self.cachedata = data 864 865 def create_cachedata(self): 866 data = [{}] 867 return data 868 869 def save_extras(self): 870 if not self.cachefile: 871 return 872 873 have_data = any(self.cachedata_extras) 874 if not have_data: 875 return 876 877 glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) 878 879 i = os.getpid() 880 lf = None 881 while not lf: 882 lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False) 883 if not lf or os.path.exists(self.cachefile + "-" + str(i)): 884 if lf: 885 bb.utils.unlockfile(lf) 886 lf = None 887 i = i + 1 888 continue 889 890 with open(self.cachefile + "-" + str(i), "wb") as f: 891 p = pickle.Pickler(f, -1) 892 p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) 893 894 bb.utils.unlockfile(lf) 895 bb.utils.unlockfile(glf) 896 897 def merge_data(self, source, dest): 898 for j in range(0,len(dest)): 899 for h in source[j]: 900 if h not in dest[j]: 901 dest[j][h] = source[j][h] 902 903 def save_merge(self): 904 if not self.cachefile: 905 return 906 907 glf = bb.utils.lockfile(self.cachefile + ".lock") 908 909 data = self.cachedata 910 911 have_data = False 912 913 for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: 914 f = os.path.join(os.path.dirname(self.cachefile), f) 915 try: 916 with open(f, "rb") as fd: 917 p = pickle.Unpickler(fd) 918 extradata, version = p.load() 919 except (IOError, EOFError): 920 os.unlink(f) 921 continue 922 923 if version != self.__class__.CACHE_VERSION: 924 os.unlink(f) 925 continue 926 927 have_data = True 928 self.merge_data(extradata, data) 929 os.unlink(f) 930 931 if have_data: 932 with open(self.cachefile, "wb") as f: 933 p = pickle.Pickler(f, -1) 934 p.dump([data, self.__class__.CACHE_VERSION]) 935 936 bb.utils.unlockfile(glf) 937 938 939class SimpleCache(object): 940 """ 941 BitBake multi-process cache implementation 942 943 Used by the codeparser & file checksum caches 944 """ 945 946 def __init__(self, version): 947 self.cachefile = None 948 self.cachedata = None 949 self.cacheversion = version 950 951 def init_cache(self, d, cache_file_name=None, defaultdata=None): 952 cachedir = (d.getVar("PERSISTENT_DIR") or 953 d.getVar("CACHE")) 954 if not cachedir: 955 return defaultdata 956 957 bb.utils.mkdirhier(cachedir) 958 self.cachefile = os.path.join(cachedir, 959 cache_file_name or self.__class__.cache_file_name) 960 logger.debug("Using cache in '%s'", self.cachefile) 961 962 glf = bb.utils.lockfile(self.cachefile + ".lock") 963 964 try: 965 with open(self.cachefile, "rb") as f: 966 p = pickle.Unpickler(f) 967 data, version = p.load() 968 except: 969 bb.utils.unlockfile(glf) 970 return defaultdata 971 972 bb.utils.unlockfile(glf) 973 974 if version != self.cacheversion: 975 return defaultdata 976 977 return data 978 979 def save(self, data): 980 if not self.cachefile: 981 return 982 983 glf = bb.utils.lockfile(self.cachefile + ".lock") 984 985 with open(self.cachefile, "wb") as f: 986 p = pickle.Pickler(f, -1) 987 p.dump([data, self.cacheversion]) 988 989 bb.utils.unlockfile(glf) 990 991 def copyfile(self, target): 992 if not self.cachefile: 993 return 994 995 glf = bb.utils.lockfile(self.cachefile + ".lock") 996 shutil.copy(self.cachefile, target) 997 bb.utils.unlockfile(glf) 998