1eb8dc403SDave Cobbley# 2eb8dc403SDave Cobbley# BitBake Cache implementation 3eb8dc403SDave Cobbley# 4eb8dc403SDave Cobbley# Caching of bitbake variables before task execution 5eb8dc403SDave Cobbley 6eb8dc403SDave Cobbley# Copyright (C) 2006 Richard Purdie 7eb8dc403SDave Cobbley# Copyright (C) 2012 Intel Corporation 8eb8dc403SDave Cobbley 9eb8dc403SDave Cobbley# but small sections based on code from bin/bitbake: 10eb8dc403SDave Cobbley# Copyright (C) 2003, 2004 Chris Larson 11eb8dc403SDave Cobbley# Copyright (C) 2003, 2004 Phil Blundell 12eb8dc403SDave Cobbley# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 13eb8dc403SDave Cobbley# Copyright (C) 2005 Holger Hans Peter Freyther 14eb8dc403SDave Cobbley# Copyright (C) 2005 ROAD GmbH 15eb8dc403SDave Cobbley# 16c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 17eb8dc403SDave Cobbley# 18eb8dc403SDave Cobbley 19eb8dc403SDave Cobbleyimport os 20eb8dc403SDave Cobbleyimport logging 21eb8dc403SDave Cobbleyimport pickle 225199d831SAndrew Geisslerfrom collections import defaultdict 235199d831SAndrew Geisslerfrom collections.abc import Mapping 24eb8dc403SDave Cobbleyimport bb.utils 255a43b434SAndrew Geisslerfrom bb import PrefixLoggerAdapter 2682c905dcSAndrew Geisslerimport re 2778b72798SAndrew Geisslerimport shutil 28eb8dc403SDave Cobbley 29eb8dc403SDave Cobbleylogger = logging.getLogger("BitBake.Cache") 30eb8dc403SDave Cobbley 31517393d9SAndrew Geissler__cache_version__ = "155" 32eb8dc403SDave Cobbley 335a43b434SAndrew Geisslerdef getCacheFile(path, filename, mc, data_hash): 345a43b434SAndrew Geissler mcspec = '' 355a43b434SAndrew Geissler if mc: 365a43b434SAndrew Geissler mcspec = ".%s" % mc 375a43b434SAndrew Geissler return os.path.join(path, filename + mcspec + "." + data_hash) 38eb8dc403SDave Cobbley 39eb8dc403SDave Cobbley# RecipeInfoCommon defines common data retrieving methods 40eb8dc403SDave Cobbley# from meta data for caches. CoreRecipeInfo as well as other 41eb8dc403SDave Cobbley# Extra RecipeInfo needs to inherit this class 42eb8dc403SDave Cobbleyclass RecipeInfoCommon(object): 43eb8dc403SDave Cobbley 44eb8dc403SDave Cobbley @classmethod 45eb8dc403SDave Cobbley def listvar(cls, var, metadata): 46eb8dc403SDave Cobbley return cls.getvar(var, metadata).split() 47eb8dc403SDave Cobbley 48eb8dc403SDave Cobbley @classmethod 49eb8dc403SDave Cobbley def intvar(cls, var, metadata): 50eb8dc403SDave Cobbley return int(cls.getvar(var, metadata) or 0) 51eb8dc403SDave Cobbley 52eb8dc403SDave Cobbley @classmethod 53eb8dc403SDave Cobbley def depvar(cls, var, metadata): 54eb8dc403SDave Cobbley return bb.utils.explode_deps(cls.getvar(var, metadata)) 55eb8dc403SDave Cobbley 56eb8dc403SDave Cobbley @classmethod 57eb8dc403SDave Cobbley def pkgvar(cls, var, packages, metadata): 58213cb269SPatrick Williams return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata)) 59eb8dc403SDave Cobbley for pkg in packages) 60eb8dc403SDave Cobbley 61eb8dc403SDave Cobbley @classmethod 62eb8dc403SDave Cobbley def taskvar(cls, var, tasks, metadata): 63213cb269SPatrick Williams return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata)) 64eb8dc403SDave Cobbley for task in tasks) 65eb8dc403SDave Cobbley 66eb8dc403SDave Cobbley @classmethod 67eb8dc403SDave Cobbley def flaglist(cls, flag, varlist, metadata, squash=False): 68eb8dc403SDave Cobbley out_dict = dict((var, metadata.getVarFlag(var, flag)) 69eb8dc403SDave Cobbley for var in varlist) 70eb8dc403SDave Cobbley if squash: 71eb8dc403SDave Cobbley return dict((k,v) for (k,v) in out_dict.items() if v) 72eb8dc403SDave Cobbley else: 73eb8dc403SDave Cobbley return out_dict 74eb8dc403SDave Cobbley 75eb8dc403SDave Cobbley @classmethod 76eb8dc403SDave Cobbley def getvar(cls, var, metadata, expand = True): 77eb8dc403SDave Cobbley return metadata.getVar(var, expand) or '' 78eb8dc403SDave Cobbley 79eb8dc403SDave Cobbley 80eb8dc403SDave Cobbleyclass CoreRecipeInfo(RecipeInfoCommon): 81eb8dc403SDave Cobbley __slots__ = () 82eb8dc403SDave Cobbley 83eb8dc403SDave Cobbley cachefile = "bb_cache.dat" 84eb8dc403SDave Cobbley 85eb8dc403SDave Cobbley def __init__(self, filename, metadata): 86eb8dc403SDave Cobbley self.file_depends = metadata.getVar('__depends', False) 87eb8dc403SDave Cobbley self.timestamp = bb.parse.cached_mtime(filename) 88eb8dc403SDave Cobbley self.variants = self.listvar('__VARIANTS', metadata) + [''] 89eb8dc403SDave Cobbley self.appends = self.listvar('__BBAPPEND', metadata) 90eb8dc403SDave Cobbley self.nocache = self.getvar('BB_DONT_CACHE', metadata) 91eb8dc403SDave Cobbley 92eb8dc403SDave Cobbley self.provides = self.depvar('PROVIDES', metadata) 93eb8dc403SDave Cobbley self.rprovides = self.depvar('RPROVIDES', metadata) 9496ff1984SBrad Bishop self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0] 95eb8dc403SDave Cobbley self.packages = self.listvar('PACKAGES', metadata) 96eb8dc403SDave Cobbley if not self.packages: 97eb8dc403SDave Cobbley self.packages.append(self.pn) 9896ff1984SBrad Bishop self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata) 99d1e89497SAndrew Geissler self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata) 10096ff1984SBrad Bishop 10196ff1984SBrad Bishop self.skipreason = self.getvar('__SKIPPED', metadata) 10296ff1984SBrad Bishop if self.skipreason: 10396ff1984SBrad Bishop self.skipped = True 10496ff1984SBrad Bishop return 10596ff1984SBrad Bishop 10696ff1984SBrad Bishop self.tasks = metadata.getVar('__BBTASKS', False) 107eb8dc403SDave Cobbley 108517393d9SAndrew Geissler self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {} 109eb8dc403SDave Cobbley self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) 110eb8dc403SDave Cobbley 111eb8dc403SDave Cobbley self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} 112eb8dc403SDave Cobbley 113eb8dc403SDave Cobbley self.skipped = False 114eb8dc403SDave Cobbley self.pe = self.getvar('PE', metadata) 115eb8dc403SDave Cobbley self.pv = self.getvar('PV', metadata) 116eb8dc403SDave Cobbley self.pr = self.getvar('PR', metadata) 117eb8dc403SDave Cobbley self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata) 118eb8dc403SDave Cobbley self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata) 119eb8dc403SDave Cobbley self.stamp = self.getvar('STAMP', metadata) 120eb8dc403SDave Cobbley self.stampclean = self.getvar('STAMPCLEAN', metadata) 121eb8dc403SDave Cobbley self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata) 122eb8dc403SDave Cobbley self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True) 123eb8dc403SDave Cobbley self.depends = self.depvar('DEPENDS', metadata) 124eb8dc403SDave Cobbley self.rdepends = self.depvar('RDEPENDS', metadata) 125eb8dc403SDave Cobbley self.rrecommends = self.depvar('RRECOMMENDS', metadata) 126eb8dc403SDave Cobbley self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata) 127eb8dc403SDave Cobbley self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata) 128eb8dc403SDave Cobbley self.inherits = self.getvar('__inherit_cache', metadata, expand=False) 129eb8dc403SDave Cobbley self.fakerootenv = self.getvar('FAKEROOTENV', metadata) 130eb8dc403SDave Cobbley self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) 13195ac1b8dSAndrew Geissler self.fakerootlogs = self.getvar('FAKEROOTLOGS', metadata) 132eb8dc403SDave Cobbley self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata) 133eb8dc403SDave Cobbley self.extradepsfunc = self.getvar('calculate_extra_depends', metadata) 134eb8dc403SDave Cobbley 135eb8dc403SDave Cobbley @classmethod 136eb8dc403SDave Cobbley def init_cacheData(cls, cachedata): 137eb8dc403SDave Cobbley # CacheData in Core RecipeInfo Class 138eb8dc403SDave Cobbley cachedata.task_deps = {} 139eb8dc403SDave Cobbley cachedata.pkg_fn = {} 140eb8dc403SDave Cobbley cachedata.pkg_pn = defaultdict(list) 141eb8dc403SDave Cobbley cachedata.pkg_pepvpr = {} 142eb8dc403SDave Cobbley cachedata.pkg_dp = {} 143eb8dc403SDave Cobbley 144eb8dc403SDave Cobbley cachedata.stamp = {} 145eb8dc403SDave Cobbley cachedata.stampclean = {} 146eb8dc403SDave Cobbley cachedata.stamp_extrainfo = {} 147eb8dc403SDave Cobbley cachedata.file_checksums = {} 148eb8dc403SDave Cobbley cachedata.fn_provides = {} 149eb8dc403SDave Cobbley cachedata.pn_provides = defaultdict(list) 150eb8dc403SDave Cobbley cachedata.all_depends = [] 151eb8dc403SDave Cobbley 152eb8dc403SDave Cobbley cachedata.deps = defaultdict(list) 153eb8dc403SDave Cobbley cachedata.packages = defaultdict(list) 154eb8dc403SDave Cobbley cachedata.providers = defaultdict(list) 155eb8dc403SDave Cobbley cachedata.rproviders = defaultdict(list) 156eb8dc403SDave Cobbley cachedata.packages_dynamic = defaultdict(list) 157eb8dc403SDave Cobbley 158eb8dc403SDave Cobbley cachedata.rundeps = defaultdict(lambda: defaultdict(list)) 159eb8dc403SDave Cobbley cachedata.runrecs = defaultdict(lambda: defaultdict(list)) 160eb8dc403SDave Cobbley cachedata.possible_world = [] 161eb8dc403SDave Cobbley cachedata.universe_target = [] 162eb8dc403SDave Cobbley cachedata.hashfn = {} 163eb8dc403SDave Cobbley 164eb8dc403SDave Cobbley cachedata.basetaskhash = {} 165eb8dc403SDave Cobbley cachedata.inherits = {} 166eb8dc403SDave Cobbley cachedata.fakerootenv = {} 167eb8dc403SDave Cobbley cachedata.fakerootnoenv = {} 168eb8dc403SDave Cobbley cachedata.fakerootdirs = {} 16995ac1b8dSAndrew Geissler cachedata.fakerootlogs = {} 170eb8dc403SDave Cobbley cachedata.extradepsfunc = {} 171eb8dc403SDave Cobbley 172eb8dc403SDave Cobbley def add_cacheData(self, cachedata, fn): 173eb8dc403SDave Cobbley cachedata.task_deps[fn] = self.task_deps 174eb8dc403SDave Cobbley cachedata.pkg_fn[fn] = self.pn 175eb8dc403SDave Cobbley cachedata.pkg_pn[self.pn].append(fn) 176eb8dc403SDave Cobbley cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr) 177eb8dc403SDave Cobbley cachedata.pkg_dp[fn] = self.defaultpref 178eb8dc403SDave Cobbley cachedata.stamp[fn] = self.stamp 179eb8dc403SDave Cobbley cachedata.stampclean[fn] = self.stampclean 180eb8dc403SDave Cobbley cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo 181eb8dc403SDave Cobbley cachedata.file_checksums[fn] = self.file_checksums 182eb8dc403SDave Cobbley 183eb8dc403SDave Cobbley provides = [self.pn] 184eb8dc403SDave Cobbley for provide in self.provides: 185eb8dc403SDave Cobbley if provide not in provides: 186eb8dc403SDave Cobbley provides.append(provide) 187eb8dc403SDave Cobbley cachedata.fn_provides[fn] = provides 188eb8dc403SDave Cobbley 189eb8dc403SDave Cobbley for provide in provides: 190eb8dc403SDave Cobbley cachedata.providers[provide].append(fn) 191eb8dc403SDave Cobbley if provide not in cachedata.pn_provides[self.pn]: 192eb8dc403SDave Cobbley cachedata.pn_provides[self.pn].append(provide) 193eb8dc403SDave Cobbley 194eb8dc403SDave Cobbley for dep in self.depends: 195eb8dc403SDave Cobbley if dep not in cachedata.deps[fn]: 196eb8dc403SDave Cobbley cachedata.deps[fn].append(dep) 197eb8dc403SDave Cobbley if dep not in cachedata.all_depends: 198eb8dc403SDave Cobbley cachedata.all_depends.append(dep) 199eb8dc403SDave Cobbley 200eb8dc403SDave Cobbley rprovides = self.rprovides 201eb8dc403SDave Cobbley for package in self.packages: 202eb8dc403SDave Cobbley cachedata.packages[package].append(fn) 203eb8dc403SDave Cobbley rprovides += self.rprovides_pkg[package] 204eb8dc403SDave Cobbley 205eb8dc403SDave Cobbley for rprovide in rprovides: 206eb8dc403SDave Cobbley if fn not in cachedata.rproviders[rprovide]: 207eb8dc403SDave Cobbley cachedata.rproviders[rprovide].append(fn) 208eb8dc403SDave Cobbley 209eb8dc403SDave Cobbley for package in self.packages_dynamic: 210eb8dc403SDave Cobbley cachedata.packages_dynamic[package].append(fn) 211eb8dc403SDave Cobbley 212eb8dc403SDave Cobbley # Build hash of runtime depends and recommends 213eb8dc403SDave Cobbley for package in self.packages: 214eb8dc403SDave Cobbley cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package] 215eb8dc403SDave Cobbley cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package] 216eb8dc403SDave Cobbley 217eb8dc403SDave Cobbley # Collect files we may need for possible world-dep 218eb8dc403SDave Cobbley # calculations 219fc113eadSAndrew Geissler if not bb.utils.to_boolean(self.not_world): 220eb8dc403SDave Cobbley cachedata.possible_world.append(fn) 22182c905dcSAndrew Geissler #else: 222d1e89497SAndrew Geissler # logger.debug2("EXCLUDE FROM WORLD: %s", fn) 223eb8dc403SDave Cobbley 224eb8dc403SDave Cobbley # create a collection of all targets for sanity checking 225eb8dc403SDave Cobbley # tasks, such as upstream versions, license, and tools for 226eb8dc403SDave Cobbley # task and image creation. 227eb8dc403SDave Cobbley cachedata.universe_target.append(self.pn) 228eb8dc403SDave Cobbley 229eb8dc403SDave Cobbley cachedata.hashfn[fn] = self.hashfilename 230eb8dc403SDave Cobbley for task, taskhash in self.basetaskhashes.items(): 23108902b01SBrad Bishop identifier = '%s:%s' % (fn, task) 232eb8dc403SDave Cobbley cachedata.basetaskhash[identifier] = taskhash 233eb8dc403SDave Cobbley 234eb8dc403SDave Cobbley cachedata.inherits[fn] = self.inherits 235eb8dc403SDave Cobbley cachedata.fakerootenv[fn] = self.fakerootenv 236eb8dc403SDave Cobbley cachedata.fakerootnoenv[fn] = self.fakerootnoenv 237eb8dc403SDave Cobbley cachedata.fakerootdirs[fn] = self.fakerootdirs 23895ac1b8dSAndrew Geissler cachedata.fakerootlogs[fn] = self.fakerootlogs 239eb8dc403SDave Cobbley cachedata.extradepsfunc[fn] = self.extradepsfunc 240eb8dc403SDave Cobbley 241517393d9SAndrew Geissler 242517393d9SAndrew Geisslerclass SiggenRecipeInfo(RecipeInfoCommon): 243517393d9SAndrew Geissler __slots__ = () 244517393d9SAndrew Geissler 245517393d9SAndrew Geissler classname = "SiggenRecipeInfo" 246517393d9SAndrew Geissler cachefile = "bb_cache_" + classname +".dat" 247517393d9SAndrew Geissler # we don't want to show this information in graph files so don't set cachefields 248517393d9SAndrew Geissler #cachefields = [] 249517393d9SAndrew Geissler 250517393d9SAndrew Geissler def __init__(self, filename, metadata): 251517393d9SAndrew Geissler self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False) 252517393d9SAndrew Geissler self.siggen_varvals = metadata.getVar("__siggen_varvals", False) 253517393d9SAndrew Geissler self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False) 254517393d9SAndrew Geissler 255517393d9SAndrew Geissler @classmethod 256517393d9SAndrew Geissler def init_cacheData(cls, cachedata): 257517393d9SAndrew Geissler cachedata.siggen_taskdeps = {} 258517393d9SAndrew Geissler cachedata.siggen_gendeps = {} 259517393d9SAndrew Geissler cachedata.siggen_varvals = {} 260517393d9SAndrew Geissler 261517393d9SAndrew Geissler def add_cacheData(self, cachedata, fn): 262517393d9SAndrew Geissler cachedata.siggen_gendeps[fn] = self.siggen_gendeps 263517393d9SAndrew Geissler cachedata.siggen_varvals[fn] = self.siggen_varvals 264517393d9SAndrew Geissler cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps 265517393d9SAndrew Geissler 266517393d9SAndrew Geissler # The siggen variable data is large and impacts: 267517393d9SAndrew Geissler # - bitbake's overall memory usage 268517393d9SAndrew Geissler # - the amount of data sent over IPC between parsing processes and the server 269517393d9SAndrew Geissler # - the size of the cache files on disk 270517393d9SAndrew Geissler # - the size of "sigdata" hash information files on disk 271517393d9SAndrew Geissler # The data consists of strings (some large) or frozenset lists of variables 272517393d9SAndrew Geissler # As such, we a) deplicate the data here and b) pass references to the object at second 273517393d9SAndrew Geissler # access (e.g. over IPC or saving into pickle). 274517393d9SAndrew Geissler 275517393d9SAndrew Geissler store = {} 276517393d9SAndrew Geissler save_map = {} 277517393d9SAndrew Geissler save_count = 1 278517393d9SAndrew Geissler restore_map = {} 279517393d9SAndrew Geissler restore_count = {} 280517393d9SAndrew Geissler 281517393d9SAndrew Geissler @classmethod 282517393d9SAndrew Geissler def reset(cls): 283517393d9SAndrew Geissler # Needs to be called before starting new streamed data in a given process 284517393d9SAndrew Geissler # (e.g. writing out the cache again) 285517393d9SAndrew Geissler cls.save_map = {} 286517393d9SAndrew Geissler cls.save_count = 1 287517393d9SAndrew Geissler cls.restore_map = {} 288517393d9SAndrew Geissler 289517393d9SAndrew Geissler @classmethod 290517393d9SAndrew Geissler def _save(cls, deps): 291517393d9SAndrew Geissler ret = [] 292517393d9SAndrew Geissler if not deps: 293517393d9SAndrew Geissler return deps 294517393d9SAndrew Geissler for dep in deps: 295517393d9SAndrew Geissler fs = deps[dep] 296517393d9SAndrew Geissler if fs is None: 297517393d9SAndrew Geissler ret.append((dep, None, None)) 298517393d9SAndrew Geissler elif fs in cls.save_map: 299517393d9SAndrew Geissler ret.append((dep, None, cls.save_map[fs])) 300517393d9SAndrew Geissler else: 301517393d9SAndrew Geissler cls.save_map[fs] = cls.save_count 302517393d9SAndrew Geissler ret.append((dep, fs, cls.save_count)) 303517393d9SAndrew Geissler cls.save_count = cls.save_count + 1 304517393d9SAndrew Geissler return ret 305517393d9SAndrew Geissler 306517393d9SAndrew Geissler @classmethod 307517393d9SAndrew Geissler def _restore(cls, deps, pid): 308517393d9SAndrew Geissler ret = {} 309517393d9SAndrew Geissler if not deps: 310517393d9SAndrew Geissler return deps 311517393d9SAndrew Geissler if pid not in cls.restore_map: 312517393d9SAndrew Geissler cls.restore_map[pid] = {} 313517393d9SAndrew Geissler map = cls.restore_map[pid] 314517393d9SAndrew Geissler for dep, fs, mapnum in deps: 315517393d9SAndrew Geissler if fs is None and mapnum is None: 316517393d9SAndrew Geissler ret[dep] = None 317517393d9SAndrew Geissler elif fs is None: 318517393d9SAndrew Geissler ret[dep] = map[mapnum] 319517393d9SAndrew Geissler else: 320517393d9SAndrew Geissler try: 321517393d9SAndrew Geissler fs = cls.store[fs] 322517393d9SAndrew Geissler except KeyError: 323517393d9SAndrew Geissler cls.store[fs] = fs 324517393d9SAndrew Geissler map[mapnum] = fs 325517393d9SAndrew Geissler ret[dep] = fs 326517393d9SAndrew Geissler return ret 327517393d9SAndrew Geissler 328517393d9SAndrew Geissler def __getstate__(self): 329517393d9SAndrew Geissler ret = {} 330517393d9SAndrew Geissler for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: 331517393d9SAndrew Geissler ret[key] = self._save(self.__dict__[key]) 332517393d9SAndrew Geissler ret['pid'] = os.getpid() 333517393d9SAndrew Geissler return ret 334517393d9SAndrew Geissler 335517393d9SAndrew Geissler def __setstate__(self, state): 336517393d9SAndrew Geissler pid = state['pid'] 337517393d9SAndrew Geissler for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: 338517393d9SAndrew Geissler setattr(self, key, self._restore(state[key], pid)) 339517393d9SAndrew Geissler 340517393d9SAndrew Geissler 341eb8dc403SDave Cobbleydef virtualfn2realfn(virtualfn): 342eb8dc403SDave Cobbley """ 343eb8dc403SDave Cobbley Convert a virtual file name to a real one + the associated subclass keyword 344eb8dc403SDave Cobbley """ 345eb8dc403SDave Cobbley mc = "" 346d1e89497SAndrew Geissler if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2: 347*ac13d5f3SPatrick Williams (_, mc, virtualfn) = virtualfn.split(':', 2) 348eb8dc403SDave Cobbley 349eb8dc403SDave Cobbley fn = virtualfn 350eb8dc403SDave Cobbley cls = "" 351eb8dc403SDave Cobbley if virtualfn.startswith('virtual:'): 352eb8dc403SDave Cobbley elems = virtualfn.split(':') 353eb8dc403SDave Cobbley cls = ":".join(elems[1:-1]) 354eb8dc403SDave Cobbley fn = elems[-1] 355eb8dc403SDave Cobbley 356eb8dc403SDave Cobbley return (fn, cls, mc) 357eb8dc403SDave Cobbley 358eb8dc403SDave Cobbleydef realfn2virtual(realfn, cls, mc): 359eb8dc403SDave Cobbley """ 360eb8dc403SDave Cobbley Convert a real filename + the associated subclass keyword to a virtual filename 361eb8dc403SDave Cobbley """ 362eb8dc403SDave Cobbley if cls: 363eb8dc403SDave Cobbley realfn = "virtual:" + cls + ":" + realfn 364eb8dc403SDave Cobbley if mc: 36515ae2509SBrad Bishop realfn = "mc:" + mc + ":" + realfn 366eb8dc403SDave Cobbley return realfn 367eb8dc403SDave Cobbley 368eb8dc403SDave Cobbleydef variant2virtual(realfn, variant): 369eb8dc403SDave Cobbley """ 370*ac13d5f3SPatrick Williams Convert a real filename + a variant to a virtual filename 371eb8dc403SDave Cobbley """ 372eb8dc403SDave Cobbley if variant == "": 373eb8dc403SDave Cobbley return realfn 374d1e89497SAndrew Geissler if variant.startswith("mc:") and variant.count(':') >= 2: 375eb8dc403SDave Cobbley elems = variant.split(":") 376eb8dc403SDave Cobbley if elems[2]: 37715ae2509SBrad Bishop return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn 37815ae2509SBrad Bishop return "mc:" + elems[1] + ":" + realfn 379eb8dc403SDave Cobbley return "virtual:" + variant + ":" + realfn 380eb8dc403SDave Cobbley 381517393d9SAndrew Geissler# 382517393d9SAndrew Geissler# Cooker calls cacheValid on its recipe list, then either calls loadCached 383517393d9SAndrew Geissler# from it's main thread or parse from separate processes to generate an up to 384517393d9SAndrew Geissler# date cache 385517393d9SAndrew Geissler# 386517393d9SAndrew Geisslerclass Cache(object): 387eb8dc403SDave Cobbley """ 388eb8dc403SDave Cobbley BitBake Cache implementation 389eb8dc403SDave Cobbley """ 3905a43b434SAndrew Geissler def __init__(self, databuilder, mc, data_hash, caches_array): 391517393d9SAndrew Geissler self.databuilder = databuilder 392517393d9SAndrew Geissler self.data = databuilder.data 393eb8dc403SDave Cobbley 394eb8dc403SDave Cobbley # Pass caches_array information into Cache Constructor 395eb8dc403SDave Cobbley # It will be used later for deciding whether we 396eb8dc403SDave Cobbley # need extra cache file dump/load support 3975a43b434SAndrew Geissler self.mc = mc 3985a43b434SAndrew Geissler self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) 399eb8dc403SDave Cobbley self.caches_array = caches_array 400517393d9SAndrew Geissler self.cachedir = self.data.getVar("CACHE") 401eb8dc403SDave Cobbley self.clean = set() 402eb8dc403SDave Cobbley self.checked = set() 403eb8dc403SDave Cobbley self.depends_cache = {} 404eb8dc403SDave Cobbley self.data_fn = None 405eb8dc403SDave Cobbley self.cacheclean = True 406eb8dc403SDave Cobbley self.data_hash = data_hash 40782c905dcSAndrew Geissler self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') 408eb8dc403SDave Cobbley 409eb8dc403SDave Cobbley if self.cachedir in [None, '']: 410517393d9SAndrew Geissler bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use") 411eb8dc403SDave Cobbley 4125a43b434SAndrew Geissler def getCacheFile(self, cachefile): 4135a43b434SAndrew Geissler return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) 4145a43b434SAndrew Geissler 4155a43b434SAndrew Geissler def prepare_cache(self, progress): 4165a43b434SAndrew Geissler loaded = 0 4175a43b434SAndrew Geissler 4185a43b434SAndrew Geissler self.cachefile = self.getCacheFile("bb_cache.dat") 4195a43b434SAndrew Geissler 420d1e89497SAndrew Geissler self.logger.debug("Cache dir: %s", self.cachedir) 421eb8dc403SDave Cobbley bb.utils.mkdirhier(self.cachedir) 422eb8dc403SDave Cobbley 423eb8dc403SDave Cobbley cache_ok = True 424eb8dc403SDave Cobbley if self.caches_array: 425eb8dc403SDave Cobbley for cache_class in self.caches_array: 4265a43b434SAndrew Geissler cachefile = self.getCacheFile(cache_class.cachefile) 4275a43b434SAndrew Geissler cache_exists = os.path.exists(cachefile) 428d1e89497SAndrew Geissler self.logger.debug2("Checking if %s exists: %r", cachefile, cache_exists) 4295a43b434SAndrew Geissler cache_ok = cache_ok and cache_exists 430eb8dc403SDave Cobbley cache_class.init_cacheData(self) 431eb8dc403SDave Cobbley if cache_ok: 4325a43b434SAndrew Geissler loaded = self.load_cachefile(progress) 433eb8dc403SDave Cobbley elif os.path.isfile(self.cachefile): 4345a43b434SAndrew Geissler self.logger.info("Out of date cache found, rebuilding...") 435eb8dc403SDave Cobbley else: 436d1e89497SAndrew Geissler self.logger.debug("Cache file %s not found, building..." % self.cachefile) 437eb8dc403SDave Cobbley 43896ff1984SBrad Bishop # We don't use the symlink, its just for debugging convinience 4395a43b434SAndrew Geissler if self.mc: 4405a43b434SAndrew Geissler symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) 4415a43b434SAndrew Geissler else: 44296ff1984SBrad Bishop symlink = os.path.join(self.cachedir, "bb_cache.dat") 4435a43b434SAndrew Geissler 44496ff1984SBrad Bishop if os.path.exists(symlink): 44596ff1984SBrad Bishop bb.utils.remove(symlink) 44696ff1984SBrad Bishop try: 44796ff1984SBrad Bishop os.symlink(os.path.basename(self.cachefile), symlink) 44896ff1984SBrad Bishop except OSError: 44996ff1984SBrad Bishop pass 45096ff1984SBrad Bishop 4515a43b434SAndrew Geissler return loaded 4525a43b434SAndrew Geissler 4535a43b434SAndrew Geissler def cachesize(self): 454eb8dc403SDave Cobbley cachesize = 0 4555a43b434SAndrew Geissler for cache_class in self.caches_array: 4565a43b434SAndrew Geissler cachefile = self.getCacheFile(cache_class.cachefile) 4575a43b434SAndrew Geissler try: 4585a43b434SAndrew Geissler with open(cachefile, "rb") as cachefile: 4595a43b434SAndrew Geissler cachesize += os.fstat(cachefile.fileno()).st_size 4605a43b434SAndrew Geissler except FileNotFoundError: 4615a43b434SAndrew Geissler pass 4625a43b434SAndrew Geissler 4635a43b434SAndrew Geissler return cachesize 4645a43b434SAndrew Geissler 4655a43b434SAndrew Geissler def load_cachefile(self, progress): 466eb8dc403SDave Cobbley previous_progress = 0 467eb8dc403SDave Cobbley 468eb8dc403SDave Cobbley for cache_class in self.caches_array: 4695a43b434SAndrew Geissler cachefile = self.getCacheFile(cache_class.cachefile) 470d1e89497SAndrew Geissler self.logger.debug('Loading cache file: %s' % cachefile) 471eb8dc403SDave Cobbley with open(cachefile, "rb") as cachefile: 472eb8dc403SDave Cobbley pickled = pickle.Unpickler(cachefile) 473eb8dc403SDave Cobbley # Check cache version information 474eb8dc403SDave Cobbley try: 475eb8dc403SDave Cobbley cache_ver = pickled.load() 476eb8dc403SDave Cobbley bitbake_ver = pickled.load() 477eb8dc403SDave Cobbley except Exception: 4785a43b434SAndrew Geissler self.logger.info('Invalid cache, rebuilding...') 4795a43b434SAndrew Geissler return 0 480eb8dc403SDave Cobbley 481eb8dc403SDave Cobbley if cache_ver != __cache_version__: 4825a43b434SAndrew Geissler self.logger.info('Cache version mismatch, rebuilding...') 4835a43b434SAndrew Geissler return 0 484eb8dc403SDave Cobbley elif bitbake_ver != bb.__version__: 4855a43b434SAndrew Geissler self.logger.info('Bitbake version mismatch, rebuilding...') 4865a43b434SAndrew Geissler return 0 487eb8dc403SDave Cobbley 488eb8dc403SDave Cobbley # Load the rest of the cache file 489eb8dc403SDave Cobbley current_progress = 0 490eb8dc403SDave Cobbley while cachefile: 491eb8dc403SDave Cobbley try: 492eb8dc403SDave Cobbley key = pickled.load() 493eb8dc403SDave Cobbley value = pickled.load() 494eb8dc403SDave Cobbley except Exception: 495eb8dc403SDave Cobbley break 496eb8dc403SDave Cobbley if not isinstance(key, str): 497eb8dc403SDave Cobbley bb.warn("%s from extras cache is not a string?" % key) 498eb8dc403SDave Cobbley break 499eb8dc403SDave Cobbley if not isinstance(value, RecipeInfoCommon): 500eb8dc403SDave Cobbley bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value) 501eb8dc403SDave Cobbley break 502eb8dc403SDave Cobbley 503eb8dc403SDave Cobbley if key in self.depends_cache: 504eb8dc403SDave Cobbley self.depends_cache[key].append(value) 505eb8dc403SDave Cobbley else: 506eb8dc403SDave Cobbley self.depends_cache[key] = [value] 507eb8dc403SDave Cobbley # only fire events on even percentage boundaries 508eb8dc403SDave Cobbley current_progress = cachefile.tell() + previous_progress 5095a43b434SAndrew Geissler progress(cachefile.tell() + previous_progress) 510eb8dc403SDave Cobbley 511eb8dc403SDave Cobbley previous_progress += current_progress 512eb8dc403SDave Cobbley 5135a43b434SAndrew Geissler return len(self.depends_cache) 514eb8dc403SDave Cobbley 515e760df85SPatrick Williams def parse(self, filename, appends, layername): 516eb8dc403SDave Cobbley """Parse the specified filename, returning the recipe information""" 517d1e89497SAndrew Geissler self.logger.debug("Parsing %s", filename) 518eb8dc403SDave Cobbley infos = [] 519e760df85SPatrick Williams datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername) 520eb8dc403SDave Cobbley depends = [] 521eb8dc403SDave Cobbley variants = [] 522eb8dc403SDave Cobbley # Process the "real" fn last so we can store variants list 523eb8dc403SDave Cobbley for variant, data in sorted(datastores.items(), 524eb8dc403SDave Cobbley key=lambda i: i[0], 525eb8dc403SDave Cobbley reverse=True): 526eb8dc403SDave Cobbley virtualfn = variant2virtual(filename, variant) 527eb8dc403SDave Cobbley variants.append(variant) 528eb8dc403SDave Cobbley depends = depends + (data.getVar("__depends", False) or []) 529eb8dc403SDave Cobbley if depends and not variant: 530eb8dc403SDave Cobbley data.setVar("__depends", depends) 531eb8dc403SDave Cobbley if virtualfn == filename: 532eb8dc403SDave Cobbley data.setVar("__VARIANTS", " ".join(variants)) 533eb8dc403SDave Cobbley info_array = [] 534eb8dc403SDave Cobbley for cache_class in self.caches_array: 535eb8dc403SDave Cobbley info = cache_class(filename, data) 536eb8dc403SDave Cobbley info_array.append(info) 537eb8dc403SDave Cobbley infos.append((virtualfn, info_array)) 538eb8dc403SDave Cobbley 539eb8dc403SDave Cobbley return infos 540eb8dc403SDave Cobbley 541517393d9SAndrew Geissler def loadCached(self, filename, appends): 542eb8dc403SDave Cobbley """Obtain the recipe information for the specified filename, 543517393d9SAndrew Geissler using cached values. 544517393d9SAndrew Geissler """ 545eb8dc403SDave Cobbley 546eb8dc403SDave Cobbley infos = [] 547eb8dc403SDave Cobbley # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] 548eb8dc403SDave Cobbley info_array = self.depends_cache[filename] 549eb8dc403SDave Cobbley for variant in info_array[0].variants: 550eb8dc403SDave Cobbley virtualfn = variant2virtual(filename, variant) 551eb8dc403SDave Cobbley infos.append((virtualfn, self.depends_cache[virtualfn])) 552eb8dc403SDave Cobbley 553517393d9SAndrew Geissler return infos 554eb8dc403SDave Cobbley 555eb8dc403SDave Cobbley def cacheValid(self, fn, appends): 556eb8dc403SDave Cobbley """ 557eb8dc403SDave Cobbley Is the cache valid for fn? 558eb8dc403SDave Cobbley Fast version, no timestamps checked. 559eb8dc403SDave Cobbley """ 560eb8dc403SDave Cobbley if fn not in self.checked: 561eb8dc403SDave Cobbley self.cacheValidUpdate(fn, appends) 562eb8dc403SDave Cobbley if fn in self.clean: 563eb8dc403SDave Cobbley return True 564eb8dc403SDave Cobbley return False 565eb8dc403SDave Cobbley 566eb8dc403SDave Cobbley def cacheValidUpdate(self, fn, appends): 567eb8dc403SDave Cobbley """ 568eb8dc403SDave Cobbley Is the cache valid for fn? 569eb8dc403SDave Cobbley Make thorough (slower) checks including timestamps. 570eb8dc403SDave Cobbley """ 571eb8dc403SDave Cobbley self.checked.add(fn) 572eb8dc403SDave Cobbley 573eb8dc403SDave Cobbley # File isn't in depends_cache 574eb8dc403SDave Cobbley if not fn in self.depends_cache: 575d1e89497SAndrew Geissler self.logger.debug2("%s is not cached", fn) 576eb8dc403SDave Cobbley return False 577eb8dc403SDave Cobbley 578eb8dc403SDave Cobbley mtime = bb.parse.cached_mtime_noerror(fn) 579eb8dc403SDave Cobbley 580eb8dc403SDave Cobbley # Check file still exists 581eb8dc403SDave Cobbley if mtime == 0: 582d1e89497SAndrew Geissler self.logger.debug2("%s no longer exists", fn) 583eb8dc403SDave Cobbley self.remove(fn) 584eb8dc403SDave Cobbley return False 585eb8dc403SDave Cobbley 586eb8dc403SDave Cobbley info_array = self.depends_cache[fn] 587eb8dc403SDave Cobbley # Check the file's timestamp 588eb8dc403SDave Cobbley if mtime != info_array[0].timestamp: 589d1e89497SAndrew Geissler self.logger.debug2("%s changed", fn) 590eb8dc403SDave Cobbley self.remove(fn) 591eb8dc403SDave Cobbley return False 592eb8dc403SDave Cobbley 593eb8dc403SDave Cobbley # Check dependencies are still valid 594eb8dc403SDave Cobbley depends = info_array[0].file_depends 595eb8dc403SDave Cobbley if depends: 596eb8dc403SDave Cobbley for f, old_mtime in depends: 597eb8dc403SDave Cobbley fmtime = bb.parse.cached_mtime_noerror(f) 598eb8dc403SDave Cobbley # Check if file still exists 599eb8dc403SDave Cobbley if old_mtime != 0 and fmtime == 0: 600d1e89497SAndrew Geissler self.logger.debug2("%s's dependency %s was removed", 601eb8dc403SDave Cobbley fn, f) 602eb8dc403SDave Cobbley self.remove(fn) 603eb8dc403SDave Cobbley return False 604eb8dc403SDave Cobbley 605eb8dc403SDave Cobbley if (fmtime != old_mtime): 606d1e89497SAndrew Geissler self.logger.debug2("%s's dependency %s changed", 607eb8dc403SDave Cobbley fn, f) 608eb8dc403SDave Cobbley self.remove(fn) 609eb8dc403SDave Cobbley return False 610eb8dc403SDave Cobbley 611eb8dc403SDave Cobbley if hasattr(info_array[0], 'file_checksums'): 612eb8dc403SDave Cobbley for _, fl in info_array[0].file_checksums.items(): 613eb8dc403SDave Cobbley fl = fl.strip() 61482c905dcSAndrew Geissler if not fl: 61582c905dcSAndrew Geissler continue 61682c905dcSAndrew Geissler # Have to be careful about spaces and colons in filenames 61782c905dcSAndrew Geissler flist = self.filelist_regex.split(fl) 61882c905dcSAndrew Geissler for f in flist: 619c9f7865aSAndrew Geissler if not f: 620eb8dc403SDave Cobbley continue 621d583833aSAndrew Geissler f, exist = f.rsplit(":", 1) 622eb8dc403SDave Cobbley if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): 623d1e89497SAndrew Geissler self.logger.debug2("%s's file checksum list file %s changed", 624eb8dc403SDave Cobbley fn, f) 625eb8dc403SDave Cobbley self.remove(fn) 626eb8dc403SDave Cobbley return False 627eb8dc403SDave Cobbley 6285a43b434SAndrew Geissler if tuple(appends) != tuple(info_array[0].appends): 629d1e89497SAndrew Geissler self.logger.debug2("appends for %s changed", fn) 630d1e89497SAndrew Geissler self.logger.debug2("%s to %s" % (str(appends), str(info_array[0].appends))) 631eb8dc403SDave Cobbley self.remove(fn) 632eb8dc403SDave Cobbley return False 633eb8dc403SDave Cobbley 634eb8dc403SDave Cobbley invalid = False 635eb8dc403SDave Cobbley for cls in info_array[0].variants: 636eb8dc403SDave Cobbley virtualfn = variant2virtual(fn, cls) 637eb8dc403SDave Cobbley self.clean.add(virtualfn) 638eb8dc403SDave Cobbley if virtualfn not in self.depends_cache: 639d1e89497SAndrew Geissler self.logger.debug2("%s is not cached", virtualfn) 640eb8dc403SDave Cobbley invalid = True 641eb8dc403SDave Cobbley elif len(self.depends_cache[virtualfn]) != len(self.caches_array): 642d1e89497SAndrew Geissler self.logger.debug2("Extra caches missing for %s?" % virtualfn) 643eb8dc403SDave Cobbley invalid = True 644eb8dc403SDave Cobbley 645eb8dc403SDave Cobbley # If any one of the variants is not present, mark as invalid for all 646eb8dc403SDave Cobbley if invalid: 647eb8dc403SDave Cobbley for cls in info_array[0].variants: 648eb8dc403SDave Cobbley virtualfn = variant2virtual(fn, cls) 649eb8dc403SDave Cobbley if virtualfn in self.clean: 650d1e89497SAndrew Geissler self.logger.debug2("Removing %s from cache", virtualfn) 651eb8dc403SDave Cobbley self.clean.remove(virtualfn) 652eb8dc403SDave Cobbley if fn in self.clean: 653d1e89497SAndrew Geissler self.logger.debug2("Marking %s as not clean", fn) 654eb8dc403SDave Cobbley self.clean.remove(fn) 655eb8dc403SDave Cobbley return False 656eb8dc403SDave Cobbley 657eb8dc403SDave Cobbley self.clean.add(fn) 658eb8dc403SDave Cobbley return True 659eb8dc403SDave Cobbley 660eb8dc403SDave Cobbley def remove(self, fn): 661eb8dc403SDave Cobbley """ 662eb8dc403SDave Cobbley Remove a fn from the cache 663eb8dc403SDave Cobbley Called from the parser in error cases 664eb8dc403SDave Cobbley """ 665eb8dc403SDave Cobbley if fn in self.depends_cache: 666d1e89497SAndrew Geissler self.logger.debug("Removing %s from cache", fn) 667eb8dc403SDave Cobbley del self.depends_cache[fn] 668eb8dc403SDave Cobbley if fn in self.clean: 669d1e89497SAndrew Geissler self.logger.debug("Marking %s as unclean", fn) 670eb8dc403SDave Cobbley self.clean.remove(fn) 671eb8dc403SDave Cobbley 672eb8dc403SDave Cobbley def sync(self): 673eb8dc403SDave Cobbley """ 674eb8dc403SDave Cobbley Save the cache 675eb8dc403SDave Cobbley Called from the parser when complete (or exiting) 676eb8dc403SDave Cobbley """ 677eb8dc403SDave Cobbley if self.cacheclean: 678d1e89497SAndrew Geissler self.logger.debug2("Cache is clean, not saving.") 679eb8dc403SDave Cobbley return 680eb8dc403SDave Cobbley 681eb8dc403SDave Cobbley for cache_class in self.caches_array: 682eb8dc403SDave Cobbley cache_class_name = cache_class.__name__ 6835a43b434SAndrew Geissler cachefile = self.getCacheFile(cache_class.cachefile) 684d1e89497SAndrew Geissler self.logger.debug2("Writing %s", cachefile) 685eb8dc403SDave Cobbley with open(cachefile, "wb") as f: 686eb8dc403SDave Cobbley p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) 687eb8dc403SDave Cobbley p.dump(__cache_version__) 688eb8dc403SDave Cobbley p.dump(bb.__version__) 689eb8dc403SDave Cobbley 690eb8dc403SDave Cobbley for key, info_array in self.depends_cache.items(): 691eb8dc403SDave Cobbley for info in info_array: 692eb8dc403SDave Cobbley if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name: 693eb8dc403SDave Cobbley p.dump(key) 694eb8dc403SDave Cobbley p.dump(info) 695eb8dc403SDave Cobbley 696eb8dc403SDave Cobbley del self.depends_cache 697517393d9SAndrew Geissler SiggenRecipeInfo.reset() 698eb8dc403SDave Cobbley 699eb8dc403SDave Cobbley @staticmethod 700eb8dc403SDave Cobbley def mtime(cachefile): 701eb8dc403SDave Cobbley return bb.parse.cached_mtime_noerror(cachefile) 702eb8dc403SDave Cobbley 703eb8dc403SDave Cobbley def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): 7045a43b434SAndrew Geissler if self.mc is not None: 7055a43b434SAndrew Geissler (fn, cls, mc) = virtualfn2realfn(filename) 7065a43b434SAndrew Geissler if mc: 707d25ed324SAndrew Geissler self.logger.error("Unexpected multiconfig %s", filename) 7085a43b434SAndrew Geissler return 7095a43b434SAndrew Geissler 7105a43b434SAndrew Geissler vfn = realfn2virtual(fn, cls, self.mc) 7115a43b434SAndrew Geissler else: 7125a43b434SAndrew Geissler vfn = filename 7135a43b434SAndrew Geissler 714eb8dc403SDave Cobbley if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): 7155a43b434SAndrew Geissler cacheData.add_from_recipeinfo(vfn, info_array) 716eb8dc403SDave Cobbley 717eb8dc403SDave Cobbley if watcher: 718eb8dc403SDave Cobbley watcher(info_array[0].file_depends) 719eb8dc403SDave Cobbley 720eb8dc403SDave Cobbley if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: 721eb8dc403SDave Cobbley if parsed: 722eb8dc403SDave Cobbley self.cacheclean = False 723eb8dc403SDave Cobbley self.depends_cache[filename] = info_array 724eb8dc403SDave Cobbley 7255a43b434SAndrew Geisslerclass MulticonfigCache(Mapping): 7265a43b434SAndrew Geissler def __init__(self, databuilder, data_hash, caches_array): 7275a43b434SAndrew Geissler def progress(p): 7285a43b434SAndrew Geissler nonlocal current_progress 7295a43b434SAndrew Geissler nonlocal previous_progress 7305a43b434SAndrew Geissler nonlocal previous_percent 7315a43b434SAndrew Geissler nonlocal cachesize 7325a43b434SAndrew Geissler 7335a43b434SAndrew Geissler current_progress = previous_progress + p 7345a43b434SAndrew Geissler 7355a43b434SAndrew Geissler if current_progress > cachesize: 7365a43b434SAndrew Geissler # we might have calculated incorrect total size because a file 7375a43b434SAndrew Geissler # might've been written out just after we checked its size 7385a43b434SAndrew Geissler cachesize = current_progress 7395a43b434SAndrew Geissler current_percent = 100 * current_progress / cachesize 7405a43b434SAndrew Geissler if current_percent > previous_percent: 7415a43b434SAndrew Geissler previous_percent = current_percent 7425a43b434SAndrew Geissler bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), 7435a43b434SAndrew Geissler databuilder.data) 7445a43b434SAndrew Geissler 7455a43b434SAndrew Geissler 7465a43b434SAndrew Geissler cachesize = 0 7475a43b434SAndrew Geissler current_progress = 0 7485a43b434SAndrew Geissler previous_progress = 0 7495a43b434SAndrew Geissler previous_percent = 0 7505a43b434SAndrew Geissler self.__caches = {} 7515a43b434SAndrew Geissler 7525a43b434SAndrew Geissler for mc, mcdata in databuilder.mcdata.items(): 7535a43b434SAndrew Geissler self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) 7545a43b434SAndrew Geissler 7555a43b434SAndrew Geissler cachesize += self.__caches[mc].cachesize() 7565a43b434SAndrew Geissler 7575a43b434SAndrew Geissler bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) 7585a43b434SAndrew Geissler loaded = 0 7595a43b434SAndrew Geissler 7605a43b434SAndrew Geissler for c in self.__caches.values(): 761517393d9SAndrew Geissler SiggenRecipeInfo.reset() 7625a43b434SAndrew Geissler loaded += c.prepare_cache(progress) 7635a43b434SAndrew Geissler previous_progress = current_progress 7645a43b434SAndrew Geissler 7655a43b434SAndrew Geissler # Note: depends cache number is corresponding to the parsing file numbers. 7665a43b434SAndrew Geissler # The same file has several caches, still regarded as one item in the cache 7675a43b434SAndrew Geissler bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) 7685a43b434SAndrew Geissler 7695a43b434SAndrew Geissler def __len__(self): 7705a43b434SAndrew Geissler return len(self.__caches) 7715a43b434SAndrew Geissler 7725a43b434SAndrew Geissler def __getitem__(self, key): 7735a43b434SAndrew Geissler return self.__caches[key] 7745a43b434SAndrew Geissler 7755a43b434SAndrew Geissler def __contains__(self, key): 7765a43b434SAndrew Geissler return key in self.__caches 7775a43b434SAndrew Geissler 7785a43b434SAndrew Geissler def __iter__(self): 7795a43b434SAndrew Geissler for k in self.__caches: 7805a43b434SAndrew Geissler yield k 7815a43b434SAndrew Geissler 782eb8dc403SDave Cobbleydef init(cooker): 783eb8dc403SDave Cobbley """ 784eb8dc403SDave Cobbley The Objective: Cache the minimum amount of data possible yet get to the 785eb8dc403SDave Cobbley stage of building packages (i.e. tryBuild) without reparsing any .bb files. 786eb8dc403SDave Cobbley 787eb8dc403SDave Cobbley To do this, we intercept getVar calls and only cache the variables we see 788eb8dc403SDave Cobbley being accessed. We rely on the cache getVar calls being made for all 789eb8dc403SDave Cobbley variables bitbake might need to use to reach this stage. For each cached 790eb8dc403SDave Cobbley file we need to track: 791eb8dc403SDave Cobbley 792eb8dc403SDave Cobbley * Its mtime 793eb8dc403SDave Cobbley * The mtimes of all its dependencies 794eb8dc403SDave Cobbley * Whether it caused a parse.SkipRecipe exception 795eb8dc403SDave Cobbley 796eb8dc403SDave Cobbley Files causing parsing errors are evicted from the cache. 797eb8dc403SDave Cobbley 798eb8dc403SDave Cobbley """ 799eb8dc403SDave Cobbley return Cache(cooker.configuration.data, cooker.configuration.data_hash) 800eb8dc403SDave Cobbley 801eb8dc403SDave Cobbley 802eb8dc403SDave Cobbleyclass CacheData(object): 803eb8dc403SDave Cobbley """ 804eb8dc403SDave Cobbley The data structures we compile from the cached data 805eb8dc403SDave Cobbley """ 806eb8dc403SDave Cobbley 807eb8dc403SDave Cobbley def __init__(self, caches_array): 808eb8dc403SDave Cobbley self.caches_array = caches_array 809eb8dc403SDave Cobbley for cache_class in self.caches_array: 810eb8dc403SDave Cobbley if not issubclass(cache_class, RecipeInfoCommon): 811eb8dc403SDave Cobbley bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class) 812eb8dc403SDave Cobbley cache_class.init_cacheData(self) 813eb8dc403SDave Cobbley 814eb8dc403SDave Cobbley # Direct cache variables 815eb8dc403SDave Cobbley self.task_queues = {} 816eb8dc403SDave Cobbley self.preferred = {} 817eb8dc403SDave Cobbley self.tasks = {} 818eb8dc403SDave Cobbley # Indirect Cache variables (set elsewhere) 819eb8dc403SDave Cobbley self.ignored_dependencies = [] 820eb8dc403SDave Cobbley self.world_target = set() 821eb8dc403SDave Cobbley self.bbfile_priority = {} 822eb8dc403SDave Cobbley 823eb8dc403SDave Cobbley def add_from_recipeinfo(self, fn, info_array): 824eb8dc403SDave Cobbley for info in info_array: 825eb8dc403SDave Cobbley info.add_cacheData(self, fn) 826eb8dc403SDave Cobbley 827eb8dc403SDave Cobbleyclass MultiProcessCache(object): 828eb8dc403SDave Cobbley """ 829eb8dc403SDave Cobbley BitBake multi-process cache implementation 830eb8dc403SDave Cobbley 831eb8dc403SDave Cobbley Used by the codeparser & file checksum caches 832eb8dc403SDave Cobbley """ 833eb8dc403SDave Cobbley 834eb8dc403SDave Cobbley def __init__(self): 835eb8dc403SDave Cobbley self.cachefile = None 836eb8dc403SDave Cobbley self.cachedata = self.create_cachedata() 837eb8dc403SDave Cobbley self.cachedata_extras = self.create_cachedata() 838eb8dc403SDave Cobbley 839c5535c91SAndrew Geissler def init_cache(self, cachedir, cache_file_name=None): 840c5535c91SAndrew Geissler if not cachedir: 841eb8dc403SDave Cobbley return 842c5535c91SAndrew Geissler 843eb8dc403SDave Cobbley bb.utils.mkdirhier(cachedir) 844eb8dc403SDave Cobbley self.cachefile = os.path.join(cachedir, 845eb8dc403SDave Cobbley cache_file_name or self.__class__.cache_file_name) 846d1e89497SAndrew Geissler logger.debug("Using cache in '%s'", self.cachefile) 847eb8dc403SDave Cobbley 848eb8dc403SDave Cobbley glf = bb.utils.lockfile(self.cachefile + ".lock") 849eb8dc403SDave Cobbley 850eb8dc403SDave Cobbley try: 851eb8dc403SDave Cobbley with open(self.cachefile, "rb") as f: 852eb8dc403SDave Cobbley p = pickle.Unpickler(f) 853eb8dc403SDave Cobbley data, version = p.load() 854eb8dc403SDave Cobbley except: 855eb8dc403SDave Cobbley bb.utils.unlockfile(glf) 856eb8dc403SDave Cobbley return 857eb8dc403SDave Cobbley 858eb8dc403SDave Cobbley bb.utils.unlockfile(glf) 859eb8dc403SDave Cobbley 860eb8dc403SDave Cobbley if version != self.__class__.CACHE_VERSION: 861eb8dc403SDave Cobbley return 862eb8dc403SDave Cobbley 863eb8dc403SDave Cobbley self.cachedata = data 864eb8dc403SDave Cobbley 865eb8dc403SDave Cobbley def create_cachedata(self): 866eb8dc403SDave Cobbley data = [{}] 867eb8dc403SDave Cobbley return data 868eb8dc403SDave Cobbley 869eb8dc403SDave Cobbley def save_extras(self): 870eb8dc403SDave Cobbley if not self.cachefile: 871eb8dc403SDave Cobbley return 872eb8dc403SDave Cobbley 873c5535c91SAndrew Geissler have_data = any(self.cachedata_extras) 874c5535c91SAndrew Geissler if not have_data: 875c5535c91SAndrew Geissler return 876c5535c91SAndrew Geissler 877eb8dc403SDave Cobbley glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) 878eb8dc403SDave Cobbley 879eb8dc403SDave Cobbley i = os.getpid() 880eb8dc403SDave Cobbley lf = None 881eb8dc403SDave Cobbley while not lf: 882eb8dc403SDave Cobbley lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False) 883eb8dc403SDave Cobbley if not lf or os.path.exists(self.cachefile + "-" + str(i)): 884eb8dc403SDave Cobbley if lf: 885eb8dc403SDave Cobbley bb.utils.unlockfile(lf) 886eb8dc403SDave Cobbley lf = None 887eb8dc403SDave Cobbley i = i + 1 888eb8dc403SDave Cobbley continue 889eb8dc403SDave Cobbley 890eb8dc403SDave Cobbley with open(self.cachefile + "-" + str(i), "wb") as f: 891eb8dc403SDave Cobbley p = pickle.Pickler(f, -1) 892eb8dc403SDave Cobbley p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) 893eb8dc403SDave Cobbley 894eb8dc403SDave Cobbley bb.utils.unlockfile(lf) 895eb8dc403SDave Cobbley bb.utils.unlockfile(glf) 896eb8dc403SDave Cobbley 897eb8dc403SDave Cobbley def merge_data(self, source, dest): 898eb8dc403SDave Cobbley for j in range(0,len(dest)): 899eb8dc403SDave Cobbley for h in source[j]: 900eb8dc403SDave Cobbley if h not in dest[j]: 901eb8dc403SDave Cobbley dest[j][h] = source[j][h] 902eb8dc403SDave Cobbley 903eb8dc403SDave Cobbley def save_merge(self): 904eb8dc403SDave Cobbley if not self.cachefile: 905eb8dc403SDave Cobbley return 906eb8dc403SDave Cobbley 907eb8dc403SDave Cobbley glf = bb.utils.lockfile(self.cachefile + ".lock") 908eb8dc403SDave Cobbley 909eb8dc403SDave Cobbley data = self.cachedata 910eb8dc403SDave Cobbley 911c5535c91SAndrew Geissler have_data = False 912c5535c91SAndrew Geissler 913eb8dc403SDave Cobbley for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: 914eb8dc403SDave Cobbley f = os.path.join(os.path.dirname(self.cachefile), f) 915eb8dc403SDave Cobbley try: 916eb8dc403SDave Cobbley with open(f, "rb") as fd: 917eb8dc403SDave Cobbley p = pickle.Unpickler(fd) 918eb8dc403SDave Cobbley extradata, version = p.load() 919eb8dc403SDave Cobbley except (IOError, EOFError): 920eb8dc403SDave Cobbley os.unlink(f) 921eb8dc403SDave Cobbley continue 922eb8dc403SDave Cobbley 923eb8dc403SDave Cobbley if version != self.__class__.CACHE_VERSION: 924eb8dc403SDave Cobbley os.unlink(f) 925eb8dc403SDave Cobbley continue 926eb8dc403SDave Cobbley 927c5535c91SAndrew Geissler have_data = True 928eb8dc403SDave Cobbley self.merge_data(extradata, data) 929eb8dc403SDave Cobbley os.unlink(f) 930eb8dc403SDave Cobbley 931c5535c91SAndrew Geissler if have_data: 932eb8dc403SDave Cobbley with open(self.cachefile, "wb") as f: 933eb8dc403SDave Cobbley p = pickle.Pickler(f, -1) 934eb8dc403SDave Cobbley p.dump([data, self.__class__.CACHE_VERSION]) 935eb8dc403SDave Cobbley 936eb8dc403SDave Cobbley bb.utils.unlockfile(glf) 93708902b01SBrad Bishop 93808902b01SBrad Bishop 93908902b01SBrad Bishopclass SimpleCache(object): 94008902b01SBrad Bishop """ 94108902b01SBrad Bishop BitBake multi-process cache implementation 94208902b01SBrad Bishop 94308902b01SBrad Bishop Used by the codeparser & file checksum caches 94408902b01SBrad Bishop """ 94508902b01SBrad Bishop 94608902b01SBrad Bishop def __init__(self, version): 94708902b01SBrad Bishop self.cachefile = None 94808902b01SBrad Bishop self.cachedata = None 94908902b01SBrad Bishop self.cacheversion = version 95008902b01SBrad Bishop 95108902b01SBrad Bishop def init_cache(self, d, cache_file_name=None, defaultdata=None): 95208902b01SBrad Bishop cachedir = (d.getVar("PERSISTENT_DIR") or 95308902b01SBrad Bishop d.getVar("CACHE")) 95408902b01SBrad Bishop if not cachedir: 95508902b01SBrad Bishop return defaultdata 95608902b01SBrad Bishop 95708902b01SBrad Bishop bb.utils.mkdirhier(cachedir) 95808902b01SBrad Bishop self.cachefile = os.path.join(cachedir, 95908902b01SBrad Bishop cache_file_name or self.__class__.cache_file_name) 960d1e89497SAndrew Geissler logger.debug("Using cache in '%s'", self.cachefile) 96108902b01SBrad Bishop 96208902b01SBrad Bishop glf = bb.utils.lockfile(self.cachefile + ".lock") 96308902b01SBrad Bishop 96408902b01SBrad Bishop try: 96508902b01SBrad Bishop with open(self.cachefile, "rb") as f: 96608902b01SBrad Bishop p = pickle.Unpickler(f) 96708902b01SBrad Bishop data, version = p.load() 96808902b01SBrad Bishop except: 96908902b01SBrad Bishop bb.utils.unlockfile(glf) 97008902b01SBrad Bishop return defaultdata 97108902b01SBrad Bishop 97208902b01SBrad Bishop bb.utils.unlockfile(glf) 97308902b01SBrad Bishop 97408902b01SBrad Bishop if version != self.cacheversion: 97508902b01SBrad Bishop return defaultdata 97608902b01SBrad Bishop 97708902b01SBrad Bishop return data 97808902b01SBrad Bishop 97908902b01SBrad Bishop def save(self, data): 98008902b01SBrad Bishop if not self.cachefile: 98108902b01SBrad Bishop return 98208902b01SBrad Bishop 98308902b01SBrad Bishop glf = bb.utils.lockfile(self.cachefile + ".lock") 98408902b01SBrad Bishop 98508902b01SBrad Bishop with open(self.cachefile, "wb") as f: 98608902b01SBrad Bishop p = pickle.Pickler(f, -1) 98708902b01SBrad Bishop p.dump([data, self.cacheversion]) 98808902b01SBrad Bishop 98908902b01SBrad Bishop bb.utils.unlockfile(glf) 99078b72798SAndrew Geissler 99178b72798SAndrew Geissler def copyfile(self, target): 99278b72798SAndrew Geissler if not self.cachefile: 99378b72798SAndrew Geissler return 99478b72798SAndrew Geissler 99578b72798SAndrew Geissler glf = bb.utils.lockfile(self.cachefile + ".lock") 99678b72798SAndrew Geissler shutil.copy(self.cachefile, target) 99778b72798SAndrew Geissler bb.utils.unlockfile(glf) 998