1eb8dc403SDave Cobbley# Local file checksum cache implementation 2eb8dc403SDave Cobbley# 3eb8dc403SDave Cobbley# Copyright (C) 2012 Intel Corporation 4eb8dc403SDave Cobbley# 5c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 6eb8dc403SDave Cobbley# 7eb8dc403SDave Cobbley 8eb8dc403SDave Cobbleyimport glob 9eb8dc403SDave Cobbleyimport operator 10eb8dc403SDave Cobbleyimport os 11eb8dc403SDave Cobbleyimport stat 12eb8dc403SDave Cobbleyimport bb.utils 13eb8dc403SDave Cobbleyimport logging 14*03907ee1SPatrick Williamsimport re 15eb8dc403SDave Cobbleyfrom bb.cache import MultiProcessCache 16eb8dc403SDave Cobbley 17eb8dc403SDave Cobbleylogger = logging.getLogger("BitBake.Cache") 18eb8dc403SDave Cobbley 19*03907ee1SPatrick Williamsfilelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') 20*03907ee1SPatrick Williams 21eb8dc403SDave Cobbley# mtime cache (non-persistent) 22eb8dc403SDave Cobbley# based upon the assumption that files do not change during bitbake run 23eb8dc403SDave Cobbleyclass FileMtimeCache(object): 24eb8dc403SDave Cobbley cache = {} 25eb8dc403SDave Cobbley 26eb8dc403SDave Cobbley def cached_mtime(self, f): 27eb8dc403SDave Cobbley if f not in self.cache: 28eb8dc403SDave Cobbley self.cache[f] = os.stat(f)[stat.ST_MTIME] 29eb8dc403SDave Cobbley return self.cache[f] 30eb8dc403SDave Cobbley 31eb8dc403SDave Cobbley def cached_mtime_noerror(self, f): 32eb8dc403SDave Cobbley if f not in self.cache: 33eb8dc403SDave Cobbley try: 34eb8dc403SDave Cobbley self.cache[f] = os.stat(f)[stat.ST_MTIME] 35eb8dc403SDave Cobbley except OSError: 36eb8dc403SDave Cobbley return 0 37eb8dc403SDave Cobbley return self.cache[f] 38eb8dc403SDave Cobbley 39eb8dc403SDave Cobbley def update_mtime(self, f): 40eb8dc403SDave Cobbley self.cache[f] = os.stat(f)[stat.ST_MTIME] 41eb8dc403SDave Cobbley return self.cache[f] 42eb8dc403SDave Cobbley 43eb8dc403SDave Cobbley def clear(self): 44eb8dc403SDave Cobbley self.cache.clear() 45eb8dc403SDave Cobbley 46eb8dc403SDave Cobbley# Checksum + mtime cache (persistent) 47eb8dc403SDave Cobbleyclass FileChecksumCache(MultiProcessCache): 48eb8dc403SDave Cobbley cache_file_name = "local_file_checksum_cache.dat" 49eb8dc403SDave Cobbley CACHE_VERSION = 1 50eb8dc403SDave Cobbley 51eb8dc403SDave Cobbley def __init__(self): 52eb8dc403SDave Cobbley self.mtime_cache = FileMtimeCache() 53eb8dc403SDave Cobbley MultiProcessCache.__init__(self) 54eb8dc403SDave Cobbley 55eb8dc403SDave Cobbley def get_checksum(self, f): 56595f6308SAndrew Geissler f = os.path.normpath(f) 57eb8dc403SDave Cobbley entry = self.cachedata[0].get(f) 58eb8dc403SDave Cobbley cmtime = self.mtime_cache.cached_mtime(f) 59eb8dc403SDave Cobbley if entry: 60eb8dc403SDave Cobbley (mtime, hashval) = entry 61eb8dc403SDave Cobbley if cmtime == mtime: 62eb8dc403SDave Cobbley return hashval 63eb8dc403SDave Cobbley else: 64eb8dc403SDave Cobbley bb.debug(2, "file %s changed mtime, recompute checksum" % f) 65eb8dc403SDave Cobbley 66eb8dc403SDave Cobbley hashval = bb.utils.md5_file(f) 67eb8dc403SDave Cobbley self.cachedata_extras[0][f] = (cmtime, hashval) 68eb8dc403SDave Cobbley return hashval 69eb8dc403SDave Cobbley 70eb8dc403SDave Cobbley def merge_data(self, source, dest): 71eb8dc403SDave Cobbley for h in source[0]: 72eb8dc403SDave Cobbley if h in dest: 73eb8dc403SDave Cobbley (smtime, _) = source[0][h] 74eb8dc403SDave Cobbley (dmtime, _) = dest[0][h] 75eb8dc403SDave Cobbley if smtime > dmtime: 76eb8dc403SDave Cobbley dest[0][h] = source[0][h] 77eb8dc403SDave Cobbley else: 78eb8dc403SDave Cobbley dest[0][h] = source[0][h] 79eb8dc403SDave Cobbley 8082c905dcSAndrew Geissler def get_checksums(self, filelist, pn, localdirsexclude): 81eb8dc403SDave Cobbley """Get checksums for a list of files""" 82eb8dc403SDave Cobbley 83eb8dc403SDave Cobbley def checksum_file(f): 84eb8dc403SDave Cobbley try: 85eb8dc403SDave Cobbley checksum = self.get_checksum(f) 86eb8dc403SDave Cobbley except OSError as e: 87eb8dc403SDave Cobbley bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) 88eb8dc403SDave Cobbley return None 89eb8dc403SDave Cobbley return checksum 90eb8dc403SDave Cobbley 91595f6308SAndrew Geissler # 92595f6308SAndrew Geissler # Changing the format of file-checksums is problematic as both OE and Bitbake have 93595f6308SAndrew Geissler # knowledge of them. We need to encode a new piece of data, the portion of the path 94595f6308SAndrew Geissler # we care about from a checksum perspective. This means that files that change subdirectory 95595f6308SAndrew Geissler # are tracked by the task hashes. To do this, we do something horrible and put a "/./" into 96595f6308SAndrew Geissler # the path. The filesystem handles it but it gives us a marker to know which subsection 97595f6308SAndrew Geissler # of the path to cache. 98595f6308SAndrew Geissler # 99eb8dc403SDave Cobbley def checksum_dir(pth): 100eb8dc403SDave Cobbley # Handle directories recursively 101220d5534SBrad Bishop if pth == "/": 102220d5534SBrad Bishop bb.fatal("Refusing to checksum /") 103595f6308SAndrew Geissler pth = pth.rstrip("/") 104eb8dc403SDave Cobbley dirchecksums = [] 10582c905dcSAndrew Geissler for root, dirs, files in os.walk(pth, topdown=True): 10682c905dcSAndrew Geissler [dirs.remove(d) for d in list(dirs) if d in localdirsexclude] 107eb8dc403SDave Cobbley for name in files: 108595f6308SAndrew Geissler fullpth = os.path.join(root, name).replace(pth, os.path.join(pth, ".")) 109eb8dc403SDave Cobbley checksum = checksum_file(fullpth) 110eb8dc403SDave Cobbley if checksum: 111eb8dc403SDave Cobbley dirchecksums.append((fullpth, checksum)) 112eb8dc403SDave Cobbley return dirchecksums 113eb8dc403SDave Cobbley 114eb8dc403SDave Cobbley checksums = [] 115*03907ee1SPatrick Williams for pth in filelist_regex.split(filelist): 116*03907ee1SPatrick Williams if not pth: 117*03907ee1SPatrick Williams continue 118*03907ee1SPatrick Williams pth = pth.strip() 119*03907ee1SPatrick Williams if not pth: 120*03907ee1SPatrick Williams continue 121eb8dc403SDave Cobbley exist = pth.split(":")[1] 122eb8dc403SDave Cobbley if exist == "False": 123eb8dc403SDave Cobbley continue 124eb8dc403SDave Cobbley pth = pth.split(":")[0] 125eb8dc403SDave Cobbley if '*' in pth: 126eb8dc403SDave Cobbley # Handle globs 127eb8dc403SDave Cobbley for f in glob.glob(pth): 128eb8dc403SDave Cobbley if os.path.isdir(f): 129eb8dc403SDave Cobbley if not os.path.islink(f): 130eb8dc403SDave Cobbley checksums.extend(checksum_dir(f)) 131eb8dc403SDave Cobbley else: 132eb8dc403SDave Cobbley checksum = checksum_file(f) 133eb8dc403SDave Cobbley if checksum: 134eb8dc403SDave Cobbley checksums.append((f, checksum)) 135eb8dc403SDave Cobbley elif os.path.isdir(pth): 136eb8dc403SDave Cobbley if not os.path.islink(pth): 137eb8dc403SDave Cobbley checksums.extend(checksum_dir(pth)) 138eb8dc403SDave Cobbley else: 139eb8dc403SDave Cobbley checksum = checksum_file(pth) 140eb8dc403SDave Cobbley if checksum: 141eb8dc403SDave Cobbley checksums.append((pth, checksum)) 142eb8dc403SDave Cobbley 143eb8dc403SDave Cobbley checksums.sort(key=operator.itemgetter(1)) 144eb8dc403SDave Cobbley return checksums 145