1# Local file checksum cache implementation 2# 3# Copyright (C) 2012 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8import glob 9import operator 10import os 11import stat 12import bb.utils 13import logging 14from bb.cache import MultiProcessCache 15 16logger = logging.getLogger("BitBake.Cache") 17 18# mtime cache (non-persistent) 19# based upon the assumption that files do not change during bitbake run 20class FileMtimeCache(object): 21 cache = {} 22 23 def cached_mtime(self, f): 24 if f not in self.cache: 25 self.cache[f] = os.stat(f)[stat.ST_MTIME] 26 return self.cache[f] 27 28 def cached_mtime_noerror(self, f): 29 if f not in self.cache: 30 try: 31 self.cache[f] = os.stat(f)[stat.ST_MTIME] 32 except OSError: 33 return 0 34 return self.cache[f] 35 36 def update_mtime(self, f): 37 self.cache[f] = os.stat(f)[stat.ST_MTIME] 38 return self.cache[f] 39 40 def clear(self): 41 self.cache.clear() 42 43# Checksum + mtime cache (persistent) 44class FileChecksumCache(MultiProcessCache): 45 cache_file_name = "local_file_checksum_cache.dat" 46 CACHE_VERSION = 1 47 48 def __init__(self): 49 self.mtime_cache = FileMtimeCache() 50 MultiProcessCache.__init__(self) 51 52 def get_checksum(self, f): 53 f = os.path.normpath(f) 54 entry = self.cachedata[0].get(f) 55 cmtime = self.mtime_cache.cached_mtime(f) 56 if entry: 57 (mtime, hashval) = entry 58 if cmtime == mtime: 59 return hashval 60 else: 61 bb.debug(2, "file %s changed mtime, recompute checksum" % f) 62 63 hashval = bb.utils.md5_file(f) 64 self.cachedata_extras[0][f] = (cmtime, hashval) 65 return hashval 66 67 def merge_data(self, source, dest): 68 for h in source[0]: 69 if h in dest: 70 (smtime, _) = source[0][h] 71 (dmtime, _) = dest[0][h] 72 if smtime > dmtime: 73 dest[0][h] = source[0][h] 74 else: 75 dest[0][h] = source[0][h] 76 77 def get_checksums(self, filelist, pn, localdirsexclude): 78 """Get checksums for a list of files""" 79 80 def checksum_file(f): 81 try: 82 checksum = self.get_checksum(f) 83 except OSError as e: 84 bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) 85 return None 86 return checksum 87 88 # 89 # Changing the format of file-checksums is problematic as both OE and Bitbake have 90 # knowledge of them. We need to encode a new piece of data, the portion of the path 91 # we care about from a checksum perspective. This means that files that change subdirectory 92 # are tracked by the task hashes. To do this, we do something horrible and put a "/./" into 93 # the path. The filesystem handles it but it gives us a marker to know which subsection 94 # of the path to cache. 95 # 96 def checksum_dir(pth): 97 # Handle directories recursively 98 if pth == "/": 99 bb.fatal("Refusing to checksum /") 100 pth = pth.rstrip("/") 101 dirchecksums = [] 102 for root, dirs, files in os.walk(pth, topdown=True): 103 [dirs.remove(d) for d in list(dirs) if d in localdirsexclude] 104 for name in files: 105 fullpth = os.path.join(root, name).replace(pth, os.path.join(pth, ".")) 106 checksum = checksum_file(fullpth) 107 if checksum: 108 dirchecksums.append((fullpth, checksum)) 109 return dirchecksums 110 111 checksums = [] 112 for pth in filelist.split(): 113 exist = pth.split(":")[1] 114 if exist == "False": 115 continue 116 pth = pth.split(":")[0] 117 if '*' in pth: 118 # Handle globs 119 for f in glob.glob(pth): 120 if os.path.isdir(f): 121 if not os.path.islink(f): 122 checksums.extend(checksum_dir(f)) 123 else: 124 checksum = checksum_file(f) 125 if checksum: 126 checksums.append((f, checksum)) 127 elif os.path.isdir(pth): 128 if not os.path.islink(pth): 129 checksums.extend(checksum_dir(pth)) 130 else: 131 checksum = checksum_file(pth) 132 if checksum: 133 checksums.append((pth, checksum)) 134 135 checksums.sort(key=operator.itemgetter(1)) 136 return checksums 137