1# 2# Copyright (C) 2003, 2004 Chris Larson 3# Copyright (C) 2003, 2004 Phil Blundell 4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 5# Copyright (C) 2005 Holger Hans Peter Freyther 6# Copyright (C) 2005 ROAD GmbH 7# Copyright (C) 2006 Richard Purdie 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import re 13import logging 14from bb import data, utils 15from collections import defaultdict 16import bb 17 18logger = logging.getLogger("BitBake.Provider") 19 20class NoProvider(bb.BBHandledException): 21 """Exception raised when no provider of a build dependency can be found""" 22 23class NoRProvider(bb.BBHandledException): 24 """Exception raised when no provider of a runtime dependency can be found""" 25 26class MultipleRProvider(bb.BBHandledException): 27 """Exception raised when multiple providers of a runtime dependency can be found""" 28 29def findProviders(cfgData, dataCache, pkg_pn = None): 30 """ 31 Convenience function to get latest and preferred providers in pkg_pn 32 """ 33 34 if not pkg_pn: 35 pkg_pn = dataCache.pkg_pn 36 37 # Need to ensure data store is expanded 38 localdata = data.createCopy(cfgData) 39 bb.data.expandKeys(localdata) 40 41 required = {} 42 preferred_versions = {} 43 latest_versions = {} 44 45 for pn in pkg_pn: 46 (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn) 47 preferred_versions[pn] = (pref_ver, pref_file) 48 latest_versions[pn] = (last_ver, last_file) 49 required[pn] = req 50 51 return (latest_versions, preferred_versions, required) 52 53def allProviders(dataCache): 54 """ 55 Find all providers for each pn 56 """ 57 all_providers = defaultdict(list) 58 for (fn, pn) in dataCache.pkg_fn.items(): 59 ver = dataCache.pkg_pepvpr[fn] 60 all_providers[pn].append((ver, fn)) 61 return all_providers 62 63def sortPriorities(pn, dataCache, pkg_pn = None): 64 """ 65 Reorder pkg_pn by file priority and default preference 66 """ 67 68 if not pkg_pn: 69 pkg_pn = dataCache.pkg_pn 70 71 files = pkg_pn[pn] 72 priorities = {} 73 for f in files: 74 priority = dataCache.bbfile_priority[f] 75 preference = dataCache.pkg_dp[f] 76 if priority not in priorities: 77 priorities[priority] = {} 78 if preference not in priorities[priority]: 79 priorities[priority][preference] = [] 80 priorities[priority][preference].append(f) 81 tmp_pn = [] 82 for pri in sorted(priorities): 83 tmp_pref = [] 84 for pref in sorted(priorities[pri]): 85 tmp_pref.extend(priorities[pri][pref]) 86 tmp_pn = [tmp_pref] + tmp_pn 87 88 return tmp_pn 89 90def versionVariableMatch(cfgData, keyword, pn): 91 """ 92 Return the value of the <keyword>_VERSION variable if set. 93 """ 94 95 # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot 96 # hence we do this manually rather than use OVERRIDES 97 ver = cfgData.getVar("%s_VERSION:pn-%s" % (keyword, pn)) 98 if not ver: 99 ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn)) 100 if not ver: 101 ver = cfgData.getVar("%s_VERSION" % keyword) 102 103 return ver 104 105def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): 106 """ 107 Check if the version pe,pv,pr is the preferred one. 108 If there is preferred version defined and ends with '%', then pv has to start with that version after removing the '%' 109 """ 110 if pr == preferred_r or preferred_r is None: 111 if pe == preferred_e or preferred_e is None: 112 if preferred_v == pv: 113 return True 114 if preferred_v is not None and preferred_v.endswith('%') and pv.startswith(preferred_v[:len(preferred_v)-1]): 115 return True 116 return False 117 118def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): 119 """ 120 Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set. 121 """ 122 123 preferred_file = None 124 preferred_ver = None 125 required = False 126 127 required_v = versionVariableMatch(cfgData, "REQUIRED", pn) 128 preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn) 129 130 itemstr = "" 131 if item: 132 itemstr = " (for item %s)" % item 133 134 if required_v is not None: 135 if preferred_v is not None: 136 logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v) 137 else: 138 logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr) 139 # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION 140 preferred_v = required_v 141 required = True 142 143 if preferred_v: 144 m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v) 145 if m: 146 if m.group(1): 147 preferred_e = m.group(1)[:-1] 148 else: 149 preferred_e = None 150 preferred_v = m.group(2) 151 if m.group(3): 152 preferred_r = m.group(3)[1:] 153 else: 154 preferred_r = None 155 else: 156 preferred_e = None 157 preferred_r = None 158 159 for file_set in pkg_pn: 160 for f in file_set: 161 pe, pv, pr = dataCache.pkg_pepvpr[f] 162 if preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r): 163 preferred_file = f 164 preferred_ver = (pe, pv, pr) 165 break 166 if preferred_file: 167 break; 168 if preferred_r: 169 pv_str = '%s-%s' % (preferred_v, preferred_r) 170 else: 171 pv_str = preferred_v 172 if not (preferred_e is None): 173 pv_str = '%s:%s' % (preferred_e, pv_str) 174 if preferred_file is None: 175 if not required: 176 logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr) 177 available_vers = [] 178 for file_set in pkg_pn: 179 for f in file_set: 180 pe, pv, pr = dataCache.pkg_pepvpr[f] 181 ver_str = pv 182 if pe: 183 ver_str = "%s:%s" % (pe, ver_str) 184 if not ver_str in available_vers: 185 available_vers.append(ver_str) 186 if available_vers: 187 available_vers.sort() 188 logger.warning("versions of %s available: %s", pn, ' '.join(available_vers)) 189 if required: 190 logger.error("required version %s of %s not available%s", pv_str, pn, itemstr) 191 else: 192 if required: 193 logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr) 194 else: 195 logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr) 196 197 return (preferred_ver, preferred_file, required) 198 199def findLatestProvider(pn, cfgData, dataCache, file_set): 200 """ 201 Return the highest version of the providers in file_set. 202 Take default preferences into account. 203 """ 204 latest = None 205 latest_p = 0 206 latest_f = None 207 for file_name in file_set: 208 pe, pv, pr = dataCache.pkg_pepvpr[file_name] 209 dp = dataCache.pkg_dp[file_name] 210 211 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pe, pv, pr)) < 0)) or (dp > latest_p): 212 latest = (pe, pv, pr) 213 latest_f = file_name 214 latest_p = dp 215 216 return (latest, latest_f) 217 218def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None): 219 """ 220 If there is a PREFERRED_VERSION, find the highest-priority bbfile 221 providing that version. If not, find the latest version provided by 222 an bbfile in the highest-priority set. 223 """ 224 225 sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn) 226 # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set 227 (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item) 228 # Find the latest version of the highest priority provider 229 (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0]) 230 231 if not required and preferred_file is None: 232 preferred_file = latest_f 233 preferred_ver = latest 234 235 return (latest, latest_f, preferred_ver, preferred_file, required) 236 237def _filterProviders(providers, item, cfgData, dataCache): 238 """ 239 Take a list of providers and filter/reorder according to the 240 environment variables 241 """ 242 eligible = [] 243 preferred_versions = {} 244 sortpkg_pn = {} 245 246 # The order of providers depends on the order of the files on the disk 247 # up to here. Sort pkg_pn to make dependency issues reproducible rather 248 # than effectively random. 249 providers.sort() 250 251 # Collate providers by PN 252 pkg_pn = {} 253 for p in providers: 254 pn = dataCache.pkg_fn[p] 255 if pn not in pkg_pn: 256 pkg_pn[pn] = [] 257 pkg_pn[pn].append(p) 258 259 logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys()))) 260 261 # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS 262 for pn in sorted(pkg_pn): 263 sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn) 264 preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item) 265 if required and preferred_file is None: 266 return eligible 267 preferred_versions[pn] = (preferred_ver, preferred_file) 268 if preferred_versions[pn][1]: 269 eligible.append(preferred_versions[pn][1]) 270 271 # Now add latest versions 272 for pn in sorted(sortpkg_pn): 273 if pn in preferred_versions and preferred_versions[pn][1]: 274 continue 275 preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0]) 276 eligible.append(preferred_versions[pn][1]) 277 278 if not eligible: 279 return eligible 280 281 # If pn == item, give it a slight default preference 282 # This means PREFERRED_PROVIDER_foobar defaults to foobar if available 283 for p in providers: 284 pn = dataCache.pkg_fn[p] 285 if pn != item: 286 continue 287 (newvers, fn) = preferred_versions[pn] 288 if not fn in eligible: 289 continue 290 eligible.remove(fn) 291 eligible = [fn] + eligible 292 293 return eligible 294 295def filterProviders(providers, item, cfgData, dataCache): 296 """ 297 Take a list of providers and filter/reorder according to the 298 environment variables 299 Takes a "normal" target item 300 """ 301 302 eligible = _filterProviders(providers, item, cfgData, dataCache) 303 304 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item) 305 if prefervar: 306 dataCache.preferred[item] = prefervar 307 308 foundUnique = False 309 if item in dataCache.preferred: 310 for p in eligible: 311 pn = dataCache.pkg_fn[p] 312 if dataCache.preferred[item] == pn: 313 logger.verbose("selecting %s to satisfy %s due to PREFERRED_PROVIDERS", pn, item) 314 eligible.remove(p) 315 eligible = [p] + eligible 316 foundUnique = True 317 break 318 319 logger.debug("sorted providers for %s are: %s", item, eligible) 320 321 return eligible, foundUnique 322 323def filterProvidersRunTime(providers, item, cfgData, dataCache): 324 """ 325 Take a list of providers and filter/reorder according to the 326 environment variables 327 Takes a "runtime" target item 328 """ 329 330 eligible = _filterProviders(providers, item, cfgData, dataCache) 331 332 # First try and match any PREFERRED_RPROVIDER entry 333 prefervar = cfgData.getVar('PREFERRED_RPROVIDER_%s' % item) 334 foundUnique = False 335 if prefervar: 336 for p in eligible: 337 pn = dataCache.pkg_fn[p] 338 if prefervar == pn: 339 logger.verbose("selecting %s to satisfy %s due to PREFERRED_RPROVIDER", pn, item) 340 eligible.remove(p) 341 eligible = [p] + eligible 342 foundUnique = True 343 numberPreferred = 1 344 break 345 346 # If we didn't find an RPROVIDER entry, try and infer the provider from PREFERRED_PROVIDER entries 347 # by looking through the provides of each eligible recipe and seeing if a PREFERRED_PROVIDER was set. 348 # This is most useful for virtual/ entries rather than having a RPROVIDER per entry. 349 if not foundUnique: 350 # Should use dataCache.preferred here? 351 preferred = [] 352 preferred_vars = [] 353 pns = {} 354 for p in eligible: 355 pns[dataCache.pkg_fn[p]] = p 356 for p in eligible: 357 pn = dataCache.pkg_fn[p] 358 provides = dataCache.pn_provides[pn] 359 for provide in provides: 360 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide) 361 #logger.debug("checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys()) 362 if prefervar in pns and pns[prefervar] not in preferred: 363 var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar) 364 logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var) 365 preferred_vars.append(var) 366 pref = pns[prefervar] 367 eligible.remove(pref) 368 eligible = [pref] + eligible 369 preferred.append(pref) 370 break 371 372 numberPreferred = len(preferred) 373 374 if numberPreferred > 1: 375 logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item)) 376 377 logger.debug("sorted runtime providers for %s are: %s", item, eligible) 378 379 return eligible, numberPreferred 380 381regexp_cache = {} 382 383def getRuntimeProviders(dataCache, rdepend): 384 """ 385 Return any providers of runtime dependency 386 """ 387 rproviders = [] 388 389 if rdepend in dataCache.rproviders: 390 rproviders += dataCache.rproviders[rdepend] 391 392 if rdepend in dataCache.packages: 393 rproviders += dataCache.packages[rdepend] 394 395 if rproviders: 396 return rproviders 397 398 # Only search dynamic packages if we can't find anything in other variables 399 for pat_key in dataCache.packages_dynamic: 400 pattern = pat_key.replace(r'+', r"\+") 401 if pattern in regexp_cache: 402 regexp = regexp_cache[pattern] 403 else: 404 try: 405 regexp = re.compile(pattern) 406 except: 407 logger.error("Error parsing regular expression '%s'", pattern) 408 raise 409 regexp_cache[pattern] = regexp 410 if regexp.match(rdepend): 411 rproviders += dataCache.packages_dynamic[pat_key] 412 logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend) 413 414 return rproviders 415 416def buildWorldTargetList(dataCache, task=None): 417 """ 418 Build package list for "bitbake world" 419 """ 420 if dataCache.world_target: 421 return 422 423 logger.debug("collating packages for \"world\"") 424 for f in dataCache.possible_world: 425 terminal = True 426 pn = dataCache.pkg_fn[f] 427 if task and task not in dataCache.task_deps[f]['tasks']: 428 logger.debug2("World build skipping %s as task %s doesn't exist", f, task) 429 terminal = False 430 431 for p in dataCache.pn_provides[pn]: 432 if p.startswith('virtual/'): 433 logger.debug2("World build skipping %s due to %s provider starting with virtual/", f, p) 434 terminal = False 435 break 436 for pf in dataCache.providers[p]: 437 if dataCache.pkg_fn[pf] != pn: 438 logger.debug2("World build skipping %s due to both us and %s providing %s", f, pf, p) 439 terminal = False 440 break 441 if terminal: 442 dataCache.world_target.add(pn) 443