1""" 2BitBake 'TaskData' implementation 3 4Task data collection and handling 5 6""" 7 8# Copyright (C) 2006 Richard Purdie 9# 10# SPDX-License-Identifier: GPL-2.0-only 11# 12 13import logging 14import re 15import bb 16 17logger = logging.getLogger("BitBake.TaskData") 18 19def re_match_strings(target, strings): 20 """ 21 Whether or not the string 'target' matches 22 any one string of the strings which can be regular expression string 23 """ 24 for name in strings: 25 if name.startswith("^") or name.endswith("$"): 26 if re.match(name, target): 27 return True 28 elif name == target: 29 return True 30 return False 31 32class TaskEntry: 33 def __init__(self): 34 self.tdepends = [] 35 self.idepends = [] 36 self.irdepends = [] 37 38class TaskData: 39 """ 40 BitBake Task Data implementation 41 """ 42 def __init__(self, halt = True, skiplist = None, allowincomplete = False): 43 self.build_targets = {} 44 self.run_targets = {} 45 46 self.external_targets = [] 47 48 self.seenfns = [] 49 self.taskentries = {} 50 51 self.depids = {} 52 self.rdepids = {} 53 54 self.consider_msgs_cache = [] 55 56 self.failed_deps = [] 57 self.failed_rdeps = [] 58 self.failed_fns = [] 59 60 self.halt = halt 61 self.allowincomplete = allowincomplete 62 63 self.skiplist = skiplist 64 65 self.mcdepends = [] 66 67 def add_tasks(self, fn, dataCache): 68 """ 69 Add tasks for a given fn to the database 70 """ 71 72 task_deps = dataCache.task_deps[fn] 73 74 if fn in self.failed_fns: 75 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") 76 77 # Check if we've already seen this fn 78 if fn in self.seenfns: 79 return 80 81 self.seenfns.append(fn) 82 83 self.add_extra_deps(fn, dataCache) 84 85 def add_mcdepends(task): 86 for dep in task_deps['mcdepends'][task].split(): 87 if len(dep.split(':')) != 5: 88 bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends')) 89 if dep not in self.mcdepends: 90 self.mcdepends.append(dep) 91 92 # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends 93 def handle_deps(task, dep_name, depends, seen): 94 if dep_name in task_deps and task in task_deps[dep_name]: 95 ids = [] 96 for dep in task_deps[dep_name][task].split(): 97 if dep: 98 parts = dep.split(":") 99 if len(parts) != 2: 100 bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name)) 101 ids.append((parts[0], parts[1])) 102 seen(parts[0]) 103 depends.extend(ids) 104 105 for task in task_deps['tasks']: 106 107 tid = "%s:%s" % (fn, task) 108 self.taskentries[tid] = TaskEntry() 109 110 # Work out task dependencies 111 parentids = [] 112 for dep in task_deps['parents'][task]: 113 if dep not in task_deps['tasks']: 114 bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep)) 115 continue 116 parentid = "%s:%s" % (fn, dep) 117 parentids.append(parentid) 118 self.taskentries[tid].tdepends.extend(parentids) 119 120 121 # Touch all intertask dependencies 122 handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target) 123 handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target) 124 125 if 'mcdepends' in task_deps and task in task_deps['mcdepends']: 126 add_mcdepends(task) 127 128 # Work out build dependencies 129 if not fn in self.depids: 130 dependids = set() 131 for depend in dataCache.deps[fn]: 132 dependids.add(depend) 133 self.depids[fn] = list(dependids) 134 logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn) 135 136 # Work out runtime dependencies 137 if not fn in self.rdepids: 138 rdependids = set() 139 rdepends = dataCache.rundeps[fn] 140 rrecs = dataCache.runrecs[fn] 141 rdependlist = [] 142 rreclist = [] 143 for package in rdepends: 144 for rdepend in rdepends[package]: 145 rdependlist.append(rdepend) 146 rdependids.add(rdepend) 147 for package in rrecs: 148 for rdepend in rrecs[package]: 149 rreclist.append(rdepend) 150 rdependids.add(rdepend) 151 if rdependlist: 152 logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn) 153 if rreclist: 154 logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn) 155 self.rdepids[fn] = list(rdependids) 156 157 for dep in self.depids[fn]: 158 self.seen_build_target(dep) 159 if dep in self.failed_deps: 160 self.fail_fn(fn) 161 return 162 for dep in self.rdepids[fn]: 163 self.seen_run_target(dep) 164 if dep in self.failed_rdeps: 165 self.fail_fn(fn) 166 return 167 168 def add_extra_deps(self, fn, dataCache): 169 func = dataCache.extradepsfunc.get(fn, None) 170 if func: 171 bb.providers.buildWorldTargetList(dataCache) 172 pn = dataCache.pkg_fn[fn] 173 params = {'deps': dataCache.deps[fn], 174 'world_target': dataCache.world_target, 175 'pkg_pn': dataCache.pkg_pn, 176 'self_pn': pn} 177 funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_') 178 paramlist = ','.join(params.keys()) 179 func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist) 180 bb.utils.better_exec(func, params) 181 182 183 def have_build_target(self, target): 184 """ 185 Have we a build target matching this name? 186 """ 187 if target in self.build_targets and self.build_targets[target]: 188 return True 189 return False 190 191 def have_runtime_target(self, target): 192 """ 193 Have we a runtime target matching this name? 194 """ 195 if target in self.run_targets and self.run_targets[target]: 196 return True 197 return False 198 199 def seen_build_target(self, name): 200 """ 201 Maintain a list of build targets 202 """ 203 if name not in self.build_targets: 204 self.build_targets[name] = [] 205 206 def add_build_target(self, fn, item): 207 """ 208 Add a build target. 209 If already present, append the provider fn to the list 210 """ 211 if item in self.build_targets: 212 if fn in self.build_targets[item]: 213 return 214 self.build_targets[item].append(fn) 215 return 216 self.build_targets[item] = [fn] 217 218 def seen_run_target(self, name): 219 """ 220 Maintain a list of runtime build targets 221 """ 222 if name not in self.run_targets: 223 self.run_targets[name] = [] 224 225 def add_runtime_target(self, fn, item): 226 """ 227 Add a runtime target. 228 If already present, append the provider fn to the list 229 """ 230 if item in self.run_targets: 231 if fn in self.run_targets[item]: 232 return 233 self.run_targets[item].append(fn) 234 return 235 self.run_targets[item] = [fn] 236 237 def mark_external_target(self, target): 238 """ 239 Mark a build target as being externally requested 240 """ 241 if target not in self.external_targets: 242 self.external_targets.append(target) 243 244 def get_unresolved_build_targets(self, dataCache): 245 """ 246 Return a list of build targets who's providers 247 are unknown. 248 """ 249 unresolved = [] 250 for target in self.build_targets: 251 if re_match_strings(target, dataCache.ignored_dependencies): 252 continue 253 if target in self.failed_deps: 254 continue 255 if not self.build_targets[target]: 256 unresolved.append(target) 257 return unresolved 258 259 def get_unresolved_run_targets(self, dataCache): 260 """ 261 Return a list of runtime targets who's providers 262 are unknown. 263 """ 264 unresolved = [] 265 for target in self.run_targets: 266 if re_match_strings(target, dataCache.ignored_dependencies): 267 continue 268 if target in self.failed_rdeps: 269 continue 270 if not self.run_targets[target]: 271 unresolved.append(target) 272 return unresolved 273 274 def get_provider(self, item): 275 """ 276 Return a list of providers of item 277 """ 278 return self.build_targets[item] 279 280 def get_dependees(self, item): 281 """ 282 Return a list of targets which depend on item 283 """ 284 dependees = [] 285 for fn in self.depids: 286 if item in self.depids[fn]: 287 dependees.append(fn) 288 return dependees 289 290 def get_rdependees(self, item): 291 """ 292 Return a list of targets which depend on runtime item 293 """ 294 dependees = [] 295 for fn in self.rdepids: 296 if item in self.rdepids[fn]: 297 dependees.append(fn) 298 return dependees 299 300 def get_reasons(self, item, runtime=False): 301 """ 302 Get the reason(s) for an item not being provided, if any 303 """ 304 reasons = [] 305 if self.skiplist: 306 for fn in self.skiplist: 307 skipitem = self.skiplist[fn] 308 if skipitem.pn == item: 309 reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason)) 310 elif runtime and item in skipitem.rprovides: 311 reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) 312 elif not runtime and item in skipitem.provides: 313 reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) 314 return reasons 315 316 def get_close_matches(self, item, provider_list): 317 import difflib 318 if self.skiplist: 319 skipped = [] 320 for fn in self.skiplist: 321 skipped.append(self.skiplist[fn].pn) 322 full_list = provider_list + skipped 323 else: 324 full_list = provider_list 325 return difflib.get_close_matches(item, full_list, cutoff=0.7) 326 327 def add_provider(self, cfgData, dataCache, item): 328 try: 329 self.add_provider_internal(cfgData, dataCache, item) 330 except bb.providers.NoProvider: 331 if self.halt: 332 raise 333 self.remove_buildtarget(item) 334 335 self.mark_external_target(item) 336 337 def add_provider_internal(self, cfgData, dataCache, item): 338 """ 339 Add the providers of item to the task data 340 Mark entries were specifically added externally as against dependencies 341 added internally during dependency resolution 342 """ 343 344 if re_match_strings(item, dataCache.ignored_dependencies): 345 return 346 347 if not item in dataCache.providers: 348 close_matches = self.get_close_matches(item, list(dataCache.providers.keys())) 349 # Is it in RuntimeProviders ? 350 all_p = bb.providers.getRuntimeProviders(dataCache, item) 351 for fn in all_p: 352 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item 353 if new not in close_matches: 354 close_matches.append(new) 355 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) 356 raise bb.providers.NoProvider(item) 357 358 if self.have_build_target(item): 359 return 360 361 all_p = dataCache.providers[item] 362 363 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) 364 eligible = [p for p in eligible if not p in self.failed_fns] 365 366 if not eligible: 367 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) 368 raise bb.providers.NoProvider(item) 369 370 if len(eligible) > 1 and not foundUnique: 371 if item not in self.consider_msgs_cache: 372 providers_list = [] 373 for fn in eligible: 374 providers_list.append(dataCache.pkg_fn[fn]) 375 bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData) 376 self.consider_msgs_cache.append(item) 377 378 for fn in eligible: 379 if fn in self.failed_fns: 380 continue 381 logger.debug2("adding %s to satisfy %s", fn, item) 382 self.add_build_target(fn, item) 383 self.add_tasks(fn, dataCache) 384 385 386 #item = dataCache.pkg_fn[fn] 387 388 def add_rprovider(self, cfgData, dataCache, item): 389 """ 390 Add the runtime providers of item to the task data 391 (takes item names from RDEPENDS/PACKAGES namespace) 392 """ 393 394 if re_match_strings(item, dataCache.ignored_dependencies): 395 return 396 397 if self.have_runtime_target(item): 398 return 399 400 all_p = bb.providers.getRuntimeProviders(dataCache, item) 401 402 if not all_p: 403 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData) 404 raise bb.providers.NoRProvider(item) 405 406 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) 407 eligible = [p for p in eligible if not p in self.failed_fns] 408 409 if not eligible: 410 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) 411 raise bb.providers.NoRProvider(item) 412 413 if len(eligible) > 1 and numberPreferred == 0: 414 if item not in self.consider_msgs_cache: 415 providers_list = [] 416 for fn in eligible: 417 providers_list.append(dataCache.pkg_fn[fn]) 418 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) 419 self.consider_msgs_cache.append(item) 420 421 if numberPreferred > 1: 422 if item not in self.consider_msgs_cache: 423 providers_list = [] 424 for fn in eligible: 425 providers_list.append(dataCache.pkg_fn[fn]) 426 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) 427 self.consider_msgs_cache.append(item) 428 raise bb.providers.MultipleRProvider(item) 429 430 # run through the list until we find one that we can build 431 for fn in eligible: 432 if fn in self.failed_fns: 433 continue 434 logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item) 435 self.add_runtime_target(fn, item) 436 self.add_tasks(fn, dataCache) 437 438 def fail_fn(self, fn, missing_list=None): 439 """ 440 Mark a file as failed (unbuildable) 441 Remove any references from build and runtime provider lists 442 443 missing_list, A list of missing requirements for this target 444 """ 445 if fn in self.failed_fns: 446 return 447 if not missing_list: 448 missing_list = [] 449 logger.debug("File '%s' is unbuildable, removing...", fn) 450 self.failed_fns.append(fn) 451 for target in self.build_targets: 452 if fn in self.build_targets[target]: 453 self.build_targets[target].remove(fn) 454 if not self.build_targets[target]: 455 self.remove_buildtarget(target, missing_list) 456 for target in self.run_targets: 457 if fn in self.run_targets[target]: 458 self.run_targets[target].remove(fn) 459 if not self.run_targets[target]: 460 self.remove_runtarget(target, missing_list) 461 462 def remove_buildtarget(self, target, missing_list=None): 463 """ 464 Mark a build target as failed (unbuildable) 465 Trigger removal of any files that have this as a dependency 466 """ 467 if not missing_list: 468 missing_list = [target] 469 else: 470 missing_list = [target] + missing_list 471 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) 472 self.failed_deps.append(target) 473 dependees = self.get_dependees(target) 474 for fn in dependees: 475 self.fail_fn(fn, missing_list) 476 for tid in self.taskentries: 477 for (idepend, idependtask) in self.taskentries[tid].idepends: 478 if idepend == target: 479 fn = tid.rsplit(":",1)[0] 480 self.fail_fn(fn, missing_list) 481 482 if self.halt and target in self.external_targets: 483 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) 484 raise bb.providers.NoProvider(target) 485 486 def remove_runtarget(self, target, missing_list=None): 487 """ 488 Mark a run target as failed (unbuildable) 489 Trigger removal of any files that have this as a dependency 490 """ 491 if not missing_list: 492 missing_list = [target] 493 else: 494 missing_list = [target] + missing_list 495 496 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) 497 self.failed_rdeps.append(target) 498 dependees = self.get_rdependees(target) 499 for fn in dependees: 500 self.fail_fn(fn, missing_list) 501 for tid in self.taskentries: 502 for (idepend, idependtask) in self.taskentries[tid].irdepends: 503 if idepend == target: 504 fn = tid.rsplit(":",1)[0] 505 self.fail_fn(fn, missing_list) 506 507 def add_unresolved(self, cfgData, dataCache): 508 """ 509 Resolve all unresolved build and runtime targets 510 """ 511 logger.info("Resolving any missing task queue dependencies") 512 while True: 513 added = 0 514 for target in self.get_unresolved_build_targets(dataCache): 515 try: 516 self.add_provider_internal(cfgData, dataCache, target) 517 added = added + 1 518 except bb.providers.NoProvider: 519 if self.halt and target in self.external_targets and not self.allowincomplete: 520 raise 521 if not self.allowincomplete: 522 self.remove_buildtarget(target) 523 for target in self.get_unresolved_run_targets(dataCache): 524 try: 525 self.add_rprovider(cfgData, dataCache, target) 526 added = added + 1 527 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): 528 self.remove_runtarget(target) 529 logger.debug("Resolved " + str(added) + " extra dependencies") 530 if added == 0: 531 break 532 # self.dump_data() 533 534 def get_providermap(self, prefix=None): 535 provmap = {} 536 for name in self.build_targets: 537 if prefix and not name.startswith(prefix): 538 continue 539 if self.have_build_target(name): 540 provider = self.get_provider(name) 541 if provider: 542 provmap[name] = provider[0] 543 return provmap 544 545 def get_mcdepends(self): 546 return self.mcdepends 547 548 def dump_data(self): 549 """ 550 Dump some debug information on the internal data structures 551 """ 552 logger.debug3("build_names:") 553 logger.debug3(", ".join(self.build_targets)) 554 555 logger.debug3("run_names:") 556 logger.debug3(", ".join(self.run_targets)) 557 558 logger.debug3("build_targets:") 559 for target in self.build_targets: 560 targets = "None" 561 if target in self.build_targets: 562 targets = self.build_targets[target] 563 logger.debug3(" %s: %s", target, targets) 564 565 logger.debug3("run_targets:") 566 for target in self.run_targets: 567 targets = "None" 568 if target in self.run_targets: 569 targets = self.run_targets[target] 570 logger.debug3(" %s: %s", target, targets) 571 572 logger.debug3("tasks:") 573 for tid in self.taskentries: 574 logger.debug3(" %s: %s %s %s", 575 tid, 576 self.taskentries[tid].idepends, 577 self.taskentries[tid].irdepends, 578 self.taskentries[tid].tdepends) 579 580 logger.debug3("dependency ids (per fn):") 581 for fn in self.depids: 582 logger.debug3(" %s: %s", fn, self.depids[fn]) 583 584 logger.debug3("runtime dependency ids (per fn):") 585 for fn in self.rdepids: 586 logger.debug3(" %s: %s", fn, self.rdepids[fn]) 587