1""" 2BitBake 'TaskData' implementation 3 4Task data collection and handling 5 6""" 7 8# Copyright (C) 2006 Richard Purdie 9# 10# SPDX-License-Identifier: GPL-2.0-only 11# 12 13import logging 14import re 15import bb 16 17logger = logging.getLogger("BitBake.TaskData") 18 19def re_match_strings(target, strings): 20 """ 21 Whether or not the string 'target' matches 22 any one string of the strings which can be regular expression string 23 """ 24 return any(name == target or re.match(name, target) 25 for name in strings) 26 27class TaskEntry: 28 def __init__(self): 29 self.tdepends = [] 30 self.idepends = [] 31 self.irdepends = [] 32 33class TaskData: 34 """ 35 BitBake Task Data implementation 36 """ 37 def __init__(self, abort = True, skiplist = None, allowincomplete = False): 38 self.build_targets = {} 39 self.run_targets = {} 40 41 self.external_targets = [] 42 43 self.seenfns = [] 44 self.taskentries = {} 45 46 self.depids = {} 47 self.rdepids = {} 48 49 self.consider_msgs_cache = [] 50 51 self.failed_deps = [] 52 self.failed_rdeps = [] 53 self.failed_fns = [] 54 55 self.abort = abort 56 self.allowincomplete = allowincomplete 57 58 self.skiplist = skiplist 59 60 self.mcdepends = [] 61 62 def add_tasks(self, fn, dataCache): 63 """ 64 Add tasks for a given fn to the database 65 """ 66 67 task_deps = dataCache.task_deps[fn] 68 69 if fn in self.failed_fns: 70 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") 71 72 # Check if we've already seen this fn 73 if fn in self.seenfns: 74 return 75 76 self.seenfns.append(fn) 77 78 self.add_extra_deps(fn, dataCache) 79 80 def add_mcdepends(task): 81 for dep in task_deps['mcdepends'][task].split(): 82 if len(dep.split(':')) != 5: 83 bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends')) 84 if dep not in self.mcdepends: 85 self.mcdepends.append(dep) 86 87 # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends 88 def handle_deps(task, dep_name, depends, seen): 89 if dep_name in task_deps and task in task_deps[dep_name]: 90 ids = [] 91 for dep in task_deps[dep_name][task].split(): 92 if dep: 93 parts = dep.split(":") 94 if len(parts) != 2: 95 bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name)) 96 ids.append((parts[0], parts[1])) 97 seen(parts[0]) 98 depends.extend(ids) 99 100 for task in task_deps['tasks']: 101 102 tid = "%s:%s" % (fn, task) 103 self.taskentries[tid] = TaskEntry() 104 105 # Work out task dependencies 106 parentids = [] 107 for dep in task_deps['parents'][task]: 108 if dep not in task_deps['tasks']: 109 bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep)) 110 continue 111 parentid = "%s:%s" % (fn, dep) 112 parentids.append(parentid) 113 self.taskentries[tid].tdepends.extend(parentids) 114 115 116 # Touch all intertask dependencies 117 handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target) 118 handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target) 119 120 if 'mcdepends' in task_deps and task in task_deps['mcdepends']: 121 add_mcdepends(task) 122 123 # Work out build dependencies 124 if not fn in self.depids: 125 dependids = set() 126 for depend in dataCache.deps[fn]: 127 dependids.add(depend) 128 self.depids[fn] = list(dependids) 129 logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn) 130 131 # Work out runtime dependencies 132 if not fn in self.rdepids: 133 rdependids = set() 134 rdepends = dataCache.rundeps[fn] 135 rrecs = dataCache.runrecs[fn] 136 rdependlist = [] 137 rreclist = [] 138 for package in rdepends: 139 for rdepend in rdepends[package]: 140 rdependlist.append(rdepend) 141 rdependids.add(rdepend) 142 for package in rrecs: 143 for rdepend in rrecs[package]: 144 rreclist.append(rdepend) 145 rdependids.add(rdepend) 146 if rdependlist: 147 logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn) 148 if rreclist: 149 logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn) 150 self.rdepids[fn] = list(rdependids) 151 152 for dep in self.depids[fn]: 153 self.seen_build_target(dep) 154 if dep in self.failed_deps: 155 self.fail_fn(fn) 156 return 157 for dep in self.rdepids[fn]: 158 self.seen_run_target(dep) 159 if dep in self.failed_rdeps: 160 self.fail_fn(fn) 161 return 162 163 def add_extra_deps(self, fn, dataCache): 164 func = dataCache.extradepsfunc.get(fn, None) 165 if func: 166 bb.providers.buildWorldTargetList(dataCache) 167 pn = dataCache.pkg_fn[fn] 168 params = {'deps': dataCache.deps[fn], 169 'world_target': dataCache.world_target, 170 'pkg_pn': dataCache.pkg_pn, 171 'self_pn': pn} 172 funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_') 173 paramlist = ','.join(params.keys()) 174 func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist) 175 bb.utils.better_exec(func, params) 176 177 178 def have_build_target(self, target): 179 """ 180 Have we a build target matching this name? 181 """ 182 if target in self.build_targets and self.build_targets[target]: 183 return True 184 return False 185 186 def have_runtime_target(self, target): 187 """ 188 Have we a runtime target matching this name? 189 """ 190 if target in self.run_targets and self.run_targets[target]: 191 return True 192 return False 193 194 def seen_build_target(self, name): 195 """ 196 Maintain a list of build targets 197 """ 198 if name not in self.build_targets: 199 self.build_targets[name] = [] 200 201 def add_build_target(self, fn, item): 202 """ 203 Add a build target. 204 If already present, append the provider fn to the list 205 """ 206 if item in self.build_targets: 207 if fn in self.build_targets[item]: 208 return 209 self.build_targets[item].append(fn) 210 return 211 self.build_targets[item] = [fn] 212 213 def seen_run_target(self, name): 214 """ 215 Maintain a list of runtime build targets 216 """ 217 if name not in self.run_targets: 218 self.run_targets[name] = [] 219 220 def add_runtime_target(self, fn, item): 221 """ 222 Add a runtime target. 223 If already present, append the provider fn to the list 224 """ 225 if item in self.run_targets: 226 if fn in self.run_targets[item]: 227 return 228 self.run_targets[item].append(fn) 229 return 230 self.run_targets[item] = [fn] 231 232 def mark_external_target(self, target): 233 """ 234 Mark a build target as being externally requested 235 """ 236 if target not in self.external_targets: 237 self.external_targets.append(target) 238 239 def get_unresolved_build_targets(self, dataCache): 240 """ 241 Return a list of build targets who's providers 242 are unknown. 243 """ 244 unresolved = [] 245 for target in self.build_targets: 246 if re_match_strings(target, dataCache.ignored_dependencies): 247 continue 248 if target in self.failed_deps: 249 continue 250 if not self.build_targets[target]: 251 unresolved.append(target) 252 return unresolved 253 254 def get_unresolved_run_targets(self, dataCache): 255 """ 256 Return a list of runtime targets who's providers 257 are unknown. 258 """ 259 unresolved = [] 260 for target in self.run_targets: 261 if re_match_strings(target, dataCache.ignored_dependencies): 262 continue 263 if target in self.failed_rdeps: 264 continue 265 if not self.run_targets[target]: 266 unresolved.append(target) 267 return unresolved 268 269 def get_provider(self, item): 270 """ 271 Return a list of providers of item 272 """ 273 return self.build_targets[item] 274 275 def get_dependees(self, item): 276 """ 277 Return a list of targets which depend on item 278 """ 279 dependees = [] 280 for fn in self.depids: 281 if item in self.depids[fn]: 282 dependees.append(fn) 283 return dependees 284 285 def get_rdependees(self, item): 286 """ 287 Return a list of targets which depend on runtime item 288 """ 289 dependees = [] 290 for fn in self.rdepids: 291 if item in self.rdepids[fn]: 292 dependees.append(fn) 293 return dependees 294 295 def get_reasons(self, item, runtime=False): 296 """ 297 Get the reason(s) for an item not being provided, if any 298 """ 299 reasons = [] 300 if self.skiplist: 301 for fn in self.skiplist: 302 skipitem = self.skiplist[fn] 303 if skipitem.pn == item: 304 reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason)) 305 elif runtime and item in skipitem.rprovides: 306 reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) 307 elif not runtime and item in skipitem.provides: 308 reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) 309 return reasons 310 311 def get_close_matches(self, item, provider_list): 312 import difflib 313 if self.skiplist: 314 skipped = [] 315 for fn in self.skiplist: 316 skipped.append(self.skiplist[fn].pn) 317 full_list = provider_list + skipped 318 else: 319 full_list = provider_list 320 return difflib.get_close_matches(item, full_list, cutoff=0.7) 321 322 def add_provider(self, cfgData, dataCache, item): 323 try: 324 self.add_provider_internal(cfgData, dataCache, item) 325 except bb.providers.NoProvider: 326 if self.abort: 327 raise 328 self.remove_buildtarget(item) 329 330 self.mark_external_target(item) 331 332 def add_provider_internal(self, cfgData, dataCache, item): 333 """ 334 Add the providers of item to the task data 335 Mark entries were specifically added externally as against dependencies 336 added internally during dependency resolution 337 """ 338 339 if re_match_strings(item, dataCache.ignored_dependencies): 340 return 341 342 if not item in dataCache.providers: 343 close_matches = self.get_close_matches(item, list(dataCache.providers.keys())) 344 # Is it in RuntimeProviders ? 345 all_p = bb.providers.getRuntimeProviders(dataCache, item) 346 for fn in all_p: 347 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item 348 if new not in close_matches: 349 close_matches.append(new) 350 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) 351 raise bb.providers.NoProvider(item) 352 353 if self.have_build_target(item): 354 return 355 356 all_p = dataCache.providers[item] 357 358 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) 359 eligible = [p for p in eligible if not p in self.failed_fns] 360 361 if not eligible: 362 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) 363 raise bb.providers.NoProvider(item) 364 365 if len(eligible) > 1 and foundUnique == False: 366 if item not in self.consider_msgs_cache: 367 providers_list = [] 368 for fn in eligible: 369 providers_list.append(dataCache.pkg_fn[fn]) 370 bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData) 371 self.consider_msgs_cache.append(item) 372 373 for fn in eligible: 374 if fn in self.failed_fns: 375 continue 376 logger.debug(2, "adding %s to satisfy %s", fn, item) 377 self.add_build_target(fn, item) 378 self.add_tasks(fn, dataCache) 379 380 381 #item = dataCache.pkg_fn[fn] 382 383 def add_rprovider(self, cfgData, dataCache, item): 384 """ 385 Add the runtime providers of item to the task data 386 (takes item names from RDEPENDS/PACKAGES namespace) 387 """ 388 389 if re_match_strings(item, dataCache.ignored_dependencies): 390 return 391 392 if self.have_runtime_target(item): 393 return 394 395 all_p = bb.providers.getRuntimeProviders(dataCache, item) 396 397 if not all_p: 398 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData) 399 raise bb.providers.NoRProvider(item) 400 401 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) 402 eligible = [p for p in eligible if not p in self.failed_fns] 403 404 if not eligible: 405 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) 406 raise bb.providers.NoRProvider(item) 407 408 if len(eligible) > 1 and numberPreferred == 0: 409 if item not in self.consider_msgs_cache: 410 providers_list = [] 411 for fn in eligible: 412 providers_list.append(dataCache.pkg_fn[fn]) 413 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) 414 self.consider_msgs_cache.append(item) 415 416 if numberPreferred > 1: 417 if item not in self.consider_msgs_cache: 418 providers_list = [] 419 for fn in eligible: 420 providers_list.append(dataCache.pkg_fn[fn]) 421 bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) 422 self.consider_msgs_cache.append(item) 423 raise bb.providers.MultipleRProvider(item) 424 425 # run through the list until we find one that we can build 426 for fn in eligible: 427 if fn in self.failed_fns: 428 continue 429 logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item) 430 self.add_runtime_target(fn, item) 431 self.add_tasks(fn, dataCache) 432 433 def fail_fn(self, fn, missing_list=None): 434 """ 435 Mark a file as failed (unbuildable) 436 Remove any references from build and runtime provider lists 437 438 missing_list, A list of missing requirements for this target 439 """ 440 if fn in self.failed_fns: 441 return 442 if not missing_list: 443 missing_list = [] 444 logger.debug(1, "File '%s' is unbuildable, removing...", fn) 445 self.failed_fns.append(fn) 446 for target in self.build_targets: 447 if fn in self.build_targets[target]: 448 self.build_targets[target].remove(fn) 449 if len(self.build_targets[target]) == 0: 450 self.remove_buildtarget(target, missing_list) 451 for target in self.run_targets: 452 if fn in self.run_targets[target]: 453 self.run_targets[target].remove(fn) 454 if len(self.run_targets[target]) == 0: 455 self.remove_runtarget(target, missing_list) 456 457 def remove_buildtarget(self, target, missing_list=None): 458 """ 459 Mark a build target as failed (unbuildable) 460 Trigger removal of any files that have this as a dependency 461 """ 462 if not missing_list: 463 missing_list = [target] 464 else: 465 missing_list = [target] + missing_list 466 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) 467 self.failed_deps.append(target) 468 dependees = self.get_dependees(target) 469 for fn in dependees: 470 self.fail_fn(fn, missing_list) 471 for tid in self.taskentries: 472 for (idepend, idependtask) in self.taskentries[tid].idepends: 473 if idepend == target: 474 fn = tid.rsplit(":",1)[0] 475 self.fail_fn(fn, missing_list) 476 477 if self.abort and target in self.external_targets: 478 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) 479 raise bb.providers.NoProvider(target) 480 481 def remove_runtarget(self, target, missing_list=None): 482 """ 483 Mark a run target as failed (unbuildable) 484 Trigger removal of any files that have this as a dependency 485 """ 486 if not missing_list: 487 missing_list = [target] 488 else: 489 missing_list = [target] + missing_list 490 491 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) 492 self.failed_rdeps.append(target) 493 dependees = self.get_rdependees(target) 494 for fn in dependees: 495 self.fail_fn(fn, missing_list) 496 for tid in self.taskentries: 497 for (idepend, idependtask) in self.taskentries[tid].irdepends: 498 if idepend == target: 499 fn = tid.rsplit(":",1)[0] 500 self.fail_fn(fn, missing_list) 501 502 def add_unresolved(self, cfgData, dataCache): 503 """ 504 Resolve all unresolved build and runtime targets 505 """ 506 logger.info("Resolving any missing task queue dependencies") 507 while True: 508 added = 0 509 for target in self.get_unresolved_build_targets(dataCache): 510 try: 511 self.add_provider_internal(cfgData, dataCache, target) 512 added = added + 1 513 except bb.providers.NoProvider: 514 if self.abort and target in self.external_targets and not self.allowincomplete: 515 raise 516 if not self.allowincomplete: 517 self.remove_buildtarget(target) 518 for target in self.get_unresolved_run_targets(dataCache): 519 try: 520 self.add_rprovider(cfgData, dataCache, target) 521 added = added + 1 522 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): 523 self.remove_runtarget(target) 524 logger.debug(1, "Resolved " + str(added) + " extra dependencies") 525 if added == 0: 526 break 527 # self.dump_data() 528 529 def get_providermap(self, prefix=None): 530 provmap = {} 531 for name in self.build_targets: 532 if prefix and not name.startswith(prefix): 533 continue 534 if self.have_build_target(name): 535 provider = self.get_provider(name) 536 if provider: 537 provmap[name] = provider[0] 538 return provmap 539 540 def get_mcdepends(self): 541 return self.mcdepends 542 543 def dump_data(self): 544 """ 545 Dump some debug information on the internal data structures 546 """ 547 logger.debug(3, "build_names:") 548 logger.debug(3, ", ".join(self.build_targets)) 549 550 logger.debug(3, "run_names:") 551 logger.debug(3, ", ".join(self.run_targets)) 552 553 logger.debug(3, "build_targets:") 554 for target in self.build_targets: 555 targets = "None" 556 if target in self.build_targets: 557 targets = self.build_targets[target] 558 logger.debug(3, " %s: %s", target, targets) 559 560 logger.debug(3, "run_targets:") 561 for target in self.run_targets: 562 targets = "None" 563 if target in self.run_targets: 564 targets = self.run_targets[target] 565 logger.debug(3, " %s: %s", target, targets) 566 567 logger.debug(3, "tasks:") 568 for tid in self.taskentries: 569 logger.debug(3, " %s: %s %s %s", 570 tid, 571 self.taskentries[tid].idepends, 572 self.taskentries[tid].irdepends, 573 self.taskentries[tid].tdepends) 574 575 logger.debug(3, "dependency ids (per fn):") 576 for fn in self.depids: 577 logger.debug(3, " %s: %s", fn, self.depids[fn]) 578 579 logger.debug(3, "runtime dependency ids (per fn):") 580 for fn in self.rdepids: 581 logger.debug(3, " %s: %s", fn, self.rdepids[fn]) 582