1# 2# Copyright (C) 2003, 2004 Chris Larson 3# Copyright (C) 2003, 2004 Phil Blundell 4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 5# Copyright (C) 2005 Holger Hans Peter Freyther 6# Copyright (C) 2005 ROAD GmbH 7# Copyright (C) 2006 - 2007 Richard Purdie 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import sys, os, glob, os.path, re, time 13import itertools 14import logging 15import multiprocessing 16import threading 17from io import StringIO, UnsupportedOperation 18from contextlib import closing 19from collections import defaultdict, namedtuple 20import bb, bb.exceptions, bb.command 21from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build 22import queue 23import signal 24import prserv.serv 25import json 26import pickle 27import codecs 28import hashserv 29 30logger = logging.getLogger("BitBake") 31collectlog = logging.getLogger("BitBake.Collection") 32buildlog = logging.getLogger("BitBake.Build") 33parselog = logging.getLogger("BitBake.Parsing") 34providerlog = logging.getLogger("BitBake.Provider") 35 36class NoSpecificMatch(bb.BBHandledException): 37 """ 38 Exception raised when no or multiple file matches are found 39 """ 40 41class NothingToBuild(Exception): 42 """ 43 Exception raised when there is nothing to build 44 """ 45 46class CollectionError(bb.BBHandledException): 47 """ 48 Exception raised when layer configuration is incorrect 49 """ 50 51class state: 52 initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7)) 53 54 @classmethod 55 def get_name(cls, code): 56 for name in dir(cls): 57 value = getattr(cls, name) 58 if type(value) == type(cls.initial) and value == code: 59 return name 60 raise ValueError("Invalid status code: %s" % code) 61 62 63class SkippedPackage: 64 def __init__(self, info = None, reason = None): 65 self.pn = None 66 self.skipreason = None 67 self.provides = None 68 self.rprovides = None 69 70 if info: 71 self.pn = info.pn 72 self.skipreason = info.skipreason 73 self.provides = info.provides 74 self.rprovides = info.packages + info.rprovides 75 for package in info.packages: 76 self.rprovides += info.rprovides_pkg[package] 77 elif reason: 78 self.skipreason = reason 79 80 81class CookerFeatures(object): 82 _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS, RECIPE_SIGGEN_INFO] = list(range(4)) 83 84 def __init__(self): 85 self._features=set() 86 87 def setFeature(self, f): 88 # validate we got a request for a feature we support 89 if f not in CookerFeatures._feature_list: 90 return 91 self._features.add(f) 92 93 def __contains__(self, f): 94 return f in self._features 95 96 def __iter__(self): 97 return self._features.__iter__() 98 99 def __next__(self): 100 return next(self._features) 101 102 103class EventWriter: 104 def __init__(self, cooker, eventfile): 105 self.cooker = cooker 106 self.eventfile = eventfile 107 self.event_queue = [] 108 109 def write_variables(self): 110 with open(self.eventfile, "a") as f: 111 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) 112 113 def send(self, event): 114 with open(self.eventfile, "a") as f: 115 try: 116 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') 117 f.write("%s\n" % json.dumps({"class": event.__module__ + "." + event.__class__.__name__, 118 "vars": str_event})) 119 except Exception as err: 120 import traceback 121 print(err, traceback.format_exc()) 122 123 124#============================================================================# 125# BBCooker 126#============================================================================# 127class BBCooker: 128 """ 129 Manages one bitbake build run 130 """ 131 132 def __init__(self, featureSet=None, server=None): 133 self.recipecaches = None 134 self.baseconfig_valid = False 135 self.parsecache_valid = False 136 self.eventlog = None 137 self.skiplist = {} 138 self.featureset = CookerFeatures() 139 if featureSet: 140 for f in featureSet: 141 self.featureset.setFeature(f) 142 143 self.orig_syspath = sys.path.copy() 144 self.orig_sysmodules = [*sys.modules] 145 146 self.configuration = bb.cookerdata.CookerConfiguration() 147 148 self.process_server = server 149 self.idleCallBackRegister = None 150 self.waitIdle = None 151 if server: 152 self.idleCallBackRegister = server.register_idle_function 153 self.waitIdle = server.wait_for_idle 154 155 bb.debug(1, "BBCooker starting %s" % time.time()) 156 157 self.configwatched = {} 158 self.parsewatched = {} 159 160 # If being called by something like tinfoil, we need to clean cached data 161 # which may now be invalid 162 bb.parse.clear_cache() 163 bb.parse.BBHandler.cached_statements = {} 164 165 self.ui_cmdline = None 166 self.hashserv = None 167 self.hashservaddr = None 168 169 # TOSTOP must not be set or our children will hang when they output 170 try: 171 fd = sys.stdout.fileno() 172 if os.isatty(fd): 173 import termios 174 tcattr = termios.tcgetattr(fd) 175 if tcattr[3] & termios.TOSTOP: 176 buildlog.info("The terminal had the TOSTOP bit set, clearing...") 177 tcattr[3] = tcattr[3] & ~termios.TOSTOP 178 termios.tcsetattr(fd, termios.TCSANOW, tcattr) 179 except UnsupportedOperation: 180 pass 181 182 self.command = bb.command.Command(self, self.process_server) 183 self.state = state.initial 184 185 self.parser = None 186 187 signal.signal(signal.SIGTERM, self.sigterm_exception) 188 # Let SIGHUP exit as SIGTERM 189 signal.signal(signal.SIGHUP, self.sigterm_exception) 190 191 bb.debug(1, "BBCooker startup complete %s" % time.time()) 192 193 def init_configdata(self): 194 if not hasattr(self, "data"): 195 self.initConfigurationData() 196 bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) 197 self.handlePRServ() 198 199 def _baseconfig_set(self, value): 200 if value and not self.baseconfig_valid: 201 bb.server.process.serverlog("Base config valid") 202 elif not value and self.baseconfig_valid: 203 bb.server.process.serverlog("Base config invalidated") 204 self.baseconfig_valid = value 205 206 def _parsecache_set(self, value): 207 if value and not self.parsecache_valid: 208 bb.server.process.serverlog("Parse cache valid") 209 elif not value and self.parsecache_valid: 210 bb.server.process.serverlog("Parse cache invalidated") 211 self.parsecache_valid = value 212 213 def add_filewatch(self, deps, configwatcher=False): 214 if configwatcher: 215 watcher = self.configwatched 216 else: 217 watcher = self.parsewatched 218 219 for i in deps: 220 f = i[0] 221 mtime = i[1] 222 watcher[f] = mtime 223 224 def sigterm_exception(self, signum, stackframe): 225 if signum == signal.SIGTERM: 226 bb.warn("Cooker received SIGTERM, shutting down...") 227 elif signum == signal.SIGHUP: 228 bb.warn("Cooker received SIGHUP, shutting down...") 229 self.state = state.forceshutdown 230 bb.event._should_exit.set() 231 232 def setFeatures(self, features): 233 # we only accept a new feature set if we're in state initial, so we can reset without problems 234 if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]: 235 raise Exception("Illegal state for feature set change") 236 original_featureset = list(self.featureset) 237 for feature in features: 238 self.featureset.setFeature(feature) 239 bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset))) 240 if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"): 241 self.reset() 242 243 def initConfigurationData(self): 244 245 self.state = state.initial 246 self.caches_array = [] 247 248 sys.path = self.orig_syspath.copy() 249 for mod in [*sys.modules]: 250 if mod not in self.orig_sysmodules: 251 del sys.modules[mod] 252 253 self.configwatched = {} 254 255 # Need to preserve BB_CONSOLELOG over resets 256 consolelog = None 257 if hasattr(self, "data"): 258 consolelog = self.data.getVar("BB_CONSOLELOG") 259 260 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: 261 self.enableDataTracking() 262 263 caches_name_array = ['bb.cache:CoreRecipeInfo'] 264 # We hardcode all known cache types in a single place, here. 265 if CookerFeatures.HOB_EXTRA_CACHES in self.featureset: 266 caches_name_array.append("bb.cache_extra:HobRecipeInfo") 267 if CookerFeatures.RECIPE_SIGGEN_INFO in self.featureset: 268 caches_name_array.append("bb.cache:SiggenRecipeInfo") 269 270 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! 271 # This is the entry point, no further check needed! 272 for var in caches_name_array: 273 try: 274 module_name, cache_name = var.split(':') 275 module = __import__(module_name, fromlist=(cache_name,)) 276 self.caches_array.append(getattr(module, cache_name)) 277 except ImportError as exc: 278 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) 279 raise bb.BBHandledException() 280 281 self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False) 282 self.databuilder.parseBaseConfiguration() 283 self.data = self.databuilder.data 284 self.data_hash = self.databuilder.data_hash 285 self.extraconfigdata = {} 286 287 eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG") 288 if not self.configuration.writeeventlog and eventlog: 289 self.setupEventLog(eventlog) 290 291 if consolelog: 292 self.data.setVar("BB_CONSOLELOG", consolelog) 293 294 self.data.setVar('BB_CMDLINE', self.ui_cmdline) 295 296 if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: 297 self.disableDataTracking() 298 299 for mc in self.databuilder.mcdata.values(): 300 self.add_filewatch(mc.getVar("__base_depends", False), configwatcher=True) 301 302 self._baseconfig_set(True) 303 self._parsecache_set(False) 304 305 def handlePRServ(self): 306 # Setup a PR Server based on the new configuration 307 try: 308 self.prhost = prserv.serv.auto_start(self.data) 309 except prserv.serv.PRServiceConfigError as e: 310 bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log") 311 312 if self.data.getVar("BB_HASHSERVE") == "auto": 313 # Create a new hash server bound to a unix domain socket 314 if not self.hashserv: 315 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" 316 upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None 317 if upstream: 318 import socket 319 try: 320 sock = socket.create_connection(upstream.split(":"), 5) 321 sock.close() 322 except socket.error as e: 323 bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s" 324 % (upstream, repr(e))) 325 326 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") 327 self.hashserv = hashserv.create_server( 328 self.hashservaddr, 329 dbfile, 330 sync=False, 331 upstream=upstream, 332 ) 333 self.hashserv.serve_as_process(log_level=logging.WARNING) 334 for mc in self.databuilder.mcdata: 335 self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) 336 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) 337 338 bb.parse.init_parser(self.data) 339 340 def enableDataTracking(self): 341 self.configuration.tracking = True 342 if hasattr(self, "data"): 343 self.data.enableTracking() 344 345 def disableDataTracking(self): 346 self.configuration.tracking = False 347 if hasattr(self, "data"): 348 self.data.disableTracking() 349 350 def revalidateCaches(self): 351 bb.parse.clear_cache() 352 353 clean = True 354 for f in self.configwatched: 355 if not bb.parse.check_mtime(f, self.configwatched[f]): 356 bb.server.process.serverlog("Found %s changed, invalid cache" % f) 357 self._baseconfig_set(False) 358 self._parsecache_set(False) 359 clean = False 360 break 361 362 if clean: 363 for f in self.parsewatched: 364 if not bb.parse.check_mtime(f, self.parsewatched[f]): 365 bb.server.process.serverlog("Found %s changed, invalid cache" % f) 366 self._parsecache_set(False) 367 clean = False 368 break 369 370 if not clean: 371 bb.parse.BBHandler.cached_statements = {} 372 373 def parseConfiguration(self): 374 self.updateCacheSync() 375 376 # Change nice level if we're asked to 377 nice = self.data.getVar("BB_NICE_LEVEL") 378 if nice: 379 curnice = os.nice(0) 380 nice = int(nice) - curnice 381 buildlog.verbose("Renice to %s " % os.nice(nice)) 382 383 if self.recipecaches: 384 del self.recipecaches 385 self.multiconfigs = self.databuilder.mcdata.keys() 386 self.recipecaches = {} 387 for mc in self.multiconfigs: 388 self.recipecaches[mc] = bb.cache.CacheData(self.caches_array) 389 390 self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS")) 391 self.collections = {} 392 for mc in self.multiconfigs: 393 self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) 394 395 self._parsecache_set(False) 396 397 def setupEventLog(self, eventlog): 398 if self.eventlog and self.eventlog[0] != eventlog: 399 bb.event.unregister_UIHhandler(self.eventlog[1]) 400 self.eventlog = None 401 if not self.eventlog or self.eventlog[0] != eventlog: 402 # we log all events to a file if so directed 403 # register the log file writer as UI Handler 404 if not os.path.exists(os.path.dirname(eventlog)): 405 bb.utils.mkdirhier(os.path.dirname(eventlog)) 406 writer = EventWriter(self, eventlog) 407 EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) 408 self.eventlog = (eventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)), writer) 409 410 def updateConfigOpts(self, options, environment, cmdline): 411 self.ui_cmdline = cmdline 412 clean = True 413 for o in options: 414 if o in ['prefile', 'postfile']: 415 # Only these options may require a reparse 416 try: 417 if getattr(self.configuration, o) == options[o]: 418 # Value is the same, no need to mark dirty 419 continue 420 except AttributeError: 421 pass 422 logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) 423 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o])) 424 clean = False 425 if hasattr(self.configuration, o): 426 setattr(self.configuration, o, options[o]) 427 428 if self.configuration.writeeventlog: 429 self.setupEventLog(self.configuration.writeeventlog) 430 431 bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel 432 bb.msg.loggerDefaultDomains = self.configuration.debug_domains 433 434 if hasattr(self, "data"): 435 origenv = bb.data.init() 436 for k in environment: 437 origenv.setVar(k, environment[k]) 438 self.data.setVar("BB_ORIGENV", origenv) 439 440 for k in bb.utils.approved_variables(): 441 if k in environment and k not in self.configuration.env: 442 logger.debug("Updating new environment variable %s to %s" % (k, environment[k])) 443 self.configuration.env[k] = environment[k] 444 clean = False 445 if k in self.configuration.env and k not in environment: 446 logger.debug("Updating environment variable %s (deleted)" % (k)) 447 del self.configuration.env[k] 448 clean = False 449 if k not in self.configuration.env and k not in environment: 450 continue 451 if environment[k] != self.configuration.env[k]: 452 logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k])) 453 self.configuration.env[k] = environment[k] 454 clean = False 455 456 # Now update all the variables not in the datastore to match 457 self.configuration.env = environment 458 459 self.revalidateCaches() 460 if not clean: 461 logger.debug("Base environment change, triggering reparse") 462 self.reset() 463 464 def showVersions(self): 465 466 (latest_versions, preferred_versions, required) = self.findProviders() 467 468 logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version") 469 logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================") 470 471 for p in sorted(self.recipecaches[''].pkg_pn): 472 preferred = preferred_versions[p] 473 latest = latest_versions[p] 474 requiredstr = "" 475 preferredstr = "" 476 if required[p]: 477 if preferred[0] is not None: 478 requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2] 479 else: 480 bb.fatal("REQUIRED_VERSION of package %s not available" % p) 481 else: 482 preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2] 483 484 lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] 485 486 if preferred == latest: 487 preferredstr = "" 488 489 logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr) 490 491 def showEnvironment(self, buildfile=None, pkgs_to_build=None): 492 """ 493 Show the outer or per-recipe environment 494 """ 495 fn = None 496 envdata = None 497 mc = '' 498 if not pkgs_to_build: 499 pkgs_to_build = [] 500 501 orig_tracking = self.configuration.tracking 502 if not orig_tracking: 503 self.enableDataTracking() 504 self.reset() 505 # reset() resets to the UI requested value so we have to redo this 506 self.enableDataTracking() 507 508 def mc_base(p): 509 if p.startswith('mc:'): 510 s = p.split(':') 511 if len(s) == 2: 512 return s[1] 513 return None 514 515 if buildfile: 516 # Parse the configuration here. We need to do it explicitly here since 517 # this showEnvironment() code path doesn't use the cache 518 self.parseConfiguration() 519 520 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) 521 fn = self.matchFile(fn, mc) 522 fn = bb.cache.realfn2virtual(fn, cls, mc) 523 elif len(pkgs_to_build) == 1: 524 mc = mc_base(pkgs_to_build[0]) 525 if not mc: 526 ignore = self.data.getVar("ASSUME_PROVIDED") or "" 527 if pkgs_to_build[0] in set(ignore.split()): 528 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) 529 530 taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True) 531 532 mc = runlist[0][0] 533 fn = runlist[0][3] 534 535 if fn: 536 try: 537 layername = self.collections[mc].calc_bbfile_priority(fn)[2] 538 envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn), layername) 539 except Exception as e: 540 parselog.exception("Unable to read %s", fn) 541 raise 542 else: 543 if not mc in self.databuilder.mcdata: 544 bb.fatal('No multiconfig named "%s" found' % mc) 545 envdata = self.databuilder.mcdata[mc] 546 data.expandKeys(envdata) 547 parse.ast.runAnonFuncs(envdata) 548 549 # Display history 550 with closing(StringIO()) as env: 551 self.data.inchistory.emit(env) 552 logger.plain(env.getvalue()) 553 554 # emit variables and shell functions 555 with closing(StringIO()) as env: 556 data.emit_env(env, envdata, True) 557 logger.plain(env.getvalue()) 558 559 # emit the metadata which isn't valid shell 560 for e in sorted(envdata.keys()): 561 if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False): 562 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) 563 564 if not orig_tracking: 565 self.disableDataTracking() 566 self.reset() 567 568 def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False): 569 """ 570 Prepare a runqueue and taskdata object for iteration over pkgs_to_build 571 """ 572 bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data) 573 574 # A task of None means use the default task 575 if task is None: 576 task = self.configuration.cmd 577 if not task.startswith("do_"): 578 task = "do_%s" % task 579 580 targetlist = self.checkPackages(pkgs_to_build, task) 581 fulltargetlist = [] 582 defaulttask_implicit = '' 583 defaulttask_explicit = False 584 wildcard = False 585 586 # Wild card expansion: 587 # Replace string such as "mc:*:bash" 588 # into "mc:A:bash mc:B:bash bash" 589 for k in targetlist: 590 if k.startswith("mc:") and k.count(':') >= 2: 591 if wildcard: 592 bb.fatal('multiconfig conflict') 593 if k.split(":")[1] == "*": 594 wildcard = True 595 for mc in self.multiconfigs: 596 if mc: 597 fulltargetlist.append(k.replace('*', mc)) 598 # implicit default task 599 else: 600 defaulttask_implicit = k.split(":")[2] 601 else: 602 fulltargetlist.append(k) 603 else: 604 defaulttask_explicit = True 605 fulltargetlist.append(k) 606 607 if not defaulttask_explicit and defaulttask_implicit != '': 608 fulltargetlist.append(defaulttask_implicit) 609 610 bb.debug(1,"Target list: %s" % (str(fulltargetlist))) 611 taskdata = {} 612 localdata = {} 613 614 for mc in self.multiconfigs: 615 taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete) 616 localdata[mc] = data.createCopy(self.databuilder.mcdata[mc]) 617 bb.data.expandKeys(localdata[mc]) 618 619 current = 0 620 runlist = [] 621 for k in fulltargetlist: 622 origk = k 623 mc = "" 624 if k.startswith("mc:") and k.count(':') >= 2: 625 mc = k.split(":")[1] 626 k = ":".join(k.split(":")[2:]) 627 ktask = task 628 if ":do_" in k: 629 k2 = k.split(":do_") 630 k = k2[0] 631 ktask = k2[1] 632 633 if mc not in self.multiconfigs: 634 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc)) 635 636 taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k) 637 current += 1 638 if not ktask.startswith("do_"): 639 ktask = "do_%s" % ktask 640 if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]: 641 # e.g. in ASSUME_PROVIDED 642 continue 643 fn = taskdata[mc].build_targets[k][0] 644 runlist.append([mc, k, ktask, fn]) 645 bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data) 646 647 havemc = False 648 for mc in self.multiconfigs: 649 if taskdata[mc].get_mcdepends(): 650 havemc = True 651 652 # No need to do check providers if there are no mcdeps or not an mc build 653 if havemc or len(self.multiconfigs) > 1: 654 seen = set() 655 new = True 656 # Make sure we can provide the multiconfig dependency 657 while new: 658 mcdeps = set() 659 # Add unresolved first, so we can get multiconfig indirect dependencies on time 660 for mc in self.multiconfigs: 661 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) 662 mcdeps |= set(taskdata[mc].get_mcdepends()) 663 new = False 664 for k in mcdeps: 665 if k in seen: 666 continue 667 l = k.split(':') 668 depmc = l[2] 669 if depmc not in self.multiconfigs: 670 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) 671 else: 672 logger.debug("Adding providers for multiconfig dependency %s" % l[3]) 673 taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) 674 seen.add(k) 675 new = True 676 677 for mc in self.multiconfigs: 678 taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) 679 680 bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data) 681 return taskdata, runlist 682 683 def prepareTreeData(self, pkgs_to_build, task): 684 """ 685 Prepare a runqueue and taskdata object for iteration over pkgs_to_build 686 """ 687 688 # We set halt to False here to prevent unbuildable targets raising 689 # an exception when we're just generating data 690 taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) 691 692 return runlist, taskdata 693 694 ######## WARNING : this function requires cache_extra to be enabled ######## 695 696 def generateTaskDepTreeData(self, pkgs_to_build, task): 697 """ 698 Create a dependency graph of pkgs_to_build including reverse dependency 699 information. 700 """ 701 if not task.startswith("do_"): 702 task = "do_%s" % task 703 704 runlist, taskdata = self.prepareTreeData(pkgs_to_build, task) 705 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) 706 rq.rqdata.prepare() 707 return self.buildDependTree(rq, taskdata) 708 709 @staticmethod 710 def add_mc_prefix(mc, pn): 711 if mc: 712 return "mc:%s:%s" % (mc, pn) 713 return pn 714 715 def buildDependTree(self, rq, taskdata): 716 seen_fns = [] 717 depend_tree = {} 718 depend_tree["depends"] = {} 719 depend_tree["tdepends"] = {} 720 depend_tree["pn"] = {} 721 depend_tree["rdepends-pn"] = {} 722 depend_tree["packages"] = {} 723 depend_tree["rdepends-pkg"] = {} 724 depend_tree["rrecs-pkg"] = {} 725 depend_tree['providermap'] = {} 726 depend_tree["layer-priorities"] = self.bbfile_config_priorities 727 728 for mc in taskdata: 729 for name, fn in list(taskdata[mc].get_providermap().items()): 730 pn = self.recipecaches[mc].pkg_fn[fn] 731 pn = self.add_mc_prefix(mc, pn) 732 if name != pn: 733 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn] 734 depend_tree['providermap'][name] = (pn, version) 735 736 for tid in rq.rqdata.runtaskentries: 737 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) 738 pn = self.recipecaches[mc].pkg_fn[taskfn] 739 pn = self.add_mc_prefix(mc, pn) 740 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] 741 if pn not in depend_tree["pn"]: 742 depend_tree["pn"][pn] = {} 743 depend_tree["pn"][pn]["filename"] = taskfn 744 depend_tree["pn"][pn]["version"] = version 745 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) 746 747 # if we have extra caches, list all attributes they bring in 748 extra_info = [] 749 for cache_class in self.caches_array: 750 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): 751 cachefields = getattr(cache_class, 'cachefields', []) 752 extra_info = extra_info + cachefields 753 754 # for all attributes stored, add them to the dependency tree 755 for ei in extra_info: 756 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] 757 758 759 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid)) 760 if not dotname in depend_tree["tdepends"]: 761 depend_tree["tdepends"][dotname] = [] 762 for dep in rq.rqdata.runtaskentries[tid].depends: 763 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep) 764 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn] 765 if depmc: 766 depmc = "mc:" + depmc + ":" 767 depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep))) 768 if taskfn not in seen_fns: 769 seen_fns.append(taskfn) 770 packages = [] 771 772 depend_tree["depends"][pn] = [] 773 for dep in taskdata[mc].depids[taskfn]: 774 depend_tree["depends"][pn].append(dep) 775 776 depend_tree["rdepends-pn"][pn] = [] 777 for rdep in taskdata[mc].rdepids[taskfn]: 778 depend_tree["rdepends-pn"][pn].append(rdep) 779 780 rdepends = self.recipecaches[mc].rundeps[taskfn] 781 for package in rdepends: 782 depend_tree["rdepends-pkg"][package] = [] 783 for rdepend in rdepends[package]: 784 depend_tree["rdepends-pkg"][package].append(rdepend) 785 packages.append(package) 786 787 rrecs = self.recipecaches[mc].runrecs[taskfn] 788 for package in rrecs: 789 depend_tree["rrecs-pkg"][package] = [] 790 for rdepend in rrecs[package]: 791 depend_tree["rrecs-pkg"][package].append(rdepend) 792 if not package in packages: 793 packages.append(package) 794 795 for package in packages: 796 if package not in depend_tree["packages"]: 797 depend_tree["packages"][package] = {} 798 depend_tree["packages"][package]["pn"] = pn 799 depend_tree["packages"][package]["filename"] = taskfn 800 depend_tree["packages"][package]["version"] = version 801 802 return depend_tree 803 804 ######## WARNING : this function requires cache_extra to be enabled ######## 805 def generatePkgDepTreeData(self, pkgs_to_build, task): 806 """ 807 Create a dependency tree of pkgs_to_build, returning the data. 808 """ 809 if not task.startswith("do_"): 810 task = "do_%s" % task 811 812 _, taskdata = self.prepareTreeData(pkgs_to_build, task) 813 814 seen_fns = [] 815 depend_tree = {} 816 depend_tree["depends"] = {} 817 depend_tree["pn"] = {} 818 depend_tree["rdepends-pn"] = {} 819 depend_tree["rdepends-pkg"] = {} 820 depend_tree["rrecs-pkg"] = {} 821 822 # if we have extra caches, list all attributes they bring in 823 extra_info = [] 824 for cache_class in self.caches_array: 825 if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'): 826 cachefields = getattr(cache_class, 'cachefields', []) 827 extra_info = extra_info + cachefields 828 829 tids = [] 830 for mc in taskdata: 831 for tid in taskdata[mc].taskentries: 832 tids.append(tid) 833 834 for tid in tids: 835 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) 836 837 pn = self.recipecaches[mc].pkg_fn[taskfn] 838 pn = self.add_mc_prefix(mc, pn) 839 840 if pn not in depend_tree["pn"]: 841 depend_tree["pn"][pn] = {} 842 depend_tree["pn"][pn]["filename"] = taskfn 843 version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn] 844 depend_tree["pn"][pn]["version"] = version 845 rdepends = self.recipecaches[mc].rundeps[taskfn] 846 rrecs = self.recipecaches[mc].runrecs[taskfn] 847 depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None) 848 849 # for all extra attributes stored, add them to the dependency tree 850 for ei in extra_info: 851 depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn] 852 853 if taskfn not in seen_fns: 854 seen_fns.append(taskfn) 855 856 depend_tree["depends"][pn] = [] 857 for dep in taskdata[mc].depids[taskfn]: 858 pn_provider = "" 859 if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]: 860 fn_provider = taskdata[mc].build_targets[dep][0] 861 pn_provider = self.recipecaches[mc].pkg_fn[fn_provider] 862 else: 863 pn_provider = dep 864 pn_provider = self.add_mc_prefix(mc, pn_provider) 865 depend_tree["depends"][pn].append(pn_provider) 866 867 depend_tree["rdepends-pn"][pn] = [] 868 for rdep in taskdata[mc].rdepids[taskfn]: 869 pn_rprovider = "" 870 if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]: 871 fn_rprovider = taskdata[mc].run_targets[rdep][0] 872 pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider] 873 else: 874 pn_rprovider = rdep 875 pn_rprovider = self.add_mc_prefix(mc, pn_rprovider) 876 depend_tree["rdepends-pn"][pn].append(pn_rprovider) 877 878 depend_tree["rdepends-pkg"].update(rdepends) 879 depend_tree["rrecs-pkg"].update(rrecs) 880 881 return depend_tree 882 883 def generateDepTreeEvent(self, pkgs_to_build, task): 884 """ 885 Create a task dependency graph of pkgs_to_build. 886 Generate an event with the result 887 """ 888 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) 889 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.data) 890 891 def generateDotGraphFiles(self, pkgs_to_build, task): 892 """ 893 Create a task dependency graph of pkgs_to_build. 894 Save the result to a set of .dot files. 895 """ 896 897 depgraph = self.generateTaskDepTreeData(pkgs_to_build, task) 898 899 with open('pn-buildlist', 'w') as f: 900 for pn in depgraph["pn"]: 901 f.write(pn + "\n") 902 logger.info("PN build list saved to 'pn-buildlist'") 903 904 # Remove old format output files to ensure no confusion with stale data 905 try: 906 os.unlink('pn-depends.dot') 907 except FileNotFoundError: 908 pass 909 try: 910 os.unlink('package-depends.dot') 911 except FileNotFoundError: 912 pass 913 try: 914 os.unlink('recipe-depends.dot') 915 except FileNotFoundError: 916 pass 917 918 with open('task-depends.dot', 'w') as f: 919 f.write("digraph depends {\n") 920 for task in sorted(depgraph["tdepends"]): 921 (pn, taskname) = task.rsplit(".", 1) 922 fn = depgraph["pn"][pn]["filename"] 923 version = depgraph["pn"][pn]["version"] 924 f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn)) 925 for dep in sorted(depgraph["tdepends"][task]): 926 f.write('"%s" -> "%s"\n' % (task, dep)) 927 f.write("}\n") 928 logger.info("Task dependencies saved to 'task-depends.dot'") 929 930 def show_appends_with_no_recipes(self): 931 appends_without_recipes = {} 932 # Determine which bbappends haven't been applied 933 for mc in self.multiconfigs: 934 # First get list of recipes, including skipped 935 recipefns = list(self.recipecaches[mc].pkg_fn.keys()) 936 recipefns.extend(self.skiplist.keys()) 937 938 # Work out list of bbappends that have been applied 939 applied_appends = [] 940 for fn in recipefns: 941 applied_appends.extend(self.collections[mc].get_file_appends(fn)) 942 943 appends_without_recipes[mc] = [] 944 for _, appendfn in self.collections[mc].bbappends: 945 if not appendfn in applied_appends: 946 appends_without_recipes[mc].append(appendfn) 947 948 msgs = [] 949 for mc in sorted(appends_without_recipes.keys()): 950 if appends_without_recipes[mc]: 951 msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', 952 '\n '.join(appends_without_recipes[mc]))) 953 954 if msgs: 955 msg = "\n".join(msgs) 956 warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ 957 False) or "no" 958 if warn_only.lower() in ("1", "yes", "true"): 959 bb.warn(msg) 960 else: 961 bb.fatal(msg) 962 963 def handlePrefProviders(self): 964 965 for mc in self.multiconfigs: 966 localdata = data.createCopy(self.databuilder.mcdata[mc]) 967 bb.data.expandKeys(localdata) 968 969 # Handle PREFERRED_PROVIDERS 970 for p in (localdata.getVar('PREFERRED_PROVIDERS') or "").split(): 971 try: 972 (providee, provider) = p.split(':') 973 except: 974 providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p) 975 continue 976 if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider: 977 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee]) 978 self.recipecaches[mc].preferred[providee] = provider 979 980 def findConfigFilePath(self, configfile): 981 """ 982 Find the location on disk of configfile and if it exists and was parsed by BitBake 983 emit the ConfigFilePathFound event with the path to the file. 984 """ 985 path = bb.cookerdata.findConfigFile(configfile, self.data) 986 if not path: 987 return 988 989 # Generate a list of parsed configuration files by searching the files 990 # listed in the __depends and __base_depends variables with a .conf suffix. 991 conffiles = [] 992 dep_files = self.data.getVar('__base_depends', False) or [] 993 dep_files = dep_files + (self.data.getVar('__depends', False) or []) 994 995 for f in dep_files: 996 if f[0].endswith(".conf"): 997 conffiles.append(f[0]) 998 999 _, conf, conffile = path.rpartition("conf/") 1000 match = os.path.join(conf, conffile) 1001 # Try and find matches for conf/conffilename.conf as we don't always 1002 # have the full path to the file. 1003 for cfg in conffiles: 1004 if cfg.endswith(match): 1005 bb.event.fire(bb.event.ConfigFilePathFound(path), 1006 self.data) 1007 break 1008 1009 def findFilesMatchingInDir(self, filepattern, directory): 1010 """ 1011 Searches for files containing the substring 'filepattern' which are children of 1012 'directory' in each BBPATH. i.e. to find all rootfs package classes available 1013 to BitBake one could call findFilesMatchingInDir(self, 'rootfs_', 'classes') 1014 or to find all machine configuration files one could call: 1015 findFilesMatchingInDir(self, '.conf', 'conf/machine') 1016 """ 1017 1018 matches = [] 1019 bbpaths = self.data.getVar('BBPATH').split(':') 1020 for path in bbpaths: 1021 dirpath = os.path.join(path, directory) 1022 if os.path.exists(dirpath): 1023 for root, dirs, files in os.walk(dirpath): 1024 for f in files: 1025 if filepattern in f: 1026 matches.append(f) 1027 1028 if matches: 1029 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) 1030 1031 def testCookerCommandEvent(self, filepattern): 1032 # Dummy command used by OEQA selftest to test tinfoil without IO 1033 matches = ["A", "B"] 1034 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) 1035 1036 def findProviders(self, mc=''): 1037 return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) 1038 1039 def findBestProvider(self, pn, mc=''): 1040 if pn in self.recipecaches[mc].providers: 1041 filenames = self.recipecaches[mc].providers[pn] 1042 eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc]) 1043 if eligible is not None: 1044 filename = eligible[0] 1045 else: 1046 filename = None 1047 return None, None, None, filename 1048 elif pn in self.recipecaches[mc].pkg_pn: 1049 (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) 1050 if required and preferred_file is None: 1051 return None, None, None, None 1052 return (latest, latest_f, preferred_ver, preferred_file) 1053 else: 1054 return None, None, None, None 1055 1056 def findConfigFiles(self, varname): 1057 """ 1058 Find config files which are appropriate values for varname. 1059 i.e. MACHINE, DISTRO 1060 """ 1061 possible = [] 1062 var = varname.lower() 1063 1064 data = self.data 1065 # iterate configs 1066 bbpaths = data.getVar('BBPATH').split(':') 1067 for path in bbpaths: 1068 confpath = os.path.join(path, "conf", var) 1069 if os.path.exists(confpath): 1070 for root, dirs, files in os.walk(confpath): 1071 # get all child files, these are appropriate values 1072 for f in files: 1073 val, sep, end = f.rpartition('.') 1074 if end == 'conf': 1075 possible.append(val) 1076 1077 if possible: 1078 bb.event.fire(bb.event.ConfigFilesFound(var, possible), self.data) 1079 1080 def findInheritsClass(self, klass): 1081 """ 1082 Find all recipes which inherit the specified class 1083 """ 1084 pkg_list = [] 1085 1086 for pfn in self.recipecaches[''].pkg_fn: 1087 inherits = self.recipecaches[''].inherits.get(pfn, None) 1088 if inherits and klass in inherits: 1089 pkg_list.append(self.recipecaches[''].pkg_fn[pfn]) 1090 1091 return pkg_list 1092 1093 def generateTargetsTree(self, klass=None, pkgs=None): 1094 """ 1095 Generate a dependency tree of buildable targets 1096 Generate an event with the result 1097 """ 1098 # if the caller hasn't specified a pkgs list default to universe 1099 if not pkgs: 1100 pkgs = ['universe'] 1101 # if inherited_class passed ensure all recipes which inherit the 1102 # specified class are included in pkgs 1103 if klass: 1104 extra_pkgs = self.findInheritsClass(klass) 1105 pkgs = pkgs + extra_pkgs 1106 1107 # generate a dependency tree for all our packages 1108 tree = self.generatePkgDepTreeData(pkgs, 'build') 1109 bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.data) 1110 1111 def interactiveMode( self ): 1112 """Drop off into a shell""" 1113 try: 1114 from bb import shell 1115 except ImportError: 1116 parselog.exception("Interactive mode not available") 1117 raise bb.BBHandledException() 1118 else: 1119 shell.start( self ) 1120 1121 1122 def handleCollections(self, collections): 1123 """Handle collections""" 1124 errors = False 1125 self.bbfile_config_priorities = [] 1126 if collections: 1127 collection_priorities = {} 1128 collection_depends = {} 1129 collection_list = collections.split() 1130 min_prio = 0 1131 for c in collection_list: 1132 bb.debug(1,'Processing %s in collection list' % (c)) 1133 1134 # Get collection priority if defined explicitly 1135 priority = self.data.getVar("BBFILE_PRIORITY_%s" % c) 1136 if priority: 1137 try: 1138 prio = int(priority) 1139 except ValueError: 1140 parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority) 1141 errors = True 1142 if min_prio == 0 or prio < min_prio: 1143 min_prio = prio 1144 collection_priorities[c] = prio 1145 else: 1146 collection_priorities[c] = None 1147 1148 # Check dependencies and store information for priority calculation 1149 deps = self.data.getVar("LAYERDEPENDS_%s" % c) 1150 if deps: 1151 try: 1152 depDict = bb.utils.explode_dep_versions2(deps) 1153 except bb.utils.VersionStringException as vse: 1154 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) 1155 for dep, oplist in list(depDict.items()): 1156 if dep in collection_list: 1157 for opstr in oplist: 1158 layerver = self.data.getVar("LAYERVERSION_%s" % dep) 1159 (op, depver) = opstr.split() 1160 if layerver: 1161 try: 1162 res = bb.utils.vercmp_string_op(layerver, depver, op) 1163 except bb.utils.VersionStringException as vse: 1164 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse))) 1165 if not res: 1166 parselog.error("Layer '%s' depends on version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep, layerver) 1167 errors = True 1168 else: 1169 parselog.error("Layer '%s' depends on version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, dep) 1170 errors = True 1171 else: 1172 parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep) 1173 errors = True 1174 collection_depends[c] = list(depDict.keys()) 1175 else: 1176 collection_depends[c] = [] 1177 1178 # Check recommends and store information for priority calculation 1179 recs = self.data.getVar("LAYERRECOMMENDS_%s" % c) 1180 if recs: 1181 try: 1182 recDict = bb.utils.explode_dep_versions2(recs) 1183 except bb.utils.VersionStringException as vse: 1184 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) 1185 for rec, oplist in list(recDict.items()): 1186 if rec in collection_list: 1187 if oplist: 1188 opstr = oplist[0] 1189 layerver = self.data.getVar("LAYERVERSION_%s" % rec) 1190 if layerver: 1191 (op, recver) = opstr.split() 1192 try: 1193 res = bb.utils.vercmp_string_op(layerver, recver, op) 1194 except bb.utils.VersionStringException as vse: 1195 bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) 1196 if not res: 1197 parselog.debug3("Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver) 1198 continue 1199 else: 1200 parselog.debug3("Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec) 1201 continue 1202 parselog.debug3("Layer '%s' recommends layer '%s', so we are adding it", c, rec) 1203 collection_depends[c].append(rec) 1204 else: 1205 parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec) 1206 1207 # Recursively work out collection priorities based on dependencies 1208 def calc_layer_priority(collection): 1209 if not collection_priorities[collection]: 1210 max_depprio = min_prio 1211 for dep in collection_depends[collection]: 1212 calc_layer_priority(dep) 1213 depprio = collection_priorities[dep] 1214 if depprio > max_depprio: 1215 max_depprio = depprio 1216 max_depprio += 1 1217 parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio) 1218 collection_priorities[collection] = max_depprio 1219 1220 # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities 1221 for c in collection_list: 1222 calc_layer_priority(c) 1223 regex = self.data.getVar("BBFILE_PATTERN_%s" % c) 1224 if regex is None: 1225 parselog.error("BBFILE_PATTERN_%s not defined" % c) 1226 errors = True 1227 continue 1228 elif regex == "": 1229 parselog.debug("BBFILE_PATTERN_%s is empty" % c) 1230 cre = re.compile('^NULL$') 1231 errors = False 1232 else: 1233 try: 1234 cre = re.compile(regex) 1235 except re.error: 1236 parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex) 1237 errors = True 1238 continue 1239 self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c])) 1240 if errors: 1241 # We've already printed the actual error(s) 1242 raise CollectionError("Errors during parsing layer configuration") 1243 1244 def buildSetVars(self): 1245 """ 1246 Setup any variables needed before starting a build 1247 """ 1248 t = time.gmtime() 1249 for mc in self.databuilder.mcdata: 1250 ds = self.databuilder.mcdata[mc] 1251 if not ds.getVar("BUILDNAME", False): 1252 ds.setVar("BUILDNAME", "${DATE}${TIME}") 1253 ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t)) 1254 ds.setVar("DATE", time.strftime('%Y%m%d', t)) 1255 ds.setVar("TIME", time.strftime('%H%M%S', t)) 1256 1257 def reset_mtime_caches(self): 1258 """ 1259 Reset mtime caches - this is particularly important when memory resident as something 1260 which is cached is not unlikely to have changed since the last invocation (e.g. a 1261 file associated with a recipe might have been modified by the user). 1262 """ 1263 build.reset_cache() 1264 bb.fetch._checksum_cache.mtime_cache.clear() 1265 siggen_cache = getattr(bb.parse.siggen, 'checksum_cache', None) 1266 if siggen_cache: 1267 bb.parse.siggen.checksum_cache.mtime_cache.clear() 1268 1269 def matchFiles(self, bf, mc=''): 1270 """ 1271 Find the .bb files which match the expression in 'buildfile'. 1272 """ 1273 if bf.startswith("/") or bf.startswith("../"): 1274 bf = os.path.abspath(bf) 1275 1276 collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} 1277 filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) 1278 try: 1279 os.stat(bf) 1280 bf = os.path.abspath(bf) 1281 return [bf] 1282 except OSError: 1283 regexp = re.compile(bf) 1284 matches = [] 1285 for f in filelist: 1286 if regexp.search(f) and os.path.isfile(f): 1287 matches.append(f) 1288 return matches 1289 1290 def matchFile(self, buildfile, mc=''): 1291 """ 1292 Find the .bb file which matches the expression in 'buildfile'. 1293 Raise an error if multiple files 1294 """ 1295 matches = self.matchFiles(buildfile, mc) 1296 if len(matches) != 1: 1297 if matches: 1298 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) 1299 if matches: 1300 for f in matches: 1301 msg += "\n %s" % f 1302 parselog.error(msg) 1303 else: 1304 parselog.error("Unable to find any recipe file matching '%s'" % buildfile) 1305 raise NoSpecificMatch 1306 return matches[0] 1307 1308 def buildFile(self, buildfile, task): 1309 """ 1310 Build the file matching regexp buildfile 1311 """ 1312 bb.event.fire(bb.event.BuildInit(), self.data) 1313 1314 # Too many people use -b because they think it's how you normally 1315 # specify a target to be built, so show a warning 1316 bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.") 1317 1318 self.buildFileInternal(buildfile, task) 1319 1320 def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False): 1321 """ 1322 Build the file matching regexp buildfile 1323 """ 1324 1325 # Parse the configuration here. We need to do it explicitly here since 1326 # buildFile() doesn't use the cache 1327 self.parseConfiguration() 1328 1329 # If we are told to do the None task then query the default task 1330 if task is None: 1331 task = self.configuration.cmd 1332 if not task.startswith("do_"): 1333 task = "do_%s" % task 1334 1335 fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) 1336 fn = self.matchFile(fn, mc) 1337 1338 self.buildSetVars() 1339 self.reset_mtime_caches() 1340 1341 bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) 1342 1343 layername = self.collections[mc].calc_bbfile_priority(fn)[2] 1344 infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername) 1345 infos = dict(infos) 1346 1347 fn = bb.cache.realfn2virtual(fn, cls, mc) 1348 try: 1349 info_array = infos[fn] 1350 except KeyError: 1351 bb.fatal("%s does not exist" % fn) 1352 1353 if info_array[0].skipped: 1354 bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason)) 1355 1356 self.recipecaches[mc].add_from_recipeinfo(fn, info_array) 1357 1358 # Tweak some variables 1359 item = info_array[0].pn 1360 self.recipecaches[mc].ignored_dependencies = set() 1361 self.recipecaches[mc].bbfile_priority[fn] = 1 1362 self.configuration.limited_deps = True 1363 1364 # Remove external dependencies 1365 self.recipecaches[mc].task_deps[fn]['depends'] = {} 1366 self.recipecaches[mc].deps[fn] = [] 1367 self.recipecaches[mc].rundeps[fn] = defaultdict(list) 1368 self.recipecaches[mc].runrecs[fn] = defaultdict(list) 1369 1370 bb.parse.siggen.setup_datacache(self.recipecaches) 1371 1372 # Invalidate task for target if force mode active 1373 if self.configuration.force: 1374 logger.verbose("Invalidate task %s, %s", task, fn) 1375 bb.parse.siggen.invalidate_task(task, fn) 1376 1377 # Setup taskdata structure 1378 taskdata = {} 1379 taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt) 1380 taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item) 1381 1382 if quietlog: 1383 rqloglevel = bb.runqueue.logger.getEffectiveLevel() 1384 bb.runqueue.logger.setLevel(logging.WARNING) 1385 1386 buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME") 1387 if fireevents: 1388 bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc]) 1389 if self.eventlog: 1390 self.eventlog[2].write_variables() 1391 bb.event.enable_heartbeat() 1392 1393 # Execute the runqueue 1394 runlist = [[mc, item, task, fn]] 1395 1396 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) 1397 1398 def buildFileIdle(server, rq, halt): 1399 1400 msg = None 1401 interrupted = 0 1402 if halt or self.state == state.forceshutdown: 1403 rq.finish_runqueue(True) 1404 msg = "Forced shutdown" 1405 interrupted = 2 1406 elif self.state == state.shutdown: 1407 rq.finish_runqueue(False) 1408 msg = "Stopped build" 1409 interrupted = 1 1410 failures = 0 1411 try: 1412 retval = rq.execute_runqueue() 1413 except runqueue.TaskFailure as exc: 1414 failures += len(exc.args) 1415 retval = False 1416 except SystemExit as exc: 1417 if quietlog: 1418 bb.runqueue.logger.setLevel(rqloglevel) 1419 return bb.server.process.idleFinish(str(exc)) 1420 1421 if not retval: 1422 if fireevents: 1423 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc]) 1424 bb.event.disable_heartbeat() 1425 # We trashed self.recipecaches above 1426 self._parsecache_set(False) 1427 self.configuration.limited_deps = False 1428 bb.parse.siggen.reset(self.data) 1429 if quietlog: 1430 bb.runqueue.logger.setLevel(rqloglevel) 1431 return bb.server.process.idleFinish(msg) 1432 if retval is True: 1433 return True 1434 return retval 1435 1436 self.idleCallBackRegister(buildFileIdle, rq) 1437 1438 def getTaskSignatures(self, target, tasks): 1439 sig = [] 1440 getAllTaskSignatures = False 1441 1442 if not tasks: 1443 tasks = ["do_build"] 1444 getAllTaskSignatures = True 1445 1446 for task in tasks: 1447 taskdata, runlist = self.buildTaskData(target, task, self.configuration.halt) 1448 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) 1449 rq.rqdata.prepare() 1450 1451 for l in runlist: 1452 mc, pn, taskname, fn = l 1453 1454 taskdep = rq.rqdata.dataCaches[mc].task_deps[fn] 1455 for t in taskdep['tasks']: 1456 if t in taskdep['nostamp'] or "setscene" in t: 1457 continue 1458 tid = bb.runqueue.build_tid(mc, fn, t) 1459 1460 if t in task or getAllTaskSignatures: 1461 try: 1462 rq.rqdata.prepare_task_hash(tid) 1463 sig.append([pn, t, rq.rqdata.get_task_unihash(tid)]) 1464 except KeyError: 1465 sig.append(self.getTaskSignatures(target, [t])[0]) 1466 1467 return sig 1468 1469 def buildTargets(self, targets, task): 1470 """ 1471 Attempt to build the targets specified 1472 """ 1473 1474 def buildTargetsIdle(server, rq, halt): 1475 msg = None 1476 interrupted = 0 1477 if halt or self.state == state.forceshutdown: 1478 bb.event._should_exit.set() 1479 rq.finish_runqueue(True) 1480 msg = "Forced shutdown" 1481 interrupted = 2 1482 elif self.state == state.shutdown: 1483 rq.finish_runqueue(False) 1484 msg = "Stopped build" 1485 interrupted = 1 1486 failures = 0 1487 try: 1488 retval = rq.execute_runqueue() 1489 except runqueue.TaskFailure as exc: 1490 failures += len(exc.args) 1491 retval = False 1492 except SystemExit as exc: 1493 return bb.server.process.idleFinish(str(exc)) 1494 1495 if not retval: 1496 try: 1497 for mc in self.multiconfigs: 1498 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc]) 1499 finally: 1500 bb.event.disable_heartbeat() 1501 return bb.server.process.idleFinish(msg) 1502 1503 if retval is True: 1504 return True 1505 return retval 1506 1507 self.reset_mtime_caches() 1508 self.buildSetVars() 1509 1510 # If we are told to do the None task then query the default task 1511 if task is None: 1512 task = self.configuration.cmd 1513 1514 if not task.startswith("do_"): 1515 task = "do_%s" % task 1516 1517 packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets] 1518 1519 bb.event.fire(bb.event.BuildInit(packages), self.data) 1520 1521 taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt) 1522 1523 buildname = self.data.getVar("BUILDNAME", False) 1524 1525 # make targets to always look as <target>:do_<task> 1526 ntargets = [] 1527 for target in runlist: 1528 if target[0]: 1529 ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2])) 1530 ntargets.append("%s:%s" % (target[1], target[2])) 1531 1532 for mc in self.multiconfigs: 1533 bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc]) 1534 if self.eventlog: 1535 self.eventlog[2].write_variables() 1536 bb.event.enable_heartbeat() 1537 1538 rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) 1539 if 'universe' in targets: 1540 rq.rqdata.warn_multi_bb = True 1541 1542 self.idleCallBackRegister(buildTargetsIdle, rq) 1543 1544 1545 def getAllKeysWithFlags(self, flaglist): 1546 def dummy_autorev(d): 1547 return 1548 1549 dump = {} 1550 # Horrible but for now we need to avoid any sideeffects of autorev being called 1551 saved = bb.fetch2.get_autorev 1552 bb.fetch2.get_autorev = dummy_autorev 1553 for k in self.data.keys(): 1554 try: 1555 expand = True 1556 flags = self.data.getVarFlags(k) 1557 if flags and "func" in flags and "python" in flags: 1558 expand = False 1559 v = self.data.getVar(k, expand) 1560 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart): 1561 dump[k] = { 1562 'v' : str(v) , 1563 'history' : self.data.varhistory.variable(k), 1564 } 1565 for d in flaglist: 1566 if flags and d in flags: 1567 dump[k][d] = flags[d] 1568 else: 1569 dump[k][d] = None 1570 except Exception as e: 1571 print(e) 1572 bb.fetch2.get_autorev = saved 1573 return dump 1574 1575 1576 def updateCacheSync(self): 1577 if self.state == state.running: 1578 return 1579 1580 if not self.baseconfig_valid: 1581 logger.debug("Reloading base configuration data") 1582 self.initConfigurationData() 1583 self.handlePRServ() 1584 1585 # This is called for all async commands when self.state != running 1586 def updateCache(self): 1587 if self.state == state.running: 1588 return 1589 1590 if self.state in (state.shutdown, state.forceshutdown, state.error): 1591 if hasattr(self.parser, 'shutdown'): 1592 self.parser.shutdown(clean=False) 1593 self.parser.final_cleanup() 1594 raise bb.BBHandledException() 1595 1596 if self.state != state.parsing: 1597 self.updateCacheSync() 1598 1599 if self.state != state.parsing and not self.parsecache_valid: 1600 bb.server.process.serverlog("Parsing started") 1601 self.parsewatched = {} 1602 1603 bb.parse.siggen.reset(self.data) 1604 self.parseConfiguration () 1605 if CookerFeatures.SEND_SANITYEVENTS in self.featureset: 1606 for mc in self.multiconfigs: 1607 bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc]) 1608 1609 for mc in self.multiconfigs: 1610 ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or "" 1611 self.recipecaches[mc].ignored_dependencies = set(ignore.split()) 1612 1613 for dep in self.configuration.extra_assume_provided: 1614 self.recipecaches[mc].ignored_dependencies.add(dep) 1615 1616 mcfilelist = {} 1617 total_masked = 0 1618 searchdirs = set() 1619 for mc in self.multiconfigs: 1620 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) 1621 1622 mcfilelist[mc] = filelist 1623 total_masked += masked 1624 searchdirs |= set(search) 1625 1626 # Add mtimes for directories searched for bb/bbappend files 1627 for dirent in searchdirs: 1628 self.add_filewatch([(dirent, bb.parse.cached_mtime_noerror(dirent))]) 1629 1630 self.parser = CookerParser(self, mcfilelist, total_masked) 1631 self._parsecache_set(True) 1632 1633 self.state = state.parsing 1634 1635 if not self.parser.parse_next(): 1636 collectlog.debug("parsing complete") 1637 if self.parser.error: 1638 raise bb.BBHandledException() 1639 self.show_appends_with_no_recipes() 1640 self.handlePrefProviders() 1641 for mc in self.multiconfigs: 1642 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data) 1643 self.state = state.running 1644 1645 # Send an event listing all stamps reachable after parsing 1646 # which the metadata may use to clean up stale data 1647 for mc in self.multiconfigs: 1648 event = bb.event.ReachableStamps(self.recipecaches[mc].stamp) 1649 bb.event.fire(event, self.databuilder.mcdata[mc]) 1650 return None 1651 1652 return True 1653 1654 def checkPackages(self, pkgs_to_build, task=None): 1655 1656 # Return a copy, don't modify the original 1657 pkgs_to_build = pkgs_to_build[:] 1658 1659 if not pkgs_to_build: 1660 raise NothingToBuild 1661 1662 ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() 1663 for pkg in pkgs_to_build.copy(): 1664 if pkg in ignore: 1665 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg) 1666 if pkg.startswith("multiconfig:"): 1667 pkgs_to_build.remove(pkg) 1668 pkgs_to_build.append(pkg.replace("multiconfig:", "mc:")) 1669 1670 if 'world' in pkgs_to_build: 1671 pkgs_to_build.remove('world') 1672 for mc in self.multiconfigs: 1673 bb.providers.buildWorldTargetList(self.recipecaches[mc], task) 1674 for t in self.recipecaches[mc].world_target: 1675 if mc: 1676 t = "mc:" + mc + ":" + t 1677 pkgs_to_build.append(t) 1678 1679 if 'universe' in pkgs_to_build: 1680 parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.") 1681 parselog.debug("collating packages for \"universe\"") 1682 pkgs_to_build.remove('universe') 1683 for mc in self.multiconfigs: 1684 for t in self.recipecaches[mc].universe_target: 1685 if task: 1686 foundtask = False 1687 for provider_fn in self.recipecaches[mc].providers[t]: 1688 if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']: 1689 foundtask = True 1690 break 1691 if not foundtask: 1692 bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task)) 1693 continue 1694 if mc: 1695 t = "mc:" + mc + ":" + t 1696 pkgs_to_build.append(t) 1697 1698 return pkgs_to_build 1699 1700 def pre_serve(self): 1701 return 1702 1703 def post_serve(self): 1704 self.shutdown(force=True) 1705 prserv.serv.auto_shutdown() 1706 if hasattr(bb.parse, "siggen"): 1707 bb.parse.siggen.exit() 1708 if self.hashserv: 1709 self.hashserv.process.terminate() 1710 self.hashserv.process.join() 1711 if hasattr(self, "data"): 1712 bb.event.fire(CookerExit(), self.data) 1713 1714 def shutdown(self, force=False): 1715 if force: 1716 self.state = state.forceshutdown 1717 bb.event._should_exit.set() 1718 else: 1719 self.state = state.shutdown 1720 1721 if self.parser: 1722 self.parser.shutdown(clean=False) 1723 self.parser.final_cleanup() 1724 1725 def finishcommand(self): 1726 if hasattr(self.parser, 'shutdown'): 1727 self.parser.shutdown(clean=False) 1728 self.parser.final_cleanup() 1729 self.state = state.initial 1730 bb.event._should_exit.clear() 1731 1732 def reset(self): 1733 if hasattr(bb.parse, "siggen"): 1734 bb.parse.siggen.exit() 1735 self.finishcommand() 1736 self.initConfigurationData() 1737 self.handlePRServ() 1738 1739 def clientComplete(self): 1740 """Called when the client is done using the server""" 1741 self.finishcommand() 1742 self.extraconfigdata = {} 1743 self.command.reset() 1744 if hasattr(self, "data"): 1745 self.databuilder.reset() 1746 self.data = self.databuilder.data 1747 # In theory tinfoil could have modified the base data before parsing, 1748 # ideally need to track if anything did modify the datastore 1749 self._parsecache_set(False) 1750 1751class CookerExit(bb.event.Event): 1752 """ 1753 Notify clients of the Cooker shutdown 1754 """ 1755 1756 def __init__(self): 1757 bb.event.Event.__init__(self) 1758 1759 1760class CookerCollectFiles(object): 1761 def __init__(self, priorities, mc=''): 1762 self.mc = mc 1763 self.bbappends = [] 1764 # Priorities is a list of tuples, with the second element as the pattern. 1765 # We need to sort the list with the longest pattern first, and so on to 1766 # the shortest. This allows nested layers to be properly evaluated. 1767 self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) 1768 1769 def calc_bbfile_priority(self, filename): 1770 for layername, _, regex, pri in self.bbfile_config_priorities: 1771 if regex.match(filename): 1772 return pri, regex, layername 1773 return 0, None, None 1774 1775 def get_bbfiles(self): 1776 """Get list of default .bb files by reading out the current directory""" 1777 path = os.getcwd() 1778 contents = os.listdir(path) 1779 bbfiles = [] 1780 for f in contents: 1781 if f.endswith(".bb"): 1782 bbfiles.append(os.path.abspath(os.path.join(path, f))) 1783 return bbfiles 1784 1785 def find_bbfiles(self, path): 1786 """Find all the .bb and .bbappend files in a directory""" 1787 found = [] 1788 for dir, dirs, files in os.walk(path): 1789 for ignored in ('SCCS', 'CVS', '.svn'): 1790 if ignored in dirs: 1791 dirs.remove(ignored) 1792 found += [os.path.join(dir, f) for f in files if (f.endswith(('.bb', '.bbappend')))] 1793 1794 return found 1795 1796 def collect_bbfiles(self, config, eventdata): 1797 """Collect all available .bb build files""" 1798 masked = 0 1799 1800 collectlog.debug("collecting .bb files") 1801 1802 files = (config.getVar( "BBFILES") or "").split() 1803 1804 # Sort files by priority 1805 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) 1806 config.setVar("BBFILES_PRIORITIZED", " ".join(files)) 1807 1808 if not files: 1809 files = self.get_bbfiles() 1810 1811 if not files: 1812 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") 1813 bb.event.fire(CookerExit(), eventdata) 1814 1815 # We need to track where we look so that we can know when the cache is invalid. There 1816 # is no nice way to do this, this is horrid. We intercept the os.listdir() 1817 # (or os.scandir() for python 3.6+) calls while we run glob(). 1818 origlistdir = os.listdir 1819 if hasattr(os, 'scandir'): 1820 origscandir = os.scandir 1821 searchdirs = [] 1822 1823 def ourlistdir(d): 1824 searchdirs.append(d) 1825 return origlistdir(d) 1826 1827 def ourscandir(d): 1828 searchdirs.append(d) 1829 return origscandir(d) 1830 1831 os.listdir = ourlistdir 1832 if hasattr(os, 'scandir'): 1833 os.scandir = ourscandir 1834 try: 1835 # Can't use set here as order is important 1836 newfiles = [] 1837 for f in files: 1838 if os.path.isdir(f): 1839 dirfiles = self.find_bbfiles(f) 1840 for g in dirfiles: 1841 if g not in newfiles: 1842 newfiles.append(g) 1843 else: 1844 globbed = glob.glob(f) 1845 if not globbed and os.path.exists(f): 1846 globbed = [f] 1847 # glob gives files in order on disk. Sort to be deterministic. 1848 for g in sorted(globbed): 1849 if g not in newfiles: 1850 newfiles.append(g) 1851 finally: 1852 os.listdir = origlistdir 1853 if hasattr(os, 'scandir'): 1854 os.scandir = origscandir 1855 1856 bbmask = config.getVar('BBMASK') 1857 1858 if bbmask: 1859 # First validate the individual regular expressions and ignore any 1860 # that do not compile 1861 bbmasks = [] 1862 for mask in bbmask.split(): 1863 # When constructing an older style single regex, it's possible for BBMASK 1864 # to end up beginning with '|', which matches and masks _everything_. 1865 if mask.startswith("|"): 1866 collectlog.warning("BBMASK contains regular expression beginning with '|', fixing: %s" % mask) 1867 mask = mask[1:] 1868 try: 1869 re.compile(mask) 1870 bbmasks.append(mask) 1871 except re.error: 1872 collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) 1873 1874 # Then validate the combined regular expressions. This should never 1875 # fail, but better safe than sorry... 1876 bbmask = "|".join(bbmasks) 1877 try: 1878 bbmask_compiled = re.compile(bbmask) 1879 except re.error: 1880 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) 1881 bbmask = None 1882 1883 bbfiles = [] 1884 bbappend = [] 1885 for f in newfiles: 1886 if bbmask and bbmask_compiled.search(f): 1887 collectlog.debug("skipping masked file %s", f) 1888 masked += 1 1889 continue 1890 if f.endswith('.bb'): 1891 bbfiles.append(f) 1892 elif f.endswith('.bbappend'): 1893 bbappend.append(f) 1894 else: 1895 collectlog.debug("skipping %s: unknown file extension", f) 1896 1897 # Build a list of .bbappend files for each .bb file 1898 for f in bbappend: 1899 base = os.path.basename(f).replace('.bbappend', '.bb') 1900 self.bbappends.append((base, f)) 1901 1902 # Find overlayed recipes 1903 # bbfiles will be in priority order which makes this easy 1904 bbfile_seen = dict() 1905 self.overlayed = defaultdict(list) 1906 for f in reversed(bbfiles): 1907 base = os.path.basename(f) 1908 if base not in bbfile_seen: 1909 bbfile_seen[base] = f 1910 else: 1911 topfile = bbfile_seen[base] 1912 self.overlayed[topfile].append(f) 1913 1914 return (bbfiles, masked, searchdirs) 1915 1916 def get_file_appends(self, fn): 1917 """ 1918 Returns a list of .bbappend files to apply to fn 1919 """ 1920 filelist = [] 1921 f = os.path.basename(fn) 1922 for b in self.bbappends: 1923 (bbappend, filename) = b 1924 if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): 1925 filelist.append(filename) 1926 return tuple(filelist) 1927 1928 def collection_priorities(self, pkgfns, fns, d): 1929 # Return the priorities of the entries in pkgfns 1930 # Also check that all the regexes in self.bbfile_config_priorities are used 1931 # (but to do that we need to ensure skipped recipes aren't counted, nor 1932 # collections in BBFILE_PATTERN_IGNORE_EMPTY) 1933 1934 priorities = {} 1935 seen = set() 1936 matched = set() 1937 1938 matched_regex = set() 1939 unmatched_regex = set() 1940 for _, _, regex, _ in self.bbfile_config_priorities: 1941 unmatched_regex.add(regex) 1942 1943 # Calculate priorities for each file 1944 for p in pkgfns: 1945 realfn, cls, mc = bb.cache.virtualfn2realfn(p) 1946 priorities[p], regex, _ = self.calc_bbfile_priority(realfn) 1947 if regex in unmatched_regex: 1948 matched_regex.add(regex) 1949 unmatched_regex.remove(regex) 1950 seen.add(realfn) 1951 if regex: 1952 matched.add(realfn) 1953 1954 if unmatched_regex: 1955 # Account for bbappend files 1956 for b in self.bbappends: 1957 (bbfile, append) = b 1958 seen.add(append) 1959 1960 # Account for skipped recipes 1961 seen.update(fns) 1962 1963 seen.difference_update(matched) 1964 1965 def already_matched(fn): 1966 for regex in matched_regex: 1967 if regex.match(fn): 1968 return True 1969 return False 1970 1971 for unmatch in unmatched_regex.copy(): 1972 for fn in seen: 1973 if unmatch.match(fn): 1974 # If the bbappend or file was already matched by another regex, skip it 1975 # e.g. for a layer within a layer, the outer regex could match, the inner 1976 # regex may match nothing and we should warn about that 1977 if already_matched(fn): 1978 continue 1979 unmatched_regex.remove(unmatch) 1980 break 1981 1982 for collection, pattern, regex, _ in self.bbfile_config_priorities: 1983 if regex in unmatched_regex: 1984 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': 1985 collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', 1986 collection, pattern)) 1987 1988 return priorities 1989 1990class ParsingFailure(Exception): 1991 def __init__(self, realexception, recipe): 1992 self.realexception = realexception 1993 self.recipe = recipe 1994 Exception.__init__(self, realexception, recipe) 1995 1996class Parser(multiprocessing.Process): 1997 def __init__(self, jobs, results, quit, profile): 1998 self.jobs = jobs 1999 self.results = results 2000 self.quit = quit 2001 multiprocessing.Process.__init__(self) 2002 self.context = bb.utils.get_context().copy() 2003 self.handlers = bb.event.get_class_handlers().copy() 2004 self.profile = profile 2005 self.queue_signals = False 2006 self.signal_received = [] 2007 self.signal_threadlock = threading.Lock() 2008 2009 def catch_sig(self, signum, frame): 2010 if self.queue_signals: 2011 self.signal_received.append(signum) 2012 else: 2013 self.handle_sig(signum, frame) 2014 2015 def handle_sig(self, signum, frame): 2016 if signum == signal.SIGTERM: 2017 signal.signal(signal.SIGTERM, signal.SIG_DFL) 2018 os.kill(os.getpid(), signal.SIGTERM) 2019 elif signum == signal.SIGINT: 2020 signal.default_int_handler(signum, frame) 2021 2022 def run(self): 2023 2024 if not self.profile: 2025 self.realrun() 2026 return 2027 2028 try: 2029 import cProfile as profile 2030 except: 2031 import profile 2032 prof = profile.Profile() 2033 try: 2034 profile.Profile.runcall(prof, self.realrun) 2035 finally: 2036 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name 2037 prof.dump_stats(logfile) 2038 2039 def realrun(self): 2040 # Signal handling here is hard. We must not terminate any process or thread holding the write 2041 # lock for the event stream as it will not be released, ever, and things will hang. 2042 # Python handles signals in the main thread/process but they can be raised from any thread and 2043 # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section 2044 # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any 2045 # new thread should also do so) and we defer handling but we handle with the local thread lock 2046 # held (a threading lock, not a multiprocessing one) so that no other thread in the process 2047 # can be in the critical section. 2048 signal.signal(signal.SIGTERM, self.catch_sig) 2049 signal.signal(signal.SIGHUP, signal.SIG_DFL) 2050 signal.signal(signal.SIGINT, self.catch_sig) 2051 bb.utils.set_process_name(multiprocessing.current_process().name) 2052 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) 2053 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) 2054 2055 pending = [] 2056 havejobs = True 2057 try: 2058 while havejobs or pending: 2059 if self.quit.is_set(): 2060 break 2061 2062 job = None 2063 try: 2064 job = self.jobs.pop() 2065 except IndexError: 2066 havejobs = False 2067 if job: 2068 result = self.parse(*job) 2069 # Clear the siggen cache after parsing to control memory usage, its huge 2070 bb.parse.siggen.postparsing_clean_cache() 2071 pending.append(result) 2072 2073 if pending: 2074 try: 2075 result = pending.pop() 2076 self.results.put(result, timeout=0.05) 2077 except queue.Full: 2078 pending.append(result) 2079 finally: 2080 self.results.close() 2081 self.results.join_thread() 2082 2083 def parse(self, mc, cache, filename, appends, layername): 2084 try: 2085 origfilter = bb.event.LogHandler.filter 2086 # Record the filename we're parsing into any events generated 2087 def parse_filter(self, record): 2088 record.taskpid = bb.event.worker_pid 2089 record.fn = filename 2090 return True 2091 2092 # Reset our environment and handlers to the original settings 2093 bb.utils.set_context(self.context.copy()) 2094 bb.event.set_class_handlers(self.handlers.copy()) 2095 bb.event.LogHandler.filter = parse_filter 2096 2097 return True, mc, cache.parse(filename, appends, layername) 2098 except Exception as exc: 2099 tb = sys.exc_info()[2] 2100 exc.recipe = filename 2101 exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) 2102 return True, None, exc 2103 # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown 2104 # and for example a worker thread doesn't just exit on its own in response to 2105 # a SystemExit event for example. 2106 except BaseException as exc: 2107 return True, None, ParsingFailure(exc, filename) 2108 finally: 2109 bb.event.LogHandler.filter = origfilter 2110 2111class CookerParser(object): 2112 def __init__(self, cooker, mcfilelist, masked): 2113 self.mcfilelist = mcfilelist 2114 self.cooker = cooker 2115 self.cfgdata = cooker.data 2116 self.cfghash = cooker.data_hash 2117 self.cfgbuilder = cooker.databuilder 2118 2119 # Accounting statistics 2120 self.parsed = 0 2121 self.cached = 0 2122 self.error = 0 2123 self.masked = masked 2124 2125 self.skipped = 0 2126 self.virtuals = 0 2127 2128 self.current = 0 2129 self.process_names = [] 2130 2131 self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array) 2132 self.fromcache = set() 2133 self.willparse = set() 2134 for mc in self.cooker.multiconfigs: 2135 for filename in self.mcfilelist[mc]: 2136 appends = self.cooker.collections[mc].get_file_appends(filename) 2137 layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] 2138 if not self.bb_caches[mc].cacheValid(filename, appends): 2139 self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername)) 2140 else: 2141 self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername)) 2142 2143 self.total = len(self.fromcache) + len(self.willparse) 2144 self.toparse = len(self.willparse) 2145 self.progress_chunk = int(max(self.toparse / 100, 1)) 2146 2147 self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or 2148 multiprocessing.cpu_count()), self.toparse) 2149 2150 bb.cache.SiggenRecipeInfo.reset() 2151 self.start() 2152 self.haveshutdown = False 2153 self.syncthread = None 2154 2155 def start(self): 2156 self.results = self.load_cached() 2157 self.processes = [] 2158 if self.toparse: 2159 bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) 2160 2161 self.parser_quit = multiprocessing.Event() 2162 self.result_queue = multiprocessing.Queue() 2163 2164 def chunkify(lst,n): 2165 return [lst[i::n] for i in range(n)] 2166 self.jobs = chunkify(list(self.willparse), self.num_processes) 2167 2168 for i in range(0, self.num_processes): 2169 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile) 2170 parser.start() 2171 self.process_names.append(parser.name) 2172 self.processes.append(parser) 2173 2174 self.results = itertools.chain(self.results, self.parse_generator()) 2175 2176 def shutdown(self, clean=True, eventmsg="Parsing halted due to errors"): 2177 if not self.toparse: 2178 return 2179 if self.haveshutdown: 2180 return 2181 self.haveshutdown = True 2182 2183 if clean: 2184 event = bb.event.ParseCompleted(self.cached, self.parsed, 2185 self.skipped, self.masked, 2186 self.virtuals, self.error, 2187 self.total) 2188 2189 bb.event.fire(event, self.cfgdata) 2190 else: 2191 bb.event.fire(bb.event.ParseError(eventmsg), self.cfgdata) 2192 bb.error("Parsing halted due to errors, see error messages above") 2193 2194 # Cleanup the queue before call process.join(), otherwise there might be 2195 # deadlocks. 2196 while True: 2197 try: 2198 self.result_queue.get(timeout=0.25) 2199 except queue.Empty: 2200 break 2201 2202 def sync_caches(): 2203 for c in self.bb_caches.values(): 2204 bb.cache.SiggenRecipeInfo.reset() 2205 c.sync() 2206 2207 self.syncthread = threading.Thread(target=sync_caches, name="SyncThread") 2208 self.syncthread.start() 2209 2210 self.parser_quit.set() 2211 2212 for process in self.processes: 2213 process.join(0.5) 2214 2215 for process in self.processes: 2216 if process.exitcode is None: 2217 os.kill(process.pid, signal.SIGINT) 2218 2219 for process in self.processes: 2220 process.join(0.5) 2221 2222 for process in self.processes: 2223 if process.exitcode is None: 2224 process.terminate() 2225 2226 for process in self.processes: 2227 process.join() 2228 # Added in 3.7, cleans up zombies 2229 if hasattr(process, "close"): 2230 process.close() 2231 2232 bb.codeparser.parser_cache_save() 2233 bb.codeparser.parser_cache_savemerge() 2234 bb.cache.SiggenRecipeInfo.reset() 2235 bb.fetch.fetcher_parse_done() 2236 if self.cooker.configuration.profile: 2237 profiles = [] 2238 for i in self.process_names: 2239 logfile = "profile-parse-%s.log" % i 2240 if os.path.exists(logfile): 2241 profiles.append(logfile) 2242 2243 pout = "profile-parse.log.processed" 2244 bb.utils.process_profilelog(profiles, pout = pout) 2245 print("Processed parsing statistics saved to %s" % (pout)) 2246 2247 def final_cleanup(self): 2248 if self.syncthread: 2249 self.syncthread.join() 2250 2251 def load_cached(self): 2252 for mc, cache, filename, appends, layername in self.fromcache: 2253 infos = cache.loadCached(filename, appends) 2254 yield False, mc, infos 2255 2256 def parse_generator(self): 2257 empty = False 2258 while self.processes or not empty: 2259 for process in self.processes.copy(): 2260 if not process.is_alive(): 2261 process.join() 2262 self.processes.remove(process) 2263 2264 if self.parsed >= self.toparse: 2265 break 2266 2267 try: 2268 result = self.result_queue.get(timeout=0.25) 2269 except queue.Empty: 2270 empty = True 2271 yield None, None, None 2272 else: 2273 empty = False 2274 yield result 2275 2276 if not (self.parsed >= self.toparse): 2277 raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None) 2278 2279 2280 def parse_next(self): 2281 result = [] 2282 parsed = None 2283 try: 2284 parsed, mc, result = next(self.results) 2285 if isinstance(result, BaseException): 2286 # Turn exceptions back into exceptions 2287 raise result 2288 if parsed is None: 2289 # Timeout, loop back through the main loop 2290 return True 2291 2292 except StopIteration: 2293 self.shutdown() 2294 return False 2295 except bb.BBHandledException as exc: 2296 self.error += 1 2297 logger.debug('Failed to parse recipe: %s' % exc.recipe) 2298 self.shutdown(clean=False) 2299 return False 2300 except ParsingFailure as exc: 2301 self.error += 1 2302 logger.error('Unable to parse %s: %s' % 2303 (exc.recipe, bb.exceptions.to_string(exc.realexception))) 2304 self.shutdown(clean=False) 2305 return False 2306 except bb.parse.ParseError as exc: 2307 self.error += 1 2308 logger.error(str(exc)) 2309 self.shutdown(clean=False, eventmsg=str(exc)) 2310 return False 2311 except bb.data_smart.ExpansionError as exc: 2312 self.error += 1 2313 bbdir = os.path.dirname(__file__) + os.sep 2314 etype, value, _ = sys.exc_info() 2315 tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback)) 2316 logger.error('ExpansionError during parsing %s', value.recipe, 2317 exc_info=(etype, value, tb)) 2318 self.shutdown(clean=False) 2319 return False 2320 except Exception as exc: 2321 self.error += 1 2322 etype, value, tb = sys.exc_info() 2323 if hasattr(value, "recipe"): 2324 logger.error('Unable to parse %s' % value.recipe, 2325 exc_info=(etype, value, exc.traceback)) 2326 else: 2327 # Most likely, an exception occurred during raising an exception 2328 import traceback 2329 logger.error('Exception during parse: %s' % traceback.format_exc()) 2330 self.shutdown(clean=False) 2331 return False 2332 2333 self.current += 1 2334 self.virtuals += len(result) 2335 if parsed: 2336 self.parsed += 1 2337 if self.parsed % self.progress_chunk == 0: 2338 bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse), 2339 self.cfgdata) 2340 else: 2341 self.cached += 1 2342 2343 for virtualfn, info_array in result: 2344 if info_array[0].skipped: 2345 self.skipped += 1 2346 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) 2347 self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], 2348 parsed=parsed, watcher = self.cooker.add_filewatch) 2349 return True 2350 2351 def reparse(self, filename): 2352 bb.cache.SiggenRecipeInfo.reset() 2353 to_reparse = set() 2354 for mc in self.cooker.multiconfigs: 2355 layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] 2356 to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename), layername)) 2357 2358 for mc, filename, appends, layername in to_reparse: 2359 infos = self.bb_caches[mc].parse(filename, appends, layername) 2360 for vfn, info_array in infos: 2361 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) 2362