1# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities 2# 3# Copyright (C) 2012-2017 Intel Corporation 4# Copyright (C) 2011 Mentor Graphics Corporation 5# Copyright (C) 2006-2012 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import logging 11import os 12import sys 13import time 14import atexit 15import re 16from collections import OrderedDict, defaultdict 17from functools import partial 18 19import bb.cache 20import bb.cooker 21import bb.providers 22import bb.taskdata 23import bb.utils 24import bb.command 25import bb.remotedata 26from bb.main import setup_bitbake, BitBakeConfigParameters 27import bb.fetch2 28 29 30# We need this in order to shut down the connection to the bitbake server, 31# otherwise the process will never properly exit 32_server_connections = [] 33def _terminate_connections(): 34 for connection in _server_connections: 35 connection.terminate() 36atexit.register(_terminate_connections) 37 38class TinfoilUIException(Exception): 39 """Exception raised when the UI returns non-zero from its main function""" 40 def __init__(self, returncode): 41 self.returncode = returncode 42 def __repr__(self): 43 return 'UI module main returned %d' % self.returncode 44 45class TinfoilCommandFailed(Exception): 46 """Exception raised when run_command fails""" 47 48class TinfoilDataStoreConnectorVarHistory: 49 def __init__(self, tinfoil, dsindex): 50 self.tinfoil = tinfoil 51 self.dsindex = dsindex 52 53 def remoteCommand(self, cmd, *args, **kwargs): 54 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs) 55 56 def emit(self, var, oval, val, o, d): 57 ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex) 58 o.write(ret) 59 60 def __getattr__(self, name): 61 if not hasattr(bb.data_smart.VariableHistory, name): 62 raise AttributeError("VariableHistory has no such method %s" % name) 63 64 newfunc = partial(self.remoteCommand, name) 65 setattr(self, name, newfunc) 66 return newfunc 67 68class TinfoilDataStoreConnectorIncHistory: 69 def __init__(self, tinfoil, dsindex): 70 self.tinfoil = tinfoil 71 self.dsindex = dsindex 72 73 def remoteCommand(self, cmd, *args, **kwargs): 74 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs) 75 76 def __getattr__(self, name): 77 if not hasattr(bb.data_smart.IncludeHistory, name): 78 raise AttributeError("IncludeHistory has no such method %s" % name) 79 80 newfunc = partial(self.remoteCommand, name) 81 setattr(self, name, newfunc) 82 return newfunc 83 84class TinfoilDataStoreConnector: 85 """ 86 Connector object used to enable access to datastore objects via tinfoil 87 Method calls are transmitted to the remote datastore for processing, if a datastore is 88 returned we return a connector object for the new store 89 """ 90 91 def __init__(self, tinfoil, dsindex): 92 self.tinfoil = tinfoil 93 self.dsindex = dsindex 94 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex) 95 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex) 96 97 def remoteCommand(self, cmd, *args, **kwargs): 98 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs) 99 if isinstance(ret, bb.command.DataStoreConnectionHandle): 100 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex) 101 return ret 102 103 def __getattr__(self, name): 104 if not hasattr(bb.data._dict_type, name): 105 raise AttributeError("Data store has no such method %s" % name) 106 107 newfunc = partial(self.remoteCommand, name) 108 setattr(self, name, newfunc) 109 return newfunc 110 111 def __iter__(self): 112 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {}) 113 for k in keys: 114 yield k 115 116class TinfoilCookerAdapter: 117 """ 118 Provide an adapter for existing code that expects to access a cooker object via Tinfoil, 119 since now Tinfoil is on the client side it no longer has direct access. 120 """ 121 122 class TinfoilCookerCollectionAdapter: 123 """ cooker.collection adapter """ 124 def __init__(self, tinfoil, mc=''): 125 self.tinfoil = tinfoil 126 self.mc = mc 127 def get_file_appends(self, fn): 128 return self.tinfoil.get_file_appends(fn, self.mc) 129 def __getattr__(self, name): 130 if name == 'overlayed': 131 return self.tinfoil.get_overlayed_recipes(self.mc) 132 elif name == 'bbappends': 133 return self.tinfoil.run_command('getAllAppends', self.mc) 134 else: 135 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 136 137 class TinfoilRecipeCacheAdapter: 138 """ cooker.recipecache adapter """ 139 def __init__(self, tinfoil, mc=''): 140 self.tinfoil = tinfoil 141 self.mc = mc 142 self._cache = {} 143 144 def get_pkg_pn_fn(self): 145 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or []) 146 pkg_fn = {} 147 for pn, fnlist in pkg_pn.items(): 148 for fn in fnlist: 149 pkg_fn[fn] = pn 150 self._cache['pkg_pn'] = pkg_pn 151 self._cache['pkg_fn'] = pkg_fn 152 153 def __getattr__(self, name): 154 # Grab these only when they are requested since they aren't always used 155 if name in self._cache: 156 return self._cache[name] 157 elif name == 'pkg_pn': 158 self.get_pkg_pn_fn() 159 return self._cache[name] 160 elif name == 'pkg_fn': 161 self.get_pkg_pn_fn() 162 return self._cache[name] 163 elif name == 'deps': 164 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or []) 165 elif name == 'rundeps': 166 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or []) 167 elif name == 'runrecs': 168 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or []) 169 elif name == 'pkg_pepvpr': 170 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {} 171 elif name == 'inherits': 172 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {} 173 elif name == 'bbfile_priority': 174 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {} 175 elif name == 'pkg_dp': 176 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {} 177 elif name == 'fn_provides': 178 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {} 179 elif name == 'packages': 180 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {} 181 elif name == 'packages_dynamic': 182 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {} 183 elif name == 'rproviders': 184 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {} 185 else: 186 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 187 188 self._cache[name] = attrvalue 189 return attrvalue 190 191 def __init__(self, tinfoil): 192 self.tinfoil = tinfoil 193 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() 194 self.collections = {} 195 self.recipecaches = {} 196 for mc in self.multiconfigs: 197 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) 198 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) 199 self._cache = {} 200 def __getattr__(self, name): 201 # Grab these only when they are requested since they aren't always used 202 if name in self._cache: 203 return self._cache[name] 204 elif name == 'skiplist': 205 attrvalue = self.tinfoil.get_skipped_recipes() 206 elif name == 'bbfile_config_priorities': 207 ret = self.tinfoil.run_command('getLayerPriorities') 208 bbfile_config_priorities = [] 209 for collection, pattern, regex, pri in ret: 210 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri)) 211 212 attrvalue = bbfile_config_priorities 213 else: 214 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 215 216 self._cache[name] = attrvalue 217 return attrvalue 218 219 def findBestProvider(self, pn): 220 return self.tinfoil.find_best_provider(pn) 221 222 223class TinfoilRecipeInfo: 224 """ 225 Provides a convenient representation of the cached information for a single recipe. 226 Some attributes are set on construction, others are read on-demand (which internally 227 may result in a remote procedure call to the bitbake server the first time). 228 Note that only information which is cached is available through this object - if 229 you need other variable values you will need to parse the recipe using 230 Tinfoil.parse_recipe(). 231 """ 232 def __init__(self, recipecache, d, pn, fn, fns): 233 self._recipecache = recipecache 234 self._d = d 235 self.pn = pn 236 self.fn = fn 237 self.fns = fns 238 self.inherit_files = recipecache.inherits[fn] 239 self.depends = recipecache.deps[fn] 240 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn] 241 self._cached_packages = None 242 self._cached_rprovides = None 243 self._cached_packages_dynamic = None 244 245 def __getattr__(self, name): 246 if name == 'alternates': 247 return [x for x in self.fns if x != self.fn] 248 elif name == 'rdepends': 249 return self._recipecache.rundeps[self.fn] 250 elif name == 'rrecommends': 251 return self._recipecache.runrecs[self.fn] 252 elif name == 'provides': 253 return self._recipecache.fn_provides[self.fn] 254 elif name == 'packages': 255 if self._cached_packages is None: 256 self._cached_packages = [] 257 for pkg, fns in self._recipecache.packages.items(): 258 if self.fn in fns: 259 self._cached_packages.append(pkg) 260 return self._cached_packages 261 elif name == 'packages_dynamic': 262 if self._cached_packages_dynamic is None: 263 self._cached_packages_dynamic = [] 264 for pkg, fns in self._recipecache.packages_dynamic.items(): 265 if self.fn in fns: 266 self._cached_packages_dynamic.append(pkg) 267 return self._cached_packages_dynamic 268 elif name == 'rprovides': 269 if self._cached_rprovides is None: 270 self._cached_rprovides = [] 271 for pkg, fns in self._recipecache.rproviders.items(): 272 if self.fn in fns: 273 self._cached_rprovides.append(pkg) 274 return self._cached_rprovides 275 else: 276 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 277 def inherits(self, only_recipe=False): 278 """ 279 Get the inherited classes for a recipe. Returns the class names only. 280 Parameters: 281 only_recipe: True to return only the classes inherited by the recipe 282 itself, False to return all classes inherited within 283 the context for the recipe (which includes globally 284 inherited classes). 285 """ 286 if only_recipe: 287 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')] 288 else: 289 global_inherit = [] 290 for clsfile in self.inherit_files: 291 if only_recipe and clsfile in global_inherit: 292 continue 293 clsname = os.path.splitext(os.path.basename(clsfile))[0] 294 yield clsname 295 def __str__(self): 296 return '%s' % self.pn 297 298 299class Tinfoil: 300 """ 301 Tinfoil - an API for scripts and utilities to query 302 BitBake internals and perform build operations. 303 """ 304 305 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True): 306 """ 307 Create a new tinfoil object. 308 Parameters: 309 output: specifies where console output should be sent. Defaults 310 to sys.stdout. 311 tracking: True to enable variable history tracking, False to 312 disable it (default). Enabling this has a minor 313 performance impact so typically it isn't enabled 314 unless you need to query variable history. 315 setup_logging: True to setup a logger so that things like 316 bb.warn() will work immediately and timeout warnings 317 are visible; False to let BitBake do this itself. 318 """ 319 self.logger = logging.getLogger('BitBake') 320 self.config_data = None 321 self.cooker = None 322 self.tracking = tracking 323 self.ui_module = None 324 self.server_connection = None 325 self.recipes_parsed = False 326 self.quiet = 0 327 self.oldhandlers = self.logger.handlers[:] 328 self.localhandlers = [] 329 if setup_logging: 330 # This is the *client-side* logger, nothing to do with 331 # logging messages from the server 332 bb.msg.logger_create('BitBake', output) 333 for handler in self.logger.handlers: 334 if handler not in self.oldhandlers: 335 self.localhandlers.append(handler) 336 337 def __enter__(self): 338 return self 339 340 def __exit__(self, type, value, traceback): 341 self.shutdown() 342 343 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None): 344 """ 345 Prepares the underlying BitBake system to be used via tinfoil. 346 This function must be called prior to calling any of the other 347 functions in the API. 348 NOTE: if you call prepare() you must absolutely call shutdown() 349 before your code terminates. You can use a "with" block to ensure 350 this happens e.g. 351 352 with bb.tinfoil.Tinfoil() as tinfoil: 353 tinfoil.prepare() 354 ... 355 356 Parameters: 357 config_only: True to read only the configuration and not load 358 the cache / parse recipes. This is useful if you just 359 want to query the value of a variable at the global 360 level or you want to do anything else that doesn't 361 involve knowing anything about the recipes in the 362 current configuration. False loads the cache / parses 363 recipes. 364 config_params: optionally specify your own configuration 365 parameters. If not specified an instance of 366 TinfoilConfigParameters will be created internally. 367 quiet: quiet level controlling console output - equivalent 368 to bitbake's -q/--quiet option. Default of 0 gives 369 the same output level as normal bitbake execution. 370 extra_features: extra features to be added to the feature 371 set requested from the server. See 372 CookerFeatures._feature_list for possible 373 features. 374 """ 375 self.quiet = quiet 376 377 if self.tracking: 378 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING] 379 else: 380 extrafeatures = [] 381 382 if extra_features: 383 extrafeatures += extra_features 384 385 if not config_params: 386 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) 387 388 if not config_only: 389 # Disable local loggers because the UI module is going to set up its own 390 for handler in self.localhandlers: 391 self.logger.handlers.remove(handler) 392 self.localhandlers = [] 393 394 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures) 395 396 self.ui_module = ui_module 397 398 # Ensure the path to bitbake's bin directory is in PATH so that things like 399 # bitbake-worker can be run (usually this is the case, but it doesn't have to be) 400 path = os.getenv('PATH').split(':') 401 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin')) 402 for entry in path: 403 if entry.endswith(os.sep): 404 entry = entry[:-1] 405 if os.path.abspath(entry) == bitbakebinpath: 406 break 407 else: 408 path.insert(0, bitbakebinpath) 409 os.environ['PATH'] = ':'.join(path) 410 411 if self.server_connection: 412 _server_connections.append(self.server_connection) 413 if config_only: 414 config_params.updateToServer(self.server_connection.connection, os.environ.copy()) 415 self.run_command('parseConfiguration') 416 else: 417 self.run_actions(config_params) 418 self.recipes_parsed = True 419 420 self.config_data = TinfoilDataStoreConnector(self, 0) 421 self.cooker = TinfoilCookerAdapter(self) 422 self.cooker_data = self.cooker.recipecaches[''] 423 else: 424 raise Exception('Failed to start bitbake server') 425 426 def run_actions(self, config_params): 427 """ 428 Run the actions specified in config_params through the UI. 429 """ 430 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params) 431 if ret: 432 raise TinfoilUIException(ret) 433 434 def parseRecipes(self): 435 """ 436 Legacy function - use parse_recipes() instead. 437 """ 438 self.parse_recipes() 439 440 def parse_recipes(self): 441 """ 442 Load information on all recipes. Normally you should specify 443 config_only=False when calling prepare() instead of using this 444 function; this function is designed for situations where you need 445 to initialise Tinfoil and use it with config_only=True first and 446 then conditionally call this function to parse recipes later. 447 """ 448 config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet) 449 self.run_actions(config_params) 450 self.recipes_parsed = True 451 452 def modified_files(self): 453 """ 454 Notify the server it needs to revalidate it's caches since the client has modified files 455 """ 456 self.run_command("revalidateCaches") 457 458 def run_command(self, command, *params, handle_events=True): 459 """ 460 Run a command on the server (as implemented in bb.command). 461 Note that there are two types of command - synchronous and 462 asynchronous; in order to receive the results of asynchronous 463 commands you will need to set an appropriate event mask 464 using set_event_mask() and listen for the result using 465 wait_event() - with the correct event mask you'll at least get 466 bb.command.CommandCompleted and possibly other events before 467 that depending on the command. 468 """ 469 if not self.server_connection: 470 raise Exception('Not connected to server (did you call .prepare()?)') 471 472 commandline = [command] 473 if params: 474 commandline.extend(params) 475 try: 476 result = self.server_connection.connection.runCommand(commandline) 477 finally: 478 while handle_events: 479 event = self.wait_event() 480 if not event: 481 break 482 if isinstance(event, logging.LogRecord): 483 if event.taskpid == 0 or event.levelno > logging.INFO: 484 self.logger.handle(event) 485 if result[1]: 486 raise TinfoilCommandFailed(result[1]) 487 return result[0] 488 489 def set_event_mask(self, eventlist): 490 """Set the event mask which will be applied within wait_event()""" 491 if not self.server_connection: 492 raise Exception('Not connected to server (did you call .prepare()?)') 493 llevel, debug_domains = bb.msg.constructLogOptions() 494 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist) 495 if not ret: 496 raise Exception('setEventMask failed') 497 498 def wait_event(self, timeout=0): 499 """ 500 Wait for an event from the server for the specified time. 501 A timeout of 0 means don't wait if there are no events in the queue. 502 Returns the next event in the queue or None if the timeout was 503 reached. Note that in order to receive any events you will 504 first need to set the internal event mask using set_event_mask() 505 (otherwise whatever event mask the UI set up will be in effect). 506 """ 507 if not self.server_connection: 508 raise Exception('Not connected to server (did you call .prepare()?)') 509 return self.server_connection.events.waitEvent(timeout) 510 511 def get_overlayed_recipes(self, mc=''): 512 """ 513 Find recipes which are overlayed (i.e. where recipes exist in multiple layers) 514 """ 515 return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) 516 517 def get_skipped_recipes(self): 518 """ 519 Find recipes which were skipped (i.e. SkipRecipe was raised 520 during parsing). 521 """ 522 return OrderedDict(self.run_command('getSkippedRecipes')) 523 524 def get_all_providers(self, mc=''): 525 return defaultdict(list, self.run_command('allProviders', mc)) 526 527 def find_providers(self, mc=''): 528 return self.run_command('findProviders', mc) 529 530 def find_best_provider(self, pn): 531 return self.run_command('findBestProvider', pn) 532 533 def get_runtime_providers(self, rdep): 534 return self.run_command('getRuntimeProviders', rdep) 535 536 def get_recipe_file(self, pn): 537 """ 538 Get the file name for the specified recipe/target. Raises 539 bb.providers.NoProvider if there is no match or the recipe was 540 skipped. 541 """ 542 best = self.find_best_provider(pn) 543 if not best or (len(best) > 3 and not best[3]): 544 skiplist = self.get_skipped_recipes() 545 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist) 546 skipreasons = taskdata.get_reasons(pn) 547 if skipreasons: 548 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons))) 549 else: 550 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) 551 return best[3] 552 553 def get_file_appends(self, fn, mc=''): 554 """ 555 Find the bbappends for a recipe file 556 """ 557 return self.run_command('getFileAppends', fn, mc) 558 559 def all_recipes(self, mc='', sort=True): 560 """ 561 Enable iterating over all recipes in the current configuration. 562 Returns an iterator over TinfoilRecipeInfo objects created on demand. 563 Parameters: 564 mc: The multiconfig, default of '' uses the main configuration. 565 sort: True to sort recipes alphabetically (default), False otherwise 566 """ 567 recipecache = self.cooker.recipecaches[mc] 568 if sort: 569 recipes = sorted(recipecache.pkg_pn.items()) 570 else: 571 recipes = recipecache.pkg_pn.items() 572 for pn, fns in recipes: 573 prov = self.find_best_provider(pn) 574 recipe = TinfoilRecipeInfo(recipecache, 575 self.config_data, 576 pn=pn, 577 fn=prov[3], 578 fns=fns) 579 yield recipe 580 581 def all_recipe_files(self, mc='', variants=True, preferred_only=False): 582 """ 583 Enable iterating over all recipe files in the current configuration. 584 Returns an iterator over file paths. 585 Parameters: 586 mc: The multiconfig, default of '' uses the main configuration. 587 variants: True to include variants of recipes created through 588 BBCLASSEXTEND (default) or False to exclude them 589 preferred_only: True to include only the preferred recipe where 590 multiple exist providing the same PN, False to list 591 all recipes 592 """ 593 recipecache = self.cooker.recipecaches[mc] 594 if preferred_only: 595 files = [] 596 for pn in recipecache.pkg_pn.keys(): 597 prov = self.find_best_provider(pn) 598 files.append(prov[3]) 599 else: 600 files = recipecache.pkg_fn.keys() 601 for fn in sorted(files): 602 if not variants and fn.startswith('virtual:'): 603 continue 604 yield fn 605 606 607 def get_recipe_info(self, pn, mc=''): 608 """ 609 Get information on a specific recipe in the current configuration by name (PN). 610 Returns a TinfoilRecipeInfo object created on demand. 611 Parameters: 612 mc: The multiconfig, default of '' uses the main configuration. 613 """ 614 recipecache = self.cooker.recipecaches[mc] 615 prov = self.find_best_provider(pn) 616 fn = prov[3] 617 if fn: 618 actual_pn = recipecache.pkg_fn[fn] 619 recipe = TinfoilRecipeInfo(recipecache, 620 self.config_data, 621 pn=actual_pn, 622 fn=fn, 623 fns=recipecache.pkg_pn[actual_pn]) 624 return recipe 625 else: 626 return None 627 628 def parse_recipe(self, pn): 629 """ 630 Parse the specified recipe and return a datastore object 631 representing the environment for the recipe. 632 """ 633 fn = self.get_recipe_file(pn) 634 return self.parse_recipe_file(fn) 635 636 def finalizeData(self): 637 """ 638 Run anonymous functions and expand keys 639 """ 640 return self._reconvert_type(self.run_command('finalizeData'), 'DataStoreConnectionHandle') 641 642 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): 643 """ 644 Parse the specified recipe file (with or without bbappends) 645 and return a datastore object representing the environment 646 for the recipe. 647 Parameters: 648 fn: recipe file to parse - can be a file path or virtual 649 specification 650 appends: True to apply bbappends, False otherwise 651 appendlist: optional list of bbappend files to apply, if you 652 want to filter them 653 """ 654 if self.tracking: 655 # Enable history tracking just for the parse operation 656 self.run_command('enableDataTracking') 657 try: 658 if appends and appendlist == []: 659 appends = False 660 if config_data: 661 config_data = bb.data.createCopy(config_data) 662 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex) 663 else: 664 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist) 665 if dscon: 666 return self._reconvert_type(dscon, 'DataStoreConnectionHandle') 667 else: 668 return None 669 finally: 670 if self.tracking: 671 self.run_command('disableDataTracking') 672 673 def build_file(self, buildfile, task, internal=True): 674 """ 675 Runs the specified task for just a single recipe (i.e. no dependencies). 676 This is equivalent to bitbake -b, except with the default internal=True 677 no warning about dependencies will be produced, normal info messages 678 from the runqueue will be silenced and BuildInit, BuildStarted and 679 BuildCompleted events will not be fired. 680 """ 681 return self.run_command('buildFile', buildfile, task, internal) 682 683 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None): 684 """ 685 Builds the specified targets. This is equivalent to a normal invocation 686 of bitbake. Has built-in event handling which is enabled by default and 687 can be extended if needed. 688 Parameters: 689 targets: 690 One or more targets to build. Can be a list or a 691 space-separated string. 692 task: 693 The task to run; if None then the value of BB_DEFAULT_TASK 694 will be used. Default None. 695 handle_events: 696 True to handle events in a similar way to normal bitbake 697 invocation with knotty; False to return immediately (on the 698 assumption that the caller will handle the events instead). 699 Default True. 700 extra_events: 701 An optional list of events to add to the event mask (if 702 handle_events=True). If you add events here you also need 703 to specify a callback function in event_callback that will 704 handle the additional events. Default None. 705 event_callback: 706 An optional function taking a single parameter which 707 will be called first upon receiving any event (if 708 handle_events=True) so that the caller can override or 709 extend the event handling. Default None. 710 """ 711 if isinstance(targets, str): 712 targets = targets.split() 713 if not task: 714 task = self.config_data.getVar('BB_DEFAULT_TASK') 715 716 if handle_events: 717 # A reasonable set of default events matching up with those we handle below 718 eventmask = [ 719 'bb.event.BuildStarted', 720 'bb.event.BuildCompleted', 721 'logging.LogRecord', 722 'bb.event.NoProvider', 723 'bb.command.CommandCompleted', 724 'bb.command.CommandFailed', 725 'bb.build.TaskStarted', 726 'bb.build.TaskFailed', 727 'bb.build.TaskSucceeded', 728 'bb.build.TaskFailedSilent', 729 'bb.build.TaskProgress', 730 'bb.runqueue.runQueueTaskStarted', 731 'bb.runqueue.sceneQueueTaskStarted', 732 'bb.event.ProcessStarted', 733 'bb.event.ProcessProgress', 734 'bb.event.ProcessFinished', 735 ] 736 if extra_events: 737 eventmask.extend(extra_events) 738 ret = self.set_event_mask(eventmask) 739 740 includelogs = self.config_data.getVar('BBINCLUDELOGS') 741 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES') 742 743 ret = self.run_command('buildTargets', targets, task) 744 if handle_events: 745 lastevent = time.time() 746 result = False 747 # Borrowed from knotty, instead somewhat hackily we use the helper 748 # as the object to store "shutdown" on 749 helper = bb.ui.uihelper.BBUIHelper() 750 helper.shutdown = 0 751 parseprogress = None 752 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet) 753 try: 754 while True: 755 try: 756 event = self.wait_event(0.25) 757 if event: 758 lastevent = time.time() 759 if event_callback and event_callback(event): 760 continue 761 if helper.eventHandler(event): 762 if isinstance(event, bb.build.TaskFailedSilent): 763 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile) 764 elif isinstance(event, bb.build.TaskFailed): 765 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter) 766 continue 767 if isinstance(event, bb.event.ProcessStarted): 768 if self.quiet > 1: 769 continue 770 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total) 771 parseprogress.start(False) 772 continue 773 if isinstance(event, bb.event.ProcessProgress): 774 if self.quiet > 1: 775 continue 776 if parseprogress: 777 parseprogress.update(event.progress) 778 else: 779 bb.warn("Got ProcessProgress event for something that never started?") 780 continue 781 if isinstance(event, bb.event.ProcessFinished): 782 if self.quiet > 1: 783 continue 784 if parseprogress: 785 parseprogress.finish() 786 parseprogress = None 787 continue 788 if isinstance(event, bb.command.CommandCompleted): 789 result = True 790 break 791 if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)): 792 self.logger.error(str(event)) 793 result = False 794 break 795 if isinstance(event, logging.LogRecord): 796 if event.taskpid == 0 or event.levelno > logging.INFO: 797 self.logger.handle(event) 798 continue 799 if isinstance(event, bb.event.NoProvider): 800 self.logger.error(str(event)) 801 result = False 802 break 803 elif helper.shutdown > 1: 804 break 805 termfilter.updateFooter() 806 if time.time() > (lastevent + (3*60)): 807 if not self.run_command('ping', handle_events=False): 808 print("\nUnable to ping server and no events, closing down...\n") 809 return False 810 except KeyboardInterrupt: 811 termfilter.clearFooter() 812 if helper.shutdown == 1: 813 print("\nSecond Keyboard Interrupt, stopping...\n") 814 ret = self.run_command("stateForceShutdown") 815 if ret and ret[2]: 816 self.logger.error("Unable to cleanly stop: %s" % ret[2]) 817 elif helper.shutdown == 0: 818 print("\nKeyboard Interrupt, closing down...\n") 819 interrupted = True 820 ret = self.run_command("stateShutdown") 821 if ret and ret[2]: 822 self.logger.error("Unable to cleanly shutdown: %s" % ret[2]) 823 helper.shutdown = helper.shutdown + 1 824 termfilter.clearFooter() 825 finally: 826 termfilter.finish() 827 if helper.failed_tasks: 828 result = False 829 return result 830 else: 831 return ret 832 833 def shutdown(self): 834 """ 835 Shut down tinfoil. Disconnects from the server and gracefully 836 releases any associated resources. You must call this function if 837 prepare() has been called, or use a with... block when you create 838 the tinfoil object which will ensure that it gets called. 839 """ 840 try: 841 if self.server_connection: 842 try: 843 self.run_command('clientComplete') 844 finally: 845 _server_connections.remove(self.server_connection) 846 bb.event.ui_queue = [] 847 self.server_connection.terminate() 848 self.server_connection = None 849 850 finally: 851 # Restore logging handlers to how it looked when we started 852 if self.oldhandlers: 853 for handler in self.logger.handlers: 854 if handler not in self.oldhandlers: 855 self.logger.handlers.remove(handler) 856 857 def _reconvert_type(self, obj, origtypename): 858 """ 859 Convert an object back to the right type, in the case 860 that marshalling has changed it (especially with xmlrpc) 861 """ 862 supported_types = { 863 'set': set, 864 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle, 865 } 866 867 origtype = supported_types.get(origtypename, None) 868 if origtype is None: 869 raise Exception('Unsupported type "%s"' % origtypename) 870 if type(obj) == origtype: 871 newobj = obj 872 elif isinstance(obj, dict): 873 # New style class 874 newobj = origtype() 875 for k,v in obj.items(): 876 setattr(newobj, k, v) 877 else: 878 # Assume we can coerce the type 879 newobj = origtype(obj) 880 881 if isinstance(newobj, bb.command.DataStoreConnectionHandle): 882 newobj = TinfoilDataStoreConnector(self, newobj.dsindex) 883 884 return newobj 885 886 887class TinfoilConfigParameters(BitBakeConfigParameters): 888 889 def __init__(self, config_only, **options): 890 self.initial_options = options 891 # Apply some sane defaults 892 if not 'parse_only' in options: 893 self.initial_options['parse_only'] = not config_only 894 #if not 'status_only' in options: 895 # self.initial_options['status_only'] = config_only 896 if not 'ui' in options: 897 self.initial_options['ui'] = 'knotty' 898 if not 'argv' in options: 899 self.initial_options['argv'] = [] 900 901 super(TinfoilConfigParameters, self).__init__() 902 903 def parseCommandLine(self, argv=None): 904 # We don't want any parameters parsed from the command line 905 opts = super(TinfoilConfigParameters, self).parseCommandLine([]) 906 for key, val in self.initial_options.items(): 907 setattr(opts[0], key, val) 908 return opts 909