1# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities 2# 3# Copyright (C) 2012-2017 Intel Corporation 4# Copyright (C) 2011 Mentor Graphics Corporation 5# Copyright (C) 2006-2012 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import logging 11import os 12import sys 13import time 14import atexit 15import re 16from collections import OrderedDict, defaultdict 17from functools import partial 18from contextlib import contextmanager 19 20import bb.cache 21import bb.cooker 22import bb.providers 23import bb.taskdata 24import bb.utils 25import bb.command 26import bb.remotedata 27from bb.main import setup_bitbake, BitBakeConfigParameters 28import bb.fetch2 29 30 31# We need this in order to shut down the connection to the bitbake server, 32# otherwise the process will never properly exit 33_server_connections = [] 34def _terminate_connections(): 35 for connection in _server_connections: 36 connection.terminate() 37atexit.register(_terminate_connections) 38 39class TinfoilUIException(Exception): 40 """Exception raised when the UI returns non-zero from its main function""" 41 def __init__(self, returncode): 42 self.returncode = returncode 43 def __repr__(self): 44 return 'UI module main returned %d' % self.returncode 45 46class TinfoilCommandFailed(Exception): 47 """Exception raised when run_command fails""" 48 49class TinfoilDataStoreConnectorVarHistory: 50 def __init__(self, tinfoil, dsindex): 51 self.tinfoil = tinfoil 52 self.dsindex = dsindex 53 54 def remoteCommand(self, cmd, *args, **kwargs): 55 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs) 56 57 def emit(self, var, oval, val, o, d): 58 ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex) 59 o.write(ret) 60 61 def __getattr__(self, name): 62 if not hasattr(bb.data_smart.VariableHistory, name): 63 raise AttributeError("VariableHistory has no such method %s" % name) 64 65 newfunc = partial(self.remoteCommand, name) 66 setattr(self, name, newfunc) 67 return newfunc 68 69class TinfoilDataStoreConnectorIncHistory: 70 def __init__(self, tinfoil, dsindex): 71 self.tinfoil = tinfoil 72 self.dsindex = dsindex 73 74 def remoteCommand(self, cmd, *args, **kwargs): 75 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs) 76 77 def __getattr__(self, name): 78 if not hasattr(bb.data_smart.IncludeHistory, name): 79 raise AttributeError("IncludeHistory has no such method %s" % name) 80 81 newfunc = partial(self.remoteCommand, name) 82 setattr(self, name, newfunc) 83 return newfunc 84 85class TinfoilDataStoreConnector: 86 """ 87 Connector object used to enable access to datastore objects via tinfoil 88 Method calls are transmitted to the remote datastore for processing, if a datastore is 89 returned we return a connector object for the new store 90 """ 91 92 def __init__(self, tinfoil, dsindex): 93 self.tinfoil = tinfoil 94 self.dsindex = dsindex 95 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex) 96 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex) 97 98 def remoteCommand(self, cmd, *args, **kwargs): 99 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs) 100 if isinstance(ret, bb.command.DataStoreConnectionHandle): 101 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex) 102 return ret 103 104 def __getattr__(self, name): 105 if not hasattr(bb.data._dict_type, name): 106 raise AttributeError("Data store has no such method %s" % name) 107 108 newfunc = partial(self.remoteCommand, name) 109 setattr(self, name, newfunc) 110 return newfunc 111 112 def __iter__(self): 113 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {}) 114 for k in keys: 115 yield k 116 117class TinfoilCookerAdapter: 118 """ 119 Provide an adapter for existing code that expects to access a cooker object via Tinfoil, 120 since now Tinfoil is on the client side it no longer has direct access. 121 """ 122 123 class TinfoilCookerCollectionAdapter: 124 """ cooker.collection adapter """ 125 def __init__(self, tinfoil, mc=''): 126 self.tinfoil = tinfoil 127 self.mc = mc 128 def get_file_appends(self, fn): 129 return self.tinfoil.get_file_appends(fn, self.mc) 130 def __getattr__(self, name): 131 if name == 'overlayed': 132 return self.tinfoil.get_overlayed_recipes(self.mc) 133 elif name == 'bbappends': 134 return self.tinfoil.run_command('getAllAppends', self.mc) 135 else: 136 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 137 138 class TinfoilRecipeCacheAdapter: 139 """ cooker.recipecache adapter """ 140 def __init__(self, tinfoil, mc=''): 141 self.tinfoil = tinfoil 142 self.mc = mc 143 self._cache = {} 144 145 def get_pkg_pn_fn(self): 146 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or []) 147 pkg_fn = {} 148 for pn, fnlist in pkg_pn.items(): 149 for fn in fnlist: 150 pkg_fn[fn] = pn 151 self._cache['pkg_pn'] = pkg_pn 152 self._cache['pkg_fn'] = pkg_fn 153 154 def __getattr__(self, name): 155 # Grab these only when they are requested since they aren't always used 156 if name in self._cache: 157 return self._cache[name] 158 elif name == 'pkg_pn': 159 self.get_pkg_pn_fn() 160 return self._cache[name] 161 elif name == 'pkg_fn': 162 self.get_pkg_pn_fn() 163 return self._cache[name] 164 elif name == 'deps': 165 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or []) 166 elif name == 'rundeps': 167 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or []) 168 elif name == 'runrecs': 169 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or []) 170 elif name == 'pkg_pepvpr': 171 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {} 172 elif name == 'inherits': 173 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {} 174 elif name == 'bbfile_priority': 175 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {} 176 elif name == 'pkg_dp': 177 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {} 178 elif name == 'fn_provides': 179 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {} 180 elif name == 'packages': 181 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {} 182 elif name == 'packages_dynamic': 183 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {} 184 elif name == 'rproviders': 185 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {} 186 else: 187 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 188 189 self._cache[name] = attrvalue 190 return attrvalue 191 192 class TinfoilSkiplistByMcAdapter: 193 def __init__(self, tinfoil): 194 self.tinfoil = tinfoil 195 196 def __getitem__(self, mc): 197 return self.tinfoil.get_skipped_recipes(mc) 198 199 def __init__(self, tinfoil): 200 self.tinfoil = tinfoil 201 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() 202 self.collections = {} 203 self.recipecaches = {} 204 self.skiplist_by_mc = self.TinfoilSkiplistByMcAdapter(tinfoil) 205 for mc in self.multiconfigs: 206 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) 207 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) 208 self._cache = {} 209 def __getattr__(self, name): 210 # Grab these only when they are requested since they aren't always used 211 if name in self._cache: 212 return self._cache[name] 213 elif name == 'bbfile_config_priorities': 214 ret = self.tinfoil.run_command('getLayerPriorities') 215 bbfile_config_priorities = [] 216 for collection, pattern, regex, pri in ret: 217 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri)) 218 219 attrvalue = bbfile_config_priorities 220 else: 221 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 222 223 self._cache[name] = attrvalue 224 return attrvalue 225 226 def findBestProvider(self, pn): 227 return self.tinfoil.find_best_provider(pn) 228 229 230class TinfoilRecipeInfo: 231 """ 232 Provides a convenient representation of the cached information for a single recipe. 233 Some attributes are set on construction, others are read on-demand (which internally 234 may result in a remote procedure call to the bitbake server the first time). 235 Note that only information which is cached is available through this object - if 236 you need other variable values you will need to parse the recipe using 237 Tinfoil.parse_recipe(). 238 """ 239 def __init__(self, recipecache, d, pn, fn, fns): 240 self._recipecache = recipecache 241 self._d = d 242 self.pn = pn 243 self.fn = fn 244 self.fns = fns 245 self.inherit_files = recipecache.inherits[fn] 246 self.depends = recipecache.deps[fn] 247 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn] 248 self._cached_packages = None 249 self._cached_rprovides = None 250 self._cached_packages_dynamic = None 251 252 def __getattr__(self, name): 253 if name == 'alternates': 254 return [x for x in self.fns if x != self.fn] 255 elif name == 'rdepends': 256 return self._recipecache.rundeps[self.fn] 257 elif name == 'rrecommends': 258 return self._recipecache.runrecs[self.fn] 259 elif name == 'provides': 260 return self._recipecache.fn_provides[self.fn] 261 elif name == 'packages': 262 if self._cached_packages is None: 263 self._cached_packages = [] 264 for pkg, fns in self._recipecache.packages.items(): 265 if self.fn in fns: 266 self._cached_packages.append(pkg) 267 return self._cached_packages 268 elif name == 'packages_dynamic': 269 if self._cached_packages_dynamic is None: 270 self._cached_packages_dynamic = [] 271 for pkg, fns in self._recipecache.packages_dynamic.items(): 272 if self.fn in fns: 273 self._cached_packages_dynamic.append(pkg) 274 return self._cached_packages_dynamic 275 elif name == 'rprovides': 276 if self._cached_rprovides is None: 277 self._cached_rprovides = [] 278 for pkg, fns in self._recipecache.rproviders.items(): 279 if self.fn in fns: 280 self._cached_rprovides.append(pkg) 281 return self._cached_rprovides 282 else: 283 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 284 def inherits(self, only_recipe=False): 285 """ 286 Get the inherited classes for a recipe. Returns the class names only. 287 Parameters: 288 only_recipe: True to return only the classes inherited by the recipe 289 itself, False to return all classes inherited within 290 the context for the recipe (which includes globally 291 inherited classes). 292 """ 293 if only_recipe: 294 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')] 295 else: 296 global_inherit = [] 297 for clsfile in self.inherit_files: 298 if only_recipe and clsfile in global_inherit: 299 continue 300 clsname = os.path.splitext(os.path.basename(clsfile))[0] 301 yield clsname 302 def __str__(self): 303 return '%s' % self.pn 304 305 306class Tinfoil: 307 """ 308 Tinfoil - an API for scripts and utilities to query 309 BitBake internals and perform build operations. 310 """ 311 312 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True): 313 """ 314 Create a new tinfoil object. 315 Parameters: 316 output: specifies where console output should be sent. Defaults 317 to sys.stdout. 318 tracking: True to enable variable history tracking, False to 319 disable it (default). Enabling this has a minor 320 performance impact so typically it isn't enabled 321 unless you need to query variable history. 322 setup_logging: True to setup a logger so that things like 323 bb.warn() will work immediately and timeout warnings 324 are visible; False to let BitBake do this itself. 325 """ 326 self.logger = logging.getLogger('BitBake') 327 self.config_data = None 328 self.cooker = None 329 self.tracking = tracking 330 self.ui_module = None 331 self.server_connection = None 332 self.recipes_parsed = False 333 self.quiet = 0 334 self.oldhandlers = self.logger.handlers[:] 335 self.localhandlers = [] 336 if setup_logging: 337 # This is the *client-side* logger, nothing to do with 338 # logging messages from the server 339 bb.msg.logger_create('BitBake', output) 340 for handler in self.logger.handlers: 341 if handler not in self.oldhandlers: 342 self.localhandlers.append(handler) 343 344 def __enter__(self): 345 return self 346 347 def __exit__(self, type, value, traceback): 348 self.shutdown() 349 350 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None): 351 """ 352 Prepares the underlying BitBake system to be used via tinfoil. 353 This function must be called prior to calling any of the other 354 functions in the API. 355 NOTE: if you call prepare() you must absolutely call shutdown() 356 before your code terminates. You can use a "with" block to ensure 357 this happens e.g. 358 359 with bb.tinfoil.Tinfoil() as tinfoil: 360 tinfoil.prepare() 361 ... 362 363 Parameters: 364 config_only: True to read only the configuration and not load 365 the cache / parse recipes. This is useful if you just 366 want to query the value of a variable at the global 367 level or you want to do anything else that doesn't 368 involve knowing anything about the recipes in the 369 current configuration. False loads the cache / parses 370 recipes. 371 config_params: optionally specify your own configuration 372 parameters. If not specified an instance of 373 TinfoilConfigParameters will be created internally. 374 quiet: quiet level controlling console output - equivalent 375 to bitbake's -q/--quiet option. Default of 0 gives 376 the same output level as normal bitbake execution. 377 extra_features: extra features to be added to the feature 378 set requested from the server. See 379 CookerFeatures._feature_list for possible 380 features. 381 """ 382 self.quiet = quiet 383 384 if self.tracking: 385 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING] 386 else: 387 extrafeatures = [] 388 389 if extra_features: 390 extrafeatures += extra_features 391 392 if not config_params: 393 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) 394 395 if not config_only: 396 # Disable local loggers because the UI module is going to set up its own 397 for handler in self.localhandlers: 398 self.logger.handlers.remove(handler) 399 self.localhandlers = [] 400 401 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures) 402 403 self.ui_module = ui_module 404 405 # Ensure the path to bitbake's bin directory is in PATH so that things like 406 # bitbake-worker can be run (usually this is the case, but it doesn't have to be) 407 path = os.getenv('PATH').split(':') 408 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin')) 409 for entry in path: 410 if entry.endswith(os.sep): 411 entry = entry[:-1] 412 if os.path.abspath(entry) == bitbakebinpath: 413 break 414 else: 415 path.insert(0, bitbakebinpath) 416 os.environ['PATH'] = ':'.join(path) 417 418 if self.server_connection: 419 _server_connections.append(self.server_connection) 420 if config_only: 421 config_params.updateToServer(self.server_connection.connection, os.environ.copy()) 422 self.run_command('parseConfiguration') 423 else: 424 self.run_actions(config_params) 425 self.recipes_parsed = True 426 427 self.config_data = TinfoilDataStoreConnector(self, 0) 428 self.cooker = TinfoilCookerAdapter(self) 429 self.cooker_data = self.cooker.recipecaches[''] 430 else: 431 raise Exception('Failed to start bitbake server') 432 433 def run_actions(self, config_params): 434 """ 435 Run the actions specified in config_params through the UI. 436 """ 437 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params) 438 if ret: 439 raise TinfoilUIException(ret) 440 441 def parseRecipes(self): 442 """ 443 Legacy function - use parse_recipes() instead. 444 """ 445 self.parse_recipes() 446 447 def parse_recipes(self): 448 """ 449 Load information on all recipes. Normally you should specify 450 config_only=False when calling prepare() instead of using this 451 function; this function is designed for situations where you need 452 to initialise Tinfoil and use it with config_only=True first and 453 then conditionally call this function to parse recipes later. 454 """ 455 config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet) 456 self.run_actions(config_params) 457 self.recipes_parsed = True 458 459 def modified_files(self): 460 """ 461 Notify the server it needs to revalidate it's caches since the client has modified files 462 """ 463 self.run_command("revalidateCaches") 464 465 def run_command(self, command, *params, handle_events=True): 466 """ 467 Run a command on the server (as implemented in bb.command). 468 Note that there are two types of command - synchronous and 469 asynchronous; in order to receive the results of asynchronous 470 commands you will need to set an appropriate event mask 471 using set_event_mask() and listen for the result using 472 wait_event() - with the correct event mask you'll at least get 473 bb.command.CommandCompleted and possibly other events before 474 that depending on the command. 475 """ 476 if not self.server_connection: 477 raise Exception('Not connected to server (did you call .prepare()?)') 478 479 commandline = [command] 480 if params: 481 commandline.extend(params) 482 try: 483 result = self.server_connection.connection.runCommand(commandline) 484 finally: 485 while handle_events: 486 event = self.wait_event() 487 if not event: 488 break 489 if isinstance(event, logging.LogRecord): 490 if event.taskpid == 0 or event.levelno > logging.INFO: 491 self.logger.handle(event) 492 if result[1]: 493 raise TinfoilCommandFailed(result[1]) 494 return result[0] 495 496 def set_event_mask(self, eventlist): 497 """Set the event mask which will be applied within wait_event()""" 498 if not self.server_connection: 499 raise Exception('Not connected to server (did you call .prepare()?)') 500 llevel, debug_domains = bb.msg.constructLogOptions() 501 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist) 502 if not ret: 503 raise Exception('setEventMask failed') 504 505 def wait_event(self, timeout=0): 506 """ 507 Wait for an event from the server for the specified time. 508 A timeout of 0 means don't wait if there are no events in the queue. 509 Returns the next event in the queue or None if the timeout was 510 reached. Note that in order to receive any events you will 511 first need to set the internal event mask using set_event_mask() 512 (otherwise whatever event mask the UI set up will be in effect). 513 """ 514 if not self.server_connection: 515 raise Exception('Not connected to server (did you call .prepare()?)') 516 return self.server_connection.events.waitEvent(timeout) 517 518 def get_overlayed_recipes(self, mc=''): 519 """ 520 Find recipes which are overlayed (i.e. where recipes exist in multiple layers) 521 """ 522 return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) 523 524 def get_skipped_recipes(self, mc=''): 525 """ 526 Find recipes which were skipped (i.e. SkipRecipe was raised 527 during parsing). 528 """ 529 return OrderedDict(self.run_command('getSkippedRecipes', mc)) 530 531 def get_all_providers(self, mc=''): 532 return defaultdict(list, self.run_command('allProviders', mc)) 533 534 def find_providers(self, mc=''): 535 return self.run_command('findProviders', mc) 536 537 def find_best_provider(self, pn): 538 return self.run_command('findBestProvider', pn) 539 540 def get_runtime_providers(self, rdep): 541 return self.run_command('getRuntimeProviders', rdep) 542 543 # TODO: teach this method about mc 544 def get_recipe_file(self, pn): 545 """ 546 Get the file name for the specified recipe/target. Raises 547 bb.providers.NoProvider if there is no match or the recipe was 548 skipped. 549 """ 550 best = self.find_best_provider(pn) 551 if not best or (len(best) > 3 and not best[3]): 552 # TODO: pass down mc 553 skiplist = self.get_skipped_recipes() 554 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist) 555 skipreasons = taskdata.get_reasons(pn) 556 if skipreasons: 557 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons))) 558 else: 559 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) 560 return best[3] 561 562 def get_file_appends(self, fn, mc=''): 563 """ 564 Find the bbappends for a recipe file 565 """ 566 return self.run_command('getFileAppends', fn, mc) 567 568 def all_recipes(self, mc='', sort=True): 569 """ 570 Enable iterating over all recipes in the current configuration. 571 Returns an iterator over TinfoilRecipeInfo objects created on demand. 572 Parameters: 573 mc: The multiconfig, default of '' uses the main configuration. 574 sort: True to sort recipes alphabetically (default), False otherwise 575 """ 576 recipecache = self.cooker.recipecaches[mc] 577 if sort: 578 recipes = sorted(recipecache.pkg_pn.items()) 579 else: 580 recipes = recipecache.pkg_pn.items() 581 for pn, fns in recipes: 582 prov = self.find_best_provider(pn) 583 recipe = TinfoilRecipeInfo(recipecache, 584 self.config_data, 585 pn=pn, 586 fn=prov[3], 587 fns=fns) 588 yield recipe 589 590 def all_recipe_files(self, mc='', variants=True, preferred_only=False): 591 """ 592 Enable iterating over all recipe files in the current configuration. 593 Returns an iterator over file paths. 594 Parameters: 595 mc: The multiconfig, default of '' uses the main configuration. 596 variants: True to include variants of recipes created through 597 BBCLASSEXTEND (default) or False to exclude them 598 preferred_only: True to include only the preferred recipe where 599 multiple exist providing the same PN, False to list 600 all recipes 601 """ 602 recipecache = self.cooker.recipecaches[mc] 603 if preferred_only: 604 files = [] 605 for pn in recipecache.pkg_pn.keys(): 606 prov = self.find_best_provider(pn) 607 files.append(prov[3]) 608 else: 609 files = recipecache.pkg_fn.keys() 610 for fn in sorted(files): 611 if not variants and fn.startswith('virtual:'): 612 continue 613 yield fn 614 615 616 def get_recipe_info(self, pn, mc=''): 617 """ 618 Get information on a specific recipe in the current configuration by name (PN). 619 Returns a TinfoilRecipeInfo object created on demand. 620 Parameters: 621 mc: The multiconfig, default of '' uses the main configuration. 622 """ 623 recipecache = self.cooker.recipecaches[mc] 624 prov = self.find_best_provider(pn) 625 fn = prov[3] 626 if fn: 627 actual_pn = recipecache.pkg_fn[fn] 628 recipe = TinfoilRecipeInfo(recipecache, 629 self.config_data, 630 pn=actual_pn, 631 fn=fn, 632 fns=recipecache.pkg_pn[actual_pn]) 633 return recipe 634 else: 635 return None 636 637 def parse_recipe(self, pn): 638 """ 639 Parse the specified recipe and return a datastore object 640 representing the environment for the recipe. 641 """ 642 fn = self.get_recipe_file(pn) 643 return self.parse_recipe_file(fn) 644 645 @contextmanager 646 def _data_tracked_if_enabled(self): 647 """ 648 A context manager to enable data tracking for a code segment if data 649 tracking was enabled for this tinfoil instance. 650 """ 651 if self.tracking: 652 # Enable history tracking just for the operation 653 self.run_command('enableDataTracking') 654 655 # Here goes the operation with the optional data tracking 656 yield 657 658 if self.tracking: 659 self.run_command('disableDataTracking') 660 661 def finalizeData(self): 662 """ 663 Run anonymous functions and expand keys 664 """ 665 with self._data_tracked_if_enabled(): 666 return self._reconvert_type(self.run_command('finalizeData'), 'DataStoreConnectionHandle') 667 668 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): 669 """ 670 Parse the specified recipe file (with or without bbappends) 671 and return a datastore object representing the environment 672 for the recipe. 673 Parameters: 674 fn: recipe file to parse - can be a file path or virtual 675 specification 676 appends: True to apply bbappends, False otherwise 677 appendlist: optional list of bbappend files to apply, if you 678 want to filter them 679 """ 680 with self._data_tracked_if_enabled(): 681 if appends and appendlist == []: 682 appends = False 683 if config_data: 684 config_data = bb.data.createCopy(config_data) 685 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex) 686 else: 687 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist) 688 if dscon: 689 return self._reconvert_type(dscon, 'DataStoreConnectionHandle') 690 else: 691 return None 692 693 def build_file(self, buildfile, task, internal=True): 694 """ 695 Runs the specified task for just a single recipe (i.e. no dependencies). 696 This is equivalent to bitbake -b, except with the default internal=True 697 no warning about dependencies will be produced, normal info messages 698 from the runqueue will be silenced and BuildInit, BuildStarted and 699 BuildCompleted events will not be fired. 700 """ 701 return self.run_command('buildFile', buildfile, task, internal) 702 703 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None): 704 """ 705 Builds the specified targets. This is equivalent to a normal invocation 706 of bitbake. Has built-in event handling which is enabled by default and 707 can be extended if needed. 708 Parameters: 709 targets: 710 One or more targets to build. Can be a list or a 711 space-separated string. 712 task: 713 The task to run; if None then the value of BB_DEFAULT_TASK 714 will be used. Default None. 715 handle_events: 716 True to handle events in a similar way to normal bitbake 717 invocation with knotty; False to return immediately (on the 718 assumption that the caller will handle the events instead). 719 Default True. 720 extra_events: 721 An optional list of events to add to the event mask (if 722 handle_events=True). If you add events here you also need 723 to specify a callback function in event_callback that will 724 handle the additional events. Default None. 725 event_callback: 726 An optional function taking a single parameter which 727 will be called first upon receiving any event (if 728 handle_events=True) so that the caller can override or 729 extend the event handling. Default None. 730 """ 731 if isinstance(targets, str): 732 targets = targets.split() 733 if not task: 734 task = self.config_data.getVar('BB_DEFAULT_TASK') 735 736 if handle_events: 737 # A reasonable set of default events matching up with those we handle below 738 eventmask = [ 739 'bb.event.BuildStarted', 740 'bb.event.BuildCompleted', 741 'logging.LogRecord', 742 'bb.event.NoProvider', 743 'bb.command.CommandCompleted', 744 'bb.command.CommandFailed', 745 'bb.build.TaskStarted', 746 'bb.build.TaskFailed', 747 'bb.build.TaskSucceeded', 748 'bb.build.TaskFailedSilent', 749 'bb.build.TaskProgress', 750 'bb.runqueue.runQueueTaskStarted', 751 'bb.runqueue.sceneQueueTaskStarted', 752 'bb.event.ProcessStarted', 753 'bb.event.ProcessProgress', 754 'bb.event.ProcessFinished', 755 ] 756 if extra_events: 757 eventmask.extend(extra_events) 758 ret = self.set_event_mask(eventmask) 759 760 includelogs = self.config_data.getVar('BBINCLUDELOGS') 761 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES') 762 763 ret = self.run_command('buildTargets', targets, task) 764 if handle_events: 765 lastevent = time.time() 766 result = False 767 # Borrowed from knotty, instead somewhat hackily we use the helper 768 # as the object to store "shutdown" on 769 helper = bb.ui.uihelper.BBUIHelper() 770 helper.shutdown = 0 771 parseprogress = None 772 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet) 773 try: 774 while True: 775 try: 776 event = self.wait_event(0.25) 777 if event: 778 lastevent = time.time() 779 if event_callback and event_callback(event): 780 continue 781 if helper.eventHandler(event): 782 if isinstance(event, bb.build.TaskFailedSilent): 783 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile) 784 elif isinstance(event, bb.build.TaskFailed): 785 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter) 786 continue 787 if isinstance(event, bb.event.ProcessStarted): 788 if self.quiet > 1: 789 continue 790 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total) 791 parseprogress.start(False) 792 continue 793 if isinstance(event, bb.event.ProcessProgress): 794 if self.quiet > 1: 795 continue 796 if parseprogress: 797 parseprogress.update(event.progress) 798 else: 799 bb.warn("Got ProcessProgress event for something that never started?") 800 continue 801 if isinstance(event, bb.event.ProcessFinished): 802 if self.quiet > 1: 803 continue 804 if parseprogress: 805 parseprogress.finish() 806 parseprogress = None 807 continue 808 if isinstance(event, bb.command.CommandCompleted): 809 result = True 810 break 811 if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)): 812 self.logger.error(str(event)) 813 result = False 814 break 815 if isinstance(event, logging.LogRecord): 816 if event.taskpid == 0 or event.levelno > logging.INFO: 817 self.logger.handle(event) 818 continue 819 if isinstance(event, bb.event.NoProvider): 820 self.logger.error(str(event)) 821 result = False 822 break 823 elif helper.shutdown > 1: 824 break 825 termfilter.updateFooter() 826 if time.time() > (lastevent + (3*60)): 827 if not self.run_command('ping', handle_events=False): 828 print("\nUnable to ping server and no events, closing down...\n") 829 return False 830 except KeyboardInterrupt: 831 termfilter.clearFooter() 832 if helper.shutdown == 1: 833 print("\nSecond Keyboard Interrupt, stopping...\n") 834 ret = self.run_command("stateForceShutdown") 835 if ret and ret[2]: 836 self.logger.error("Unable to cleanly stop: %s" % ret[2]) 837 elif helper.shutdown == 0: 838 print("\nKeyboard Interrupt, closing down...\n") 839 interrupted = True 840 ret = self.run_command("stateShutdown") 841 if ret and ret[2]: 842 self.logger.error("Unable to cleanly shutdown: %s" % ret[2]) 843 helper.shutdown = helper.shutdown + 1 844 termfilter.clearFooter() 845 finally: 846 termfilter.finish() 847 if helper.failed_tasks: 848 result = False 849 return result 850 else: 851 return ret 852 853 def shutdown(self): 854 """ 855 Shut down tinfoil. Disconnects from the server and gracefully 856 releases any associated resources. You must call this function if 857 prepare() has been called, or use a with... block when you create 858 the tinfoil object which will ensure that it gets called. 859 """ 860 try: 861 if self.server_connection: 862 try: 863 self.run_command('clientComplete') 864 finally: 865 _server_connections.remove(self.server_connection) 866 bb.event.ui_queue = [] 867 self.server_connection.terminate() 868 self.server_connection = None 869 870 finally: 871 # Restore logging handlers to how it looked when we started 872 if self.oldhandlers: 873 for handler in self.logger.handlers: 874 if handler not in self.oldhandlers: 875 self.logger.handlers.remove(handler) 876 877 def _reconvert_type(self, obj, origtypename): 878 """ 879 Convert an object back to the right type, in the case 880 that marshalling has changed it (especially with xmlrpc) 881 """ 882 supported_types = { 883 'set': set, 884 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle, 885 } 886 887 origtype = supported_types.get(origtypename, None) 888 if origtype is None: 889 raise Exception('Unsupported type "%s"' % origtypename) 890 if type(obj) == origtype: 891 newobj = obj 892 elif isinstance(obj, dict): 893 # New style class 894 newobj = origtype() 895 for k,v in obj.items(): 896 setattr(newobj, k, v) 897 else: 898 # Assume we can coerce the type 899 newobj = origtype(obj) 900 901 if isinstance(newobj, bb.command.DataStoreConnectionHandle): 902 newobj = TinfoilDataStoreConnector(self, newobj.dsindex) 903 904 return newobj 905 906 907class TinfoilConfigParameters(BitBakeConfigParameters): 908 909 def __init__(self, config_only, **options): 910 self.initial_options = options 911 # Apply some sane defaults 912 if not 'parse_only' in options: 913 self.initial_options['parse_only'] = not config_only 914 #if not 'status_only' in options: 915 # self.initial_options['status_only'] = config_only 916 if not 'ui' in options: 917 self.initial_options['ui'] = 'knotty' 918 if not 'argv' in options: 919 self.initial_options['argv'] = [] 920 921 super(TinfoilConfigParameters, self).__init__() 922 923 def parseCommandLine(self, argv=None): 924 # We don't want any parameters parsed from the command line 925 opts = super(TinfoilConfigParameters, self).parseCommandLine([]) 926 for key, val in self.initial_options.items(): 927 setattr(opts[0], key, val) 928 return opts 929