1# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities 2# 3# Copyright (C) 2012-2017 Intel Corporation 4# Copyright (C) 2011 Mentor Graphics Corporation 5# Copyright (C) 2006-2012 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import logging 11import os 12import sys 13import time 14import atexit 15import re 16from collections import OrderedDict, defaultdict 17from functools import partial 18 19import bb.cache 20import bb.cooker 21import bb.providers 22import bb.taskdata 23import bb.utils 24import bb.command 25import bb.remotedata 26from bb.main import setup_bitbake, BitBakeConfigParameters 27import bb.fetch2 28 29 30# We need this in order to shut down the connection to the bitbake server, 31# otherwise the process will never properly exit 32_server_connections = [] 33def _terminate_connections(): 34 for connection in _server_connections: 35 connection.terminate() 36atexit.register(_terminate_connections) 37 38class TinfoilUIException(Exception): 39 """Exception raised when the UI returns non-zero from its main function""" 40 def __init__(self, returncode): 41 self.returncode = returncode 42 def __repr__(self): 43 return 'UI module main returned %d' % self.returncode 44 45class TinfoilCommandFailed(Exception): 46 """Exception raised when run_command fails""" 47 48class TinfoilDataStoreConnectorVarHistory: 49 def __init__(self, tinfoil, dsindex): 50 self.tinfoil = tinfoil 51 self.dsindex = dsindex 52 53 def remoteCommand(self, cmd, *args, **kwargs): 54 return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs) 55 56 def emit(self, var, oval, val, o, d): 57 ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex) 58 o.write(ret) 59 60 def __getattr__(self, name): 61 if not hasattr(bb.data_smart.VariableHistory, name): 62 raise AttributeError("VariableHistory has no such method %s" % name) 63 64 newfunc = partial(self.remoteCommand, name) 65 setattr(self, name, newfunc) 66 return newfunc 67 68class TinfoilDataStoreConnectorIncHistory: 69 def __init__(self, tinfoil, dsindex): 70 self.tinfoil = tinfoil 71 self.dsindex = dsindex 72 73 def remoteCommand(self, cmd, *args, **kwargs): 74 return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs) 75 76 def __getattr__(self, name): 77 if not hasattr(bb.data_smart.IncludeHistory, name): 78 raise AttributeError("IncludeHistory has no such method %s" % name) 79 80 newfunc = partial(self.remoteCommand, name) 81 setattr(self, name, newfunc) 82 return newfunc 83 84class TinfoilDataStoreConnector: 85 """ 86 Connector object used to enable access to datastore objects via tinfoil 87 Method calls are transmitted to the remote datastore for processing, if a datastore is 88 returned we return a connector object for the new store 89 """ 90 91 def __init__(self, tinfoil, dsindex): 92 self.tinfoil = tinfoil 93 self.dsindex = dsindex 94 self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex) 95 self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex) 96 97 def remoteCommand(self, cmd, *args, **kwargs): 98 ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs) 99 if isinstance(ret, bb.command.DataStoreConnectionHandle): 100 return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex) 101 return ret 102 103 def __getattr__(self, name): 104 if not hasattr(bb.data._dict_type, name): 105 raise AttributeError("Data store has no such method %s" % name) 106 107 newfunc = partial(self.remoteCommand, name) 108 setattr(self, name, newfunc) 109 return newfunc 110 111 def __iter__(self): 112 keys = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, "keys", [], {}) 113 for k in keys: 114 yield k 115 116class TinfoilCookerAdapter: 117 """ 118 Provide an adapter for existing code that expects to access a cooker object via Tinfoil, 119 since now Tinfoil is on the client side it no longer has direct access. 120 """ 121 122 class TinfoilCookerCollectionAdapter: 123 """ cooker.collection adapter """ 124 def __init__(self, tinfoil, mc=''): 125 self.tinfoil = tinfoil 126 self.mc = mc 127 def get_file_appends(self, fn): 128 return self.tinfoil.get_file_appends(fn, self.mc) 129 def __getattr__(self, name): 130 if name == 'overlayed': 131 return self.tinfoil.get_overlayed_recipes(self.mc) 132 elif name == 'bbappends': 133 return self.tinfoil.run_command('getAllAppends', self.mc) 134 else: 135 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 136 137 class TinfoilRecipeCacheAdapter: 138 """ cooker.recipecache adapter """ 139 def __init__(self, tinfoil, mc=''): 140 self.tinfoil = tinfoil 141 self.mc = mc 142 self._cache = {} 143 144 def get_pkg_pn_fn(self): 145 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or []) 146 pkg_fn = {} 147 for pn, fnlist in pkg_pn.items(): 148 for fn in fnlist: 149 pkg_fn[fn] = pn 150 self._cache['pkg_pn'] = pkg_pn 151 self._cache['pkg_fn'] = pkg_fn 152 153 def __getattr__(self, name): 154 # Grab these only when they are requested since they aren't always used 155 if name in self._cache: 156 return self._cache[name] 157 elif name == 'pkg_pn': 158 self.get_pkg_pn_fn() 159 return self._cache[name] 160 elif name == 'pkg_fn': 161 self.get_pkg_pn_fn() 162 return self._cache[name] 163 elif name == 'deps': 164 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or []) 165 elif name == 'rundeps': 166 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or []) 167 elif name == 'runrecs': 168 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or []) 169 elif name == 'pkg_pepvpr': 170 attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {} 171 elif name == 'inherits': 172 attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {} 173 elif name == 'bbfile_priority': 174 attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {} 175 elif name == 'pkg_dp': 176 attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {} 177 elif name == 'fn_provides': 178 attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {} 179 elif name == 'packages': 180 attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {} 181 elif name == 'packages_dynamic': 182 attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {} 183 elif name == 'rproviders': 184 attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {} 185 else: 186 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 187 188 self._cache[name] = attrvalue 189 return attrvalue 190 191 class TinfoilSkiplistByMcAdapter: 192 def __init__(self, tinfoil): 193 self.tinfoil = tinfoil 194 195 def __getitem__(self, mc): 196 return self.tinfoil.get_skipped_recipes(mc) 197 198 def __init__(self, tinfoil): 199 self.tinfoil = tinfoil 200 self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() 201 self.collections = {} 202 self.recipecaches = {} 203 self.skiplist_by_mc = self.TinfoilSkiplistByMcAdapter(tinfoil) 204 for mc in self.multiconfigs: 205 self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) 206 self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) 207 self._cache = {} 208 def __getattr__(self, name): 209 # Grab these only when they are requested since they aren't always used 210 if name in self._cache: 211 return self._cache[name] 212 elif name == 'bbfile_config_priorities': 213 ret = self.tinfoil.run_command('getLayerPriorities') 214 bbfile_config_priorities = [] 215 for collection, pattern, regex, pri in ret: 216 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri)) 217 218 attrvalue = bbfile_config_priorities 219 else: 220 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 221 222 self._cache[name] = attrvalue 223 return attrvalue 224 225 def findBestProvider(self, pn): 226 return self.tinfoil.find_best_provider(pn) 227 228 229class TinfoilRecipeInfo: 230 """ 231 Provides a convenient representation of the cached information for a single recipe. 232 Some attributes are set on construction, others are read on-demand (which internally 233 may result in a remote procedure call to the bitbake server the first time). 234 Note that only information which is cached is available through this object - if 235 you need other variable values you will need to parse the recipe using 236 Tinfoil.parse_recipe(). 237 """ 238 def __init__(self, recipecache, d, pn, fn, fns): 239 self._recipecache = recipecache 240 self._d = d 241 self.pn = pn 242 self.fn = fn 243 self.fns = fns 244 self.inherit_files = recipecache.inherits[fn] 245 self.depends = recipecache.deps[fn] 246 (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn] 247 self._cached_packages = None 248 self._cached_rprovides = None 249 self._cached_packages_dynamic = None 250 251 def __getattr__(self, name): 252 if name == 'alternates': 253 return [x for x in self.fns if x != self.fn] 254 elif name == 'rdepends': 255 return self._recipecache.rundeps[self.fn] 256 elif name == 'rrecommends': 257 return self._recipecache.runrecs[self.fn] 258 elif name == 'provides': 259 return self._recipecache.fn_provides[self.fn] 260 elif name == 'packages': 261 if self._cached_packages is None: 262 self._cached_packages = [] 263 for pkg, fns in self._recipecache.packages.items(): 264 if self.fn in fns: 265 self._cached_packages.append(pkg) 266 return self._cached_packages 267 elif name == 'packages_dynamic': 268 if self._cached_packages_dynamic is None: 269 self._cached_packages_dynamic = [] 270 for pkg, fns in self._recipecache.packages_dynamic.items(): 271 if self.fn in fns: 272 self._cached_packages_dynamic.append(pkg) 273 return self._cached_packages_dynamic 274 elif name == 'rprovides': 275 if self._cached_rprovides is None: 276 self._cached_rprovides = [] 277 for pkg, fns in self._recipecache.rproviders.items(): 278 if self.fn in fns: 279 self._cached_rprovides.append(pkg) 280 return self._cached_rprovides 281 else: 282 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) 283 def inherits(self, only_recipe=False): 284 """ 285 Get the inherited classes for a recipe. Returns the class names only. 286 Parameters: 287 only_recipe: True to return only the classes inherited by the recipe 288 itself, False to return all classes inherited within 289 the context for the recipe (which includes globally 290 inherited classes). 291 """ 292 if only_recipe: 293 global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')] 294 else: 295 global_inherit = [] 296 for clsfile in self.inherit_files: 297 if only_recipe and clsfile in global_inherit: 298 continue 299 clsname = os.path.splitext(os.path.basename(clsfile))[0] 300 yield clsname 301 def __str__(self): 302 return '%s' % self.pn 303 304 305class Tinfoil: 306 """ 307 Tinfoil - an API for scripts and utilities to query 308 BitBake internals and perform build operations. 309 """ 310 311 def __init__(self, output=sys.stdout, tracking=False, setup_logging=True): 312 """ 313 Create a new tinfoil object. 314 Parameters: 315 output: specifies where console output should be sent. Defaults 316 to sys.stdout. 317 tracking: True to enable variable history tracking, False to 318 disable it (default). Enabling this has a minor 319 performance impact so typically it isn't enabled 320 unless you need to query variable history. 321 setup_logging: True to setup a logger so that things like 322 bb.warn() will work immediately and timeout warnings 323 are visible; False to let BitBake do this itself. 324 """ 325 self.logger = logging.getLogger('BitBake') 326 self.config_data = None 327 self.cooker = None 328 self.tracking = tracking 329 self.ui_module = None 330 self.server_connection = None 331 self.recipes_parsed = False 332 self.quiet = 0 333 self.oldhandlers = self.logger.handlers[:] 334 self.localhandlers = [] 335 if setup_logging: 336 # This is the *client-side* logger, nothing to do with 337 # logging messages from the server 338 bb.msg.logger_create('BitBake', output) 339 for handler in self.logger.handlers: 340 if handler not in self.oldhandlers: 341 self.localhandlers.append(handler) 342 343 def __enter__(self): 344 return self 345 346 def __exit__(self, type, value, traceback): 347 self.shutdown() 348 349 def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None): 350 """ 351 Prepares the underlying BitBake system to be used via tinfoil. 352 This function must be called prior to calling any of the other 353 functions in the API. 354 NOTE: if you call prepare() you must absolutely call shutdown() 355 before your code terminates. You can use a "with" block to ensure 356 this happens e.g. 357 358 with bb.tinfoil.Tinfoil() as tinfoil: 359 tinfoil.prepare() 360 ... 361 362 Parameters: 363 config_only: True to read only the configuration and not load 364 the cache / parse recipes. This is useful if you just 365 want to query the value of a variable at the global 366 level or you want to do anything else that doesn't 367 involve knowing anything about the recipes in the 368 current configuration. False loads the cache / parses 369 recipes. 370 config_params: optionally specify your own configuration 371 parameters. If not specified an instance of 372 TinfoilConfigParameters will be created internally. 373 quiet: quiet level controlling console output - equivalent 374 to bitbake's -q/--quiet option. Default of 0 gives 375 the same output level as normal bitbake execution. 376 extra_features: extra features to be added to the feature 377 set requested from the server. See 378 CookerFeatures._feature_list for possible 379 features. 380 """ 381 self.quiet = quiet 382 383 if self.tracking: 384 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING] 385 else: 386 extrafeatures = [] 387 388 if extra_features: 389 extrafeatures += extra_features 390 391 if not config_params: 392 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) 393 394 if not config_only: 395 # Disable local loggers because the UI module is going to set up its own 396 for handler in self.localhandlers: 397 self.logger.handlers.remove(handler) 398 self.localhandlers = [] 399 400 self.server_connection, ui_module = setup_bitbake(config_params, extrafeatures) 401 402 self.ui_module = ui_module 403 404 # Ensure the path to bitbake's bin directory is in PATH so that things like 405 # bitbake-worker can be run (usually this is the case, but it doesn't have to be) 406 path = os.getenv('PATH').split(':') 407 bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin')) 408 for entry in path: 409 if entry.endswith(os.sep): 410 entry = entry[:-1] 411 if os.path.abspath(entry) == bitbakebinpath: 412 break 413 else: 414 path.insert(0, bitbakebinpath) 415 os.environ['PATH'] = ':'.join(path) 416 417 if self.server_connection: 418 _server_connections.append(self.server_connection) 419 if config_only: 420 config_params.updateToServer(self.server_connection.connection, os.environ.copy()) 421 self.run_command('parseConfiguration') 422 else: 423 self.run_actions(config_params) 424 self.recipes_parsed = True 425 426 self.config_data = TinfoilDataStoreConnector(self, 0) 427 self.cooker = TinfoilCookerAdapter(self) 428 self.cooker_data = self.cooker.recipecaches[''] 429 else: 430 raise Exception('Failed to start bitbake server') 431 432 def run_actions(self, config_params): 433 """ 434 Run the actions specified in config_params through the UI. 435 """ 436 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params) 437 if ret: 438 raise TinfoilUIException(ret) 439 440 def parseRecipes(self): 441 """ 442 Legacy function - use parse_recipes() instead. 443 """ 444 self.parse_recipes() 445 446 def parse_recipes(self): 447 """ 448 Load information on all recipes. Normally you should specify 449 config_only=False when calling prepare() instead of using this 450 function; this function is designed for situations where you need 451 to initialise Tinfoil and use it with config_only=True first and 452 then conditionally call this function to parse recipes later. 453 """ 454 config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet) 455 self.run_actions(config_params) 456 self.recipes_parsed = True 457 458 def modified_files(self): 459 """ 460 Notify the server it needs to revalidate it's caches since the client has modified files 461 """ 462 self.run_command("revalidateCaches") 463 464 def run_command(self, command, *params, handle_events=True): 465 """ 466 Run a command on the server (as implemented in bb.command). 467 Note that there are two types of command - synchronous and 468 asynchronous; in order to receive the results of asynchronous 469 commands you will need to set an appropriate event mask 470 using set_event_mask() and listen for the result using 471 wait_event() - with the correct event mask you'll at least get 472 bb.command.CommandCompleted and possibly other events before 473 that depending on the command. 474 """ 475 if not self.server_connection: 476 raise Exception('Not connected to server (did you call .prepare()?)') 477 478 commandline = [command] 479 if params: 480 commandline.extend(params) 481 try: 482 result = self.server_connection.connection.runCommand(commandline) 483 finally: 484 while handle_events: 485 event = self.wait_event() 486 if not event: 487 break 488 if isinstance(event, logging.LogRecord): 489 if event.taskpid == 0 or event.levelno > logging.INFO: 490 self.logger.handle(event) 491 if result[1]: 492 raise TinfoilCommandFailed(result[1]) 493 return result[0] 494 495 def set_event_mask(self, eventlist): 496 """Set the event mask which will be applied within wait_event()""" 497 if not self.server_connection: 498 raise Exception('Not connected to server (did you call .prepare()?)') 499 llevel, debug_domains = bb.msg.constructLogOptions() 500 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist) 501 if not ret: 502 raise Exception('setEventMask failed') 503 504 def wait_event(self, timeout=0): 505 """ 506 Wait for an event from the server for the specified time. 507 A timeout of 0 means don't wait if there are no events in the queue. 508 Returns the next event in the queue or None if the timeout was 509 reached. Note that in order to receive any events you will 510 first need to set the internal event mask using set_event_mask() 511 (otherwise whatever event mask the UI set up will be in effect). 512 """ 513 if not self.server_connection: 514 raise Exception('Not connected to server (did you call .prepare()?)') 515 return self.server_connection.events.waitEvent(timeout) 516 517 def get_overlayed_recipes(self, mc=''): 518 """ 519 Find recipes which are overlayed (i.e. where recipes exist in multiple layers) 520 """ 521 return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) 522 523 def get_skipped_recipes(self, mc=''): 524 """ 525 Find recipes which were skipped (i.e. SkipRecipe was raised 526 during parsing). 527 """ 528 return OrderedDict(self.run_command('getSkippedRecipes', mc)) 529 530 def get_all_providers(self, mc=''): 531 return defaultdict(list, self.run_command('allProviders', mc)) 532 533 def find_providers(self, mc=''): 534 return self.run_command('findProviders', mc) 535 536 def find_best_provider(self, pn): 537 return self.run_command('findBestProvider', pn) 538 539 def get_runtime_providers(self, rdep): 540 return self.run_command('getRuntimeProviders', rdep) 541 542 # TODO: teach this method about mc 543 def get_recipe_file(self, pn): 544 """ 545 Get the file name for the specified recipe/target. Raises 546 bb.providers.NoProvider if there is no match or the recipe was 547 skipped. 548 """ 549 best = self.find_best_provider(pn) 550 if not best or (len(best) > 3 and not best[3]): 551 # TODO: pass down mc 552 skiplist = self.get_skipped_recipes() 553 taskdata = bb.taskdata.TaskData(None, skiplist=skiplist) 554 skipreasons = taskdata.get_reasons(pn) 555 if skipreasons: 556 raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons))) 557 else: 558 raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) 559 return best[3] 560 561 def get_file_appends(self, fn, mc=''): 562 """ 563 Find the bbappends for a recipe file 564 """ 565 return self.run_command('getFileAppends', fn, mc) 566 567 def all_recipes(self, mc='', sort=True): 568 """ 569 Enable iterating over all recipes in the current configuration. 570 Returns an iterator over TinfoilRecipeInfo objects created on demand. 571 Parameters: 572 mc: The multiconfig, default of '' uses the main configuration. 573 sort: True to sort recipes alphabetically (default), False otherwise 574 """ 575 recipecache = self.cooker.recipecaches[mc] 576 if sort: 577 recipes = sorted(recipecache.pkg_pn.items()) 578 else: 579 recipes = recipecache.pkg_pn.items() 580 for pn, fns in recipes: 581 prov = self.find_best_provider(pn) 582 recipe = TinfoilRecipeInfo(recipecache, 583 self.config_data, 584 pn=pn, 585 fn=prov[3], 586 fns=fns) 587 yield recipe 588 589 def all_recipe_files(self, mc='', variants=True, preferred_only=False): 590 """ 591 Enable iterating over all recipe files in the current configuration. 592 Returns an iterator over file paths. 593 Parameters: 594 mc: The multiconfig, default of '' uses the main configuration. 595 variants: True to include variants of recipes created through 596 BBCLASSEXTEND (default) or False to exclude them 597 preferred_only: True to include only the preferred recipe where 598 multiple exist providing the same PN, False to list 599 all recipes 600 """ 601 recipecache = self.cooker.recipecaches[mc] 602 if preferred_only: 603 files = [] 604 for pn in recipecache.pkg_pn.keys(): 605 prov = self.find_best_provider(pn) 606 files.append(prov[3]) 607 else: 608 files = recipecache.pkg_fn.keys() 609 for fn in sorted(files): 610 if not variants and fn.startswith('virtual:'): 611 continue 612 yield fn 613 614 615 def get_recipe_info(self, pn, mc=''): 616 """ 617 Get information on a specific recipe in the current configuration by name (PN). 618 Returns a TinfoilRecipeInfo object created on demand. 619 Parameters: 620 mc: The multiconfig, default of '' uses the main configuration. 621 """ 622 recipecache = self.cooker.recipecaches[mc] 623 prov = self.find_best_provider(pn) 624 fn = prov[3] 625 if fn: 626 actual_pn = recipecache.pkg_fn[fn] 627 recipe = TinfoilRecipeInfo(recipecache, 628 self.config_data, 629 pn=actual_pn, 630 fn=fn, 631 fns=recipecache.pkg_pn[actual_pn]) 632 return recipe 633 else: 634 return None 635 636 def parse_recipe(self, pn): 637 """ 638 Parse the specified recipe and return a datastore object 639 representing the environment for the recipe. 640 """ 641 fn = self.get_recipe_file(pn) 642 return self.parse_recipe_file(fn) 643 644 def finalizeData(self): 645 """ 646 Run anonymous functions and expand keys 647 """ 648 return self._reconvert_type(self.run_command('finalizeData'), 'DataStoreConnectionHandle') 649 650 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): 651 """ 652 Parse the specified recipe file (with or without bbappends) 653 and return a datastore object representing the environment 654 for the recipe. 655 Parameters: 656 fn: recipe file to parse - can be a file path or virtual 657 specification 658 appends: True to apply bbappends, False otherwise 659 appendlist: optional list of bbappend files to apply, if you 660 want to filter them 661 """ 662 if self.tracking: 663 # Enable history tracking just for the parse operation 664 self.run_command('enableDataTracking') 665 try: 666 if appends and appendlist == []: 667 appends = False 668 if config_data: 669 config_data = bb.data.createCopy(config_data) 670 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist, config_data.dsindex) 671 else: 672 dscon = self.run_command('parseRecipeFile', fn, appends, appendlist) 673 if dscon: 674 return self._reconvert_type(dscon, 'DataStoreConnectionHandle') 675 else: 676 return None 677 finally: 678 if self.tracking: 679 self.run_command('disableDataTracking') 680 681 def build_file(self, buildfile, task, internal=True): 682 """ 683 Runs the specified task for just a single recipe (i.e. no dependencies). 684 This is equivalent to bitbake -b, except with the default internal=True 685 no warning about dependencies will be produced, normal info messages 686 from the runqueue will be silenced and BuildInit, BuildStarted and 687 BuildCompleted events will not be fired. 688 """ 689 return self.run_command('buildFile', buildfile, task, internal) 690 691 def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None): 692 """ 693 Builds the specified targets. This is equivalent to a normal invocation 694 of bitbake. Has built-in event handling which is enabled by default and 695 can be extended if needed. 696 Parameters: 697 targets: 698 One or more targets to build. Can be a list or a 699 space-separated string. 700 task: 701 The task to run; if None then the value of BB_DEFAULT_TASK 702 will be used. Default None. 703 handle_events: 704 True to handle events in a similar way to normal bitbake 705 invocation with knotty; False to return immediately (on the 706 assumption that the caller will handle the events instead). 707 Default True. 708 extra_events: 709 An optional list of events to add to the event mask (if 710 handle_events=True). If you add events here you also need 711 to specify a callback function in event_callback that will 712 handle the additional events. Default None. 713 event_callback: 714 An optional function taking a single parameter which 715 will be called first upon receiving any event (if 716 handle_events=True) so that the caller can override or 717 extend the event handling. Default None. 718 """ 719 if isinstance(targets, str): 720 targets = targets.split() 721 if not task: 722 task = self.config_data.getVar('BB_DEFAULT_TASK') 723 724 if handle_events: 725 # A reasonable set of default events matching up with those we handle below 726 eventmask = [ 727 'bb.event.BuildStarted', 728 'bb.event.BuildCompleted', 729 'logging.LogRecord', 730 'bb.event.NoProvider', 731 'bb.command.CommandCompleted', 732 'bb.command.CommandFailed', 733 'bb.build.TaskStarted', 734 'bb.build.TaskFailed', 735 'bb.build.TaskSucceeded', 736 'bb.build.TaskFailedSilent', 737 'bb.build.TaskProgress', 738 'bb.runqueue.runQueueTaskStarted', 739 'bb.runqueue.sceneQueueTaskStarted', 740 'bb.event.ProcessStarted', 741 'bb.event.ProcessProgress', 742 'bb.event.ProcessFinished', 743 ] 744 if extra_events: 745 eventmask.extend(extra_events) 746 ret = self.set_event_mask(eventmask) 747 748 includelogs = self.config_data.getVar('BBINCLUDELOGS') 749 loglines = self.config_data.getVar('BBINCLUDELOGS_LINES') 750 751 ret = self.run_command('buildTargets', targets, task) 752 if handle_events: 753 lastevent = time.time() 754 result = False 755 # Borrowed from knotty, instead somewhat hackily we use the helper 756 # as the object to store "shutdown" on 757 helper = bb.ui.uihelper.BBUIHelper() 758 helper.shutdown = 0 759 parseprogress = None 760 termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet) 761 try: 762 while True: 763 try: 764 event = self.wait_event(0.25) 765 if event: 766 lastevent = time.time() 767 if event_callback and event_callback(event): 768 continue 769 if helper.eventHandler(event): 770 if isinstance(event, bb.build.TaskFailedSilent): 771 self.logger.warning("Logfile for failed setscene task is %s" % event.logfile) 772 elif isinstance(event, bb.build.TaskFailed): 773 bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter) 774 continue 775 if isinstance(event, bb.event.ProcessStarted): 776 if self.quiet > 1: 777 continue 778 parseprogress = bb.ui.knotty.new_progress(event.processname, event.total) 779 parseprogress.start(False) 780 continue 781 if isinstance(event, bb.event.ProcessProgress): 782 if self.quiet > 1: 783 continue 784 if parseprogress: 785 parseprogress.update(event.progress) 786 else: 787 bb.warn("Got ProcessProgress event for something that never started?") 788 continue 789 if isinstance(event, bb.event.ProcessFinished): 790 if self.quiet > 1: 791 continue 792 if parseprogress: 793 parseprogress.finish() 794 parseprogress = None 795 continue 796 if isinstance(event, bb.command.CommandCompleted): 797 result = True 798 break 799 if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)): 800 self.logger.error(str(event)) 801 result = False 802 break 803 if isinstance(event, logging.LogRecord): 804 if event.taskpid == 0 or event.levelno > logging.INFO: 805 self.logger.handle(event) 806 continue 807 if isinstance(event, bb.event.NoProvider): 808 self.logger.error(str(event)) 809 result = False 810 break 811 elif helper.shutdown > 1: 812 break 813 termfilter.updateFooter() 814 if time.time() > (lastevent + (3*60)): 815 if not self.run_command('ping', handle_events=False): 816 print("\nUnable to ping server and no events, closing down...\n") 817 return False 818 except KeyboardInterrupt: 819 termfilter.clearFooter() 820 if helper.shutdown == 1: 821 print("\nSecond Keyboard Interrupt, stopping...\n") 822 ret = self.run_command("stateForceShutdown") 823 if ret and ret[2]: 824 self.logger.error("Unable to cleanly stop: %s" % ret[2]) 825 elif helper.shutdown == 0: 826 print("\nKeyboard Interrupt, closing down...\n") 827 interrupted = True 828 ret = self.run_command("stateShutdown") 829 if ret and ret[2]: 830 self.logger.error("Unable to cleanly shutdown: %s" % ret[2]) 831 helper.shutdown = helper.shutdown + 1 832 termfilter.clearFooter() 833 finally: 834 termfilter.finish() 835 if helper.failed_tasks: 836 result = False 837 return result 838 else: 839 return ret 840 841 def shutdown(self): 842 """ 843 Shut down tinfoil. Disconnects from the server and gracefully 844 releases any associated resources. You must call this function if 845 prepare() has been called, or use a with... block when you create 846 the tinfoil object which will ensure that it gets called. 847 """ 848 try: 849 if self.server_connection: 850 try: 851 self.run_command('clientComplete') 852 finally: 853 _server_connections.remove(self.server_connection) 854 bb.event.ui_queue = [] 855 self.server_connection.terminate() 856 self.server_connection = None 857 858 finally: 859 # Restore logging handlers to how it looked when we started 860 if self.oldhandlers: 861 for handler in self.logger.handlers: 862 if handler not in self.oldhandlers: 863 self.logger.handlers.remove(handler) 864 865 def _reconvert_type(self, obj, origtypename): 866 """ 867 Convert an object back to the right type, in the case 868 that marshalling has changed it (especially with xmlrpc) 869 """ 870 supported_types = { 871 'set': set, 872 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle, 873 } 874 875 origtype = supported_types.get(origtypename, None) 876 if origtype is None: 877 raise Exception('Unsupported type "%s"' % origtypename) 878 if type(obj) == origtype: 879 newobj = obj 880 elif isinstance(obj, dict): 881 # New style class 882 newobj = origtype() 883 for k,v in obj.items(): 884 setattr(newobj, k, v) 885 else: 886 # Assume we can coerce the type 887 newobj = origtype(obj) 888 889 if isinstance(newobj, bb.command.DataStoreConnectionHandle): 890 newobj = TinfoilDataStoreConnector(self, newobj.dsindex) 891 892 return newobj 893 894 895class TinfoilConfigParameters(BitBakeConfigParameters): 896 897 def __init__(self, config_only, **options): 898 self.initial_options = options 899 # Apply some sane defaults 900 if not 'parse_only' in options: 901 self.initial_options['parse_only'] = not config_only 902 #if not 'status_only' in options: 903 # self.initial_options['status_only'] = config_only 904 if not 'ui' in options: 905 self.initial_options['ui'] = 'knotty' 906 if not 'argv' in options: 907 self.initial_options['argv'] = [] 908 909 super(TinfoilConfigParameters, self).__init__() 910 911 def parseCommandLine(self, argv=None): 912 # We don't want any parameters parsed from the command line 913 opts = super(TinfoilConfigParameters, self).parseCommandLine([]) 914 for key, val in self.initial_options.items(): 915 setattr(opts[0], key, val) 916 return opts 917