1# 2# Copyright (C) 2003, 2004 Chris Larson 3# Copyright (C) 2003, 2004 Phil Blundell 4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 5# Copyright (C) 2005 Holger Hans Peter Freyther 6# Copyright (C) 2005 ROAD GmbH 7# Copyright (C) 2006 Richard Purdie 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import os 13import sys 14import logging 15import argparse 16import warnings 17import fcntl 18import time 19import traceback 20import datetime 21 22import bb 23from bb import event 24import bb.msg 25from bb import cooker 26from bb import ui 27from bb import server 28from bb import cookerdata 29 30import bb.server.process 31import bb.server.xmlrpcclient 32 33logger = logging.getLogger("BitBake") 34 35class BBMainException(Exception): 36 pass 37 38class BBMainFatal(bb.BBHandledException): 39 pass 40 41def present_options(optionlist): 42 if len(optionlist) > 1: 43 return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]]) 44 else: 45 return optionlist[0] 46 47class BitbakeHelpFormatter(argparse.HelpFormatter): 48 def _get_help_string(self, action): 49 # We need to do this here rather than in the text we supply to 50 # add_option() because we don't want to call list_extension_modules() 51 # on every execution (since it imports all of the modules) 52 # Note also that we modify option.help rather than the returned text 53 # - this is so that we don't have to re-format the text ourselves 54 if action.dest == 'ui': 55 valid_uis = list_extension_modules(bb.ui, 'main') 56 return action.help.replace('@CHOICES@', present_options(valid_uis)) 57 58 return action.help 59 60def list_extension_modules(pkg, checkattr): 61 """ 62 Lists extension modules in a specific Python package 63 (e.g. UIs, servers). NOTE: Calling this function will import all of the 64 submodules of the specified module in order to check for the specified 65 attribute; this can have unusual side-effects. As a result, this should 66 only be called when displaying help text or error messages. 67 Parameters: 68 pkg: previously imported Python package to list 69 checkattr: attribute to look for in module to determine if it's valid 70 as the type of extension you are looking for 71 """ 72 import pkgutil 73 pkgdir = os.path.dirname(pkg.__file__) 74 75 modules = [] 76 for _, modulename, _ in pkgutil.iter_modules([pkgdir]): 77 if os.path.isdir(os.path.join(pkgdir, modulename)): 78 # ignore directories 79 continue 80 try: 81 module = __import__(pkg.__name__, fromlist=[modulename]) 82 except: 83 # If we can't import it, it's not valid 84 continue 85 module_if = getattr(module, modulename) 86 if getattr(module_if, 'hidden_extension', False): 87 continue 88 if not checkattr or hasattr(module_if, checkattr): 89 modules.append(modulename) 90 return modules 91 92def import_extension_module(pkg, modulename, checkattr): 93 try: 94 # Dynamically load the UI based on the ui name. Although we 95 # suggest a fixed set this allows you to have flexibility in which 96 # ones are available. 97 module = __import__(pkg.__name__, fromlist=[modulename]) 98 return getattr(module, modulename) 99 except AttributeError: 100 modules = present_options(list_extension_modules(pkg, checkattr)) 101 raise BBMainException('FATAL: Unable to import extension module "%s" from %s. ' 102 'Valid extension modules: %s' % (modulename, pkg.__name__, modules)) 103 104# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others""" 105warnlog = logging.getLogger("BitBake.Warnings") 106_warnings_showwarning = warnings.showwarning 107def _showwarning(message, category, filename, lineno, file=None, line=None): 108 if file is not None: 109 if _warnings_showwarning is not None: 110 _warnings_showwarning(message, category, filename, lineno, file, line) 111 else: 112 s = warnings.formatwarning(message, category, filename, lineno) 113 warnlog.warning(s) 114 115warnings.showwarning = _showwarning 116 117def create_bitbake_parser(): 118 parser = argparse.ArgumentParser( 119 description="""\ 120 It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which 121 will provide the layer, BBFILES and other configuration information. 122 """, 123 formatter_class=BitbakeHelpFormatter, 124 allow_abbrev=False, 125 add_help=False, # help is manually added below in a specific argument group 126 ) 127 128 general_group = parser.add_argument_group('General options') 129 task_group = parser.add_argument_group('Task control options') 130 exec_group = parser.add_argument_group('Execution control options') 131 logging_group = parser.add_argument_group('Logging/output control options') 132 server_group = parser.add_argument_group('Server options') 133 config_group = parser.add_argument_group('Configuration options') 134 135 general_group.add_argument("targets", nargs="*", metavar="recipename/target", 136 help="Execute the specified task (default is 'build') for these target " 137 "recipes (.bb files).") 138 139 general_group.add_argument("-s", "--show-versions", action="store_true", 140 help="Show current and preferred versions of all recipes.") 141 142 general_group.add_argument("-e", "--environment", action="store_true", 143 dest="show_environment", 144 help="Show the global or per-recipe environment complete with information" 145 " about where variables were set/changed.") 146 147 general_group.add_argument("-g", "--graphviz", action="store_true", dest="dot_graph", 148 help="Save dependency tree information for the specified " 149 "targets in the dot syntax.") 150 151 # @CHOICES@ is substituted out by BitbakeHelpFormatter above 152 general_group.add_argument("-u", "--ui", 153 default=os.environ.get('BITBAKE_UI', 'knotty'), 154 help="The user interface to use (@CHOICES@ - default %(default)s).") 155 156 general_group.add_argument("--version", action="store_true", 157 help="Show programs version and exit.") 158 159 general_group.add_argument('-h', '--help', action='help', 160 help='Show this help message and exit.') 161 162 163 task_group.add_argument("-f", "--force", action="store_true", 164 help="Force the specified targets/task to run (invalidating any " 165 "existing stamp file).") 166 167 task_group.add_argument("-c", "--cmd", 168 help="Specify the task to execute. The exact options available " 169 "depend on the metadata. Some examples might be 'compile'" 170 " or 'populate_sysroot' or 'listtasks' may give a list of " 171 "the tasks available.") 172 173 task_group.add_argument("-C", "--clear-stamp", dest="invalidate_stamp", 174 help="Invalidate the stamp for the specified task such as 'compile' " 175 "and then run the default task for the specified target(s).") 176 177 task_group.add_argument("--runall", action="append", default=[], 178 help="Run the specified task for any recipe in the taskgraph of the " 179 "specified target (even if it wouldn't otherwise have run).") 180 181 task_group.add_argument("--runonly", action="append", 182 help="Run only the specified task within the taskgraph of the " 183 "specified targets (and any task dependencies those tasks may have).") 184 185 task_group.add_argument("--no-setscene", action="store_true", 186 dest="nosetscene", 187 help="Do not run any setscene tasks. sstate will be ignored and " 188 "everything needed, built.") 189 190 task_group.add_argument("--skip-setscene", action="store_true", 191 dest="skipsetscene", 192 help="Skip setscene tasks if they would be executed. Tasks previously " 193 "restored from sstate will be kept, unlike --no-setscene.") 194 195 task_group.add_argument("--setscene-only", action="store_true", 196 dest="setsceneonly", 197 help="Only run setscene tasks, don't run any real tasks.") 198 199 200 exec_group.add_argument("-n", "--dry-run", action="store_true", 201 help="Don't execute, just go through the motions.") 202 203 exec_group.add_argument("-p", "--parse-only", action="store_true", 204 help="Quit after parsing the BB recipes.") 205 206 exec_group.add_argument("-k", "--continue", action="store_false", dest="halt", 207 help="Continue as much as possible after an error. While the target that " 208 "failed and anything depending on it cannot be built, as much as " 209 "possible will be built before stopping.") 210 211 exec_group.add_argument("-P", "--profile", action="store_true", 212 help="Profile the command and save reports.") 213 214 exec_group.add_argument("-S", "--dump-signatures", action="append", 215 default=[], metavar="SIGNATURE_HANDLER", 216 help="Dump out the signature construction information, with no task " 217 "execution. The SIGNATURE_HANDLER parameter is passed to the " 218 "handler. Two common values are none and printdiff but the handler " 219 "may define more/less. none means only dump the signature, printdiff" 220 " means recursively compare the dumped signature with the most recent" 221 " one in a local build or sstate cache (can be used to find out why tasks re-run" 222 " when that is not expected)") 223 224 exec_group.add_argument("--revisions-changed", action="store_true", 225 help="Set the exit code depending on whether upstream floating " 226 "revisions have changed or not.") 227 228 exec_group.add_argument("-b", "--buildfile", 229 help="Execute tasks from a specific .bb recipe directly. WARNING: Does " 230 "not handle any dependencies from other recipes.") 231 232 logging_group.add_argument("-D", "--debug", action="count", default=0, 233 help="Increase the debug level. You can specify this " 234 "more than once. -D sets the debug level to 1, " 235 "where only bb.debug(1, ...) messages are printed " 236 "to stdout; -DD sets the debug level to 2, where " 237 "both bb.debug(1, ...) and bb.debug(2, ...) " 238 "messages are printed; etc. Without -D, no debug " 239 "messages are printed. Note that -D only affects " 240 "output to stdout. All debug messages are written " 241 "to ${T}/log.do_taskname, regardless of the debug " 242 "level.") 243 244 logging_group.add_argument("-l", "--log-domains", action="append", dest="debug_domains", 245 default=[], 246 help="Show debug logging for the specified logging domains.") 247 248 logging_group.add_argument("-v", "--verbose", action="store_true", 249 help="Enable tracing of shell tasks (with 'set -x'). " 250 "Also print bb.note(...) messages to stdout (in " 251 "addition to writing them to ${T}/log.do_<task>).") 252 253 logging_group.add_argument("-q", "--quiet", action="count", default=0, 254 help="Output less log message data to the terminal. You can specify this " 255 "more than once.") 256 257 logging_group.add_argument("-w", "--write-log", dest="writeeventlog", 258 default=os.environ.get("BBEVENTLOG"), 259 help="Writes the event log of the build to a bitbake event json file. " 260 "Use '' (empty string) to assign the name automatically.") 261 262 263 server_group.add_argument("-B", "--bind", default=False, 264 help="The name/address for the bitbake xmlrpc server to bind to.") 265 266 server_group.add_argument("-T", "--idle-timeout", type=float, dest="server_timeout", 267 default=os.getenv("BB_SERVER_TIMEOUT"), 268 help="Set timeout to unload bitbake server due to inactivity, " 269 "set to -1 means no unload, " 270 "default: Environment variable BB_SERVER_TIMEOUT.") 271 272 server_group.add_argument("--remote-server", 273 default=os.environ.get("BBSERVER"), 274 help="Connect to the specified server.") 275 276 server_group.add_argument("-m", "--kill-server", action="store_true", 277 help="Terminate any running bitbake server.") 278 279 server_group.add_argument("--token", dest="xmlrpctoken", 280 default=os.environ.get("BBTOKEN"), 281 help="Specify the connection token to be used when connecting " 282 "to a remote server.") 283 284 server_group.add_argument("--observe-only", action="store_true", 285 help="Connect to a server as an observing-only client.") 286 287 server_group.add_argument("--status-only", action="store_true", 288 help="Check the status of the remote bitbake server.") 289 290 server_group.add_argument("--server-only", action="store_true", 291 help="Run bitbake without a UI, only starting a server " 292 "(cooker) process.") 293 294 295 config_group.add_argument("-r", "--read", action="append", dest="prefile", default=[], 296 help="Read the specified file before bitbake.conf.") 297 298 config_group.add_argument("-R", "--postread", action="append", dest="postfile", default=[], 299 help="Read the specified file after bitbake.conf.") 300 301 302 config_group.add_argument("-I", "--ignore-deps", action="append", 303 dest="extra_assume_provided", default=[], 304 help="Assume these dependencies don't exist and are already provided " 305 "(equivalent to ASSUME_PROVIDED). Useful to make dependency " 306 "graphs more appealing.") 307 308 return parser 309 310 311class BitBakeConfigParameters(cookerdata.ConfigParameters): 312 def parseCommandLine(self, argv=sys.argv): 313 parser = create_bitbake_parser() 314 options = parser.parse_intermixed_args(argv[1:]) 315 316 if options.version: 317 print("BitBake Build Tool Core version %s" % bb.__version__) 318 sys.exit(0) 319 320 if options.quiet and options.verbose: 321 parser.error("options --quiet and --verbose are mutually exclusive") 322 323 if options.quiet and options.debug: 324 parser.error("options --quiet and --debug are mutually exclusive") 325 326 # use configuration files from environment variables 327 if "BBPRECONF" in os.environ: 328 options.prefile.append(os.environ["BBPRECONF"]) 329 330 if "BBPOSTCONF" in os.environ: 331 options.postfile.append(os.environ["BBPOSTCONF"]) 332 333 # fill in proper log name if not supplied 334 if options.writeeventlog is not None and len(options.writeeventlog) == 0: 335 from datetime import datetime 336 eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S") 337 options.writeeventlog = eventlog 338 339 if options.bind: 340 try: 341 #Checking that the port is a number and is a ':' delimited value 342 (host, port) = options.bind.split(':') 343 port = int(port) 344 except (ValueError,IndexError): 345 raise BBMainException("FATAL: Malformed host:port bind parameter") 346 options.xmlrpcinterface = (host, port) 347 else: 348 options.xmlrpcinterface = (None, 0) 349 350 return options, options.targets 351 352 353def bitbake_main(configParams, configuration): 354 355 # Python multiprocessing requires /dev/shm on Linux 356 if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK): 357 raise BBMainException("FATAL: /dev/shm does not exist or is not writable") 358 359 # Unbuffer stdout to avoid log truncation in the event 360 # of an unorderly exit as well as to provide timely 361 # updates to log files for use with tail 362 try: 363 if sys.stdout.name == '<stdout>': 364 # Reopen with O_SYNC (unbuffered) 365 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL) 366 fl |= os.O_SYNC 367 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl) 368 except: 369 pass 370 371 if configParams.server_only and configParams.remote_server: 372 raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % 373 ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ 374 else "the '--remote-server' option")) 375 376 if configParams.observe_only and not (configParams.remote_server or configParams.bind): 377 raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " 378 "connecting to a server.\n") 379 380 if "BBDEBUG" in os.environ: 381 level = int(os.environ["BBDEBUG"]) 382 if level > configParams.debug: 383 configParams.debug = level 384 385 bb.msg.init_msgconfig(configParams.verbose, configParams.debug, 386 configParams.debug_domains) 387 388 server_connection, ui_module = setup_bitbake(configParams) 389 # No server connection 390 if server_connection is None: 391 if configParams.status_only: 392 return 1 393 if configParams.kill_server: 394 return 0 395 396 if not configParams.server_only: 397 if configParams.status_only: 398 server_connection.terminate() 399 return 0 400 401 try: 402 for event in bb.event.ui_queue: 403 server_connection.events.queue_event(event) 404 bb.event.ui_queue = [] 405 406 return ui_module.main(server_connection.connection, server_connection.events, 407 configParams) 408 finally: 409 server_connection.terminate() 410 else: 411 return 0 412 413 return 1 414 415def timestamp(): 416 return datetime.datetime.now().strftime('%H:%M:%S.%f') 417 418def setup_bitbake(configParams, extrafeatures=None): 419 # Ensure logging messages get sent to the UI as events 420 handler = bb.event.LogHandler() 421 if not configParams.status_only: 422 # In status only mode there are no logs and no UI 423 logger.addHandler(handler) 424 425 if configParams.dump_signatures: 426 if extrafeatures is None: 427 extrafeatures = [] 428 extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO) 429 430 if configParams.server_only: 431 featureset = [] 432 ui_module = None 433 else: 434 ui_module = import_extension_module(bb.ui, configParams.ui, 'main') 435 # Collect the feature set for the UI 436 featureset = getattr(ui_module, "featureSet", []) 437 438 if extrafeatures: 439 for feature in extrafeatures: 440 if not feature in featureset: 441 featureset.append(feature) 442 443 server_connection = None 444 445 # Clear away any spurious environment variables while we stoke up the cooker 446 # (done after import_extension_module() above since for example import gi triggers env var usage) 447 cleanedvars = bb.utils.clean_environment() 448 449 if configParams.remote_server: 450 # Connect to a remote XMLRPC server 451 server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset, 452 configParams.observe_only, configParams.xmlrpctoken) 453 else: 454 retries = 8 455 while retries: 456 try: 457 topdir, lock, lockfile = lockBitbake() 458 sockname = topdir + "/bitbake.sock" 459 if lock: 460 if configParams.status_only or configParams.kill_server: 461 logger.info("bitbake server is not running.") 462 lock.close() 463 return None, None 464 # we start a server with a given featureset 465 logger.info("Starting bitbake server...") 466 # Clear the event queue since we already displayed messages 467 bb.event.ui_queue = [] 468 server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface, configParams.profile) 469 470 else: 471 logger.info("Reconnecting to bitbake server...") 472 if not os.path.exists(sockname): 473 logger.info("Previous bitbake instance shutting down?, waiting to retry... (%s)" % timestamp()) 474 procs = bb.server.process.get_lockfile_process_msg(lockfile) 475 if procs: 476 logger.info("Processes holding bitbake.lock (missing socket %s):\n%s" % (sockname, procs)) 477 logger.info("Directory listing: %s" % (str(os.listdir(topdir)))) 478 i = 0 479 lock = None 480 # Wait for 5s or until we can get the lock 481 while not lock and i < 50: 482 time.sleep(0.1) 483 _, lock, _ = lockBitbake() 484 i += 1 485 if lock: 486 bb.utils.unlockfile(lock) 487 raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?") 488 if not configParams.server_only: 489 server_connection = bb.server.process.connectProcessServer(sockname, featureset) 490 491 if server_connection or configParams.server_only: 492 break 493 except BBMainFatal: 494 raise 495 except (Exception, bb.server.process.ProcessTimeout, SystemExit) as e: 496 # SystemExit does not inherit from the Exception class, needs to be included explicitly 497 if not retries: 498 raise 499 retries -= 1 500 tryno = 8 - retries 501 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)): 502 logger.info("Retrying server connection (#%d)... (%s)" % (tryno, timestamp())) 503 else: 504 logger.info("Retrying server connection (#%d)... (%s, %s)" % (tryno, traceback.format_exc(), timestamp())) 505 506 if not retries: 507 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).") 508 bb.event.print_ui_queue() 509 if retries < 5: 510 time.sleep(5) 511 512 if configParams.kill_server: 513 server_connection.connection.terminateServer() 514 server_connection.terminate() 515 bb.event.ui_queue = [] 516 logger.info("Terminated bitbake server.") 517 return None, None 518 519 # Restore the environment in case the UI needs it 520 for k in cleanedvars: 521 os.environ[k] = cleanedvars[k] 522 523 logger.removeHandler(handler) 524 525 return server_connection, ui_module 526 527def lockBitbake(): 528 topdir = bb.cookerdata.findTopdir() 529 if not topdir: 530 bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?") 531 raise BBMainFatal 532 lockfile = topdir + "/bitbake.lock" 533 return topdir, bb.utils.lockfile(lockfile, False, False), lockfile 534 535