1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import datetime 24import bb 25import bb.msg 26import bb.process 27import bb.progress 28from io import StringIO 29from bb import data, event, utils 30 31bblogger = logging.getLogger('BitBake') 32logger = logging.getLogger('BitBake.Build') 33 34verboseShellLogging = False 35verboseStdoutLogging = False 36 37__mtime_cache = {} 38 39def cached_mtime_noerror(f): 40 if f not in __mtime_cache: 41 try: 42 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 43 except OSError: 44 return 0 45 return __mtime_cache[f] 46 47def reset_cache(): 48 global __mtime_cache 49 __mtime_cache = {} 50 51# When we execute a Python function, we'd like certain things 52# in all namespaces, hence we add them to __builtins__. 53# If we do not do this and use the exec globals, they will 54# not be available to subfunctions. 55if hasattr(__builtins__, '__setitem__'): 56 builtins = __builtins__ 57else: 58 builtins = __builtins__.__dict__ 59 60builtins['bb'] = bb 61builtins['os'] = os 62 63class TaskBase(event.Event): 64 """Base class for task events""" 65 66 def __init__(self, t, fn, logfile, d): 67 self._task = t 68 self._fn = fn 69 self._package = d.getVar("PF") 70 self._mc = d.getVar("BB_CURRENT_MC") 71 self.taskfile = d.getVar("FILE") 72 self.taskname = self._task 73 self.logfile = logfile 74 self.time = time.time() 75 self.pn = d.getVar("PN") 76 self.pv = d.getVar("PV") 77 event.Event.__init__(self) 78 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 79 80 def getTask(self): 81 return self._task 82 83 def setTask(self, task): 84 self._task = task 85 86 def getDisplayName(self): 87 return bb.event.getName(self)[4:] 88 89 task = property(getTask, setTask, None, "task property") 90 91class TaskStarted(TaskBase): 92 """Task execution started""" 93 def __init__(self, t, fn, logfile, taskflags, d): 94 super(TaskStarted, self).__init__(t, fn, logfile, d) 95 self.taskflags = taskflags 96 97class TaskSucceeded(TaskBase): 98 """Task execution completed""" 99 100class TaskFailed(TaskBase): 101 """Task execution failed""" 102 103 def __init__(self, task, fn, logfile, metadata, errprinted = False): 104 self.errprinted = errprinted 105 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 106 107class TaskFailedSilent(TaskBase): 108 """Task execution failed (silently)""" 109 def getDisplayName(self): 110 # Don't need to tell the user it was silent 111 return "Failed" 112 113class TaskInvalid(TaskBase): 114 115 def __init__(self, task, fn, metadata): 116 super(TaskInvalid, self).__init__(task, fn, None, metadata) 117 self._message = "No such task '%s'" % task 118 119class TaskProgress(event.Event): 120 """ 121 Task made some progress that could be reported to the user, usually in 122 the form of a progress bar or similar. 123 NOTE: this class does not inherit from TaskBase since it doesn't need 124 to - it's fired within the task context itself, so we don't have any of 125 the context information that you do in the case of the other events. 126 The event PID can be used to determine which task it came from. 127 The progress value is normally 0-100, but can also be negative 128 indicating that progress has been made but we aren't able to determine 129 how much. 130 The rate is optional, this is simply an extra string to display to the 131 user if specified. 132 """ 133 def __init__(self, progress, rate=None): 134 self.progress = progress 135 self.rate = rate 136 event.Event.__init__(self) 137 138 139class LogTee(object): 140 def __init__(self, logger, outfile): 141 self.outfile = outfile 142 self.logger = logger 143 self.name = self.outfile.name 144 145 def write(self, string): 146 self.logger.plain(string) 147 self.outfile.write(string) 148 149 def __enter__(self): 150 self.outfile.__enter__() 151 return self 152 153 def __exit__(self, *excinfo): 154 self.outfile.__exit__(*excinfo) 155 156 def __repr__(self): 157 return '<LogTee {0}>'.format(self.name) 158 159 def flush(self): 160 self.outfile.flush() 161 162 163class StdoutNoopContextManager: 164 """ 165 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 166 """ 167 def __enter__(self): 168 return sys.stdout 169 170 def __exit__(self, *exc_info): 171 pass 172 173 def write(self, string): 174 return sys.stdout.write(string) 175 176 def flush(self): 177 sys.stdout.flush() 178 179 @property 180 def name(self): 181 if "name" in dir(sys.stdout): 182 return sys.stdout.name 183 return "<mem>" 184 185 186def exec_func(func, d, dirs = None): 187 """Execute a BB 'function'""" 188 189 try: 190 oldcwd = os.getcwd() 191 except: 192 oldcwd = None 193 194 flags = d.getVarFlags(func) 195 cleandirs = flags.get('cleandirs') if flags else None 196 if cleandirs: 197 for cdir in d.expand(cleandirs).split(): 198 bb.utils.remove(cdir, True) 199 bb.utils.mkdirhier(cdir) 200 if cdir == oldcwd: 201 os.chdir(cdir) 202 203 if flags and dirs is None: 204 dirs = flags.get('dirs') 205 if dirs: 206 dirs = d.expand(dirs).split() 207 208 if dirs: 209 for adir in dirs: 210 bb.utils.mkdirhier(adir) 211 adir = dirs[-1] 212 else: 213 adir = None 214 215 body = d.getVar(func, False) 216 if not body: 217 if body is None: 218 logger.warning("Function %s doesn't exist", func) 219 return 220 221 ispython = flags.get('python') 222 223 lockflag = flags.get('lockfiles') 224 if lockflag: 225 lockfiles = [f for f in d.expand(lockflag).split()] 226 else: 227 lockfiles = None 228 229 tempdir = d.getVar('T') 230 231 # or func allows items to be executed outside of the normal 232 # task set, such as buildhistory 233 task = d.getVar('BB_RUNTASK') or func 234 if task == func: 235 taskfunc = task 236 else: 237 taskfunc = "%s.%s" % (task, func) 238 239 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 240 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 241 runfile = os.path.join(tempdir, runfn) 242 bb.utils.mkdirhier(os.path.dirname(runfile)) 243 244 # Setup the courtesy link to the runfn, only for tasks 245 # we create the link 'just' before the run script is created 246 # if we create it after, and if the run script fails, then the 247 # link won't be created as an exception would be fired. 248 if task == func: 249 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 250 if runlink: 251 bb.utils.remove(runlink) 252 253 try: 254 os.symlink(runfn, runlink) 255 except OSError: 256 pass 257 258 with bb.utils.fileslocked(lockfiles): 259 if ispython: 260 exec_func_python(func, d, runfile, cwd=adir) 261 else: 262 exec_func_shell(func, d, runfile, cwd=adir) 263 264 try: 265 curcwd = os.getcwd() 266 except: 267 curcwd = None 268 269 if oldcwd and curcwd != oldcwd: 270 try: 271 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 272 os.chdir(oldcwd) 273 except: 274 pass 275 276_functionfmt = """ 277{function}(d) 278""" 279logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 280def exec_func_python(func, d, runfile, cwd=None): 281 """Execute a python BB 'function'""" 282 283 code = _functionfmt.format(function=func) 284 bb.utils.mkdirhier(os.path.dirname(runfile)) 285 with open(runfile, 'w') as script: 286 bb.data.emit_func_python(func, script, d) 287 288 if cwd: 289 try: 290 olddir = os.getcwd() 291 except OSError as e: 292 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 293 olddir = None 294 os.chdir(cwd) 295 296 bb.debug(2, "Executing python function %s" % func) 297 298 try: 299 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 300 fn = d.getVarFlag(func, "filename", False) 301 lineno = int(d.getVarFlag(func, "lineno", False)) 302 bb.methodpool.insert_method(func, text, fn, lineno - 1) 303 304 if verboseStdoutLogging: 305 sys.stdout.flush() 306 sys.stderr.flush() 307 currout = sys.stdout 308 currerr = sys.stderr 309 sys.stderr = sys.stdout = execio = StringIO() 310 comp = utils.better_compile(code, func, "exec_func_python() autogenerated") 311 utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") 312 finally: 313 if verboseStdoutLogging: 314 execio.flush() 315 logger.plain("%s" % execio.getvalue()) 316 sys.stdout = currout 317 sys.stderr = currerr 318 execio.close() 319 # We want any stdout/stderr to be printed before any other log messages to make debugging 320 # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. 321 sys.stdout.flush() 322 sys.stderr.flush() 323 bb.debug(2, "Python function %s finished" % func) 324 325 if cwd and olddir: 326 try: 327 os.chdir(olddir) 328 except OSError as e: 329 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 330 331def shell_trap_code(): 332 return '''#!/bin/sh\n 333__BITBAKE_LAST_LINE=0 334 335# Emit a useful diagnostic if something fails: 336bb_sh_exit_handler() { 337 ret=$? 338 if [ "$ret" != 0 ]; then 339 echo "WARNING: exit code $ret from a shell command." 340 fi 341 exit $ret 342} 343 344bb_bash_exit_handler() { 345 ret=$? 346 { set +x; } > /dev/null 347 trap "" DEBUG 348 if [ "$ret" != 0 ]; then 349 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 350 351 echo "WARNING: Backtrace (BB generated script): " 352 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 353 if [ "$i" -eq 1 ]; then 354 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 355 else 356 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 357 fi 358 done 359 fi 360 exit $ret 361} 362 363bb_bash_debug_handler() { 364 local line=${BASH_LINENO[0]} 365 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 366 if [ "$line" -eq 1 ]; then 367 return 368 fi 369 370 # Track the line number of commands as they execute. This is so we can have access to the failing line number 371 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 372 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 373 __BITBAKE_LAST_LINE=$line 374 fi 375} 376 377case $BASH_VERSION in 378"") trap 'bb_sh_exit_handler' 0 379 set -e 380 ;; 381*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 382 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 383 set -e 384 shopt -s extdebug 385 ;; 386esac 387''' 388 389def create_progress_handler(func, progress, logfile, d): 390 if progress == 'percent': 391 # Use default regex 392 return bb.progress.BasicProgressHandler(d, outfile=logfile) 393 elif progress.startswith('percent:'): 394 # Use specified regex 395 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 396 elif progress.startswith('outof:'): 397 # Use specified regex 398 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 399 elif progress.startswith("custom:"): 400 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 401 import functools 402 from types import ModuleType 403 404 parts = progress.split(":", 2) 405 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 406 if cls: 407 def resolve(x, y): 408 if not x: 409 return None 410 if isinstance(x, ModuleType): 411 return getattr(x, y, None) 412 return x.get(y) 413 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 414 if not cls_obj: 415 # Fall-back on __builtins__ 416 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 417 if cls_obj: 418 return cls_obj(d, outfile=logfile, otherargs=otherargs) 419 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 420 else: 421 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 422 423 return logfile 424 425def exec_func_shell(func, d, runfile, cwd=None): 426 """Execute a shell function from the metadata 427 428 Note on directory behavior. The 'dirs' varflag should contain a list 429 of the directories you need created prior to execution. The last 430 item in the list is where we will chdir/cd to. 431 """ 432 433 # Don't let the emitted shell script override PWD 434 d.delVarFlag('PWD', 'export') 435 436 with open(runfile, 'w') as script: 437 script.write(shell_trap_code()) 438 439 bb.data.emit_func(func, script, d) 440 441 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 442 script.write("set -x\n") 443 if cwd: 444 script.write("cd '%s'\n" % cwd) 445 script.write("%s\n" % func) 446 script.write(''' 447# cleanup 448ret=$? 449trap '' 0 450exit $ret 451''') 452 453 os.chmod(runfile, 0o775) 454 455 cmd = runfile 456 if d.getVarFlag(func, 'fakeroot', False): 457 fakerootcmd = d.getVar('FAKEROOT') 458 if fakerootcmd: 459 cmd = [fakerootcmd, runfile] 460 461 # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either 462 # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively", 463 # ie. inside another function, in which case stdout is already being captured so we don't 464 # want to Tee here as output would be printed twice, and out of order. 465 if verboseStdoutLogging and sys.stdout == sys.__stdout__: 466 logfile = LogTee(logger, StdoutNoopContextManager()) 467 else: 468 logfile = StdoutNoopContextManager() 469 470 progress = d.getVarFlag(func, 'progress') 471 if progress: 472 try: 473 logfile = create_progress_handler(func, progress, logfile, d) 474 except: 475 from traceback import format_exc 476 logger.error("Failed to create progress handler") 477 logger.error(format_exc()) 478 raise 479 480 fifobuffer = bytearray() 481 def readfifo(data): 482 nonlocal fifobuffer 483 fifobuffer.extend(data) 484 while fifobuffer: 485 message, token, nextmsg = fifobuffer.partition(b"\00") 486 if token: 487 splitval = message.split(b' ', 1) 488 cmd = splitval[0].decode("utf-8") 489 if len(splitval) > 1: 490 value = splitval[1].decode("utf-8") 491 else: 492 value = '' 493 if cmd == 'bbplain': 494 bb.plain(value) 495 elif cmd == 'bbnote': 496 bb.note(value) 497 elif cmd == 'bbverbnote': 498 bb.verbnote(value) 499 elif cmd == 'bbwarn': 500 bb.warn(value) 501 elif cmd == 'bberror': 502 bb.error(value) 503 elif cmd == 'bbfatal': 504 # The caller will call exit themselves, so bb.error() is 505 # what we want here rather than bb.fatal() 506 bb.error(value) 507 elif cmd == 'bbfatal_log': 508 bb.error(value, forcelog=True) 509 elif cmd == 'bbdebug': 510 splitval = value.split(' ', 1) 511 level = int(splitval[0]) 512 value = splitval[1] 513 bb.debug(level, value) 514 else: 515 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 516 fifobuffer = nextmsg 517 else: 518 break 519 520 tempdir = d.getVar('T') 521 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 522 if os.path.exists(fifopath): 523 os.unlink(fifopath) 524 os.mkfifo(fifopath) 525 with open(fifopath, 'r+b', buffering=0) as fifo: 526 try: 527 bb.debug(2, "Executing shell function %s" % func) 528 with open(os.devnull, 'r+') as stdin, logfile: 529 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 530 except bb.process.ExecutionError as exe: 531 # Find the backtrace that the shell trap generated 532 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 533 stdout_lines = (exe.stdout or "").split("\n") 534 backtrace_start_line = None 535 for i, line in enumerate(reversed(stdout_lines)): 536 if backtrace_marker_regex.search(line): 537 backtrace_start_line = len(stdout_lines) - i 538 break 539 540 # Read the backtrace frames, starting at the location we just found 541 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 542 r"?P<lineno>\d+)") 543 backtrace_frames = [] 544 if backtrace_start_line: 545 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 546 match = backtrace_entry_regex.search(line) 547 if match: 548 backtrace_frames.append(match.groupdict()) 549 550 with open(runfile, "r") as script: 551 script_lines = [line.rstrip() for line in script.readlines()] 552 553 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 554 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 555 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 556 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 557 better_frames = [] 558 # Skip the very last frame since it's just the call to the shell task in the body of the script 559 for frame in backtrace_frames[:-1]: 560 # Check whether the frame corresponds to a function defined in the script vs external script. 561 if os.path.samefile(frame["file"], runfile): 562 # Search backwards from the frame lineno to locate the comment that BB injected 563 i = int(frame["lineno"]) - 1 564 while i >= 0: 565 match = script_metadata_comment_regex.match(script_lines[i]) 566 if match: 567 # Calculate the relative line in the function itself 568 relative_line_in_function = int(frame["lineno"]) - i - 2 569 # Calculate line in the function as declared in the metadata 570 metadata_function_line = relative_line_in_function + int(match["lineno"]) 571 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 572 frameno=frame["frameno"], 573 funcname=frame["funcname"], 574 file=match["file"], 575 lineno=metadata_function_line 576 )) 577 break 578 i -= 1 579 else: 580 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 581 582 if better_frames: 583 better_frames = ("\t{0}".format(frame) for frame in better_frames) 584 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 585 raise 586 finally: 587 os.unlink(fifopath) 588 589 bb.debug(2, "Shell function %s finished" % func) 590 591def _task_data(fn, task, d): 592 localdata = bb.data.createCopy(d) 593 localdata.setVar('BB_FILENAME', fn) 594 localdata.setVar('OVERRIDES', 'task-%s:%s' % 595 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 596 bb.data.expandKeys(localdata) 597 return localdata 598 599def _exec_task(fn, task, d, quieterr): 600 """Execute a BB 'task' 601 602 Execution of a task involves a bit more setup than executing a function, 603 running it with its own local metadata, and with some useful variables set. 604 """ 605 if not d.getVarFlag(task, 'task', False): 606 event.fire(TaskInvalid(task, fn, d), d) 607 logger.error("No such task: %s" % task) 608 return 1 609 610 logger.debug("Executing task %s", task) 611 612 localdata = _task_data(fn, task, d) 613 tempdir = localdata.getVar('T') 614 if not tempdir: 615 bb.fatal("T variable not set, unable to build") 616 617 # Change nice level if we're asked to 618 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 619 if nice: 620 curnice = os.nice(0) 621 nice = int(nice) - curnice 622 newnice = os.nice(nice) 623 logger.debug("Renice to %s " % newnice) 624 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 625 if ionice: 626 try: 627 cls, prio = ionice.split(".", 1) 628 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 629 except: 630 bb.warn("Invalid ionice level %s" % ionice) 631 632 bb.utils.mkdirhier(tempdir) 633 634 # Determine the logfile to generate 635 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 636 logbase = logfmt.format(task=task, pid=os.getpid()) 637 638 # Document the order of the tasks... 639 logorder = os.path.join(tempdir, 'log.task_order') 640 try: 641 with open(logorder, 'a') as logorderfile: 642 timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") 643 logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) 644 except OSError: 645 logger.exception("Opening log file '%s'", logorder) 646 pass 647 648 # Setup the courtesy link to the logfn 649 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 650 logfn = os.path.join(tempdir, logbase) 651 if loglink: 652 bb.utils.remove(loglink) 653 654 try: 655 os.symlink(logbase, loglink) 656 except OSError: 657 pass 658 659 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 660 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 661 662 class ErrorCheckHandler(logging.Handler): 663 def __init__(self): 664 self.triggered = False 665 logging.Handler.__init__(self, logging.ERROR) 666 def emit(self, record): 667 if getattr(record, 'forcelog', False): 668 self.triggered = False 669 else: 670 self.triggered = True 671 672 # Handle logfiles 673 try: 674 bb.utils.mkdirhier(os.path.dirname(logfn)) 675 logfile = open(logfn, 'w') 676 except OSError: 677 logger.exception("Opening log file '%s'", logfn) 678 pass 679 680 # Dup the existing fds so we dont lose them 681 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 682 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 683 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 684 685 # Replace those fds with our own 686 with open('/dev/null', 'r') as si: 687 os.dup2(si.fileno(), osi[1]) 688 os.dup2(logfile.fileno(), oso[1]) 689 os.dup2(logfile.fileno(), ose[1]) 690 691 # Ensure Python logging goes to the logfile 692 handler = logging.StreamHandler(logfile) 693 handler.setFormatter(logformatter) 694 # Always enable full debug output into task logfiles 695 handler.setLevel(logging.DEBUG - 2) 696 bblogger.addHandler(handler) 697 698 errchk = ErrorCheckHandler() 699 bblogger.addHandler(errchk) 700 701 localdata.setVar('BB_LOGFILE', logfn) 702 localdata.setVar('BB_RUNTASK', task) 703 localdata.setVar('BB_TASK_LOGGER', bblogger) 704 705 flags = localdata.getVarFlags(task) 706 707 try: 708 try: 709 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 710 711 for func in (prefuncs or '').split(): 712 exec_func(func, localdata) 713 exec_func(task, localdata) 714 for func in (postfuncs or '').split(): 715 exec_func(func, localdata) 716 finally: 717 # Need to flush and close the logs before sending events where the 718 # UI may try to look at the logs. 719 sys.stdout.flush() 720 sys.stderr.flush() 721 722 bblogger.removeHandler(handler) 723 724 # Restore the backup fds 725 os.dup2(osi[0], osi[1]) 726 os.dup2(oso[0], oso[1]) 727 os.dup2(ose[0], ose[1]) 728 729 # Close the backup fds 730 os.close(osi[0]) 731 os.close(oso[0]) 732 os.close(ose[0]) 733 734 logfile.close() 735 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 736 logger.debug2("Zero size logfn %s, removing", logfn) 737 bb.utils.remove(logfn) 738 bb.utils.remove(loglink) 739 except (Exception, SystemExit) as exc: 740 handled = False 741 if isinstance(exc, bb.BBHandledException): 742 handled = True 743 744 if quieterr: 745 if not handled: 746 logger.warning(repr(exc)) 747 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 748 else: 749 errprinted = errchk.triggered 750 # If the output is already on stdout, we've printed the information in the 751 # logs once already so don't duplicate 752 if verboseStdoutLogging or handled: 753 errprinted = True 754 if not handled: 755 logger.error(repr(exc)) 756 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 757 return 1 758 759 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 760 761 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 762 make_stamp(task, localdata) 763 764 return 0 765 766def exec_task(fn, task, d, profile = False): 767 try: 768 quieterr = False 769 if d.getVarFlag(task, "quieterrors", False) is not None: 770 quieterr = True 771 772 if profile: 773 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 774 try: 775 import cProfile as profile 776 except: 777 import profile 778 prof = profile.Profile() 779 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 780 prof.dump_stats(profname) 781 bb.utils.process_profilelog(profname) 782 783 return ret 784 else: 785 return _exec_task(fn, task, d, quieterr) 786 787 except Exception: 788 from traceback import format_exc 789 if not quieterr: 790 logger.error("Build of %s failed" % (task)) 791 logger.error(format_exc()) 792 failedevent = TaskFailed(task, None, d, True) 793 event.fire(failedevent, d) 794 return 1 795 796def _get_cleanmask(taskname, mcfn): 797 """ 798 Internal stamp helper function to generate stamp cleaning mask 799 Returns the stamp path+filename 800 801 In the bitbake core, d can be a CacheData and file_name will be set. 802 When called in task context, d will be a data store, file_name will not be set 803 """ 804 cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn) 805 taskflagname = taskname.replace("_setscene", "") 806 if cleanmask: 807 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 808 return [] 809 810def clean_stamp_mcfn(task, mcfn): 811 cleanmask = _get_cleanmask(task, mcfn) 812 for mask in cleanmask: 813 for name in glob.glob(mask): 814 # Preserve sigdata files in the stamps directory 815 if "sigdata" in name or "sigbasedata" in name: 816 continue 817 # Preserve taint files in the stamps directory 818 if name.endswith('.taint'): 819 continue 820 os.unlink(name) 821 822def clean_stamp(task, d): 823 mcfn = d.getVar('BB_FILENAME') 824 clean_stamp_mcfn(task, mcfn) 825 826def make_stamp_mcfn(task, mcfn): 827 828 basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn) 829 830 stampdir = os.path.dirname(basestamp) 831 if cached_mtime_noerror(stampdir) == 0: 832 bb.utils.mkdirhier(stampdir) 833 834 clean_stamp_mcfn(task, mcfn) 835 836 # Remove the file and recreate to force timestamp 837 # change on broken NFS filesystems 838 if basestamp: 839 bb.utils.remove(basestamp) 840 open(basestamp, "w").close() 841 842def make_stamp(task, d): 843 """ 844 Creates/updates a stamp for a given task 845 """ 846 mcfn = d.getVar('BB_FILENAME') 847 848 make_stamp_mcfn(task, mcfn) 849 850 # If we're in task context, write out a signature file for each task 851 # as it completes 852 if not task.endswith("_setscene"): 853 stampbase = bb.parse.siggen.stampfile_base(mcfn) 854 bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True) 855 856 857def find_stale_stamps(task, mcfn): 858 current = bb.parse.siggen.stampfile_mcfn(task, mcfn) 859 current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn) 860 cleanmask = _get_cleanmask(task, mcfn) 861 found = [] 862 for mask in cleanmask: 863 for name in glob.glob(mask): 864 if "sigdata" in name or "sigbasedata" in name: 865 continue 866 if name.endswith('.taint'): 867 continue 868 if name == current or name == current2: 869 continue 870 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 871 found.append(name) 872 return found 873 874def write_taint(task, d): 875 """ 876 Creates a "taint" file which will force the specified task and its 877 dependents to be re-run the next time by influencing the value of its 878 taskhash. 879 """ 880 mcfn = d.getVar('BB_FILENAME') 881 bb.parse.siggen.invalidate_task(task, mcfn) 882 883def add_tasks(tasklist, d): 884 task_deps = d.getVar('_task_deps', False) 885 if not task_deps: 886 task_deps = {} 887 if not 'tasks' in task_deps: 888 task_deps['tasks'] = [] 889 if not 'parents' in task_deps: 890 task_deps['parents'] = {} 891 892 for task in tasklist: 893 task = d.expand(task) 894 895 d.setVarFlag(task, 'task', 1) 896 897 if not task in task_deps['tasks']: 898 task_deps['tasks'].append(task) 899 900 flags = d.getVarFlags(task) 901 def getTask(name): 902 if not name in task_deps: 903 task_deps[name] = {} 904 if name in flags: 905 deptask = d.expand(flags[name]) 906 if name in ['noexec', 'fakeroot', 'nostamp']: 907 if deptask != '1': 908 bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " 909 "will result in the flag not being set. See YP bug #13808.".format(name)) 910 911 task_deps[name][task] = deptask 912 getTask('mcdepends') 913 getTask('depends') 914 getTask('rdepends') 915 getTask('deptask') 916 getTask('rdeptask') 917 getTask('recrdeptask') 918 getTask('recideptask') 919 getTask('nostamp') 920 getTask('fakeroot') 921 getTask('noexec') 922 getTask('umask') 923 task_deps['parents'][task] = [] 924 if 'deps' in flags: 925 for dep in flags['deps']: 926 # Check and warn for "addtask task after foo" while foo does not exist 927 #if not dep in tasklist: 928 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 929 dep = d.expand(dep) 930 task_deps['parents'][task].append(dep) 931 932 # don't assume holding a reference 933 d.setVar('_task_deps', task_deps) 934 935def addtask(task, before, after, d): 936 if task[:3] != "do_": 937 task = "do_" + task 938 939 d.setVarFlag(task, "task", 1) 940 bbtasks = d.getVar('__BBTASKS', False) or [] 941 if task not in bbtasks: 942 bbtasks.append(task) 943 d.setVar('__BBTASKS', bbtasks) 944 945 existing = d.getVarFlag(task, "deps", False) or [] 946 if after is not None: 947 # set up deps for function 948 for entry in after.split(): 949 if entry not in existing: 950 existing.append(entry) 951 d.setVarFlag(task, "deps", existing) 952 if before is not None: 953 # set up things that depend on this func 954 for entry in before.split(): 955 existing = d.getVarFlag(entry, "deps", False) or [] 956 if task not in existing: 957 d.setVarFlag(entry, "deps", [task] + existing) 958 959def deltask(task, d): 960 if task[:3] != "do_": 961 task = "do_" + task 962 963 bbtasks = d.getVar('__BBTASKS', False) or [] 964 if task in bbtasks: 965 bbtasks.remove(task) 966 d.delVarFlag(task, 'task') 967 d.setVar('__BBTASKS', bbtasks) 968 969 d.delVarFlag(task, 'deps') 970 for bbtask in d.getVar('__BBTASKS', False) or []: 971 deps = d.getVarFlag(bbtask, 'deps', False) or [] 972 if task in deps: 973 deps.remove(task) 974 d.setVarFlag(bbtask, 'deps', deps) 975 976def preceedtask(task, with_recrdeptasks, d): 977 """ 978 Returns a set of tasks in the current recipe which were specified as 979 precondition by the task itself ("after") or which listed themselves 980 as precondition ("before"). Preceeding tasks specified via the 981 "recrdeptask" are included in the result only if requested. Beware 982 that this may lead to the task itself being listed. 983 """ 984 preceed = set() 985 986 # Ignore tasks which don't exist 987 tasks = d.getVar('__BBTASKS', False) 988 if task not in tasks: 989 return preceed 990 991 preceed.update(d.getVarFlag(task, 'deps') or []) 992 if with_recrdeptasks: 993 recrdeptask = d.getVarFlag(task, 'recrdeptask') 994 if recrdeptask: 995 preceed.update(recrdeptask.split()) 996 return preceed 997 998def tasksbetween(task_start, task_end, d): 999 """ 1000 Return the list of tasks between two tasks in the current recipe, 1001 where task_start is to start at and task_end is the task to end at 1002 (and task_end has a dependency chain back to task_start). 1003 """ 1004 outtasks = [] 1005 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 1006 def follow_chain(task, endtask, chain=None): 1007 if not chain: 1008 chain = [] 1009 if task in chain: 1010 bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) 1011 chain.append(task) 1012 for othertask in tasks: 1013 if othertask == task: 1014 continue 1015 if task == endtask: 1016 for ctask in chain: 1017 if ctask not in outtasks: 1018 outtasks.append(ctask) 1019 else: 1020 deps = d.getVarFlag(othertask, 'deps', False) 1021 if task in deps: 1022 follow_chain(othertask, endtask, chain) 1023 chain.pop() 1024 follow_chain(task_start, task_end) 1025 return outtasks 1026