1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import datetime 24import bb 25import bb.msg 26import bb.process 27import bb.progress 28from io import StringIO 29from bb import data, event, utils 30 31bblogger = logging.getLogger('BitBake') 32logger = logging.getLogger('BitBake.Build') 33 34verboseShellLogging = False 35verboseStdoutLogging = False 36 37__mtime_cache = {} 38 39def cached_mtime_noerror(f): 40 if f not in __mtime_cache: 41 try: 42 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 43 except OSError: 44 return 0 45 return __mtime_cache[f] 46 47def reset_cache(): 48 global __mtime_cache 49 __mtime_cache = {} 50 51# When we execute a Python function, we'd like certain things 52# in all namespaces, hence we add them to __builtins__. 53# If we do not do this and use the exec globals, they will 54# not be available to subfunctions. 55if hasattr(__builtins__, '__setitem__'): 56 builtins = __builtins__ 57else: 58 builtins = __builtins__.__dict__ 59 60builtins['bb'] = bb 61builtins['os'] = os 62 63class TaskBase(event.Event): 64 """Base class for task events""" 65 66 def __init__(self, t, fn, logfile, d): 67 self._task = t 68 self._fn = fn 69 self._package = d.getVar("PF") 70 self._mc = d.getVar("BB_CURRENT_MC") 71 self.taskfile = d.getVar("FILE") 72 self.taskname = self._task 73 self.logfile = logfile 74 self.time = time.time() 75 self.pn = d.getVar("PN") 76 self.pv = d.getVar("PV") 77 event.Event.__init__(self) 78 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 79 80 def getTask(self): 81 return self._task 82 83 def setTask(self, task): 84 self._task = task 85 86 def getDisplayName(self): 87 return bb.event.getName(self)[4:] 88 89 task = property(getTask, setTask, None, "task property") 90 91class TaskStarted(TaskBase): 92 """Task execution started""" 93 def __init__(self, t, fn, logfile, taskflags, d): 94 super(TaskStarted, self).__init__(t, fn, logfile, d) 95 self.taskflags = taskflags 96 97class TaskSucceeded(TaskBase): 98 """Task execution completed""" 99 100class TaskFailed(TaskBase): 101 """Task execution failed""" 102 103 def __init__(self, task, fn, logfile, metadata, errprinted = False): 104 self.errprinted = errprinted 105 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 106 107class TaskFailedSilent(TaskBase): 108 """Task execution failed (silently)""" 109 def getDisplayName(self): 110 # Don't need to tell the user it was silent 111 return "Failed" 112 113class TaskInvalid(TaskBase): 114 115 def __init__(self, task, fn, metadata): 116 super(TaskInvalid, self).__init__(task, fn, None, metadata) 117 self._message = "No such task '%s'" % task 118 119class TaskProgress(event.Event): 120 """ 121 Task made some progress that could be reported to the user, usually in 122 the form of a progress bar or similar. 123 NOTE: this class does not inherit from TaskBase since it doesn't need 124 to - it's fired within the task context itself, so we don't have any of 125 the context information that you do in the case of the other events. 126 The event PID can be used to determine which task it came from. 127 The progress value is normally 0-100, but can also be negative 128 indicating that progress has been made but we aren't able to determine 129 how much. 130 The rate is optional, this is simply an extra string to display to the 131 user if specified. 132 """ 133 def __init__(self, progress, rate=None): 134 self.progress = progress 135 self.rate = rate 136 event.Event.__init__(self) 137 138 139class LogTee(object): 140 def __init__(self, logger, outfile): 141 self.outfile = outfile 142 self.logger = logger 143 self.name = self.outfile.name 144 145 def write(self, string): 146 self.logger.plain(string) 147 self.outfile.write(string) 148 149 def __enter__(self): 150 self.outfile.__enter__() 151 return self 152 153 def __exit__(self, *excinfo): 154 self.outfile.__exit__(*excinfo) 155 156 def __repr__(self): 157 return '<LogTee {0}>'.format(self.name) 158 159 def flush(self): 160 self.outfile.flush() 161 162 163class StdoutNoopContextManager: 164 """ 165 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 166 """ 167 def __enter__(self): 168 return sys.stdout 169 170 def __exit__(self, *exc_info): 171 pass 172 173 def write(self, string): 174 return sys.stdout.write(string) 175 176 def flush(self): 177 sys.stdout.flush() 178 179 @property 180 def name(self): 181 if "name" in dir(sys.stdout): 182 return sys.stdout.name 183 return "<mem>" 184 185 186def exec_func(func, d, dirs = None): 187 """Execute a BB 'function'""" 188 189 try: 190 oldcwd = os.getcwd() 191 except: 192 oldcwd = None 193 194 flags = d.getVarFlags(func) 195 cleandirs = flags.get('cleandirs') if flags else None 196 if cleandirs: 197 for cdir in d.expand(cleandirs).split(): 198 bb.utils.remove(cdir, True) 199 bb.utils.mkdirhier(cdir) 200 201 if flags and dirs is None: 202 dirs = flags.get('dirs') 203 if dirs: 204 dirs = d.expand(dirs).split() 205 206 if dirs: 207 for adir in dirs: 208 bb.utils.mkdirhier(adir) 209 adir = dirs[-1] 210 else: 211 adir = None 212 213 body = d.getVar(func, False) 214 if not body: 215 if body is None: 216 logger.warning("Function %s doesn't exist", func) 217 return 218 219 ispython = flags.get('python') 220 221 lockflag = flags.get('lockfiles') 222 if lockflag: 223 lockfiles = [f for f in d.expand(lockflag).split()] 224 else: 225 lockfiles = None 226 227 tempdir = d.getVar('T') 228 229 # or func allows items to be executed outside of the normal 230 # task set, such as buildhistory 231 task = d.getVar('BB_RUNTASK') or func 232 if task == func: 233 taskfunc = task 234 else: 235 taskfunc = "%s.%s" % (task, func) 236 237 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 238 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 239 runfile = os.path.join(tempdir, runfn) 240 bb.utils.mkdirhier(os.path.dirname(runfile)) 241 242 # Setup the courtesy link to the runfn, only for tasks 243 # we create the link 'just' before the run script is created 244 # if we create it after, and if the run script fails, then the 245 # link won't be created as an exception would be fired. 246 if task == func: 247 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 248 if runlink: 249 bb.utils.remove(runlink) 250 251 try: 252 os.symlink(runfn, runlink) 253 except OSError: 254 pass 255 256 with bb.utils.fileslocked(lockfiles): 257 if ispython: 258 exec_func_python(func, d, runfile, cwd=adir) 259 else: 260 exec_func_shell(func, d, runfile, cwd=adir) 261 262 try: 263 curcwd = os.getcwd() 264 except: 265 curcwd = None 266 267 if oldcwd and curcwd != oldcwd: 268 try: 269 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 270 os.chdir(oldcwd) 271 except: 272 pass 273 274_functionfmt = """ 275{function}(d) 276""" 277logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 278def exec_func_python(func, d, runfile, cwd=None): 279 """Execute a python BB 'function'""" 280 281 code = _functionfmt.format(function=func) 282 bb.utils.mkdirhier(os.path.dirname(runfile)) 283 with open(runfile, 'w') as script: 284 bb.data.emit_func_python(func, script, d) 285 286 if cwd: 287 try: 288 olddir = os.getcwd() 289 except OSError as e: 290 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 291 olddir = None 292 os.chdir(cwd) 293 294 bb.debug(2, "Executing python function %s" % func) 295 296 try: 297 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 298 fn = d.getVarFlag(func, "filename", False) 299 lineno = int(d.getVarFlag(func, "lineno", False)) 300 bb.methodpool.insert_method(func, text, fn, lineno - 1) 301 302 if verboseStdoutLogging: 303 sys.stdout.flush() 304 sys.stderr.flush() 305 currout = sys.stdout 306 currerr = sys.stderr 307 sys.stderr = sys.stdout = execio = StringIO() 308 comp = utils.better_compile(code, func, "exec_func_python() autogenerated") 309 utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") 310 finally: 311 if verboseStdoutLogging: 312 execio.flush() 313 logger.plain("%s" % execio.getvalue()) 314 sys.stdout = currout 315 sys.stderr = currerr 316 execio.close() 317 # We want any stdout/stderr to be printed before any other log messages to make debugging 318 # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. 319 sys.stdout.flush() 320 sys.stderr.flush() 321 bb.debug(2, "Python function %s finished" % func) 322 323 if cwd and olddir: 324 try: 325 os.chdir(olddir) 326 except OSError as e: 327 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 328 329def shell_trap_code(): 330 return '''#!/bin/sh\n 331__BITBAKE_LAST_LINE=0 332 333# Emit a useful diagnostic if something fails: 334bb_sh_exit_handler() { 335 ret=$? 336 if [ "$ret" != 0 ]; then 337 echo "WARNING: exit code $ret from a shell command." 338 fi 339 exit $ret 340} 341 342bb_bash_exit_handler() { 343 ret=$? 344 { set +x; } > /dev/null 345 trap "" DEBUG 346 if [ "$ret" != 0 ]; then 347 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 348 349 echo "WARNING: Backtrace (BB generated script): " 350 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 351 if [ "$i" -eq 1 ]; then 352 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 353 else 354 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 355 fi 356 done 357 fi 358 exit $ret 359} 360 361bb_bash_debug_handler() { 362 local line=${BASH_LINENO[0]} 363 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 364 if [ "$line" -eq 1 ]; then 365 return 366 fi 367 368 # Track the line number of commands as they execute. This is so we can have access to the failing line number 369 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 370 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 371 __BITBAKE_LAST_LINE=$line 372 fi 373} 374 375case $BASH_VERSION in 376"") trap 'bb_sh_exit_handler' 0 377 set -e 378 ;; 379*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 380 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 381 set -e 382 shopt -s extdebug 383 ;; 384esac 385''' 386 387def create_progress_handler(func, progress, logfile, d): 388 if progress == 'percent': 389 # Use default regex 390 return bb.progress.BasicProgressHandler(d, outfile=logfile) 391 elif progress.startswith('percent:'): 392 # Use specified regex 393 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 394 elif progress.startswith('outof:'): 395 # Use specified regex 396 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 397 elif progress.startswith("custom:"): 398 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 399 import functools 400 from types import ModuleType 401 402 parts = progress.split(":", 2) 403 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 404 if cls: 405 def resolve(x, y): 406 if not x: 407 return None 408 if isinstance(x, ModuleType): 409 return getattr(x, y, None) 410 return x.get(y) 411 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 412 if not cls_obj: 413 # Fall-back on __builtins__ 414 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 415 if cls_obj: 416 return cls_obj(d, outfile=logfile, otherargs=otherargs) 417 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 418 else: 419 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 420 421 return logfile 422 423def exec_func_shell(func, d, runfile, cwd=None): 424 """Execute a shell function from the metadata 425 426 Note on directory behavior. The 'dirs' varflag should contain a list 427 of the directories you need created prior to execution. The last 428 item in the list is where we will chdir/cd to. 429 """ 430 431 # Don't let the emitted shell script override PWD 432 d.delVarFlag('PWD', 'export') 433 434 with open(runfile, 'w') as script: 435 script.write(shell_trap_code()) 436 437 bb.data.emit_func(func, script, d) 438 439 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 440 script.write("set -x\n") 441 if cwd: 442 script.write("cd '%s'\n" % cwd) 443 script.write("%s\n" % func) 444 script.write(''' 445# cleanup 446ret=$? 447trap '' 0 448exit $ret 449''') 450 451 os.chmod(runfile, 0o775) 452 453 cmd = runfile 454 if d.getVarFlag(func, 'fakeroot', False): 455 fakerootcmd = d.getVar('FAKEROOT') 456 if fakerootcmd: 457 cmd = [fakerootcmd, runfile] 458 459 # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either 460 # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively", 461 # ie. inside another function, in which case stdout is already being captured so we don't 462 # want to Tee here as output would be printed twice, and out of order. 463 if verboseStdoutLogging and sys.stdout == sys.__stdout__: 464 logfile = LogTee(logger, StdoutNoopContextManager()) 465 else: 466 logfile = StdoutNoopContextManager() 467 468 progress = d.getVarFlag(func, 'progress') 469 if progress: 470 try: 471 logfile = create_progress_handler(func, progress, logfile, d) 472 except: 473 from traceback import format_exc 474 logger.error("Failed to create progress handler") 475 logger.error(format_exc()) 476 raise 477 478 fifobuffer = bytearray() 479 def readfifo(data): 480 nonlocal fifobuffer 481 fifobuffer.extend(data) 482 while fifobuffer: 483 message, token, nextmsg = fifobuffer.partition(b"\00") 484 if token: 485 splitval = message.split(b' ', 1) 486 cmd = splitval[0].decode("utf-8") 487 if len(splitval) > 1: 488 value = splitval[1].decode("utf-8") 489 else: 490 value = '' 491 if cmd == 'bbplain': 492 bb.plain(value) 493 elif cmd == 'bbnote': 494 bb.note(value) 495 elif cmd == 'bbverbnote': 496 bb.verbnote(value) 497 elif cmd == 'bbwarn': 498 bb.warn(value) 499 elif cmd == 'bberror': 500 bb.error(value) 501 elif cmd == 'bbfatal': 502 # The caller will call exit themselves, so bb.error() is 503 # what we want here rather than bb.fatal() 504 bb.error(value) 505 elif cmd == 'bbfatal_log': 506 bb.error(value, forcelog=True) 507 elif cmd == 'bbdebug': 508 splitval = value.split(' ', 1) 509 level = int(splitval[0]) 510 value = splitval[1] 511 bb.debug(level, value) 512 else: 513 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 514 fifobuffer = nextmsg 515 else: 516 break 517 518 tempdir = d.getVar('T') 519 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 520 if os.path.exists(fifopath): 521 os.unlink(fifopath) 522 os.mkfifo(fifopath) 523 with open(fifopath, 'r+b', buffering=0) as fifo: 524 try: 525 bb.debug(2, "Executing shell function %s" % func) 526 with open(os.devnull, 'r+') as stdin, logfile: 527 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 528 except bb.process.ExecutionError as exe: 529 # Find the backtrace that the shell trap generated 530 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 531 stdout_lines = (exe.stdout or "").split("\n") 532 backtrace_start_line = None 533 for i, line in enumerate(reversed(stdout_lines)): 534 if backtrace_marker_regex.search(line): 535 backtrace_start_line = len(stdout_lines) - i 536 break 537 538 # Read the backtrace frames, starting at the location we just found 539 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 540 r"?P<lineno>\d+)") 541 backtrace_frames = [] 542 if backtrace_start_line: 543 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 544 match = backtrace_entry_regex.search(line) 545 if match: 546 backtrace_frames.append(match.groupdict()) 547 548 with open(runfile, "r") as script: 549 script_lines = [line.rstrip() for line in script.readlines()] 550 551 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 552 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 553 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 554 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 555 better_frames = [] 556 # Skip the very last frame since it's just the call to the shell task in the body of the script 557 for frame in backtrace_frames[:-1]: 558 # Check whether the frame corresponds to a function defined in the script vs external script. 559 if os.path.samefile(frame["file"], runfile): 560 # Search backwards from the frame lineno to locate the comment that BB injected 561 i = int(frame["lineno"]) - 1 562 while i >= 0: 563 match = script_metadata_comment_regex.match(script_lines[i]) 564 if match: 565 # Calculate the relative line in the function itself 566 relative_line_in_function = int(frame["lineno"]) - i - 2 567 # Calculate line in the function as declared in the metadata 568 metadata_function_line = relative_line_in_function + int(match["lineno"]) 569 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 570 frameno=frame["frameno"], 571 funcname=frame["funcname"], 572 file=match["file"], 573 lineno=metadata_function_line 574 )) 575 break 576 i -= 1 577 else: 578 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 579 580 if better_frames: 581 better_frames = ("\t{0}".format(frame) for frame in better_frames) 582 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 583 raise 584 finally: 585 os.unlink(fifopath) 586 587 bb.debug(2, "Shell function %s finished" % func) 588 589def _task_data(fn, task, d): 590 localdata = bb.data.createCopy(d) 591 localdata.setVar('BB_FILENAME', fn) 592 localdata.setVar('OVERRIDES', 'task-%s:%s' % 593 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 594 bb.data.expandKeys(localdata) 595 return localdata 596 597def _exec_task(fn, task, d, quieterr): 598 """Execute a BB 'task' 599 600 Execution of a task involves a bit more setup than executing a function, 601 running it with its own local metadata, and with some useful variables set. 602 """ 603 if not d.getVarFlag(task, 'task', False): 604 event.fire(TaskInvalid(task, fn, d), d) 605 logger.error("No such task: %s" % task) 606 return 1 607 608 logger.debug("Executing task %s", task) 609 610 localdata = _task_data(fn, task, d) 611 tempdir = localdata.getVar('T') 612 if not tempdir: 613 bb.fatal("T variable not set, unable to build") 614 615 # Change nice level if we're asked to 616 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 617 if nice: 618 curnice = os.nice(0) 619 nice = int(nice) - curnice 620 newnice = os.nice(nice) 621 logger.debug("Renice to %s " % newnice) 622 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 623 if ionice: 624 try: 625 cls, prio = ionice.split(".", 1) 626 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 627 except: 628 bb.warn("Invalid ionice level %s" % ionice) 629 630 bb.utils.mkdirhier(tempdir) 631 632 # Determine the logfile to generate 633 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 634 logbase = logfmt.format(task=task, pid=os.getpid()) 635 636 # Document the order of the tasks... 637 logorder = os.path.join(tempdir, 'log.task_order') 638 try: 639 with open(logorder, 'a') as logorderfile: 640 timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") 641 logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) 642 except OSError: 643 logger.exception("Opening log file '%s'", logorder) 644 pass 645 646 # Setup the courtesy link to the logfn 647 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 648 logfn = os.path.join(tempdir, logbase) 649 if loglink: 650 bb.utils.remove(loglink) 651 652 try: 653 os.symlink(logbase, loglink) 654 except OSError: 655 pass 656 657 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 658 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 659 660 class ErrorCheckHandler(logging.Handler): 661 def __init__(self): 662 self.triggered = False 663 logging.Handler.__init__(self, logging.ERROR) 664 def emit(self, record): 665 if getattr(record, 'forcelog', False): 666 self.triggered = False 667 else: 668 self.triggered = True 669 670 # Handle logfiles 671 try: 672 bb.utils.mkdirhier(os.path.dirname(logfn)) 673 logfile = open(logfn, 'w') 674 except OSError: 675 logger.exception("Opening log file '%s'", logfn) 676 pass 677 678 # Dup the existing fds so we dont lose them 679 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 680 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 681 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 682 683 # Replace those fds with our own 684 with open('/dev/null', 'r') as si: 685 os.dup2(si.fileno(), osi[1]) 686 os.dup2(logfile.fileno(), oso[1]) 687 os.dup2(logfile.fileno(), ose[1]) 688 689 # Ensure Python logging goes to the logfile 690 handler = logging.StreamHandler(logfile) 691 handler.setFormatter(logformatter) 692 # Always enable full debug output into task logfiles 693 handler.setLevel(logging.DEBUG - 2) 694 bblogger.addHandler(handler) 695 696 errchk = ErrorCheckHandler() 697 bblogger.addHandler(errchk) 698 699 localdata.setVar('BB_LOGFILE', logfn) 700 localdata.setVar('BB_RUNTASK', task) 701 localdata.setVar('BB_TASK_LOGGER', bblogger) 702 703 flags = localdata.getVarFlags(task) 704 705 try: 706 try: 707 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 708 709 for func in (prefuncs or '').split(): 710 exec_func(func, localdata) 711 exec_func(task, localdata) 712 for func in (postfuncs or '').split(): 713 exec_func(func, localdata) 714 finally: 715 # Need to flush and close the logs before sending events where the 716 # UI may try to look at the logs. 717 sys.stdout.flush() 718 sys.stderr.flush() 719 720 bblogger.removeHandler(handler) 721 722 # Restore the backup fds 723 os.dup2(osi[0], osi[1]) 724 os.dup2(oso[0], oso[1]) 725 os.dup2(ose[0], ose[1]) 726 727 # Close the backup fds 728 os.close(osi[0]) 729 os.close(oso[0]) 730 os.close(ose[0]) 731 732 logfile.close() 733 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 734 logger.debug2("Zero size logfn %s, removing", logfn) 735 bb.utils.remove(logfn) 736 bb.utils.remove(loglink) 737 except (Exception, SystemExit) as exc: 738 handled = False 739 if isinstance(exc, bb.BBHandledException): 740 handled = True 741 742 if quieterr: 743 if not handled: 744 logger.warning(repr(exc)) 745 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 746 else: 747 errprinted = errchk.triggered 748 # If the output is already on stdout, we've printed the information in the 749 # logs once already so don't duplicate 750 if verboseStdoutLogging or handled: 751 errprinted = True 752 if not handled: 753 logger.error(repr(exc)) 754 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 755 return 1 756 757 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 758 759 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 760 make_stamp(task, localdata) 761 762 return 0 763 764def exec_task(fn, task, d, profile = False): 765 try: 766 quieterr = False 767 if d.getVarFlag(task, "quieterrors", False) is not None: 768 quieterr = True 769 770 if profile: 771 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 772 try: 773 import cProfile as profile 774 except: 775 import profile 776 prof = profile.Profile() 777 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 778 prof.dump_stats(profname) 779 bb.utils.process_profilelog(profname) 780 781 return ret 782 else: 783 return _exec_task(fn, task, d, quieterr) 784 785 except Exception: 786 from traceback import format_exc 787 if not quieterr: 788 logger.error("Build of %s failed" % (task)) 789 logger.error(format_exc()) 790 failedevent = TaskFailed(task, None, d, True) 791 event.fire(failedevent, d) 792 return 1 793 794def _get_cleanmask(taskname, mcfn): 795 """ 796 Internal stamp helper function to generate stamp cleaning mask 797 Returns the stamp path+filename 798 799 In the bitbake core, d can be a CacheData and file_name will be set. 800 When called in task context, d will be a data store, file_name will not be set 801 """ 802 cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn) 803 taskflagname = taskname.replace("_setscene", "") 804 if cleanmask: 805 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 806 return [] 807 808def clean_stamp_mcfn(task, mcfn): 809 cleanmask = _get_cleanmask(task, mcfn) 810 for mask in cleanmask: 811 for name in glob.glob(mask): 812 # Preserve sigdata files in the stamps directory 813 if "sigdata" in name or "sigbasedata" in name: 814 continue 815 # Preserve taint files in the stamps directory 816 if name.endswith('.taint'): 817 continue 818 os.unlink(name) 819 820def clean_stamp(task, d): 821 mcfn = d.getVar('BB_FILENAME') 822 clean_stamp_mcfn(task, mcfn) 823 824def make_stamp_mcfn(task, mcfn): 825 826 basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn) 827 828 stampdir = os.path.dirname(basestamp) 829 if cached_mtime_noerror(stampdir) == 0: 830 bb.utils.mkdirhier(stampdir) 831 832 clean_stamp_mcfn(task, mcfn) 833 834 # Remove the file and recreate to force timestamp 835 # change on broken NFS filesystems 836 if basestamp: 837 bb.utils.remove(basestamp) 838 open(basestamp, "w").close() 839 840def make_stamp(task, d): 841 """ 842 Creates/updates a stamp for a given task 843 """ 844 mcfn = d.getVar('BB_FILENAME') 845 846 make_stamp_mcfn(task, mcfn) 847 848 # If we're in task context, write out a signature file for each task 849 # as it completes 850 if not task.endswith("_setscene"): 851 stampbase = bb.parse.siggen.stampfile_base(mcfn) 852 bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True) 853 854 855def find_stale_stamps(task, mcfn): 856 current = bb.parse.siggen.stampfile_mcfn(task, mcfn) 857 current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn) 858 cleanmask = _get_cleanmask(task, mcfn) 859 found = [] 860 for mask in cleanmask: 861 for name in glob.glob(mask): 862 if "sigdata" in name or "sigbasedata" in name: 863 continue 864 if name.endswith('.taint'): 865 continue 866 if name == current or name == current2: 867 continue 868 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 869 found.append(name) 870 return found 871 872def write_taint(task, d): 873 """ 874 Creates a "taint" file which will force the specified task and its 875 dependents to be re-run the next time by influencing the value of its 876 taskhash. 877 """ 878 mcfn = d.getVar('BB_FILENAME') 879 bb.parse.siggen.invalidate_task(task, mcfn) 880 881def add_tasks(tasklist, d): 882 task_deps = d.getVar('_task_deps', False) 883 if not task_deps: 884 task_deps = {} 885 if not 'tasks' in task_deps: 886 task_deps['tasks'] = [] 887 if not 'parents' in task_deps: 888 task_deps['parents'] = {} 889 890 for task in tasklist: 891 task = d.expand(task) 892 893 d.setVarFlag(task, 'task', 1) 894 895 if not task in task_deps['tasks']: 896 task_deps['tasks'].append(task) 897 898 flags = d.getVarFlags(task) 899 def getTask(name): 900 if not name in task_deps: 901 task_deps[name] = {} 902 if name in flags: 903 deptask = d.expand(flags[name]) 904 if name in ['noexec', 'fakeroot', 'nostamp']: 905 if deptask != '1': 906 bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " 907 "will result in the flag not being set. See YP bug #13808.".format(name)) 908 909 task_deps[name][task] = deptask 910 getTask('mcdepends') 911 getTask('depends') 912 getTask('rdepends') 913 getTask('deptask') 914 getTask('rdeptask') 915 getTask('recrdeptask') 916 getTask('recideptask') 917 getTask('nostamp') 918 getTask('fakeroot') 919 getTask('noexec') 920 getTask('umask') 921 task_deps['parents'][task] = [] 922 if 'deps' in flags: 923 for dep in flags['deps']: 924 # Check and warn for "addtask task after foo" while foo does not exist 925 #if not dep in tasklist: 926 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 927 dep = d.expand(dep) 928 task_deps['parents'][task].append(dep) 929 930 # don't assume holding a reference 931 d.setVar('_task_deps', task_deps) 932 933def addtask(task, before, after, d): 934 if task[:3] != "do_": 935 task = "do_" + task 936 937 d.setVarFlag(task, "task", 1) 938 bbtasks = d.getVar('__BBTASKS', False) or [] 939 if task not in bbtasks: 940 bbtasks.append(task) 941 d.setVar('__BBTASKS', bbtasks) 942 943 existing = d.getVarFlag(task, "deps", False) or [] 944 if after is not None: 945 # set up deps for function 946 for entry in after.split(): 947 if entry not in existing: 948 existing.append(entry) 949 d.setVarFlag(task, "deps", existing) 950 if before is not None: 951 # set up things that depend on this func 952 for entry in before.split(): 953 existing = d.getVarFlag(entry, "deps", False) or [] 954 if task not in existing: 955 d.setVarFlag(entry, "deps", [task] + existing) 956 957def deltask(task, d): 958 if task[:3] != "do_": 959 task = "do_" + task 960 961 bbtasks = d.getVar('__BBTASKS', False) or [] 962 if task in bbtasks: 963 bbtasks.remove(task) 964 d.delVarFlag(task, 'task') 965 d.setVar('__BBTASKS', bbtasks) 966 967 d.delVarFlag(task, 'deps') 968 for bbtask in d.getVar('__BBTASKS', False) or []: 969 deps = d.getVarFlag(bbtask, 'deps', False) or [] 970 if task in deps: 971 deps.remove(task) 972 d.setVarFlag(bbtask, 'deps', deps) 973 974def preceedtask(task, with_recrdeptasks, d): 975 """ 976 Returns a set of tasks in the current recipe which were specified as 977 precondition by the task itself ("after") or which listed themselves 978 as precondition ("before"). Preceeding tasks specified via the 979 "recrdeptask" are included in the result only if requested. Beware 980 that this may lead to the task itself being listed. 981 """ 982 preceed = set() 983 984 # Ignore tasks which don't exist 985 tasks = d.getVar('__BBTASKS', False) 986 if task not in tasks: 987 return preceed 988 989 preceed.update(d.getVarFlag(task, 'deps') or []) 990 if with_recrdeptasks: 991 recrdeptask = d.getVarFlag(task, 'recrdeptask') 992 if recrdeptask: 993 preceed.update(recrdeptask.split()) 994 return preceed 995 996def tasksbetween(task_start, task_end, d): 997 """ 998 Return the list of tasks between two tasks in the current recipe, 999 where task_start is to start at and task_end is the task to end at 1000 (and task_end has a dependency chain back to task_start). 1001 """ 1002 outtasks = [] 1003 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 1004 def follow_chain(task, endtask, chain=None): 1005 if not chain: 1006 chain = [] 1007 if task in chain: 1008 bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) 1009 chain.append(task) 1010 for othertask in tasks: 1011 if othertask == task: 1012 continue 1013 if task == endtask: 1014 for ctask in chain: 1015 if ctask not in outtasks: 1016 outtasks.append(ctask) 1017 else: 1018 deps = d.getVarFlag(othertask, 'deps', False) 1019 if task in deps: 1020 follow_chain(othertask, endtask, chain) 1021 chain.pop() 1022 follow_chain(task_start, task_end) 1023 return outtasks 1024