1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import datetime 24import bb 25import bb.msg 26import bb.process 27import bb.progress 28from bb import data, event, utils 29 30bblogger = logging.getLogger('BitBake') 31logger = logging.getLogger('BitBake.Build') 32 33verboseShellLogging = False 34verboseStdoutLogging = False 35 36__mtime_cache = {} 37 38def cached_mtime_noerror(f): 39 if f not in __mtime_cache: 40 try: 41 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 42 except OSError: 43 return 0 44 return __mtime_cache[f] 45 46def reset_cache(): 47 global __mtime_cache 48 __mtime_cache = {} 49 50# When we execute a Python function, we'd like certain things 51# in all namespaces, hence we add them to __builtins__. 52# If we do not do this and use the exec globals, they will 53# not be available to subfunctions. 54if hasattr(__builtins__, '__setitem__'): 55 builtins = __builtins__ 56else: 57 builtins = __builtins__.__dict__ 58 59builtins['bb'] = bb 60builtins['os'] = os 61 62class TaskBase(event.Event): 63 """Base class for task events""" 64 65 def __init__(self, t, fn, logfile, d): 66 self._task = t 67 self._fn = fn 68 self._package = d.getVar("PF") 69 self._mc = d.getVar("BB_CURRENT_MC") 70 self.taskfile = d.getVar("FILE") 71 self.taskname = self._task 72 self.logfile = logfile 73 self.time = time.time() 74 self.pn = d.getVar("PN") 75 self.pv = d.getVar("PV") 76 event.Event.__init__(self) 77 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 78 79 def getTask(self): 80 return self._task 81 82 def setTask(self, task): 83 self._task = task 84 85 def getDisplayName(self): 86 return bb.event.getName(self)[4:] 87 88 task = property(getTask, setTask, None, "task property") 89 90class TaskStarted(TaskBase): 91 """Task execution started""" 92 def __init__(self, t, fn, logfile, taskflags, d): 93 super(TaskStarted, self).__init__(t, fn, logfile, d) 94 self.taskflags = taskflags 95 96class TaskSucceeded(TaskBase): 97 """Task execution completed""" 98 99class TaskFailed(TaskBase): 100 """Task execution failed""" 101 102 def __init__(self, task, fn, logfile, metadata, errprinted = False): 103 self.errprinted = errprinted 104 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 105 106class TaskFailedSilent(TaskBase): 107 """Task execution failed (silently)""" 108 def getDisplayName(self): 109 # Don't need to tell the user it was silent 110 return "Failed" 111 112class TaskInvalid(TaskBase): 113 114 def __init__(self, task, fn, metadata): 115 super(TaskInvalid, self).__init__(task, fn, None, metadata) 116 self._message = "No such task '%s'" % task 117 118class TaskProgress(event.Event): 119 """ 120 Task made some progress that could be reported to the user, usually in 121 the form of a progress bar or similar. 122 NOTE: this class does not inherit from TaskBase since it doesn't need 123 to - it's fired within the task context itself, so we don't have any of 124 the context information that you do in the case of the other events. 125 The event PID can be used to determine which task it came from. 126 The progress value is normally 0-100, but can also be negative 127 indicating that progress has been made but we aren't able to determine 128 how much. 129 The rate is optional, this is simply an extra string to display to the 130 user if specified. 131 """ 132 def __init__(self, progress, rate=None): 133 self.progress = progress 134 self.rate = rate 135 event.Event.__init__(self) 136 137 138class LogTee(object): 139 def __init__(self, logger, outfile): 140 self.outfile = outfile 141 self.logger = logger 142 self.name = self.outfile.name 143 144 def write(self, string): 145 self.logger.plain(string) 146 self.outfile.write(string) 147 148 def __enter__(self): 149 self.outfile.__enter__() 150 return self 151 152 def __exit__(self, *excinfo): 153 self.outfile.__exit__(*excinfo) 154 155 def __repr__(self): 156 return '<LogTee {0}>'.format(self.name) 157 158 def flush(self): 159 self.outfile.flush() 160 161 162class StdoutNoopContextManager: 163 """ 164 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 165 """ 166 def __enter__(self): 167 return sys.stdout 168 169 def __exit__(self, *exc_info): 170 pass 171 172 def write(self, string): 173 return sys.stdout.write(string) 174 175 def flush(self): 176 sys.stdout.flush() 177 178 @property 179 def name(self): 180 return sys.stdout.name 181 182 183def exec_func(func, d, dirs = None): 184 """Execute a BB 'function'""" 185 186 try: 187 oldcwd = os.getcwd() 188 except: 189 oldcwd = None 190 191 flags = d.getVarFlags(func) 192 cleandirs = flags.get('cleandirs') if flags else None 193 if cleandirs: 194 for cdir in d.expand(cleandirs).split(): 195 bb.utils.remove(cdir, True) 196 bb.utils.mkdirhier(cdir) 197 198 if flags and dirs is None: 199 dirs = flags.get('dirs') 200 if dirs: 201 dirs = d.expand(dirs).split() 202 203 if dirs: 204 for adir in dirs: 205 bb.utils.mkdirhier(adir) 206 adir = dirs[-1] 207 else: 208 adir = None 209 210 body = d.getVar(func, False) 211 if not body: 212 if body is None: 213 logger.warning("Function %s doesn't exist", func) 214 return 215 216 ispython = flags.get('python') 217 218 lockflag = flags.get('lockfiles') 219 if lockflag: 220 lockfiles = [f for f in d.expand(lockflag).split()] 221 else: 222 lockfiles = None 223 224 tempdir = d.getVar('T') 225 226 # or func allows items to be executed outside of the normal 227 # task set, such as buildhistory 228 task = d.getVar('BB_RUNTASK') or func 229 if task == func: 230 taskfunc = task 231 else: 232 taskfunc = "%s.%s" % (task, func) 233 234 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 235 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 236 runfile = os.path.join(tempdir, runfn) 237 bb.utils.mkdirhier(os.path.dirname(runfile)) 238 239 # Setup the courtesy link to the runfn, only for tasks 240 # we create the link 'just' before the run script is created 241 # if we create it after, and if the run script fails, then the 242 # link won't be created as an exception would be fired. 243 if task == func: 244 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 245 if runlink: 246 bb.utils.remove(runlink) 247 248 try: 249 os.symlink(runfn, runlink) 250 except OSError: 251 pass 252 253 with bb.utils.fileslocked(lockfiles): 254 if ispython: 255 exec_func_python(func, d, runfile, cwd=adir) 256 else: 257 exec_func_shell(func, d, runfile, cwd=adir) 258 259 try: 260 curcwd = os.getcwd() 261 except: 262 curcwd = None 263 264 if oldcwd and curcwd != oldcwd: 265 try: 266 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 267 os.chdir(oldcwd) 268 except: 269 pass 270 271_functionfmt = """ 272{function}(d) 273""" 274logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 275def exec_func_python(func, d, runfile, cwd=None): 276 """Execute a python BB 'function'""" 277 278 code = _functionfmt.format(function=func) 279 bb.utils.mkdirhier(os.path.dirname(runfile)) 280 with open(runfile, 'w') as script: 281 bb.data.emit_func_python(func, script, d) 282 283 if cwd: 284 try: 285 olddir = os.getcwd() 286 except OSError as e: 287 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 288 olddir = None 289 os.chdir(cwd) 290 291 bb.debug(2, "Executing python function %s" % func) 292 293 try: 294 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 295 fn = d.getVarFlag(func, "filename", False) 296 lineno = int(d.getVarFlag(func, "lineno", False)) 297 bb.methodpool.insert_method(func, text, fn, lineno - 1) 298 299 comp = utils.better_compile(code, func, "exec_func_python() autogenerated") 300 utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") 301 finally: 302 # We want any stdout/stderr to be printed before any other log messages to make debugging 303 # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. 304 sys.stdout.flush() 305 sys.stderr.flush() 306 bb.debug(2, "Python function %s finished" % func) 307 308 if cwd and olddir: 309 try: 310 os.chdir(olddir) 311 except OSError as e: 312 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 313 314def shell_trap_code(): 315 return '''#!/bin/sh\n 316__BITBAKE_LAST_LINE=0 317 318# Emit a useful diagnostic if something fails: 319bb_sh_exit_handler() { 320 ret=$? 321 if [ "$ret" != 0 ]; then 322 echo "WARNING: exit code $ret from a shell command." 323 fi 324 exit $ret 325} 326 327bb_bash_exit_handler() { 328 ret=$? 329 { set +x; } > /dev/null 330 trap "" DEBUG 331 if [ "$ret" != 0 ]; then 332 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 333 334 echo "WARNING: Backtrace (BB generated script): " 335 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 336 if [ "$i" -eq 1 ]; then 337 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 338 else 339 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 340 fi 341 done 342 fi 343 exit $ret 344} 345 346bb_bash_debug_handler() { 347 local line=${BASH_LINENO[0]} 348 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 349 if [ "$line" -eq 1 ]; then 350 return 351 fi 352 353 # Track the line number of commands as they execute. This is so we can have access to the failing line number 354 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 355 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 356 __BITBAKE_LAST_LINE=$line 357 fi 358} 359 360case $BASH_VERSION in 361"") trap 'bb_sh_exit_handler' 0 362 set -e 363 ;; 364*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 365 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 366 set -e 367 shopt -s extdebug 368 ;; 369esac 370''' 371 372def create_progress_handler(func, progress, logfile, d): 373 if progress == 'percent': 374 # Use default regex 375 return bb.progress.BasicProgressHandler(d, outfile=logfile) 376 elif progress.startswith('percent:'): 377 # Use specified regex 378 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 379 elif progress.startswith('outof:'): 380 # Use specified regex 381 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 382 elif progress.startswith("custom:"): 383 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 384 import functools 385 from types import ModuleType 386 387 parts = progress.split(":", 2) 388 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 389 if cls: 390 def resolve(x, y): 391 if not x: 392 return None 393 if isinstance(x, ModuleType): 394 return getattr(x, y, None) 395 return x.get(y) 396 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 397 if not cls_obj: 398 # Fall-back on __builtins__ 399 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 400 if cls_obj: 401 return cls_obj(d, outfile=logfile, otherargs=otherargs) 402 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 403 else: 404 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 405 406 return logfile 407 408def exec_func_shell(func, d, runfile, cwd=None): 409 """Execute a shell function from the metadata 410 411 Note on directory behavior. The 'dirs' varflag should contain a list 412 of the directories you need created prior to execution. The last 413 item in the list is where we will chdir/cd to. 414 """ 415 416 # Don't let the emitted shell script override PWD 417 d.delVarFlag('PWD', 'export') 418 419 with open(runfile, 'w') as script: 420 script.write(shell_trap_code()) 421 422 bb.data.emit_func(func, script, d) 423 424 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 425 script.write("set -x\n") 426 if cwd: 427 script.write("cd '%s'\n" % cwd) 428 script.write("%s\n" % func) 429 script.write(''' 430# cleanup 431ret=$? 432trap '' 0 433exit $ret 434''') 435 436 os.chmod(runfile, 0o775) 437 438 cmd = runfile 439 if d.getVarFlag(func, 'fakeroot', False): 440 fakerootcmd = d.getVar('FAKEROOT') 441 if fakerootcmd: 442 cmd = [fakerootcmd, runfile] 443 444 if verboseStdoutLogging: 445 logfile = LogTee(logger, StdoutNoopContextManager()) 446 else: 447 logfile = StdoutNoopContextManager() 448 449 progress = d.getVarFlag(func, 'progress') 450 if progress: 451 try: 452 logfile = create_progress_handler(func, progress, logfile, d) 453 except: 454 from traceback import format_exc 455 logger.error("Failed to create progress handler") 456 logger.error(format_exc()) 457 raise 458 459 fifobuffer = bytearray() 460 def readfifo(data): 461 nonlocal fifobuffer 462 fifobuffer.extend(data) 463 while fifobuffer: 464 message, token, nextmsg = fifobuffer.partition(b"\00") 465 if token: 466 splitval = message.split(b' ', 1) 467 cmd = splitval[0].decode("utf-8") 468 if len(splitval) > 1: 469 value = splitval[1].decode("utf-8") 470 else: 471 value = '' 472 if cmd == 'bbplain': 473 bb.plain(value) 474 elif cmd == 'bbnote': 475 bb.note(value) 476 elif cmd == 'bbverbnote': 477 bb.verbnote(value) 478 elif cmd == 'bbwarn': 479 bb.warn(value) 480 elif cmd == 'bberror': 481 bb.error(value) 482 elif cmd == 'bbfatal': 483 # The caller will call exit themselves, so bb.error() is 484 # what we want here rather than bb.fatal() 485 bb.error(value) 486 elif cmd == 'bbfatal_log': 487 bb.error(value, forcelog=True) 488 elif cmd == 'bbdebug': 489 splitval = value.split(' ', 1) 490 level = int(splitval[0]) 491 value = splitval[1] 492 bb.debug(level, value) 493 else: 494 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 495 fifobuffer = nextmsg 496 else: 497 break 498 499 tempdir = d.getVar('T') 500 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 501 if os.path.exists(fifopath): 502 os.unlink(fifopath) 503 os.mkfifo(fifopath) 504 with open(fifopath, 'r+b', buffering=0) as fifo: 505 try: 506 bb.debug(2, "Executing shell function %s" % func) 507 with open(os.devnull, 'r+') as stdin, logfile: 508 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 509 except bb.process.ExecutionError as exe: 510 # Find the backtrace that the shell trap generated 511 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 512 stdout_lines = (exe.stdout or "").split("\n") 513 backtrace_start_line = None 514 for i, line in enumerate(reversed(stdout_lines)): 515 if backtrace_marker_regex.search(line): 516 backtrace_start_line = len(stdout_lines) - i 517 break 518 519 # Read the backtrace frames, starting at the location we just found 520 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 521 r"?P<lineno>\d+)") 522 backtrace_frames = [] 523 if backtrace_start_line: 524 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 525 match = backtrace_entry_regex.search(line) 526 if match: 527 backtrace_frames.append(match.groupdict()) 528 529 with open(runfile, "r") as script: 530 script_lines = [line.rstrip() for line in script.readlines()] 531 532 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 533 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 534 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 535 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 536 better_frames = [] 537 # Skip the very last frame since it's just the call to the shell task in the body of the script 538 for frame in backtrace_frames[:-1]: 539 # Check whether the frame corresponds to a function defined in the script vs external script. 540 if os.path.samefile(frame["file"], runfile): 541 # Search backwards from the frame lineno to locate the comment that BB injected 542 i = int(frame["lineno"]) - 1 543 while i >= 0: 544 match = script_metadata_comment_regex.match(script_lines[i]) 545 if match: 546 # Calculate the relative line in the function itself 547 relative_line_in_function = int(frame["lineno"]) - i - 2 548 # Calculate line in the function as declared in the metadata 549 metadata_function_line = relative_line_in_function + int(match["lineno"]) 550 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 551 frameno=frame["frameno"], 552 funcname=frame["funcname"], 553 file=match["file"], 554 lineno=metadata_function_line 555 )) 556 break 557 i -= 1 558 else: 559 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 560 561 if better_frames: 562 better_frames = ("\t{0}".format(frame) for frame in better_frames) 563 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 564 raise 565 finally: 566 os.unlink(fifopath) 567 568 bb.debug(2, "Shell function %s finished" % func) 569 570def _task_data(fn, task, d): 571 localdata = bb.data.createCopy(d) 572 localdata.setVar('BB_FILENAME', fn) 573 localdata.setVar('OVERRIDES', 'task-%s:%s' % 574 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 575 bb.data.expandKeys(localdata) 576 return localdata 577 578def _exec_task(fn, task, d, quieterr): 579 """Execute a BB 'task' 580 581 Execution of a task involves a bit more setup than executing a function, 582 running it with its own local metadata, and with some useful variables set. 583 """ 584 if not d.getVarFlag(task, 'task', False): 585 event.fire(TaskInvalid(task, fn, d), d) 586 logger.error("No such task: %s" % task) 587 return 1 588 589 logger.debug("Executing task %s", task) 590 591 localdata = _task_data(fn, task, d) 592 tempdir = localdata.getVar('T') 593 if not tempdir: 594 bb.fatal("T variable not set, unable to build") 595 596 # Change nice level if we're asked to 597 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 598 if nice: 599 curnice = os.nice(0) 600 nice = int(nice) - curnice 601 newnice = os.nice(nice) 602 logger.debug("Renice to %s " % newnice) 603 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 604 if ionice: 605 try: 606 cls, prio = ionice.split(".", 1) 607 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 608 except: 609 bb.warn("Invalid ionice level %s" % ionice) 610 611 bb.utils.mkdirhier(tempdir) 612 613 # Determine the logfile to generate 614 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 615 logbase = logfmt.format(task=task, pid=os.getpid()) 616 617 # Document the order of the tasks... 618 logorder = os.path.join(tempdir, 'log.task_order') 619 try: 620 with open(logorder, 'a') as logorderfile: 621 timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") 622 logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) 623 except OSError: 624 logger.exception("Opening log file '%s'", logorder) 625 pass 626 627 # Setup the courtesy link to the logfn 628 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 629 logfn = os.path.join(tempdir, logbase) 630 if loglink: 631 bb.utils.remove(loglink) 632 633 try: 634 os.symlink(logbase, loglink) 635 except OSError: 636 pass 637 638 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 639 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 640 641 class ErrorCheckHandler(logging.Handler): 642 def __init__(self): 643 self.triggered = False 644 logging.Handler.__init__(self, logging.ERROR) 645 def emit(self, record): 646 if getattr(record, 'forcelog', False): 647 self.triggered = False 648 else: 649 self.triggered = True 650 651 # Handle logfiles 652 try: 653 bb.utils.mkdirhier(os.path.dirname(logfn)) 654 logfile = open(logfn, 'w') 655 except OSError: 656 logger.exception("Opening log file '%s'", logfn) 657 pass 658 659 # Dup the existing fds so we dont lose them 660 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 661 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 662 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 663 664 # Replace those fds with our own 665 with open('/dev/null', 'r') as si: 666 os.dup2(si.fileno(), osi[1]) 667 os.dup2(logfile.fileno(), oso[1]) 668 os.dup2(logfile.fileno(), ose[1]) 669 670 # Ensure Python logging goes to the logfile 671 handler = logging.StreamHandler(logfile) 672 handler.setFormatter(logformatter) 673 # Always enable full debug output into task logfiles 674 handler.setLevel(logging.DEBUG - 2) 675 bblogger.addHandler(handler) 676 677 errchk = ErrorCheckHandler() 678 bblogger.addHandler(errchk) 679 680 localdata.setVar('BB_LOGFILE', logfn) 681 localdata.setVar('BB_RUNTASK', task) 682 localdata.setVar('BB_TASK_LOGGER', bblogger) 683 684 flags = localdata.getVarFlags(task) 685 686 try: 687 try: 688 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 689 690 for func in (prefuncs or '').split(): 691 exec_func(func, localdata) 692 exec_func(task, localdata) 693 for func in (postfuncs or '').split(): 694 exec_func(func, localdata) 695 finally: 696 # Need to flush and close the logs before sending events where the 697 # UI may try to look at the logs. 698 sys.stdout.flush() 699 sys.stderr.flush() 700 701 bblogger.removeHandler(handler) 702 703 # Restore the backup fds 704 os.dup2(osi[0], osi[1]) 705 os.dup2(oso[0], oso[1]) 706 os.dup2(ose[0], ose[1]) 707 708 # Close the backup fds 709 os.close(osi[0]) 710 os.close(oso[0]) 711 os.close(ose[0]) 712 713 logfile.close() 714 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 715 logger.debug2("Zero size logfn %s, removing", logfn) 716 bb.utils.remove(logfn) 717 bb.utils.remove(loglink) 718 except (Exception, SystemExit) as exc: 719 handled = False 720 if isinstance(exc, bb.BBHandledException): 721 handled = True 722 723 if quieterr: 724 if not handled: 725 logger.warning(repr(exc)) 726 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 727 else: 728 errprinted = errchk.triggered 729 # If the output is already on stdout, we've printed the information in the 730 # logs once already so don't duplicate 731 if verboseStdoutLogging or handled: 732 errprinted = True 733 if not handled: 734 logger.error(repr(exc)) 735 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 736 return 1 737 738 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 739 740 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 741 make_stamp(task, localdata) 742 743 return 0 744 745def exec_task(fn, task, d, profile = False): 746 try: 747 quieterr = False 748 if d.getVarFlag(task, "quieterrors", False) is not None: 749 quieterr = True 750 751 if profile: 752 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 753 try: 754 import cProfile as profile 755 except: 756 import profile 757 prof = profile.Profile() 758 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 759 prof.dump_stats(profname) 760 bb.utils.process_profilelog(profname) 761 762 return ret 763 else: 764 return _exec_task(fn, task, d, quieterr) 765 766 except Exception: 767 from traceback import format_exc 768 if not quieterr: 769 logger.error("Build of %s failed" % (task)) 770 logger.error(format_exc()) 771 failedevent = TaskFailed(task, None, d, True) 772 event.fire(failedevent, d) 773 return 1 774 775def _get_cleanmask(taskname, mcfn): 776 """ 777 Internal stamp helper function to generate stamp cleaning mask 778 Returns the stamp path+filename 779 780 In the bitbake core, d can be a CacheData and file_name will be set. 781 When called in task context, d will be a data store, file_name will not be set 782 """ 783 cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn) 784 taskflagname = taskname.replace("_setscene", "") 785 if cleanmask: 786 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 787 return [] 788 789def clean_stamp_mcfn(task, mcfn): 790 cleanmask = _get_cleanmask(task, mcfn) 791 for mask in cleanmask: 792 for name in glob.glob(mask): 793 # Preserve sigdata files in the stamps directory 794 if "sigdata" in name or "sigbasedata" in name: 795 continue 796 # Preserve taint files in the stamps directory 797 if name.endswith('.taint'): 798 continue 799 os.unlink(name) 800 801def clean_stamp(task, d): 802 mcfn = d.getVar('BB_FILENAME') 803 clean_stamp_mcfn(task, mcfn) 804 805def make_stamp_mcfn(task, mcfn): 806 807 basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn) 808 809 stampdir = os.path.dirname(basestamp) 810 if cached_mtime_noerror(stampdir) == 0: 811 bb.utils.mkdirhier(stampdir) 812 813 clean_stamp_mcfn(task, mcfn) 814 815 # Remove the file and recreate to force timestamp 816 # change on broken NFS filesystems 817 if basestamp: 818 bb.utils.remove(basestamp) 819 open(basestamp, "w").close() 820 821def make_stamp(task, d): 822 """ 823 Creates/updates a stamp for a given task 824 """ 825 mcfn = d.getVar('BB_FILENAME') 826 827 make_stamp_mcfn(task, mcfn) 828 829 # If we're in task context, write out a signature file for each task 830 # as it completes 831 if not task.endswith("_setscene"): 832 stampbase = bb.parse.siggen.stampfile_base(mcfn) 833 bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True) 834 835 836def find_stale_stamps(task, mcfn): 837 current = bb.parse.siggen.stampfile_mcfn(task, mcfn) 838 current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn) 839 cleanmask = _get_cleanmask(task, mcfn) 840 found = [] 841 for mask in cleanmask: 842 for name in glob.glob(mask): 843 if "sigdata" in name or "sigbasedata" in name: 844 continue 845 if name.endswith('.taint'): 846 continue 847 if name == current or name == current2: 848 continue 849 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 850 found.append(name) 851 return found 852 853def write_taint(task, d): 854 """ 855 Creates a "taint" file which will force the specified task and its 856 dependents to be re-run the next time by influencing the value of its 857 taskhash. 858 """ 859 mcfn = d.getVar('BB_FILENAME') 860 bb.parse.siggen.invalidate_task(task, mcfn) 861 862def add_tasks(tasklist, d): 863 task_deps = d.getVar('_task_deps', False) 864 if not task_deps: 865 task_deps = {} 866 if not 'tasks' in task_deps: 867 task_deps['tasks'] = [] 868 if not 'parents' in task_deps: 869 task_deps['parents'] = {} 870 871 for task in tasklist: 872 task = d.expand(task) 873 874 d.setVarFlag(task, 'task', 1) 875 876 if not task in task_deps['tasks']: 877 task_deps['tasks'].append(task) 878 879 flags = d.getVarFlags(task) 880 def getTask(name): 881 if not name in task_deps: 882 task_deps[name] = {} 883 if name in flags: 884 deptask = d.expand(flags[name]) 885 if name in ['noexec', 'fakeroot', 'nostamp']: 886 if deptask != '1': 887 bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " 888 "will result in the flag not being set. See YP bug #13808.".format(name)) 889 890 task_deps[name][task] = deptask 891 getTask('mcdepends') 892 getTask('depends') 893 getTask('rdepends') 894 getTask('deptask') 895 getTask('rdeptask') 896 getTask('recrdeptask') 897 getTask('recideptask') 898 getTask('nostamp') 899 getTask('fakeroot') 900 getTask('noexec') 901 getTask('umask') 902 task_deps['parents'][task] = [] 903 if 'deps' in flags: 904 for dep in flags['deps']: 905 # Check and warn for "addtask task after foo" while foo does not exist 906 #if not dep in tasklist: 907 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 908 dep = d.expand(dep) 909 task_deps['parents'][task].append(dep) 910 911 # don't assume holding a reference 912 d.setVar('_task_deps', task_deps) 913 914def addtask(task, before, after, d): 915 if task[:3] != "do_": 916 task = "do_" + task 917 918 d.setVarFlag(task, "task", 1) 919 bbtasks = d.getVar('__BBTASKS', False) or [] 920 if task not in bbtasks: 921 bbtasks.append(task) 922 d.setVar('__BBTASKS', bbtasks) 923 924 existing = d.getVarFlag(task, "deps", False) or [] 925 if after is not None: 926 # set up deps for function 927 for entry in after.split(): 928 if entry not in existing: 929 existing.append(entry) 930 d.setVarFlag(task, "deps", existing) 931 if before is not None: 932 # set up things that depend on this func 933 for entry in before.split(): 934 existing = d.getVarFlag(entry, "deps", False) or [] 935 if task not in existing: 936 d.setVarFlag(entry, "deps", [task] + existing) 937 938def deltask(task, d): 939 if task[:3] != "do_": 940 task = "do_" + task 941 942 bbtasks = d.getVar('__BBTASKS', False) or [] 943 if task in bbtasks: 944 bbtasks.remove(task) 945 d.delVarFlag(task, 'task') 946 d.setVar('__BBTASKS', bbtasks) 947 948 d.delVarFlag(task, 'deps') 949 for bbtask in d.getVar('__BBTASKS', False) or []: 950 deps = d.getVarFlag(bbtask, 'deps', False) or [] 951 if task in deps: 952 deps.remove(task) 953 d.setVarFlag(bbtask, 'deps', deps) 954 955def preceedtask(task, with_recrdeptasks, d): 956 """ 957 Returns a set of tasks in the current recipe which were specified as 958 precondition by the task itself ("after") or which listed themselves 959 as precondition ("before"). Preceeding tasks specified via the 960 "recrdeptask" are included in the result only if requested. Beware 961 that this may lead to the task itself being listed. 962 """ 963 preceed = set() 964 965 # Ignore tasks which don't exist 966 tasks = d.getVar('__BBTASKS', False) 967 if task not in tasks: 968 return preceed 969 970 preceed.update(d.getVarFlag(task, 'deps') or []) 971 if with_recrdeptasks: 972 recrdeptask = d.getVarFlag(task, 'recrdeptask') 973 if recrdeptask: 974 preceed.update(recrdeptask.split()) 975 return preceed 976 977def tasksbetween(task_start, task_end, d): 978 """ 979 Return the list of tasks between two tasks in the current recipe, 980 where task_start is to start at and task_end is the task to end at 981 (and task_end has a dependency chain back to task_start). 982 """ 983 outtasks = [] 984 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 985 def follow_chain(task, endtask, chain=None): 986 if not chain: 987 chain = [] 988 if task in chain: 989 bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) 990 chain.append(task) 991 for othertask in tasks: 992 if othertask == task: 993 continue 994 if task == endtask: 995 for ctask in chain: 996 if ctask not in outtasks: 997 outtasks.append(ctask) 998 else: 999 deps = d.getVarFlag(othertask, 'deps', False) 1000 if task in deps: 1001 follow_chain(othertask, endtask, chain) 1002 chain.pop() 1003 follow_chain(task_start, task_end) 1004 return outtasks 1005