1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import bb 24import bb.msg 25import bb.process 26import bb.progress 27from bb import data, event, utils 28 29bblogger = logging.getLogger('BitBake') 30logger = logging.getLogger('BitBake.Build') 31 32verboseShellLogging = False 33verboseStdoutLogging = False 34 35__mtime_cache = {} 36 37def cached_mtime_noerror(f): 38 if f not in __mtime_cache: 39 try: 40 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 41 except OSError: 42 return 0 43 return __mtime_cache[f] 44 45def reset_cache(): 46 global __mtime_cache 47 __mtime_cache = {} 48 49# When we execute a Python function, we'd like certain things 50# in all namespaces, hence we add them to __builtins__. 51# If we do not do this and use the exec globals, they will 52# not be available to subfunctions. 53if hasattr(__builtins__, '__setitem__'): 54 builtins = __builtins__ 55else: 56 builtins = __builtins__.__dict__ 57 58builtins['bb'] = bb 59builtins['os'] = os 60 61class TaskBase(event.Event): 62 """Base class for task events""" 63 64 def __init__(self, t, fn, logfile, d): 65 self._task = t 66 self._fn = fn 67 self._package = d.getVar("PF") 68 self._mc = d.getVar("BB_CURRENT_MC") 69 self.taskfile = d.getVar("FILE") 70 self.taskname = self._task 71 self.logfile = logfile 72 self.time = time.time() 73 self.pn = d.getVar("PN") 74 self.pv = d.getVar("PV") 75 event.Event.__init__(self) 76 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 77 78 def getTask(self): 79 return self._task 80 81 def setTask(self, task): 82 self._task = task 83 84 def getDisplayName(self): 85 return bb.event.getName(self)[4:] 86 87 task = property(getTask, setTask, None, "task property") 88 89class TaskStarted(TaskBase): 90 """Task execution started""" 91 def __init__(self, t, fn, logfile, taskflags, d): 92 super(TaskStarted, self).__init__(t, fn, logfile, d) 93 self.taskflags = taskflags 94 95class TaskSucceeded(TaskBase): 96 """Task execution completed""" 97 98class TaskFailed(TaskBase): 99 """Task execution failed""" 100 101 def __init__(self, task, fn, logfile, metadata, errprinted = False): 102 self.errprinted = errprinted 103 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 104 105class TaskFailedSilent(TaskBase): 106 """Task execution failed (silently)""" 107 def getDisplayName(self): 108 # Don't need to tell the user it was silent 109 return "Failed" 110 111class TaskInvalid(TaskBase): 112 113 def __init__(self, task, fn, metadata): 114 super(TaskInvalid, self).__init__(task, fn, None, metadata) 115 self._message = "No such task '%s'" % task 116 117class TaskProgress(event.Event): 118 """ 119 Task made some progress that could be reported to the user, usually in 120 the form of a progress bar or similar. 121 NOTE: this class does not inherit from TaskBase since it doesn't need 122 to - it's fired within the task context itself, so we don't have any of 123 the context information that you do in the case of the other events. 124 The event PID can be used to determine which task it came from. 125 The progress value is normally 0-100, but can also be negative 126 indicating that progress has been made but we aren't able to determine 127 how much. 128 The rate is optional, this is simply an extra string to display to the 129 user if specified. 130 """ 131 def __init__(self, progress, rate=None): 132 self.progress = progress 133 self.rate = rate 134 event.Event.__init__(self) 135 136 137class LogTee(object): 138 def __init__(self, logger, outfile): 139 self.outfile = outfile 140 self.logger = logger 141 self.name = self.outfile.name 142 143 def write(self, string): 144 self.logger.plain(string) 145 self.outfile.write(string) 146 147 def __enter__(self): 148 self.outfile.__enter__() 149 return self 150 151 def __exit__(self, *excinfo): 152 self.outfile.__exit__(*excinfo) 153 154 def __repr__(self): 155 return '<LogTee {0}>'.format(self.name) 156 157 def flush(self): 158 self.outfile.flush() 159 160 161class StdoutNoopContextManager: 162 """ 163 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 164 """ 165 def __enter__(self): 166 return sys.stdout 167 168 def __exit__(self, *exc_info): 169 pass 170 171 def write(self, string): 172 return sys.stdout.write(string) 173 174 def flush(self): 175 sys.stdout.flush() 176 177 @property 178 def name(self): 179 return sys.stdout.name 180 181 182def exec_func(func, d, dirs = None): 183 """Execute a BB 'function'""" 184 185 try: 186 oldcwd = os.getcwd() 187 except: 188 oldcwd = None 189 190 flags = d.getVarFlags(func) 191 cleandirs = flags.get('cleandirs') if flags else None 192 if cleandirs: 193 for cdir in d.expand(cleandirs).split(): 194 bb.utils.remove(cdir, True) 195 bb.utils.mkdirhier(cdir) 196 197 if flags and dirs is None: 198 dirs = flags.get('dirs') 199 if dirs: 200 dirs = d.expand(dirs).split() 201 202 if dirs: 203 for adir in dirs: 204 bb.utils.mkdirhier(adir) 205 adir = dirs[-1] 206 else: 207 adir = None 208 209 body = d.getVar(func, False) 210 if not body: 211 if body is None: 212 logger.warning("Function %s doesn't exist", func) 213 return 214 215 ispython = flags.get('python') 216 217 lockflag = flags.get('lockfiles') 218 if lockflag: 219 lockfiles = [f for f in d.expand(lockflag).split()] 220 else: 221 lockfiles = None 222 223 tempdir = d.getVar('T') 224 225 # or func allows items to be executed outside of the normal 226 # task set, such as buildhistory 227 task = d.getVar('BB_RUNTASK') or func 228 if task == func: 229 taskfunc = task 230 else: 231 taskfunc = "%s.%s" % (task, func) 232 233 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 234 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 235 runfile = os.path.join(tempdir, runfn) 236 bb.utils.mkdirhier(os.path.dirname(runfile)) 237 238 # Setup the courtesy link to the runfn, only for tasks 239 # we create the link 'just' before the run script is created 240 # if we create it after, and if the run script fails, then the 241 # link won't be created as an exception would be fired. 242 if task == func: 243 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 244 if runlink: 245 bb.utils.remove(runlink) 246 247 try: 248 os.symlink(runfn, runlink) 249 except OSError: 250 pass 251 252 with bb.utils.fileslocked(lockfiles): 253 if ispython: 254 exec_func_python(func, d, runfile, cwd=adir) 255 else: 256 exec_func_shell(func, d, runfile, cwd=adir) 257 258 try: 259 curcwd = os.getcwd() 260 except: 261 curcwd = None 262 263 if oldcwd and curcwd != oldcwd: 264 try: 265 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 266 os.chdir(oldcwd) 267 except: 268 pass 269 270_functionfmt = """ 271{function}(d) 272""" 273logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 274def exec_func_python(func, d, runfile, cwd=None): 275 """Execute a python BB 'function'""" 276 277 code = _functionfmt.format(function=func) 278 bb.utils.mkdirhier(os.path.dirname(runfile)) 279 with open(runfile, 'w') as script: 280 bb.data.emit_func_python(func, script, d) 281 282 if cwd: 283 try: 284 olddir = os.getcwd() 285 except OSError as e: 286 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 287 olddir = None 288 os.chdir(cwd) 289 290 bb.debug(2, "Executing python function %s" % func) 291 292 try: 293 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 294 fn = d.getVarFlag(func, "filename", False) 295 lineno = int(d.getVarFlag(func, "lineno", False)) 296 bb.methodpool.insert_method(func, text, fn, lineno - 1) 297 298 comp = utils.better_compile(code, func, "exec_func_python() autogenerated") 299 utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") 300 finally: 301 # We want any stdout/stderr to be printed before any other log messages to make debugging 302 # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. 303 sys.stdout.flush() 304 sys.stderr.flush() 305 bb.debug(2, "Python function %s finished" % func) 306 307 if cwd and olddir: 308 try: 309 os.chdir(olddir) 310 except OSError as e: 311 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 312 313def shell_trap_code(): 314 return '''#!/bin/sh\n 315__BITBAKE_LAST_LINE=0 316 317# Emit a useful diagnostic if something fails: 318bb_sh_exit_handler() { 319 ret=$? 320 if [ "$ret" != 0 ]; then 321 echo "WARNING: exit code $ret from a shell command." 322 fi 323 exit $ret 324} 325 326bb_bash_exit_handler() { 327 ret=$? 328 { set +x; } > /dev/null 329 trap "" DEBUG 330 if [ "$ret" != 0 ]; then 331 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 332 333 echo "WARNING: Backtrace (BB generated script): " 334 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 335 if [ "$i" -eq 1 ]; then 336 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 337 else 338 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 339 fi 340 done 341 fi 342 exit $ret 343} 344 345bb_bash_debug_handler() { 346 local line=${BASH_LINENO[0]} 347 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 348 if [ "$line" -eq 1 ]; then 349 return 350 fi 351 352 # Track the line number of commands as they execute. This is so we can have access to the failing line number 353 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 354 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 355 __BITBAKE_LAST_LINE=$line 356 fi 357} 358 359case $BASH_VERSION in 360"") trap 'bb_sh_exit_handler' 0 361 set -e 362 ;; 363*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 364 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 365 set -e 366 shopt -s extdebug 367 ;; 368esac 369''' 370 371def create_progress_handler(func, progress, logfile, d): 372 if progress == 'percent': 373 # Use default regex 374 return bb.progress.BasicProgressHandler(d, outfile=logfile) 375 elif progress.startswith('percent:'): 376 # Use specified regex 377 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 378 elif progress.startswith('outof:'): 379 # Use specified regex 380 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 381 elif progress.startswith("custom:"): 382 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 383 import functools 384 from types import ModuleType 385 386 parts = progress.split(":", 2) 387 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 388 if cls: 389 def resolve(x, y): 390 if not x: 391 return None 392 if isinstance(x, ModuleType): 393 return getattr(x, y, None) 394 return x.get(y) 395 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 396 if not cls_obj: 397 # Fall-back on __builtins__ 398 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 399 if cls_obj: 400 return cls_obj(d, outfile=logfile, otherargs=otherargs) 401 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 402 else: 403 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 404 405 return logfile 406 407def exec_func_shell(func, d, runfile, cwd=None): 408 """Execute a shell function from the metadata 409 410 Note on directory behavior. The 'dirs' varflag should contain a list 411 of the directories you need created prior to execution. The last 412 item in the list is where we will chdir/cd to. 413 """ 414 415 # Don't let the emitted shell script override PWD 416 d.delVarFlag('PWD', 'export') 417 418 with open(runfile, 'w') as script: 419 script.write(shell_trap_code()) 420 421 bb.data.emit_func(func, script, d) 422 423 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 424 script.write("set -x\n") 425 if cwd: 426 script.write("cd '%s'\n" % cwd) 427 script.write("%s\n" % func) 428 script.write(''' 429# cleanup 430ret=$? 431trap '' 0 432exit $ret 433''') 434 435 os.chmod(runfile, 0o775) 436 437 cmd = runfile 438 if d.getVarFlag(func, 'fakeroot', False): 439 fakerootcmd = d.getVar('FAKEROOT') 440 if fakerootcmd: 441 cmd = [fakerootcmd, runfile] 442 443 if verboseStdoutLogging: 444 logfile = LogTee(logger, StdoutNoopContextManager()) 445 else: 446 logfile = StdoutNoopContextManager() 447 448 progress = d.getVarFlag(func, 'progress') 449 if progress: 450 try: 451 logfile = create_progress_handler(func, progress, logfile, d) 452 except: 453 from traceback import format_exc 454 logger.error("Failed to create progress handler") 455 logger.error(format_exc()) 456 raise 457 458 fifobuffer = bytearray() 459 def readfifo(data): 460 nonlocal fifobuffer 461 fifobuffer.extend(data) 462 while fifobuffer: 463 message, token, nextmsg = fifobuffer.partition(b"\00") 464 if token: 465 splitval = message.split(b' ', 1) 466 cmd = splitval[0].decode("utf-8") 467 if len(splitval) > 1: 468 value = splitval[1].decode("utf-8") 469 else: 470 value = '' 471 if cmd == 'bbplain': 472 bb.plain(value) 473 elif cmd == 'bbnote': 474 bb.note(value) 475 elif cmd == 'bbverbnote': 476 bb.verbnote(value) 477 elif cmd == 'bbwarn': 478 bb.warn(value) 479 elif cmd == 'bberror': 480 bb.error(value) 481 elif cmd == 'bbfatal': 482 # The caller will call exit themselves, so bb.error() is 483 # what we want here rather than bb.fatal() 484 bb.error(value) 485 elif cmd == 'bbfatal_log': 486 bb.error(value, forcelog=True) 487 elif cmd == 'bbdebug': 488 splitval = value.split(' ', 1) 489 level = int(splitval[0]) 490 value = splitval[1] 491 bb.debug(level, value) 492 else: 493 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 494 fifobuffer = nextmsg 495 else: 496 break 497 498 tempdir = d.getVar('T') 499 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 500 if os.path.exists(fifopath): 501 os.unlink(fifopath) 502 os.mkfifo(fifopath) 503 with open(fifopath, 'r+b', buffering=0) as fifo: 504 try: 505 bb.debug(2, "Executing shell function %s" % func) 506 with open(os.devnull, 'r+') as stdin, logfile: 507 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 508 except bb.process.ExecutionError as exe: 509 # Find the backtrace that the shell trap generated 510 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 511 stdout_lines = (exe.stdout or "").split("\n") 512 backtrace_start_line = None 513 for i, line in enumerate(reversed(stdout_lines)): 514 if backtrace_marker_regex.search(line): 515 backtrace_start_line = len(stdout_lines) - i 516 break 517 518 # Read the backtrace frames, starting at the location we just found 519 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 520 r"?P<lineno>\d+)") 521 backtrace_frames = [] 522 if backtrace_start_line: 523 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 524 match = backtrace_entry_regex.search(line) 525 if match: 526 backtrace_frames.append(match.groupdict()) 527 528 with open(runfile, "r") as script: 529 script_lines = [line.rstrip() for line in script.readlines()] 530 531 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 532 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 533 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 534 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 535 better_frames = [] 536 # Skip the very last frame since it's just the call to the shell task in the body of the script 537 for frame in backtrace_frames[:-1]: 538 # Check whether the frame corresponds to a function defined in the script vs external script. 539 if os.path.samefile(frame["file"], runfile): 540 # Search backwards from the frame lineno to locate the comment that BB injected 541 i = int(frame["lineno"]) - 1 542 while i >= 0: 543 match = script_metadata_comment_regex.match(script_lines[i]) 544 if match: 545 # Calculate the relative line in the function itself 546 relative_line_in_function = int(frame["lineno"]) - i - 2 547 # Calculate line in the function as declared in the metadata 548 metadata_function_line = relative_line_in_function + int(match["lineno"]) 549 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 550 frameno=frame["frameno"], 551 funcname=frame["funcname"], 552 file=match["file"], 553 lineno=metadata_function_line 554 )) 555 break 556 i -= 1 557 else: 558 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 559 560 if better_frames: 561 better_frames = ("\t{0}".format(frame) for frame in better_frames) 562 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 563 raise 564 finally: 565 os.unlink(fifopath) 566 567 bb.debug(2, "Shell function %s finished" % func) 568 569def _task_data(fn, task, d): 570 localdata = bb.data.createCopy(d) 571 localdata.setVar('BB_FILENAME', fn) 572 localdata.setVar('OVERRIDES', 'task-%s:%s' % 573 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 574 localdata.finalize() 575 bb.data.expandKeys(localdata) 576 return localdata 577 578def _exec_task(fn, task, d, quieterr): 579 """Execute a BB 'task' 580 581 Execution of a task involves a bit more setup than executing a function, 582 running it with its own local metadata, and with some useful variables set. 583 """ 584 if not d.getVarFlag(task, 'task', False): 585 event.fire(TaskInvalid(task, fn, d), d) 586 logger.error("No such task: %s" % task) 587 return 1 588 589 logger.debug("Executing task %s", task) 590 591 localdata = _task_data(fn, task, d) 592 tempdir = localdata.getVar('T') 593 if not tempdir: 594 bb.fatal("T variable not set, unable to build") 595 596 # Change nice level if we're asked to 597 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 598 if nice: 599 curnice = os.nice(0) 600 nice = int(nice) - curnice 601 newnice = os.nice(nice) 602 logger.debug("Renice to %s " % newnice) 603 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 604 if ionice: 605 try: 606 cls, prio = ionice.split(".", 1) 607 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 608 except: 609 bb.warn("Invalid ionice level %s" % ionice) 610 611 bb.utils.mkdirhier(tempdir) 612 613 # Determine the logfile to generate 614 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 615 logbase = logfmt.format(task=task, pid=os.getpid()) 616 617 # Document the order of the tasks... 618 logorder = os.path.join(tempdir, 'log.task_order') 619 try: 620 with open(logorder, 'a') as logorderfile: 621 logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) 622 except OSError: 623 logger.exception("Opening log file '%s'", logorder) 624 pass 625 626 # Setup the courtesy link to the logfn 627 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 628 logfn = os.path.join(tempdir, logbase) 629 if loglink: 630 bb.utils.remove(loglink) 631 632 try: 633 os.symlink(logbase, loglink) 634 except OSError: 635 pass 636 637 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 638 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 639 640 class ErrorCheckHandler(logging.Handler): 641 def __init__(self): 642 self.triggered = False 643 logging.Handler.__init__(self, logging.ERROR) 644 def emit(self, record): 645 if getattr(record, 'forcelog', False): 646 self.triggered = False 647 else: 648 self.triggered = True 649 650 # Handle logfiles 651 try: 652 bb.utils.mkdirhier(os.path.dirname(logfn)) 653 logfile = open(logfn, 'w') 654 except OSError: 655 logger.exception("Opening log file '%s'", logfn) 656 pass 657 658 # Dup the existing fds so we dont lose them 659 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 660 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 661 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 662 663 # Replace those fds with our own 664 with open('/dev/null', 'r') as si: 665 os.dup2(si.fileno(), osi[1]) 666 os.dup2(logfile.fileno(), oso[1]) 667 os.dup2(logfile.fileno(), ose[1]) 668 669 # Ensure Python logging goes to the logfile 670 handler = logging.StreamHandler(logfile) 671 handler.setFormatter(logformatter) 672 # Always enable full debug output into task logfiles 673 handler.setLevel(logging.DEBUG - 2) 674 bblogger.addHandler(handler) 675 676 errchk = ErrorCheckHandler() 677 bblogger.addHandler(errchk) 678 679 localdata.setVar('BB_LOGFILE', logfn) 680 localdata.setVar('BB_RUNTASK', task) 681 localdata.setVar('BB_TASK_LOGGER', bblogger) 682 683 flags = localdata.getVarFlags(task) 684 685 try: 686 try: 687 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 688 689 for func in (prefuncs or '').split(): 690 exec_func(func, localdata) 691 exec_func(task, localdata) 692 for func in (postfuncs or '').split(): 693 exec_func(func, localdata) 694 finally: 695 # Need to flush and close the logs before sending events where the 696 # UI may try to look at the logs. 697 sys.stdout.flush() 698 sys.stderr.flush() 699 700 bblogger.removeHandler(handler) 701 702 # Restore the backup fds 703 os.dup2(osi[0], osi[1]) 704 os.dup2(oso[0], oso[1]) 705 os.dup2(ose[0], ose[1]) 706 707 # Close the backup fds 708 os.close(osi[0]) 709 os.close(oso[0]) 710 os.close(ose[0]) 711 712 logfile.close() 713 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 714 logger.debug2("Zero size logfn %s, removing", logfn) 715 bb.utils.remove(logfn) 716 bb.utils.remove(loglink) 717 except (Exception, SystemExit) as exc: 718 handled = False 719 if isinstance(exc, bb.BBHandledException): 720 handled = True 721 722 if quieterr: 723 if not handled: 724 logger.warning(repr(exc)) 725 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 726 else: 727 errprinted = errchk.triggered 728 # If the output is already on stdout, we've printed the information in the 729 # logs once already so don't duplicate 730 if verboseStdoutLogging or handled: 731 errprinted = True 732 if not handled: 733 logger.error(repr(exc)) 734 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 735 return 1 736 737 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 738 739 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 740 make_stamp(task, localdata) 741 742 return 0 743 744def exec_task(fn, task, d, profile = False): 745 try: 746 quieterr = False 747 if d.getVarFlag(task, "quieterrors", False) is not None: 748 quieterr = True 749 750 if profile: 751 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 752 try: 753 import cProfile as profile 754 except: 755 import profile 756 prof = profile.Profile() 757 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 758 prof.dump_stats(profname) 759 bb.utils.process_profilelog(profname) 760 761 return ret 762 else: 763 return _exec_task(fn, task, d, quieterr) 764 765 except Exception: 766 from traceback import format_exc 767 if not quieterr: 768 logger.error("Build of %s failed" % (task)) 769 logger.error(format_exc()) 770 failedevent = TaskFailed(task, None, d, True) 771 event.fire(failedevent, d) 772 return 1 773 774def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False): 775 """ 776 Internal stamp helper function 777 Makes sure the stamp directory exists 778 Returns the stamp path+filename 779 780 In the bitbake core, d can be a CacheData and file_name will be set. 781 When called in task context, d will be a data store, file_name will not be set 782 """ 783 taskflagname = taskname 784 if taskname.endswith("_setscene") and taskname != "do_setscene": 785 taskflagname = taskname.replace("_setscene", "") 786 787 if file_name: 788 stamp = d.stamp[file_name] 789 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 790 else: 791 stamp = d.getVar('STAMP') 792 file_name = d.getVar('BB_FILENAME') 793 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 794 795 if baseonly: 796 return stamp 797 if noextra: 798 extrainfo = "" 799 800 if not stamp: 801 return 802 803 stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) 804 805 stampdir = os.path.dirname(stamp) 806 if cached_mtime_noerror(stampdir) == 0: 807 bb.utils.mkdirhier(stampdir) 808 809 return stamp 810 811def stamp_cleanmask_internal(taskname, d, file_name): 812 """ 813 Internal stamp helper function to generate stamp cleaning mask 814 Returns the stamp path+filename 815 816 In the bitbake core, d can be a CacheData and file_name will be set. 817 When called in task context, d will be a data store, file_name will not be set 818 """ 819 taskflagname = taskname 820 if taskname.endswith("_setscene") and taskname != "do_setscene": 821 taskflagname = taskname.replace("_setscene", "") 822 823 if file_name: 824 stamp = d.stampclean[file_name] 825 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 826 else: 827 stamp = d.getVar('STAMPCLEAN') 828 file_name = d.getVar('BB_FILENAME') 829 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 830 831 if not stamp: 832 return [] 833 834 cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) 835 836 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 837 838def make_stamp(task, d, file_name = None): 839 """ 840 Creates/updates a stamp for a given task 841 (d can be a data dict or dataCache) 842 """ 843 cleanmask = stamp_cleanmask_internal(task, d, file_name) 844 for mask in cleanmask: 845 for name in glob.glob(mask): 846 # Preserve sigdata files in the stamps directory 847 if "sigdata" in name or "sigbasedata" in name: 848 continue 849 # Preserve taint files in the stamps directory 850 if name.endswith('.taint'): 851 continue 852 os.unlink(name) 853 854 stamp = stamp_internal(task, d, file_name) 855 # Remove the file and recreate to force timestamp 856 # change on broken NFS filesystems 857 if stamp: 858 bb.utils.remove(stamp) 859 open(stamp, "w").close() 860 861 # If we're in task context, write out a signature file for each task 862 # as it completes 863 if not task.endswith("_setscene") and task != "do_setscene" and not file_name: 864 stampbase = stamp_internal(task, d, None, True) 865 file_name = d.getVar('BB_FILENAME') 866 bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) 867 868def find_stale_stamps(task, d, file_name=None): 869 current = stamp_internal(task, d, file_name) 870 current2 = stamp_internal(task + "_setscene", d, file_name) 871 cleanmask = stamp_cleanmask_internal(task, d, file_name) 872 found = [] 873 for mask in cleanmask: 874 for name in glob.glob(mask): 875 if "sigdata" in name or "sigbasedata" in name: 876 continue 877 if name.endswith('.taint'): 878 continue 879 if name == current or name == current2: 880 continue 881 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 882 found.append(name) 883 return found 884 885def del_stamp(task, d, file_name = None): 886 """ 887 Removes a stamp for a given task 888 (d can be a data dict or dataCache) 889 """ 890 stamp = stamp_internal(task, d, file_name) 891 bb.utils.remove(stamp) 892 893def write_taint(task, d, file_name = None): 894 """ 895 Creates a "taint" file which will force the specified task and its 896 dependents to be re-run the next time by influencing the value of its 897 taskhash. 898 (d can be a data dict or dataCache) 899 """ 900 import uuid 901 if file_name: 902 taintfn = d.stamp[file_name] + '.' + task + '.taint' 903 else: 904 taintfn = d.getVar('STAMP') + '.' + task + '.taint' 905 bb.utils.mkdirhier(os.path.dirname(taintfn)) 906 # The specific content of the taint file is not really important, 907 # we just need it to be random, so a random UUID is used 908 with open(taintfn, 'w') as taintf: 909 taintf.write(str(uuid.uuid4())) 910 911def stampfile(taskname, d, file_name = None, noextra=False): 912 """ 913 Return the stamp for a given task 914 (d can be a data dict or dataCache) 915 """ 916 return stamp_internal(taskname, d, file_name, noextra=noextra) 917 918def add_tasks(tasklist, d): 919 task_deps = d.getVar('_task_deps', False) 920 if not task_deps: 921 task_deps = {} 922 if not 'tasks' in task_deps: 923 task_deps['tasks'] = [] 924 if not 'parents' in task_deps: 925 task_deps['parents'] = {} 926 927 for task in tasklist: 928 task = d.expand(task) 929 930 d.setVarFlag(task, 'task', 1) 931 932 if not task in task_deps['tasks']: 933 task_deps['tasks'].append(task) 934 935 flags = d.getVarFlags(task) 936 def getTask(name): 937 if not name in task_deps: 938 task_deps[name] = {} 939 if name in flags: 940 deptask = d.expand(flags[name]) 941 if name in ['noexec', 'fakeroot', 'nostamp']: 942 if deptask != '1': 943 bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " 944 "will result in the flag not being set. See YP bug #13808.".format(name)) 945 946 task_deps[name][task] = deptask 947 getTask('mcdepends') 948 getTask('depends') 949 getTask('rdepends') 950 getTask('deptask') 951 getTask('rdeptask') 952 getTask('recrdeptask') 953 getTask('recideptask') 954 getTask('nostamp') 955 getTask('fakeroot') 956 getTask('noexec') 957 getTask('umask') 958 task_deps['parents'][task] = [] 959 if 'deps' in flags: 960 for dep in flags['deps']: 961 # Check and warn for "addtask task after foo" while foo does not exist 962 #if not dep in tasklist: 963 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 964 dep = d.expand(dep) 965 task_deps['parents'][task].append(dep) 966 967 # don't assume holding a reference 968 d.setVar('_task_deps', task_deps) 969 970def addtask(task, before, after, d): 971 if task[:3] != "do_": 972 task = "do_" + task 973 974 d.setVarFlag(task, "task", 1) 975 bbtasks = d.getVar('__BBTASKS', False) or [] 976 if task not in bbtasks: 977 bbtasks.append(task) 978 d.setVar('__BBTASKS', bbtasks) 979 980 existing = d.getVarFlag(task, "deps", False) or [] 981 if after is not None: 982 # set up deps for function 983 for entry in after.split(): 984 if entry not in existing: 985 existing.append(entry) 986 d.setVarFlag(task, "deps", existing) 987 if before is not None: 988 # set up things that depend on this func 989 for entry in before.split(): 990 existing = d.getVarFlag(entry, "deps", False) or [] 991 if task not in existing: 992 d.setVarFlag(entry, "deps", [task] + existing) 993 994def deltask(task, d): 995 if task[:3] != "do_": 996 task = "do_" + task 997 998 bbtasks = d.getVar('__BBTASKS', False) or [] 999 if task in bbtasks: 1000 bbtasks.remove(task) 1001 d.delVarFlag(task, 'task') 1002 d.setVar('__BBTASKS', bbtasks) 1003 1004 d.delVarFlag(task, 'deps') 1005 for bbtask in d.getVar('__BBTASKS', False) or []: 1006 deps = d.getVarFlag(bbtask, 'deps', False) or [] 1007 if task in deps: 1008 deps.remove(task) 1009 d.setVarFlag(bbtask, 'deps', deps) 1010 1011def preceedtask(task, with_recrdeptasks, d): 1012 """ 1013 Returns a set of tasks in the current recipe which were specified as 1014 precondition by the task itself ("after") or which listed themselves 1015 as precondition ("before"). Preceeding tasks specified via the 1016 "recrdeptask" are included in the result only if requested. Beware 1017 that this may lead to the task itself being listed. 1018 """ 1019 preceed = set() 1020 1021 # Ignore tasks which don't exist 1022 tasks = d.getVar('__BBTASKS', False) 1023 if task not in tasks: 1024 return preceed 1025 1026 preceed.update(d.getVarFlag(task, 'deps') or []) 1027 if with_recrdeptasks: 1028 recrdeptask = d.getVarFlag(task, 'recrdeptask') 1029 if recrdeptask: 1030 preceed.update(recrdeptask.split()) 1031 return preceed 1032 1033def tasksbetween(task_start, task_end, d): 1034 """ 1035 Return the list of tasks between two tasks in the current recipe, 1036 where task_start is to start at and task_end is the task to end at 1037 (and task_end has a dependency chain back to task_start). 1038 """ 1039 outtasks = [] 1040 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 1041 def follow_chain(task, endtask, chain=None): 1042 if not chain: 1043 chain = [] 1044 if task in chain: 1045 bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) 1046 chain.append(task) 1047 for othertask in tasks: 1048 if othertask == task: 1049 continue 1050 if task == endtask: 1051 for ctask in chain: 1052 if ctask not in outtasks: 1053 outtasks.append(ctask) 1054 else: 1055 deps = d.getVarFlag(othertask, 'deps', False) 1056 if task in deps: 1057 follow_chain(othertask, endtask, chain) 1058 chain.pop() 1059 follow_chain(task_start, task_end) 1060 return outtasks 1061