1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import bb 24import bb.msg 25import bb.process 26import bb.progress 27from bb import data, event, utils 28 29bblogger = logging.getLogger('BitBake') 30logger = logging.getLogger('BitBake.Build') 31 32verboseShellLogging = False 33verboseStdoutLogging = False 34 35__mtime_cache = {} 36 37def cached_mtime_noerror(f): 38 if f not in __mtime_cache: 39 try: 40 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 41 except OSError: 42 return 0 43 return __mtime_cache[f] 44 45def reset_cache(): 46 global __mtime_cache 47 __mtime_cache = {} 48 49# When we execute a Python function, we'd like certain things 50# in all namespaces, hence we add them to __builtins__. 51# If we do not do this and use the exec globals, they will 52# not be available to subfunctions. 53if hasattr(__builtins__, '__setitem__'): 54 builtins = __builtins__ 55else: 56 builtins = __builtins__.__dict__ 57 58builtins['bb'] = bb 59builtins['os'] = os 60 61class TaskBase(event.Event): 62 """Base class for task events""" 63 64 def __init__(self, t, fn, logfile, d): 65 self._task = t 66 self._fn = fn 67 self._package = d.getVar("PF") 68 self._mc = d.getVar("BB_CURRENT_MC") 69 self.taskfile = d.getVar("FILE") 70 self.taskname = self._task 71 self.logfile = logfile 72 self.time = time.time() 73 self.pn = d.getVar("PN") 74 self.pv = d.getVar("PV") 75 event.Event.__init__(self) 76 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 77 78 def getTask(self): 79 return self._task 80 81 def setTask(self, task): 82 self._task = task 83 84 def getDisplayName(self): 85 return bb.event.getName(self)[4:] 86 87 task = property(getTask, setTask, None, "task property") 88 89class TaskStarted(TaskBase): 90 """Task execution started""" 91 def __init__(self, t, fn, logfile, taskflags, d): 92 super(TaskStarted, self).__init__(t, fn, logfile, d) 93 self.taskflags = taskflags 94 95class TaskSucceeded(TaskBase): 96 """Task execution completed""" 97 98class TaskFailed(TaskBase): 99 """Task execution failed""" 100 101 def __init__(self, task, fn, logfile, metadata, errprinted = False): 102 self.errprinted = errprinted 103 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 104 105class TaskFailedSilent(TaskBase): 106 """Task execution failed (silently)""" 107 def getDisplayName(self): 108 # Don't need to tell the user it was silent 109 return "Failed" 110 111class TaskInvalid(TaskBase): 112 113 def __init__(self, task, fn, metadata): 114 super(TaskInvalid, self).__init__(task, fn, None, metadata) 115 self._message = "No such task '%s'" % task 116 117class TaskProgress(event.Event): 118 """ 119 Task made some progress that could be reported to the user, usually in 120 the form of a progress bar or similar. 121 NOTE: this class does not inherit from TaskBase since it doesn't need 122 to - it's fired within the task context itself, so we don't have any of 123 the context information that you do in the case of the other events. 124 The event PID can be used to determine which task it came from. 125 The progress value is normally 0-100, but can also be negative 126 indicating that progress has been made but we aren't able to determine 127 how much. 128 The rate is optional, this is simply an extra string to display to the 129 user if specified. 130 """ 131 def __init__(self, progress, rate=None): 132 self.progress = progress 133 self.rate = rate 134 event.Event.__init__(self) 135 136 137class LogTee(object): 138 def __init__(self, logger, outfile): 139 self.outfile = outfile 140 self.logger = logger 141 self.name = self.outfile.name 142 143 def write(self, string): 144 self.logger.plain(string) 145 self.outfile.write(string) 146 147 def __enter__(self): 148 self.outfile.__enter__() 149 return self 150 151 def __exit__(self, *excinfo): 152 self.outfile.__exit__(*excinfo) 153 154 def __repr__(self): 155 return '<LogTee {0}>'.format(self.name) 156 157 def flush(self): 158 self.outfile.flush() 159 160 161class StdoutNoopContextManager: 162 """ 163 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 164 """ 165 def __enter__(self): 166 return sys.stdout 167 168 def __exit__(self, *exc_info): 169 pass 170 171 def write(self, string): 172 return sys.stdout.write(string) 173 174 def flush(self): 175 sys.stdout.flush() 176 177 @property 178 def name(self): 179 return sys.stdout.name 180 181 182def exec_func(func, d, dirs = None): 183 """Execute a BB 'function'""" 184 185 try: 186 oldcwd = os.getcwd() 187 except: 188 oldcwd = None 189 190 flags = d.getVarFlags(func) 191 cleandirs = flags.get('cleandirs') if flags else None 192 if cleandirs: 193 for cdir in d.expand(cleandirs).split(): 194 bb.utils.remove(cdir, True) 195 bb.utils.mkdirhier(cdir) 196 197 if flags and dirs is None: 198 dirs = flags.get('dirs') 199 if dirs: 200 dirs = d.expand(dirs).split() 201 202 if dirs: 203 for adir in dirs: 204 bb.utils.mkdirhier(adir) 205 adir = dirs[-1] 206 else: 207 adir = None 208 209 body = d.getVar(func, False) 210 if not body: 211 if body is None: 212 logger.warning("Function %s doesn't exist", func) 213 return 214 215 ispython = flags.get('python') 216 217 lockflag = flags.get('lockfiles') 218 if lockflag: 219 lockfiles = [f for f in d.expand(lockflag).split()] 220 else: 221 lockfiles = None 222 223 tempdir = d.getVar('T') 224 225 # or func allows items to be executed outside of the normal 226 # task set, such as buildhistory 227 task = d.getVar('BB_RUNTASK') or func 228 if task == func: 229 taskfunc = task 230 else: 231 taskfunc = "%s.%s" % (task, func) 232 233 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 234 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 235 runfile = os.path.join(tempdir, runfn) 236 bb.utils.mkdirhier(os.path.dirname(runfile)) 237 238 # Setup the courtesy link to the runfn, only for tasks 239 # we create the link 'just' before the run script is created 240 # if we create it after, and if the run script fails, then the 241 # link won't be created as an exception would be fired. 242 if task == func: 243 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 244 if runlink: 245 bb.utils.remove(runlink) 246 247 try: 248 os.symlink(runfn, runlink) 249 except OSError: 250 pass 251 252 with bb.utils.fileslocked(lockfiles): 253 if ispython: 254 exec_func_python(func, d, runfile, cwd=adir) 255 else: 256 exec_func_shell(func, d, runfile, cwd=adir) 257 258 try: 259 curcwd = os.getcwd() 260 except: 261 curcwd = None 262 263 if oldcwd and curcwd != oldcwd: 264 try: 265 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 266 os.chdir(oldcwd) 267 except: 268 pass 269 270_functionfmt = """ 271{function}(d) 272""" 273logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 274def exec_func_python(func, d, runfile, cwd=None): 275 """Execute a python BB 'function'""" 276 277 code = _functionfmt.format(function=func) 278 bb.utils.mkdirhier(os.path.dirname(runfile)) 279 with open(runfile, 'w') as script: 280 bb.data.emit_func_python(func, script, d) 281 282 if cwd: 283 try: 284 olddir = os.getcwd() 285 except OSError as e: 286 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 287 olddir = None 288 os.chdir(cwd) 289 290 bb.debug(2, "Executing python function %s" % func) 291 292 try: 293 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 294 fn = d.getVarFlag(func, "filename", False) 295 lineno = int(d.getVarFlag(func, "lineno", False)) 296 bb.methodpool.insert_method(func, text, fn, lineno - 1) 297 298 comp = utils.better_compile(code, func, "exec_python_func() autogenerated") 299 utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated") 300 finally: 301 bb.debug(2, "Python function %s finished" % func) 302 303 if cwd and olddir: 304 try: 305 os.chdir(olddir) 306 except OSError as e: 307 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 308 309def shell_trap_code(): 310 return '''#!/bin/sh\n 311__BITBAKE_LAST_LINE=0 312 313# Emit a useful diagnostic if something fails: 314bb_sh_exit_handler() { 315 ret=$? 316 if [ "$ret" != 0 ]; then 317 echo "WARNING: exit code $ret from a shell command." 318 fi 319 exit $ret 320} 321 322bb_bash_exit_handler() { 323 ret=$? 324 { set +x; } > /dev/null 325 trap "" DEBUG 326 if [ "$ret" != 0 ]; then 327 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 328 329 echo "WARNING: Backtrace (BB generated script): " 330 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 331 if [ "$i" -eq 1 ]; then 332 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 333 else 334 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 335 fi 336 done 337 fi 338 exit $ret 339} 340 341bb_bash_debug_handler() { 342 local line=${BASH_LINENO[0]} 343 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 344 if [ "$line" -eq 1 ]; then 345 return 346 fi 347 348 # Track the line number of commands as they execute. This is so we can have access to the failing line number 349 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 350 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 351 __BITBAKE_LAST_LINE=$line 352 fi 353} 354 355case $BASH_VERSION in 356"") trap 'bb_sh_exit_handler' 0 357 set -e 358 ;; 359*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 360 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 361 set -e 362 shopt -s extdebug 363 ;; 364esac 365''' 366 367def create_progress_handler(func, progress, logfile, d): 368 if progress == 'percent': 369 # Use default regex 370 return bb.progress.BasicProgressHandler(d, outfile=logfile) 371 elif progress.startswith('percent:'): 372 # Use specified regex 373 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 374 elif progress.startswith('outof:'): 375 # Use specified regex 376 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 377 elif progress.startswith("custom:"): 378 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 379 import functools 380 from types import ModuleType 381 382 parts = progress.split(":", 2) 383 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 384 if cls: 385 def resolve(x, y): 386 if not x: 387 return None 388 if isinstance(x, ModuleType): 389 return getattr(x, y, None) 390 return x.get(y) 391 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 392 if not cls_obj: 393 # Fall-back on __builtins__ 394 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 395 if cls_obj: 396 return cls_obj(d, outfile=logfile, otherargs=otherargs) 397 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 398 else: 399 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 400 401 return logfile 402 403def exec_func_shell(func, d, runfile, cwd=None): 404 """Execute a shell function from the metadata 405 406 Note on directory behavior. The 'dirs' varflag should contain a list 407 of the directories you need created prior to execution. The last 408 item in the list is where we will chdir/cd to. 409 """ 410 411 # Don't let the emitted shell script override PWD 412 d.delVarFlag('PWD', 'export') 413 414 with open(runfile, 'w') as script: 415 script.write(shell_trap_code()) 416 417 bb.data.emit_func(func, script, d) 418 419 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 420 script.write("set -x\n") 421 if cwd: 422 script.write("cd '%s'\n" % cwd) 423 script.write("%s\n" % func) 424 script.write(''' 425# cleanup 426ret=$? 427trap '' 0 428exit $ret 429''') 430 431 os.chmod(runfile, 0o775) 432 433 cmd = runfile 434 if d.getVarFlag(func, 'fakeroot', False): 435 fakerootcmd = d.getVar('FAKEROOT') 436 if fakerootcmd: 437 cmd = [fakerootcmd, runfile] 438 439 if verboseStdoutLogging: 440 logfile = LogTee(logger, StdoutNoopContextManager()) 441 else: 442 logfile = StdoutNoopContextManager() 443 444 progress = d.getVarFlag(func, 'progress') 445 if progress: 446 try: 447 logfile = create_progress_handler(func, progress, logfile, d) 448 except: 449 from traceback import format_exc 450 logger.error("Failed to create progress handler") 451 logger.error(format_exc()) 452 raise 453 454 fifobuffer = bytearray() 455 def readfifo(data): 456 nonlocal fifobuffer 457 fifobuffer.extend(data) 458 while fifobuffer: 459 message, token, nextmsg = fifobuffer.partition(b"\00") 460 if token: 461 splitval = message.split(b' ', 1) 462 cmd = splitval[0].decode("utf-8") 463 if len(splitval) > 1: 464 value = splitval[1].decode("utf-8") 465 else: 466 value = '' 467 if cmd == 'bbplain': 468 bb.plain(value) 469 elif cmd == 'bbnote': 470 bb.note(value) 471 elif cmd == 'bbverbnote': 472 bb.verbnote(value) 473 elif cmd == 'bbwarn': 474 bb.warn(value) 475 elif cmd == 'bberror': 476 bb.error(value) 477 elif cmd == 'bbfatal': 478 # The caller will call exit themselves, so bb.error() is 479 # what we want here rather than bb.fatal() 480 bb.error(value) 481 elif cmd == 'bbfatal_log': 482 bb.error(value, forcelog=True) 483 elif cmd == 'bbdebug': 484 splitval = value.split(' ', 1) 485 level = int(splitval[0]) 486 value = splitval[1] 487 bb.debug(level, value) 488 else: 489 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 490 fifobuffer = nextmsg 491 else: 492 break 493 494 tempdir = d.getVar('T') 495 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 496 if os.path.exists(fifopath): 497 os.unlink(fifopath) 498 os.mkfifo(fifopath) 499 with open(fifopath, 'r+b', buffering=0) as fifo: 500 try: 501 bb.debug(2, "Executing shell function %s" % func) 502 with open(os.devnull, 'r+') as stdin, logfile: 503 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 504 except bb.process.ExecutionError as exe: 505 # Find the backtrace that the shell trap generated 506 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 507 stdout_lines = (exe.stdout or "").split("\n") 508 backtrace_start_line = None 509 for i, line in enumerate(reversed(stdout_lines)): 510 if backtrace_marker_regex.search(line): 511 backtrace_start_line = len(stdout_lines) - i 512 break 513 514 # Read the backtrace frames, starting at the location we just found 515 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 516 r"?P<lineno>\d+)") 517 backtrace_frames = [] 518 if backtrace_start_line: 519 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 520 match = backtrace_entry_regex.search(line) 521 if match: 522 backtrace_frames.append(match.groupdict()) 523 524 with open(runfile, "r") as script: 525 script_lines = [line.rstrip() for line in script.readlines()] 526 527 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 528 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 529 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 530 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 531 better_frames = [] 532 # Skip the very last frame since it's just the call to the shell task in the body of the script 533 for frame in backtrace_frames[:-1]: 534 # Check whether the frame corresponds to a function defined in the script vs external script. 535 if os.path.samefile(frame["file"], runfile): 536 # Search backwards from the frame lineno to locate the comment that BB injected 537 i = int(frame["lineno"]) - 1 538 while i >= 0: 539 match = script_metadata_comment_regex.match(script_lines[i]) 540 if match: 541 # Calculate the relative line in the function itself 542 relative_line_in_function = int(frame["lineno"]) - i - 2 543 # Calculate line in the function as declared in the metadata 544 metadata_function_line = relative_line_in_function + int(match["lineno"]) 545 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 546 frameno=frame["frameno"], 547 funcname=frame["funcname"], 548 file=match["file"], 549 lineno=metadata_function_line 550 )) 551 break 552 i -= 1 553 else: 554 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 555 556 if better_frames: 557 better_frames = ("\t{0}".format(frame) for frame in better_frames) 558 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 559 raise 560 finally: 561 os.unlink(fifopath) 562 563 bb.debug(2, "Shell function %s finished" % func) 564 565def _task_data(fn, task, d): 566 localdata = bb.data.createCopy(d) 567 localdata.setVar('BB_FILENAME', fn) 568 localdata.setVar('BB_CURRENTTASK', task[3:]) 569 localdata.setVar('OVERRIDES', 'task-%s:%s' % 570 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 571 localdata.finalize() 572 bb.data.expandKeys(localdata) 573 return localdata 574 575def _exec_task(fn, task, d, quieterr): 576 """Execute a BB 'task' 577 578 Execution of a task involves a bit more setup than executing a function, 579 running it with its own local metadata, and with some useful variables set. 580 """ 581 if not d.getVarFlag(task, 'task', False): 582 event.fire(TaskInvalid(task, d), d) 583 logger.error("No such task: %s" % task) 584 return 1 585 586 logger.debug("Executing task %s", task) 587 588 localdata = _task_data(fn, task, d) 589 tempdir = localdata.getVar('T') 590 if not tempdir: 591 bb.fatal("T variable not set, unable to build") 592 593 # Change nice level if we're asked to 594 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 595 if nice: 596 curnice = os.nice(0) 597 nice = int(nice) - curnice 598 newnice = os.nice(nice) 599 logger.debug("Renice to %s " % newnice) 600 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 601 if ionice: 602 try: 603 cls, prio = ionice.split(".", 1) 604 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 605 except: 606 bb.warn("Invalid ionice level %s" % ionice) 607 608 bb.utils.mkdirhier(tempdir) 609 610 # Determine the logfile to generate 611 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 612 logbase = logfmt.format(task=task, pid=os.getpid()) 613 614 # Document the order of the tasks... 615 logorder = os.path.join(tempdir, 'log.task_order') 616 try: 617 with open(logorder, 'a') as logorderfile: 618 logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) 619 except OSError: 620 logger.exception("Opening log file '%s'", logorder) 621 pass 622 623 # Setup the courtesy link to the logfn 624 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 625 logfn = os.path.join(tempdir, logbase) 626 if loglink: 627 bb.utils.remove(loglink) 628 629 try: 630 os.symlink(logbase, loglink) 631 except OSError: 632 pass 633 634 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 635 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 636 637 class ErrorCheckHandler(logging.Handler): 638 def __init__(self): 639 self.triggered = False 640 logging.Handler.__init__(self, logging.ERROR) 641 def emit(self, record): 642 if getattr(record, 'forcelog', False): 643 self.triggered = False 644 else: 645 self.triggered = True 646 647 # Handle logfiles 648 try: 649 bb.utils.mkdirhier(os.path.dirname(logfn)) 650 logfile = open(logfn, 'w') 651 except OSError: 652 logger.exception("Opening log file '%s'", logfn) 653 pass 654 655 # Dup the existing fds so we dont lose them 656 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 657 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 658 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 659 660 # Replace those fds with our own 661 with open('/dev/null', 'r') as si: 662 os.dup2(si.fileno(), osi[1]) 663 os.dup2(logfile.fileno(), oso[1]) 664 os.dup2(logfile.fileno(), ose[1]) 665 666 # Ensure Python logging goes to the logfile 667 handler = logging.StreamHandler(logfile) 668 handler.setFormatter(logformatter) 669 # Always enable full debug output into task logfiles 670 handler.setLevel(logging.DEBUG - 2) 671 bblogger.addHandler(handler) 672 673 errchk = ErrorCheckHandler() 674 bblogger.addHandler(errchk) 675 676 localdata.setVar('BB_LOGFILE', logfn) 677 localdata.setVar('BB_RUNTASK', task) 678 localdata.setVar('BB_TASK_LOGGER', bblogger) 679 680 flags = localdata.getVarFlags(task) 681 682 try: 683 try: 684 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 685 except (bb.BBHandledException, SystemExit): 686 return 1 687 688 try: 689 for func in (prefuncs or '').split(): 690 exec_func(func, localdata) 691 exec_func(task, localdata) 692 for func in (postfuncs or '').split(): 693 exec_func(func, localdata) 694 except bb.BBHandledException: 695 event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata) 696 return 1 697 except Exception as exc: 698 if quieterr: 699 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 700 else: 701 errprinted = errchk.triggered 702 logger.error(str(exc)) 703 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 704 return 1 705 finally: 706 sys.stdout.flush() 707 sys.stderr.flush() 708 709 bblogger.removeHandler(handler) 710 711 # Restore the backup fds 712 os.dup2(osi[0], osi[1]) 713 os.dup2(oso[0], oso[1]) 714 os.dup2(ose[0], ose[1]) 715 716 # Close the backup fds 717 os.close(osi[0]) 718 os.close(oso[0]) 719 os.close(ose[0]) 720 721 logfile.close() 722 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 723 logger.debug2("Zero size logfn %s, removing", logfn) 724 bb.utils.remove(logfn) 725 bb.utils.remove(loglink) 726 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 727 728 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 729 make_stamp(task, localdata) 730 731 return 0 732 733def exec_task(fn, task, d, profile = False): 734 try: 735 quieterr = False 736 if d.getVarFlag(task, "quieterrors", False) is not None: 737 quieterr = True 738 739 if profile: 740 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 741 try: 742 import cProfile as profile 743 except: 744 import profile 745 prof = profile.Profile() 746 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 747 prof.dump_stats(profname) 748 bb.utils.process_profilelog(profname) 749 750 return ret 751 else: 752 return _exec_task(fn, task, d, quieterr) 753 754 except Exception: 755 from traceback import format_exc 756 if not quieterr: 757 logger.error("Build of %s failed" % (task)) 758 logger.error(format_exc()) 759 failedevent = TaskFailed(task, None, d, True) 760 event.fire(failedevent, d) 761 return 1 762 763def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False): 764 """ 765 Internal stamp helper function 766 Makes sure the stamp directory exists 767 Returns the stamp path+filename 768 769 In the bitbake core, d can be a CacheData and file_name will be set. 770 When called in task context, d will be a data store, file_name will not be set 771 """ 772 taskflagname = taskname 773 if taskname.endswith("_setscene") and taskname != "do_setscene": 774 taskflagname = taskname.replace("_setscene", "") 775 776 if file_name: 777 stamp = d.stamp[file_name] 778 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 779 else: 780 stamp = d.getVar('STAMP') 781 file_name = d.getVar('BB_FILENAME') 782 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 783 784 if baseonly: 785 return stamp 786 if noextra: 787 extrainfo = "" 788 789 if not stamp: 790 return 791 792 stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) 793 794 stampdir = os.path.dirname(stamp) 795 if cached_mtime_noerror(stampdir) == 0: 796 bb.utils.mkdirhier(stampdir) 797 798 return stamp 799 800def stamp_cleanmask_internal(taskname, d, file_name): 801 """ 802 Internal stamp helper function to generate stamp cleaning mask 803 Returns the stamp path+filename 804 805 In the bitbake core, d can be a CacheData and file_name will be set. 806 When called in task context, d will be a data store, file_name will not be set 807 """ 808 taskflagname = taskname 809 if taskname.endswith("_setscene") and taskname != "do_setscene": 810 taskflagname = taskname.replace("_setscene", "") 811 812 if file_name: 813 stamp = d.stampclean[file_name] 814 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 815 else: 816 stamp = d.getVar('STAMPCLEAN') 817 file_name = d.getVar('BB_FILENAME') 818 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 819 820 if not stamp: 821 return [] 822 823 cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) 824 825 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 826 827def make_stamp(task, d, file_name = None): 828 """ 829 Creates/updates a stamp for a given task 830 (d can be a data dict or dataCache) 831 """ 832 cleanmask = stamp_cleanmask_internal(task, d, file_name) 833 for mask in cleanmask: 834 for name in glob.glob(mask): 835 # Preserve sigdata files in the stamps directory 836 if "sigdata" in name or "sigbasedata" in name: 837 continue 838 # Preserve taint files in the stamps directory 839 if name.endswith('.taint'): 840 continue 841 os.unlink(name) 842 843 stamp = stamp_internal(task, d, file_name) 844 # Remove the file and recreate to force timestamp 845 # change on broken NFS filesystems 846 if stamp: 847 bb.utils.remove(stamp) 848 open(stamp, "w").close() 849 850 # If we're in task context, write out a signature file for each task 851 # as it completes 852 if not task.endswith("_setscene") and task != "do_setscene" and not file_name: 853 stampbase = stamp_internal(task, d, None, True) 854 file_name = d.getVar('BB_FILENAME') 855 bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) 856 857def find_stale_stamps(task, d, file_name=None): 858 current = stamp_internal(task, d, file_name) 859 current2 = stamp_internal(task + "_setscene", d, file_name) 860 cleanmask = stamp_cleanmask_internal(task, d, file_name) 861 found = [] 862 for mask in cleanmask: 863 for name in glob.glob(mask): 864 if "sigdata" in name or "sigbasedata" in name: 865 continue 866 if name.endswith('.taint'): 867 continue 868 if name == current or name == current2: 869 continue 870 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 871 found.append(name) 872 return found 873 874def del_stamp(task, d, file_name = None): 875 """ 876 Removes a stamp for a given task 877 (d can be a data dict or dataCache) 878 """ 879 stamp = stamp_internal(task, d, file_name) 880 bb.utils.remove(stamp) 881 882def write_taint(task, d, file_name = None): 883 """ 884 Creates a "taint" file which will force the specified task and its 885 dependents to be re-run the next time by influencing the value of its 886 taskhash. 887 (d can be a data dict or dataCache) 888 """ 889 import uuid 890 if file_name: 891 taintfn = d.stamp[file_name] + '.' + task + '.taint' 892 else: 893 taintfn = d.getVar('STAMP') + '.' + task + '.taint' 894 bb.utils.mkdirhier(os.path.dirname(taintfn)) 895 # The specific content of the taint file is not really important, 896 # we just need it to be random, so a random UUID is used 897 with open(taintfn, 'w') as taintf: 898 taintf.write(str(uuid.uuid4())) 899 900def stampfile(taskname, d, file_name = None, noextra=False): 901 """ 902 Return the stamp for a given task 903 (d can be a data dict or dataCache) 904 """ 905 return stamp_internal(taskname, d, file_name, noextra=noextra) 906 907def add_tasks(tasklist, d): 908 task_deps = d.getVar('_task_deps', False) 909 if not task_deps: 910 task_deps = {} 911 if not 'tasks' in task_deps: 912 task_deps['tasks'] = [] 913 if not 'parents' in task_deps: 914 task_deps['parents'] = {} 915 916 for task in tasklist: 917 task = d.expand(task) 918 919 d.setVarFlag(task, 'task', 1) 920 921 if not task in task_deps['tasks']: 922 task_deps['tasks'].append(task) 923 924 flags = d.getVarFlags(task) 925 def getTask(name): 926 if not name in task_deps: 927 task_deps[name] = {} 928 if name in flags: 929 deptask = d.expand(flags[name]) 930 task_deps[name][task] = deptask 931 getTask('mcdepends') 932 getTask('depends') 933 getTask('rdepends') 934 getTask('deptask') 935 getTask('rdeptask') 936 getTask('recrdeptask') 937 getTask('recideptask') 938 getTask('nostamp') 939 getTask('fakeroot') 940 getTask('noexec') 941 getTask('umask') 942 task_deps['parents'][task] = [] 943 if 'deps' in flags: 944 for dep in flags['deps']: 945 # Check and warn for "addtask task after foo" while foo does not exist 946 #if not dep in tasklist: 947 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 948 dep = d.expand(dep) 949 task_deps['parents'][task].append(dep) 950 951 # don't assume holding a reference 952 d.setVar('_task_deps', task_deps) 953 954def addtask(task, before, after, d): 955 if task[:3] != "do_": 956 task = "do_" + task 957 958 d.setVarFlag(task, "task", 1) 959 bbtasks = d.getVar('__BBTASKS', False) or [] 960 if task not in bbtasks: 961 bbtasks.append(task) 962 d.setVar('__BBTASKS', bbtasks) 963 964 existing = d.getVarFlag(task, "deps", False) or [] 965 if after is not None: 966 # set up deps for function 967 for entry in after.split(): 968 if entry not in existing: 969 existing.append(entry) 970 d.setVarFlag(task, "deps", existing) 971 if before is not None: 972 # set up things that depend on this func 973 for entry in before.split(): 974 existing = d.getVarFlag(entry, "deps", False) or [] 975 if task not in existing: 976 d.setVarFlag(entry, "deps", [task] + existing) 977 978def deltask(task, d): 979 if task[:3] != "do_": 980 task = "do_" + task 981 982 bbtasks = d.getVar('__BBTASKS', False) or [] 983 if task in bbtasks: 984 bbtasks.remove(task) 985 d.delVarFlag(task, 'task') 986 d.setVar('__BBTASKS', bbtasks) 987 988 d.delVarFlag(task, 'deps') 989 for bbtask in d.getVar('__BBTASKS', False) or []: 990 deps = d.getVarFlag(bbtask, 'deps', False) or [] 991 if task in deps: 992 deps.remove(task) 993 d.setVarFlag(bbtask, 'deps', deps) 994 995def preceedtask(task, with_recrdeptasks, d): 996 """ 997 Returns a set of tasks in the current recipe which were specified as 998 precondition by the task itself ("after") or which listed themselves 999 as precondition ("before"). Preceeding tasks specified via the 1000 "recrdeptask" are included in the result only if requested. Beware 1001 that this may lead to the task itself being listed. 1002 """ 1003 preceed = set() 1004 1005 # Ignore tasks which don't exist 1006 tasks = d.getVar('__BBTASKS', False) 1007 if task not in tasks: 1008 return preceed 1009 1010 preceed.update(d.getVarFlag(task, 'deps') or []) 1011 if with_recrdeptasks: 1012 recrdeptask = d.getVarFlag(task, 'recrdeptask') 1013 if recrdeptask: 1014 preceed.update(recrdeptask.split()) 1015 return preceed 1016 1017def tasksbetween(task_start, task_end, d): 1018 """ 1019 Return the list of tasks between two tasks in the current recipe, 1020 where task_start is to start at and task_end is the task to end at 1021 (and task_end has a dependency chain back to task_start). 1022 """ 1023 outtasks = [] 1024 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 1025 def follow_chain(task, endtask, chain=None): 1026 if not chain: 1027 chain = [] 1028 chain.append(task) 1029 for othertask in tasks: 1030 if othertask == task: 1031 continue 1032 if task == endtask: 1033 for ctask in chain: 1034 if ctask not in outtasks: 1035 outtasks.append(ctask) 1036 else: 1037 deps = d.getVarFlag(othertask, 'deps', False) 1038 if task in deps: 1039 follow_chain(othertask, endtask, chain) 1040 chain.pop() 1041 follow_chain(task_start, task_end) 1042 return outtasks 1043