1# 2# Copyright (c) 2013-2014 Intel Corporation 3# 4# SPDX-License-Identifier: MIT 5# 6 7# DESCRIPTION 8# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest 9# It provides a class and methods for running commands on the host in a convienent way for tests. 10 11 12 13import os 14import sys 15import signal 16import subprocess 17import threading 18import time 19import logging 20from oeqa.utils import CommandError 21from oeqa.utils import ftools 22import re 23import contextlib 24# Export test doesn't require bb 25try: 26 import bb 27except ImportError: 28 pass 29 30class Command(object): 31 def __init__(self, command, bg=False, timeout=None, data=None, output_log=None, **options): 32 33 self.defaultopts = { 34 "stdout": subprocess.PIPE, 35 "stderr": subprocess.STDOUT, 36 "stdin": None, 37 "shell": False, 38 "bufsize": -1, 39 } 40 41 self.cmd = command 42 self.bg = bg 43 self.timeout = timeout 44 self.data = data 45 46 self.options = dict(self.defaultopts) 47 if isinstance(self.cmd, str): 48 self.options["shell"] = True 49 if self.data: 50 self.options['stdin'] = subprocess.PIPE 51 self.options.update(options) 52 53 self.status = None 54 # We collect chunks of output before joining them at the end. 55 self._output_chunks = [] 56 self._error_chunks = [] 57 self.output = None 58 self.error = None 59 self.threads = [] 60 61 self.output_log = output_log 62 self.log = logging.getLogger("utils.commands") 63 64 def run(self): 65 self.process = subprocess.Popen(self.cmd, **self.options) 66 67 def readThread(output, stream, logfunc): 68 if logfunc: 69 for line in stream: 70 output.append(line) 71 logfunc(line.decode("utf-8", errors='replace').rstrip()) 72 else: 73 output.append(stream.read()) 74 75 def readStderrThread(): 76 readThread(self._error_chunks, self.process.stderr, self.output_log.error if self.output_log else None) 77 78 def readStdoutThread(): 79 readThread(self._output_chunks, self.process.stdout, self.output_log.info if self.output_log else None) 80 81 def writeThread(): 82 try: 83 self.process.stdin.write(self.data) 84 self.process.stdin.close() 85 except OSError as ex: 86 # It's not an error when the command does not consume all 87 # of our data. subprocess.communicate() also ignores that. 88 if ex.errno != EPIPE: 89 raise 90 91 # We write in a separate thread because then we can read 92 # without worrying about deadlocks. The additional thread is 93 # expected to terminate by itself and we mark it as a daemon, 94 # so even it should happen to not terminate for whatever 95 # reason, the main process will still exit, which will then 96 # kill the write thread. 97 if self.data: 98 threading.Thread(target=writeThread, daemon=True).start() 99 if self.process.stderr: 100 thread = threading.Thread(target=readStderrThread) 101 thread.start() 102 self.threads.append(thread) 103 if self.output_log: 104 self.output_log.info('Running: %s' % self.cmd) 105 thread = threading.Thread(target=readStdoutThread) 106 thread.start() 107 self.threads.append(thread) 108 109 self.log.debug("Running command '%s'" % self.cmd) 110 111 if not self.bg: 112 if self.timeout is None: 113 for thread in self.threads: 114 thread.join() 115 else: 116 deadline = time.time() + self.timeout 117 for thread in self.threads: 118 timeout = deadline - time.time() 119 if timeout < 0: 120 timeout = 0 121 thread.join(timeout) 122 self.stop() 123 124 def stop(self): 125 for thread in self.threads: 126 if thread.isAlive(): 127 self.process.terminate() 128 # let's give it more time to terminate gracefully before killing it 129 thread.join(5) 130 if thread.isAlive(): 131 self.process.kill() 132 thread.join() 133 134 def finalize_output(data): 135 if not data: 136 data = "" 137 else: 138 data = b"".join(data) 139 data = data.decode("utf-8", errors='replace').rstrip() 140 return data 141 142 self.output = finalize_output(self._output_chunks) 143 self._output_chunks = None 144 # self.error used to be a byte string earlier, probably unintentionally. 145 # Now it is a normal string, just like self.output. 146 self.error = finalize_output(self._error_chunks) 147 self._error_chunks = None 148 # At this point we know that the process has closed stdout/stderr, so 149 # it is safe and necessary to wait for the actual process completion. 150 self.status = self.process.wait() 151 self.process.stdout.close() 152 if self.process.stderr: 153 self.process.stderr.close() 154 155 self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) 156 # logging the complete output is insane 157 # bitbake -e output is really big 158 # and makes the log file useless 159 if self.status: 160 lout = "\n".join(self.output.splitlines()[-20:]) 161 self.log.debug("Last 20 lines:\n%s" % lout) 162 163 164class Result(object): 165 pass 166 167 168def runCmd(command, ignore_status=False, timeout=None, assert_error=True, 169 native_sysroot=None, limit_exc_output=0, output_log=None, **options): 170 result = Result() 171 172 if native_sysroot: 173 extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ 174 (native_sysroot, native_sysroot, native_sysroot) 175 extra_libpaths = "%s/lib:%s/usr/lib" % \ 176 (native_sysroot, native_sysroot) 177 nenv = dict(options.get('env', os.environ)) 178 nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') 179 nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '') 180 options['env'] = nenv 181 182 cmd = Command(command, timeout=timeout, output_log=output_log, **options) 183 cmd.run() 184 185 result.command = command 186 result.status = cmd.status 187 result.output = cmd.output 188 result.error = cmd.error 189 result.pid = cmd.process.pid 190 191 if result.status and not ignore_status: 192 exc_output = result.output 193 if limit_exc_output > 0: 194 split = result.output.splitlines() 195 if len(split) > limit_exc_output: 196 exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \ 197 '\n'.join(split[-limit_exc_output:]) 198 if assert_error: 199 raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) 200 else: 201 raise CommandError(result.status, command, exc_output) 202 203 return result 204 205 206def bitbake(command, ignore_status=False, timeout=None, postconfig=None, output_log=None, **options): 207 208 if postconfig: 209 postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') 210 ftools.write_file(postconfig_file, postconfig) 211 extra_args = "-R %s" % postconfig_file 212 else: 213 extra_args = "" 214 215 if isinstance(command, str): 216 cmd = "bitbake " + extra_args + " " + command 217 else: 218 cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] 219 220 try: 221 return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options) 222 finally: 223 if postconfig: 224 os.remove(postconfig_file) 225 226 227def get_bb_env(target=None, postconfig=None): 228 if target: 229 return bitbake("-e %s" % target, postconfig=postconfig).output 230 else: 231 return bitbake("-e", postconfig=postconfig).output 232 233def get_bb_vars(variables=None, target=None, postconfig=None): 234 """Get values of multiple bitbake variables""" 235 bbenv = get_bb_env(target, postconfig=postconfig) 236 237 if variables is not None: 238 variables = list(variables) 239 var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$') 240 unset_re = re.compile(r'^unset (?P<var>\w+)$') 241 lastline = None 242 values = {} 243 for line in bbenv.splitlines(): 244 match = var_re.match(line) 245 val = None 246 if match: 247 val = match.group('value') 248 else: 249 match = unset_re.match(line) 250 if match: 251 # Handle [unexport] variables 252 if lastline.startswith('# "'): 253 val = lastline.split('"')[1] 254 if val: 255 var = match.group('var') 256 if variables is None: 257 values[var] = val 258 else: 259 if var in variables: 260 values[var] = val 261 variables.remove(var) 262 # Stop after all required variables have been found 263 if not variables: 264 break 265 lastline = line 266 if variables: 267 # Fill in missing values 268 for var in variables: 269 values[var] = None 270 return values 271 272def get_bb_var(var, target=None, postconfig=None): 273 return get_bb_vars([var], target, postconfig)[var] 274 275def get_test_layer(): 276 layers = get_bb_var("BBLAYERS").split() 277 testlayer = None 278 for l in layers: 279 if '~' in l: 280 l = os.path.expanduser(l) 281 if "/meta-selftest" in l and os.path.isdir(l): 282 testlayer = l 283 break 284 return testlayer 285 286def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'): 287 os.makedirs(os.path.join(templayerdir, 'conf')) 288 with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: 289 f.write('BBPATH .= ":${LAYERDIR}"\n') 290 f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec) 291 f.write(' ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec) 292 f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername) 293 f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername) 294 f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority)) 295 f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername) 296 f.write('LAYERSERIES_COMPAT_%s = "${LAYERSERIES_COMPAT_core}"\n' % templayername) 297 298@contextlib.contextmanager 299def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): 300 """ 301 launch_cmd means directly run the command, don't need set rootfs or env vars. 302 """ 303 304 import bb.tinfoil 305 import bb.build 306 307 # Need a non-'BitBake' logger to capture the runner output 308 targetlogger = logging.getLogger('TargetRunner') 309 targetlogger.setLevel(logging.DEBUG) 310 handler = logging.StreamHandler(sys.stdout) 311 targetlogger.addHandler(handler) 312 313 tinfoil = bb.tinfoil.Tinfoil() 314 tinfoil.prepare(config_only=False, quiet=True) 315 try: 316 tinfoil.logger.setLevel(logging.WARNING) 317 import oeqa.targetcontrol 318 recipedata = tinfoil.parse_recipe(pn) 319 recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") 320 recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") 321 # Tell QemuTarget() whether need find rootfs/kernel or not 322 if launch_cmd: 323 recipedata.setVar("FIND_ROOTFS", '0') 324 else: 325 recipedata.setVar("FIND_ROOTFS", '1') 326 327 for key, value in overrides.items(): 328 recipedata.setVar(key, value) 329 330 logdir = recipedata.getVar("TEST_LOG_DIR") 331 332 qemu = oeqa.targetcontrol.QemuTarget(recipedata, targetlogger, image_fstype) 333 finally: 334 # We need to shut down tinfoil early here in case we actually want 335 # to run tinfoil-using utilities with the running QEMU instance. 336 # Luckily QemuTarget doesn't need it after the constructor. 337 tinfoil.shutdown() 338 339 try: 340 qemu.deploy() 341 try: 342 qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) 343 except Exception as e: 344 msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir 345 if os.path.exists(qemu.qemurunnerlog): 346 with open(qemu.qemurunnerlog, 'r') as f: 347 msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read()) 348 raise Exception(msg) 349 350 yield qemu 351 352 finally: 353 targetlogger.removeHandler(handler) 354 try: 355 qemu.stop() 356 except: 357 pass 358 359def updateEnv(env_file): 360 """ 361 Source a file and update environment. 362 """ 363 364 cmd = ". %s; env -0" % env_file 365 result = runCmd(cmd) 366 367 for line in result.output.split("\0"): 368 (key, _, value) = line.partition("=") 369 os.environ[key] = value 370