1#
2# Copyright (c) 2013-2014 Intel Corporation
3#
4# SPDX-License-Identifier: MIT
5#
6
7# DESCRIPTION
8# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest
9# It provides a class and methods for running commands on the host in a convienent way for tests.
10
11
12
13import os
14import sys
15import signal
16import subprocess
17import threading
18import time
19import logging
20from oeqa.utils import CommandError
21from oeqa.utils import ftools
22import re
23import contextlib
24# Export test doesn't require bb
25try:
26    import bb
27except ImportError:
28    pass
29
30class Command(object):
31    def __init__(self, command, bg=False, timeout=None, data=None, output_log=None, **options):
32
33        self.defaultopts = {
34            "stdout": subprocess.PIPE,
35            "stderr": subprocess.STDOUT,
36            "stdin": None,
37            "shell": False,
38            "bufsize": -1,
39        }
40
41        self.cmd = command
42        self.bg = bg
43        self.timeout = timeout
44        self.data = data
45
46        self.options = dict(self.defaultopts)
47        if isinstance(self.cmd, str):
48            self.options["shell"] = True
49        if self.data:
50            self.options['stdin'] = subprocess.PIPE
51        self.options.update(options)
52
53        self.status = None
54        # We collect chunks of output before joining them at the end.
55        self._output_chunks = []
56        self._error_chunks = []
57        self.output = None
58        self.error = None
59        self.threads = []
60
61        self.output_log = output_log
62        self.log = logging.getLogger("utils.commands")
63
64    def run(self):
65        self.process = subprocess.Popen(self.cmd, **self.options)
66
67        def readThread(output, stream, logfunc):
68            if logfunc:
69                for line in stream:
70                    output.append(line)
71                    logfunc(line.decode("utf-8", errors='replace').rstrip())
72            else:
73                output.append(stream.read())
74
75        def readStderrThread():
76            readThread(self._error_chunks, self.process.stderr, self.output_log.error if self.output_log else None)
77
78        def readStdoutThread():
79            readThread(self._output_chunks, self.process.stdout, self.output_log.info if self.output_log else None)
80
81        def writeThread():
82            try:
83                self.process.stdin.write(self.data)
84                self.process.stdin.close()
85            except OSError as ex:
86                # It's not an error when the command does not consume all
87                # of our data. subprocess.communicate() also ignores that.
88                if ex.errno != EPIPE:
89                    raise
90
91        # We write in a separate thread because then we can read
92        # without worrying about deadlocks. The additional thread is
93        # expected to terminate by itself and we mark it as a daemon,
94        # so even it should happen to not terminate for whatever
95        # reason, the main process will still exit, which will then
96        # kill the write thread.
97        if self.data:
98            threading.Thread(target=writeThread, daemon=True).start()
99        if self.process.stderr:
100            thread = threading.Thread(target=readStderrThread)
101            thread.start()
102            self.threads.append(thread)
103        if self.output_log:
104            self.output_log.info('Running: %s' % self.cmd)
105        thread = threading.Thread(target=readStdoutThread)
106        thread.start()
107        self.threads.append(thread)
108
109        self.log.debug("Running command '%s'" % self.cmd)
110
111        if not self.bg:
112            if self.timeout is None:
113                for thread in self.threads:
114                    thread.join()
115            else:
116                deadline = time.time() + self.timeout
117                for thread in self.threads:
118                    timeout = deadline - time.time()
119                    if timeout < 0:
120                        timeout = 0
121                    thread.join(timeout)
122            self.stop()
123
124    def stop(self):
125        for thread in self.threads:
126            if thread.isAlive():
127                self.process.terminate()
128            # let's give it more time to terminate gracefully before killing it
129            thread.join(5)
130            if thread.isAlive():
131                self.process.kill()
132                thread.join()
133
134        def finalize_output(data):
135            if not data:
136                data = ""
137            else:
138                data = b"".join(data)
139                data = data.decode("utf-8", errors='replace').rstrip()
140            return data
141
142        self.output = finalize_output(self._output_chunks)
143        self._output_chunks = None
144        # self.error used to be a byte string earlier, probably unintentionally.
145        # Now it is a normal string, just like self.output.
146        self.error = finalize_output(self._error_chunks)
147        self._error_chunks = None
148        # At this point we know that the process has closed stdout/stderr, so
149        # it is safe and necessary to wait for the actual process completion.
150        self.status = self.process.wait()
151        self.process.stdout.close()
152        if self.process.stderr:
153            self.process.stderr.close()
154
155        self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
156        # logging the complete output is insane
157        # bitbake -e output is really big
158        # and makes the log file useless
159        if self.status:
160            lout = "\n".join(self.output.splitlines()[-20:])
161            self.log.debug("Last 20 lines:\n%s" % lout)
162
163
164class Result(object):
165    pass
166
167
168def runCmd(command, ignore_status=False, timeout=None, assert_error=True,
169          native_sysroot=None, limit_exc_output=0, output_log=None, **options):
170    result = Result()
171
172    if native_sysroot:
173        extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \
174                      (native_sysroot, native_sysroot, native_sysroot)
175        extra_libpaths = "%s/lib:%s/usr/lib" % \
176                         (native_sysroot, native_sysroot)
177        nenv = dict(options.get('env', os.environ))
178        nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '')
179        nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '')
180        options['env'] = nenv
181
182    cmd = Command(command, timeout=timeout, output_log=output_log, **options)
183    cmd.run()
184
185    result.command = command
186    result.status = cmd.status
187    result.output = cmd.output
188    result.error = cmd.error
189    result.pid = cmd.process.pid
190
191    if result.status and not ignore_status:
192        exc_output = result.output
193        if limit_exc_output > 0:
194            split = result.output.splitlines()
195            if len(split) > limit_exc_output:
196                exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \
197                             '\n'.join(split[-limit_exc_output:])
198        if assert_error:
199            raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
200        else:
201            raise CommandError(result.status, command, exc_output)
202
203    return result
204
205
206def bitbake(command, ignore_status=False, timeout=None, postconfig=None, output_log=None, **options):
207
208    if postconfig:
209        postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf')
210        ftools.write_file(postconfig_file, postconfig)
211        extra_args = "-R %s" % postconfig_file
212    else:
213        extra_args = ""
214
215    if isinstance(command, str):
216        cmd = "bitbake " + extra_args + " " + command
217    else:
218        cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]]
219
220    try:
221        return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options)
222    finally:
223        if postconfig:
224            os.remove(postconfig_file)
225
226
227def get_bb_env(target=None, postconfig=None):
228    if target:
229        return bitbake("-e %s" % target, postconfig=postconfig).output
230    else:
231        return bitbake("-e", postconfig=postconfig).output
232
233def get_bb_vars(variables=None, target=None, postconfig=None):
234    """Get values of multiple bitbake variables"""
235    bbenv = get_bb_env(target, postconfig=postconfig)
236
237    if variables is not None:
238        variables = list(variables)
239    var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$')
240    unset_re = re.compile(r'^unset (?P<var>\w+)$')
241    lastline = None
242    values = {}
243    for line in bbenv.splitlines():
244        match = var_re.match(line)
245        val = None
246        if match:
247            val = match.group('value')
248        else:
249            match = unset_re.match(line)
250            if match:
251                # Handle [unexport] variables
252                if lastline.startswith('#   "'):
253                    val = lastline.split('"')[1]
254        if val:
255            var = match.group('var')
256            if variables is None:
257                values[var] = val
258            else:
259                if var in variables:
260                    values[var] = val
261                    variables.remove(var)
262                # Stop after all required variables have been found
263                if not variables:
264                    break
265        lastline = line
266    if variables:
267        # Fill in missing values
268        for var in variables:
269            values[var] = None
270    return values
271
272def get_bb_var(var, target=None, postconfig=None):
273    return get_bb_vars([var], target, postconfig)[var]
274
275def get_test_layer():
276    layers = get_bb_var("BBLAYERS").split()
277    testlayer = None
278    for l in layers:
279        if '~' in l:
280            l = os.path.expanduser(l)
281        if "/meta-selftest" in l and os.path.isdir(l):
282            testlayer = l
283            break
284    return testlayer
285
286def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
287    os.makedirs(os.path.join(templayerdir, 'conf'))
288    with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
289        f.write('BBPATH .= ":${LAYERDIR}"\n')
290        f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec)
291        f.write('            ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec)
292        f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername)
293        f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername)
294        f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority))
295        f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername)
296        f.write('LAYERSERIES_COMPAT_%s = "${LAYERSERIES_COMPAT_core}"\n' % templayername)
297
298@contextlib.contextmanager
299def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True):
300    """
301    launch_cmd means directly run the command, don't need set rootfs or env vars.
302    """
303
304    import bb.tinfoil
305    import bb.build
306
307    # Need a non-'BitBake' logger to capture the runner output
308    targetlogger = logging.getLogger('TargetRunner')
309    targetlogger.setLevel(logging.DEBUG)
310    handler = logging.StreamHandler(sys.stdout)
311    targetlogger.addHandler(handler)
312
313    tinfoil = bb.tinfoil.Tinfoil()
314    tinfoil.prepare(config_only=False, quiet=True)
315    try:
316        tinfoil.logger.setLevel(logging.WARNING)
317        import oeqa.targetcontrol
318        recipedata = tinfoil.parse_recipe(pn)
319        recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
320        recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
321        # Tell QemuTarget() whether need find rootfs/kernel or not
322        if launch_cmd:
323            recipedata.setVar("FIND_ROOTFS", '0')
324        else:
325            recipedata.setVar("FIND_ROOTFS", '1')
326
327        for key, value in overrides.items():
328            recipedata.setVar(key, value)
329
330        logdir = recipedata.getVar("TEST_LOG_DIR")
331
332        qemu = oeqa.targetcontrol.QemuTarget(recipedata, targetlogger, image_fstype)
333    finally:
334        # We need to shut down tinfoil early here in case we actually want
335        # to run tinfoil-using utilities with the running QEMU instance.
336        # Luckily QemuTarget doesn't need it after the constructor.
337        tinfoil.shutdown()
338
339    try:
340        qemu.deploy()
341        try:
342            qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes)
343        except Exception as e:
344            msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir
345            if os.path.exists(qemu.qemurunnerlog):
346                with open(qemu.qemurunnerlog, 'r') as f:
347                    msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read())
348            raise Exception(msg)
349
350        yield qemu
351
352    finally:
353        targetlogger.removeHandler(handler)
354        try:
355            qemu.stop()
356        except:
357            pass
358
359def updateEnv(env_file):
360    """
361    Source a file and update environment.
362    """
363
364    cmd = ". %s; env -0" % env_file
365    result = runCmd(cmd)
366
367    for line in result.output.split("\0"):
368        (key, _, value) = line.partition("=")
369        os.environ[key] = value
370