1#
2# Copyright (C) 2017 Intel Corporation
3#
4# SPDX-License-Identifier: MIT
5#
6
7import os
8import time
9import glob
10import sys
11import importlib
12import subprocess
13import unittest
14from random import choice
15
16import oeqa
17import oe
18import bb.utils
19import bb.tinfoil
20
21from oeqa.core.context import OETestContext, OETestContextExecutor
22from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
23
24from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer
25
26OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"]
27
28def get_oeselftest_metadata(args):
29    result = {}
30    raw_args = vars(args)
31    for metadata in OESELFTEST_METADATA:
32        if metadata in raw_args:
33            result[metadata] = raw_args[metadata]
34
35    return result
36
37class NonConcurrentTestSuite(unittest.TestSuite):
38    def __init__(self, suite, processes, setupfunc, removefunc, bb_vars):
39        super().__init__([suite])
40        self.processes = processes
41        self.suite = suite
42        self.setupfunc = setupfunc
43        self.removefunc = removefunc
44        self.bb_vars = bb_vars
45
46    def run(self, result):
47        (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite)
48        ret = super().run(result)
49        os.chdir(builddir)
50        if newbuilddir and ret.wasSuccessful() and self.removefunc:
51            self.removefunc(newbuilddir)
52
53def removebuilddir(d):
54    delay = 5
55    while delay and (os.path.exists(d + "/bitbake.lock") or os.path.exists(d + "/cache/hashserv.db-wal")):
56        time.sleep(1)
57        delay = delay - 1
58    # Deleting these directories takes a lot of time, use autobuilder
59    # clobberdir if its available
60    clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
61    if os.path.exists(clobberdir):
62        try:
63            subprocess.check_call([clobberdir, d])
64            return
65        except subprocess.CalledProcessError:
66            pass
67    bb.utils.prunedir(d, ionice=True)
68
69class OESelftestTestContext(OETestContext):
70    def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None):
71        super(OESelftestTestContext, self).__init__(td, logger)
72
73        self.config_paths = config_paths
74        self.newbuilddir = newbuilddir
75
76        if keep_builddir:
77            self.removebuilddir = None
78        else:
79            self.removebuilddir = removebuilddir
80
81    def set_variables(self, vars):
82        self.bb_vars = vars
83
84    def setup_builddir(self, suffix, selftestdir, suite):
85        sstatedir = self.bb_vars['SSTATE_DIR']
86
87        builddir = os.environ['BUILDDIR']
88        if not selftestdir:
89            selftestdir = get_test_layer(self.bb_vars['BBLAYERS'])
90        if self.newbuilddir:
91            newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix)
92        else:
93            newbuilddir = builddir + suffix
94        newselftestdir = newbuilddir + "/meta-selftest"
95
96        if os.path.exists(newbuilddir):
97            self.logger.error("Build directory %s already exists, aborting" % newbuilddir)
98            sys.exit(1)
99
100        bb.utils.mkdirhier(newbuilddir)
101        oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
102        oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
103        oe.path.copytree(selftestdir, newselftestdir)
104
105        subprocess.check_output("git init && git add * && git commit -a -m 'initial'", cwd=newselftestdir, shell=True)
106
107        # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow
108        subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True)
109
110        # Relative paths in BBLAYERS only works when the new build dir share the same ascending node
111        if self.newbuilddir:
112            bblayers = subprocess.check_output("bitbake-getvar --value BBLAYERS | tail -1", cwd=builddir, shell=True, text=True)
113            if '..' in bblayers:
114                bblayers_abspath = [os.path.abspath(path) for path in bblayers.split()]
115                with open("%s/conf/bblayers.conf" % newbuilddir, "a") as f:
116                    newbblayers = "# new bblayers to be used by selftest in the new build dir '%s'\n" % newbuilddir
117                    newbblayers += 'BBLAYERS = "%s"\n' % ' '.join(bblayers_abspath)
118                    f.write(newbblayers)
119
120        for e in os.environ:
121            if builddir + "/" in os.environ[e]:
122                os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/")
123            if os.environ[e].endswith(builddir):
124                os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
125
126        # Set SSTATE_DIR to match the parent SSTATE_DIR
127        subprocess.check_output("echo 'SSTATE_DIR ?= \"%s\"' >> %s/conf/local.conf" % (sstatedir, newbuilddir), cwd=newbuilddir, shell=True)
128
129        os.chdir(newbuilddir)
130
131        def patch_test(t):
132            if not hasattr(t, "tc"):
133                return
134            cp = t.tc.config_paths
135            for p in cp:
136                if selftestdir in cp[p] and newselftestdir not in cp[p]:
137                    cp[p] = cp[p].replace(selftestdir, newselftestdir)
138                if builddir in cp[p] and newbuilddir not in cp[p]:
139                    cp[p] = cp[p].replace(builddir, newbuilddir)
140
141        def patch_suite(s):
142            for x in s:
143                if isinstance(x, unittest.TestSuite):
144                    patch_suite(x)
145                else:
146                    patch_test(x)
147
148        patch_suite(suite)
149
150        return (builddir, newbuilddir)
151
152    def prepareSuite(self, suites, processes):
153        if processes:
154            from oeqa.core.utils.concurrencytest import ConcurrentTestSuite
155
156            return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars)
157        else:
158            return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars)
159
160    def runTests(self, processes=None, machine=None, skips=[]):
161        return super(OESelftestTestContext, self).runTests(processes, skips)
162
163    def listTests(self, display_type, machine=None):
164        return super(OESelftestTestContext, self).listTests(display_type)
165
166class OESelftestTestContextExecutor(OETestContextExecutor):
167    _context_class = OESelftestTestContext
168    _script_executor = 'oe-selftest'
169
170    name = 'oe-selftest'
171    help = 'oe-selftest test component'
172    description = 'Executes selftest tests'
173
174    def register_commands(self, logger, parser):
175        group = parser.add_mutually_exclusive_group(required=True)
176
177        group.add_argument('-a', '--run-all-tests', default=False,
178                action="store_true", dest="run_all_tests",
179                help='Run all (unhidden) tests')
180        group.add_argument('-r', '--run-tests', required=False, action='store',
181                nargs='+', dest="run_tests", default=None,
182                help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
183
184        group.add_argument('-m', '--list-modules', required=False,
185                action="store_true", default=False,
186                help='List all available test modules.')
187        group.add_argument('--list-classes', required=False,
188                action="store_true", default=False,
189                help='List all available test classes.')
190        group.add_argument('-l', '--list-tests', required=False,
191                action="store_true", default=False,
192                help='List all available tests.')
193
194        parser.add_argument('-R', '--skip-tests', required=False, action='store',
195                nargs='+', dest="skips", default=None,
196                help='Skip the tests specified. Format should be <module>[.<class>[.<test_method>]]')
197        parser.add_argument('-j', '--num-processes', dest='processes', action='store',
198                type=int, help="number of processes to execute in parallel with")
199
200        parser.add_argument('-t', '--select-tag', dest="select_tags",
201                action='append', default=None,
202                help='Filter all (unhidden) tests to any that match any of the specified tag(s).')
203        parser.add_argument('-T', '--exclude-tag', dest="exclude_tags",
204                action='append', default=None,
205                help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)')
206
207        parser.add_argument('-K', '--keep-builddir', action='store_true',
208                help='Keep the test build directory even if all tests pass')
209
210        parser.add_argument('-B', '--newbuilddir', help='New build directory to use for tests.')
211        parser.add_argument('-v', '--verbose', action='store_true')
212        parser.set_defaults(func=self.run)
213
214    def _get_cases_paths(self, bbpath):
215        cases_paths = []
216        for layer in bbpath:
217            cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases')
218            if os.path.isdir(cases_dir):
219                cases_paths.append(cases_dir)
220        return cases_paths
221
222    def _process_args(self, logger, args):
223        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
224        args.test_data_file = None
225        args.CASES_PATHS = None
226
227        bbvars = get_bb_vars()
228        logdir = os.environ.get("BUILDDIR")
229        if 'LOG_DIR' in bbvars:
230            logdir = bbvars['LOG_DIR']
231        bb.utils.mkdirhier(logdir)
232        args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time)
233
234        super(OESelftestTestContextExecutor, self)._process_args(logger, args)
235
236        if args.list_modules:
237            args.list_tests = 'module'
238        elif args.list_classes:
239            args.list_tests = 'class'
240        elif args.list_tests:
241            args.list_tests = 'name'
242
243        self.tc_kwargs['init']['td'] = bbvars
244
245        builddir = os.environ.get("BUILDDIR")
246        self.tc_kwargs['init']['config_paths'] = {}
247        self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"])
248        self.tc_kwargs['init']['config_paths']['builddir'] = builddir
249        self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf")
250        self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf")
251        self.tc_kwargs['init']['newbuilddir'] = args.newbuilddir
252        self.tc_kwargs['init']['keep_builddir'] = args.keep_builddir
253
254        def tag_filter(tags):
255            if args.exclude_tags:
256                if any(tag in args.exclude_tags for tag in tags):
257                    return True
258            if args.select_tags:
259                if not tags or not any(tag in args.select_tags for tag in tags):
260                    return True
261            return False
262
263        if args.select_tags or args.exclude_tags:
264            self.tc_kwargs['load']['tags_filter'] = tag_filter
265
266        self.tc_kwargs['run']['skips'] = args.skips
267        self.tc_kwargs['run']['processes'] = args.processes
268
269    def _pre_run(self):
270        def _check_required_env_variables(vars):
271            for var in vars:
272                if not os.environ.get(var):
273                    self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var)
274                    raise OEQAPreRun
275
276        def _check_presence_meta_selftest():
277            builddir = os.environ.get("BUILDDIR")
278            if os.getcwd() != builddir:
279                self.tc.logger.info("Changing cwd to %s" % builddir)
280                os.chdir(builddir)
281
282            if not "meta-selftest" in self.tc.td["BBLAYERS"]:
283                self.tc.logger.info("meta-selftest layer not found in BBLAYERS, adding it")
284                meta_selftestdir = os.path.join(
285                    self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest')
286                if os.path.isdir(meta_selftestdir):
287                    runCmd("bitbake-layers add-layer %s" % meta_selftestdir)
288                    # reload data is needed because a meta-selftest layer was add
289                    self.tc.td = get_bb_vars()
290                    self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"])
291                else:
292                    self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir)
293                    raise OEQAPreRun
294
295        def _add_layer_libs():
296            bbpath = self.tc.td['BBPATH'].split(':')
297            layer_libdirs = [p for p in (os.path.join(l, 'lib') \
298                    for l in bbpath) if os.path.exists(p)]
299            if layer_libdirs:
300                self.tc.logger.info("Adding layer libraries:")
301                for l in layer_libdirs:
302                    self.tc.logger.info("\t%s" % l)
303
304                sys.path.extend(layer_libdirs)
305                importlib.reload(oeqa.selftest)
306
307        _check_required_env_variables(["BUILDDIR"])
308        _check_presence_meta_selftest()
309
310        if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]:
311            self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.")
312            raise OEQAPreRun
313
314        if "rm_work.bbclass" in self.tc.td["BBINCLUDED"]:
315            self.tc.logger.error("You have rm_work enabled which isn't recommended while running oe-selftest. Please disable it before continuing.")
316            raise OEQAPreRun
317
318        if "PRSERV_HOST" in self.tc.td:
319            self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest")
320            raise OEQAPreRun
321
322        if "SANITY_TESTED_DISTROS" in self.tc.td:
323            self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest")
324            raise OEQAPreRun
325
326        _add_layer_libs()
327
328        self.tc.logger.info("Checking base configuration is valid/parsable")
329
330        with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
331            tinfoil.prepare(quiet=2, config_only=True)
332            d = tinfoil.config_data
333            vars = {}
334            vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR'))
335            vars['BBLAYERS'] = str(d.getVar('BBLAYERS'))
336        self.tc.set_variables(vars)
337
338    def get_json_result_dir(self, args):
339        json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa')
340        if "OEQA_JSON_RESULT_DIR" in self.tc.td:
341            json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"]
342
343        return json_result_dir
344
345    def get_configuration(self, args):
346        import platform
347        from oeqa.utils.metadata import metadata_from_bb
348        metadata = metadata_from_bb()
349        oeselftest_metadata = get_oeselftest_metadata(args)
350        configuration = {'TEST_TYPE': 'oeselftest',
351                        'STARTTIME': args.test_start_time,
352                        'MACHINE': self.tc.td["MACHINE"],
353                        'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
354                        'HOST_NAME': metadata['hostname'],
355                        'LAYERS': metadata['layers'],
356                        'OESELFTEST_METADATA': oeselftest_metadata}
357        return configuration
358
359    def get_result_id(self, configuration):
360        return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME'])
361
362    def _internal_run(self, logger, args):
363        self.module_paths = self._get_cases_paths(
364                self.tc_kwargs['init']['td']['BBPATH'].split(':'))
365
366        self.tc = self._context_class(**self.tc_kwargs['init'])
367        try:
368            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
369        except OEQATestNotFound as ex:
370            logger.error(ex)
371            sys.exit(1)
372
373        if args.list_tests:
374            rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
375        else:
376            self._pre_run()
377            rc = self.tc.runTests(**self.tc_kwargs['run'])
378            configuration = self.get_configuration(args)
379            rc.logDetails(self.get_json_result_dir(args),
380                          configuration,
381                          self.get_result_id(configuration))
382            rc.logSummary(self.name)
383
384        return rc
385
386    def run(self, logger, args):
387        self._process_args(logger, args)
388
389        rc = None
390        try:
391             rc = self._internal_run(logger, args)
392        finally:
393            config_paths = self.tc_kwargs['init']['config_paths']
394
395            output_link = os.path.join(os.path.dirname(args.output_log),
396                    "%s-results.log" % self.name)
397            if os.path.lexists(output_link):
398                os.unlink(output_link)
399            os.symlink(args.output_log, output_link)
400
401        return rc
402
403_executor_class = OESelftestTestContextExecutor
404