1# 2# Copyright (C) 2017 Intel Corporation 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import time 9import glob 10import sys 11import importlib 12import subprocess 13import unittest 14from random import choice 15 16import oeqa 17import oe 18import bb.utils 19import bb.tinfoil 20 21from oeqa.core.context import OETestContext, OETestContextExecutor 22from oeqa.core.exception import OEQAPreRun, OEQATestNotFound 23 24from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer 25 26OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"] 27 28def get_oeselftest_metadata(args): 29 result = {} 30 raw_args = vars(args) 31 for metadata in OESELFTEST_METADATA: 32 if metadata in raw_args: 33 result[metadata] = raw_args[metadata] 34 35 return result 36 37class NonConcurrentTestSuite(unittest.TestSuite): 38 def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): 39 super().__init__([suite]) 40 self.processes = processes 41 self.suite = suite 42 self.setupfunc = setupfunc 43 self.removefunc = removefunc 44 self.bb_vars = bb_vars 45 46 def run(self, result): 47 (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite) 48 ret = super().run(result) 49 os.chdir(builddir) 50 if newbuilddir and ret.wasSuccessful() and self.removefunc: 51 self.removefunc(newbuilddir) 52 53def removebuilddir(d): 54 delay = 5 55 while delay and (os.path.exists(d + "/bitbake.lock") or os.path.exists(d + "/cache/hashserv.db-wal")): 56 time.sleep(1) 57 delay = delay - 1 58 # Deleting these directories takes a lot of time, use autobuilder 59 # clobberdir if its available 60 clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir") 61 if os.path.exists(clobberdir): 62 try: 63 subprocess.check_call([clobberdir, d]) 64 return 65 except subprocess.CalledProcessError: 66 pass 67 bb.utils.prunedir(d, ionice=True) 68 69class OESelftestTestContext(OETestContext): 70 def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None): 71 super(OESelftestTestContext, self).__init__(td, logger) 72 73 self.config_paths = config_paths 74 self.newbuilddir = newbuilddir 75 76 if keep_builddir: 77 self.removebuilddir = None 78 else: 79 self.removebuilddir = removebuilddir 80 81 def set_variables(self, vars): 82 self.bb_vars = vars 83 84 def setup_builddir(self, suffix, selftestdir, suite): 85 sstatedir = self.bb_vars['SSTATE_DIR'] 86 87 builddir = os.environ['BUILDDIR'] 88 if not selftestdir: 89 selftestdir = get_test_layer(self.bb_vars['BBLAYERS']) 90 if self.newbuilddir: 91 newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix) 92 else: 93 newbuilddir = builddir + suffix 94 newselftestdir = newbuilddir + "/meta-selftest" 95 96 if os.path.exists(newbuilddir): 97 self.logger.error("Build directory %s already exists, aborting" % newbuilddir) 98 sys.exit(1) 99 100 bb.utils.mkdirhier(newbuilddir) 101 oe.path.copytree(builddir + "/conf", newbuilddir + "/conf") 102 oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") 103 oe.path.copytree(selftestdir, newselftestdir) 104 105 # if the last line of local.conf in newbuilddir is not empty and does not end with newline then add one 106 localconf_path = newbuilddir + "/conf/local.conf" 107 with open(localconf_path, "r+", encoding="utf-8") as f: 108 last_line = f.readlines()[-1] 109 if last_line and not last_line.endswith("\n"): 110 f.write("\n") 111 112 subprocess.check_output("git init && git add * && git commit -a -m 'initial'", cwd=newselftestdir, shell=True) 113 114 # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow 115 subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) 116 117 # Relative paths in BBLAYERS only works when the new build dir share the same ascending node 118 if self.newbuilddir: 119 bblayers = subprocess.check_output("bitbake-getvar --value BBLAYERS | tail -1", cwd=builddir, shell=True, text=True) 120 if '..' in bblayers: 121 bblayers_abspath = [os.path.abspath(path) for path in bblayers.split()] 122 with open("%s/conf/bblayers.conf" % newbuilddir, "a") as f: 123 newbblayers = "# new bblayers to be used by selftest in the new build dir '%s'\n" % newbuilddir 124 newbblayers += 'unset BBLAYERS\n' 125 newbblayers += 'BBLAYERS = "%s"\n' % ' '.join(bblayers_abspath) 126 f.write(newbblayers) 127 128 # Rewrite builddir paths seen in environment variables 129 for e in os.environ: 130 # Rewrite paths that absolutely point inside builddir 131 # (e.g $builddir/conf/ would be rewritten but not $builddir/../bitbake/) 132 if builddir + "/" in os.environ[e] and builddir + "/" in os.path.abspath(os.environ[e]): 133 os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/") 134 if os.environ[e].endswith(builddir): 135 os.environ[e] = os.environ[e].replace(builddir, newbuilddir) 136 137 # Set SSTATE_DIR to match the parent SSTATE_DIR 138 subprocess.check_output("echo 'SSTATE_DIR ?= \"%s\"' >> %s/conf/local.conf" % (sstatedir, newbuilddir), cwd=newbuilddir, shell=True) 139 140 os.chdir(newbuilddir) 141 142 def patch_test(t): 143 if not hasattr(t, "tc"): 144 return 145 cp = t.tc.config_paths 146 for p in cp: 147 if selftestdir in cp[p] and newselftestdir not in cp[p]: 148 cp[p] = cp[p].replace(selftestdir, newselftestdir) 149 if builddir in cp[p] and newbuilddir not in cp[p]: 150 cp[p] = cp[p].replace(builddir, newbuilddir) 151 152 def patch_suite(s): 153 for x in s: 154 if isinstance(x, unittest.TestSuite): 155 patch_suite(x) 156 else: 157 patch_test(x) 158 159 patch_suite(suite) 160 161 return (builddir, newbuilddir) 162 163 def prepareSuite(self, suites, processes): 164 if processes: 165 from oeqa.core.utils.concurrencytest import ConcurrentTestSuite 166 167 return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) 168 else: 169 return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) 170 171 def runTests(self, processes=None, machine=None, skips=[]): 172 return super(OESelftestTestContext, self).runTests(processes, skips) 173 174 def listTests(self, display_type, machine=None): 175 return super(OESelftestTestContext, self).listTests(display_type) 176 177class OESelftestTestContextExecutor(OETestContextExecutor): 178 _context_class = OESelftestTestContext 179 _script_executor = 'oe-selftest' 180 181 name = 'oe-selftest' 182 help = 'oe-selftest test component' 183 description = 'Executes selftest tests' 184 185 def register_commands(self, logger, parser): 186 group = parser.add_mutually_exclusive_group(required=True) 187 188 group.add_argument('-a', '--run-all-tests', default=False, 189 action="store_true", dest="run_all_tests", 190 help='Run all (unhidden) tests') 191 group.add_argument('-r', '--run-tests', required=False, action='store', 192 nargs='+', dest="run_tests", default=None, 193 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') 194 195 group.add_argument('-m', '--list-modules', required=False, 196 action="store_true", default=False, 197 help='List all available test modules.') 198 group.add_argument('--list-classes', required=False, 199 action="store_true", default=False, 200 help='List all available test classes.') 201 group.add_argument('-l', '--list-tests', required=False, 202 action="store_true", default=False, 203 help='List all available tests.') 204 205 parser.add_argument('-R', '--skip-tests', required=False, action='store', 206 nargs='+', dest="skips", default=None, 207 help='Skip the tests specified. Format should be <module>[.<class>[.<test_method>]]') 208 209 def check_parallel_support(parameter): 210 if not parameter.isdigit(): 211 import argparse 212 raise argparse.ArgumentTypeError("argument -j/--num-processes: invalid int value: '%s' " % str(parameter)) 213 214 processes = int(parameter) 215 if processes: 216 try: 217 import testtools, subunit 218 except ImportError: 219 print("Failed to import testtools or subunit, the testcases will run serially") 220 processes = None 221 return processes 222 223 parser.add_argument('-j', '--num-processes', dest='processes', action='store', 224 type=check_parallel_support, help="number of processes to execute in parallel with") 225 226 parser.add_argument('-t', '--select-tag', dest="select_tags", 227 action='append', default=None, 228 help='Filter all (unhidden) tests to any that match any of the specified tag(s).') 229 parser.add_argument('-T', '--exclude-tag', dest="exclude_tags", 230 action='append', default=None, 231 help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)') 232 233 parser.add_argument('-K', '--keep-builddir', action='store_true', 234 help='Keep the test build directory even if all tests pass') 235 236 parser.add_argument('-B', '--newbuilddir', help='New build directory to use for tests.') 237 parser.add_argument('-v', '--verbose', action='store_true') 238 parser.set_defaults(func=self.run) 239 240 def _get_cases_paths(self, bbpath): 241 cases_paths = [] 242 for layer in bbpath: 243 cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases') 244 if os.path.isdir(cases_dir): 245 cases_paths.append(cases_dir) 246 return cases_paths 247 248 def _process_args(self, logger, args): 249 args.test_start_time = time.strftime("%Y%m%d%H%M%S") 250 args.test_data_file = None 251 args.CASES_PATHS = None 252 253 bbvars = get_bb_vars() 254 logdir = os.environ.get("BUILDDIR") 255 if 'LOG_DIR' in bbvars: 256 logdir = bbvars['LOG_DIR'] 257 bb.utils.mkdirhier(logdir) 258 args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time) 259 260 super(OESelftestTestContextExecutor, self)._process_args(logger, args) 261 262 if args.list_modules: 263 args.list_tests = 'module' 264 elif args.list_classes: 265 args.list_tests = 'class' 266 elif args.list_tests: 267 args.list_tests = 'name' 268 269 self.tc_kwargs['init']['td'] = bbvars 270 271 builddir = os.environ.get("BUILDDIR") 272 self.tc_kwargs['init']['config_paths'] = {} 273 self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"]) 274 self.tc_kwargs['init']['config_paths']['builddir'] = builddir 275 self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf") 276 self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf") 277 self.tc_kwargs['init']['newbuilddir'] = args.newbuilddir 278 self.tc_kwargs['init']['keep_builddir'] = args.keep_builddir 279 280 def tag_filter(tags): 281 if args.exclude_tags: 282 if any(tag in args.exclude_tags for tag in tags): 283 return True 284 if args.select_tags: 285 if not tags or not any(tag in args.select_tags for tag in tags): 286 return True 287 return False 288 289 if args.select_tags or args.exclude_tags: 290 self.tc_kwargs['load']['tags_filter'] = tag_filter 291 292 self.tc_kwargs['run']['skips'] = args.skips 293 self.tc_kwargs['run']['processes'] = args.processes 294 295 def _pre_run(self): 296 def _check_required_env_variables(vars): 297 for var in vars: 298 if not os.environ.get(var): 299 self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var) 300 raise OEQAPreRun 301 302 def _check_presence_meta_selftest(): 303 builddir = os.environ.get("BUILDDIR") 304 if os.getcwd() != builddir: 305 self.tc.logger.info("Changing cwd to %s" % builddir) 306 os.chdir(builddir) 307 308 if not "meta-selftest" in self.tc.td["BBLAYERS"]: 309 self.tc.logger.info("meta-selftest layer not found in BBLAYERS, adding it") 310 meta_selftestdir = os.path.join( 311 self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') 312 if os.path.isdir(meta_selftestdir): 313 runCmd("bitbake-layers add-layer %s" % meta_selftestdir) 314 # reload data is needed because a meta-selftest layer was add 315 self.tc.td = get_bb_vars() 316 self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"]) 317 else: 318 self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) 319 raise OEQAPreRun 320 321 def _add_layer_libs(): 322 bbpath = self.tc.td['BBPATH'].split(':') 323 layer_libdirs = [p for p in (os.path.join(l, 'lib') \ 324 for l in bbpath) if os.path.exists(p)] 325 if layer_libdirs: 326 self.tc.logger.info("Adding layer libraries:") 327 for l in layer_libdirs: 328 self.tc.logger.info("\t%s" % l) 329 330 sys.path.extend(layer_libdirs) 331 importlib.reload(oeqa.selftest) 332 333 _check_required_env_variables(["BUILDDIR"]) 334 _check_presence_meta_selftest() 335 336 if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]: 337 self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.") 338 raise OEQAPreRun 339 340 if "rm_work.bbclass" in self.tc.td["BBINCLUDED"]: 341 self.tc.logger.error("You have rm_work enabled which isn't recommended while running oe-selftest. Please disable it before continuing.") 342 raise OEQAPreRun 343 344 if "PRSERV_HOST" in self.tc.td: 345 self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest") 346 raise OEQAPreRun 347 348 if "SANITY_TESTED_DISTROS" in self.tc.td: 349 self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest") 350 raise OEQAPreRun 351 352 _add_layer_libs() 353 354 self.tc.logger.info("Checking base configuration is valid/parsable") 355 356 with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: 357 tinfoil.prepare(quiet=2, config_only=True) 358 d = tinfoil.config_data 359 vars = {} 360 vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR')) 361 vars['BBLAYERS'] = str(d.getVar('BBLAYERS')) 362 self.tc.set_variables(vars) 363 364 def get_json_result_dir(self, args): 365 json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') 366 if "OEQA_JSON_RESULT_DIR" in self.tc.td: 367 json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"] 368 369 return json_result_dir 370 371 def get_configuration(self, args): 372 import platform 373 from oeqa.utils.metadata import metadata_from_bb 374 metadata = metadata_from_bb() 375 oeselftest_metadata = get_oeselftest_metadata(args) 376 configuration = {'TEST_TYPE': 'oeselftest', 377 'STARTTIME': args.test_start_time, 378 'MACHINE': self.tc.td["MACHINE"], 379 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 380 'HOST_NAME': metadata['hostname'], 381 'LAYERS': metadata['layers'], 382 'OESELFTEST_METADATA': oeselftest_metadata} 383 return configuration 384 385 def get_result_id(self, configuration): 386 return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME']) 387 388 def _internal_run(self, logger, args): 389 self.module_paths = self._get_cases_paths( 390 self.tc_kwargs['init']['td']['BBPATH'].split(':')) 391 392 self.tc = self._context_class(**self.tc_kwargs['init']) 393 try: 394 self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) 395 except OEQATestNotFound as ex: 396 logger.error(ex) 397 sys.exit(1) 398 399 if args.list_tests: 400 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list']) 401 else: 402 self._pre_run() 403 rc = self.tc.runTests(**self.tc_kwargs['run']) 404 configuration = self.get_configuration(args) 405 rc.logDetails(self.get_json_result_dir(args), 406 configuration, 407 self.get_result_id(configuration)) 408 rc.logSummary(self.name) 409 410 return rc 411 412 def run(self, logger, args): 413 self._process_args(logger, args) 414 415 rc = None 416 try: 417 rc = self._internal_run(logger, args) 418 finally: 419 config_paths = self.tc_kwargs['init']['config_paths'] 420 421 output_link = os.path.join(os.path.dirname(args.output_log), 422 "%s-results.log" % self.name) 423 if os.path.lexists(output_link): 424 os.unlink(output_link) 425 os.symlink(args.output_log, output_link) 426 427 return rc 428 429_executor_class = OESelftestTestContextExecutor 430