1# 2# Copyright (C) 2017 Intel Corporation 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import time 9import glob 10import sys 11import importlib 12import subprocess 13import unittest 14from random import choice 15 16import oeqa 17import oe 18import bb.utils 19import bb.tinfoil 20 21from oeqa.core.context import OETestContext, OETestContextExecutor 22from oeqa.core.exception import OEQAPreRun, OEQATestNotFound 23 24from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer 25 26OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"] 27 28def get_oeselftest_metadata(args): 29 result = {} 30 raw_args = vars(args) 31 for metadata in OESELFTEST_METADATA: 32 if metadata in raw_args: 33 result[metadata] = raw_args[metadata] 34 35 return result 36 37class NonConcurrentTestSuite(unittest.TestSuite): 38 def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): 39 super().__init__([suite]) 40 self.processes = processes 41 self.suite = suite 42 self.setupfunc = setupfunc 43 self.removefunc = removefunc 44 self.bb_vars = bb_vars 45 46 def run(self, result): 47 (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite) 48 ret = super().run(result) 49 os.chdir(builddir) 50 if newbuilddir and ret.wasSuccessful() and self.removefunc: 51 self.removefunc(newbuilddir) 52 53def removebuilddir(d): 54 delay = 5 55 while delay and (os.path.exists(d + "/bitbake.lock") or os.path.exists(d + "/cache/hashserv.db-wal")): 56 time.sleep(1) 57 delay = delay - 1 58 # Deleting these directories takes a lot of time, use autobuilder 59 # clobberdir if its available 60 clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir") 61 if os.path.exists(clobberdir): 62 try: 63 subprocess.check_call([clobberdir, d]) 64 return 65 except subprocess.CalledProcessError: 66 pass 67 bb.utils.prunedir(d, ionice=True) 68 69class OESelftestTestContext(OETestContext): 70 def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None): 71 super(OESelftestTestContext, self).__init__(td, logger) 72 73 self.config_paths = config_paths 74 self.newbuilddir = newbuilddir 75 76 if keep_builddir: 77 self.removebuilddir = None 78 else: 79 self.removebuilddir = removebuilddir 80 81 def set_variables(self, vars): 82 self.bb_vars = vars 83 84 def setup_builddir(self, suffix, selftestdir, suite): 85 sstatedir = self.bb_vars['SSTATE_DIR'] 86 87 builddir = os.environ['BUILDDIR'] 88 if not selftestdir: 89 selftestdir = get_test_layer(self.bb_vars['BBLAYERS']) 90 if self.newbuilddir: 91 newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix) 92 else: 93 newbuilddir = builddir + suffix 94 newselftestdir = newbuilddir + "/meta-selftest" 95 96 if os.path.exists(newbuilddir): 97 self.logger.error("Build directory %s already exists, aborting" % newbuilddir) 98 sys.exit(1) 99 100 bb.utils.mkdirhier(newbuilddir) 101 oe.path.copytree(builddir + "/conf", newbuilddir + "/conf") 102 oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") 103 oe.path.copytree(selftestdir, newselftestdir) 104 105 subprocess.check_output("git init && git add * && git commit -a -m 'initial'", cwd=newselftestdir, shell=True) 106 107 # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow 108 subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) 109 110 # Relative paths in BBLAYERS only works when the new build dir share the same ascending node 111 if self.newbuilddir: 112 bblayers = subprocess.check_output("bitbake-getvar --value BBLAYERS | tail -1", cwd=builddir, shell=True, text=True) 113 if '..' in bblayers: 114 bblayers_abspath = [os.path.abspath(path) for path in bblayers.split()] 115 with open("%s/conf/bblayers.conf" % newbuilddir, "a") as f: 116 newbblayers = "# new bblayers to be used by selftest in the new build dir '%s'\n" % newbuilddir 117 newbblayers += 'unset BBLAYERS\n' 118 newbblayers += 'BBLAYERS = "%s"\n' % ' '.join(bblayers_abspath) 119 f.write(newbblayers) 120 121 # Rewrite builddir paths seen in environment variables 122 for e in os.environ: 123 # Rewrite paths that absolutely point inside builddir 124 # (e.g $builddir/conf/ would be rewritten but not $builddir/../bitbake/) 125 if builddir + "/" in os.environ[e] and builddir + "/" in os.path.abspath(os.environ[e]): 126 os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/") 127 if os.environ[e].endswith(builddir): 128 os.environ[e] = os.environ[e].replace(builddir, newbuilddir) 129 130 # Set SSTATE_DIR to match the parent SSTATE_DIR 131 subprocess.check_output("echo 'SSTATE_DIR ?= \"%s\"' >> %s/conf/local.conf" % (sstatedir, newbuilddir), cwd=newbuilddir, shell=True) 132 133 os.chdir(newbuilddir) 134 135 def patch_test(t): 136 if not hasattr(t, "tc"): 137 return 138 cp = t.tc.config_paths 139 for p in cp: 140 if selftestdir in cp[p] and newselftestdir not in cp[p]: 141 cp[p] = cp[p].replace(selftestdir, newselftestdir) 142 if builddir in cp[p] and newbuilddir not in cp[p]: 143 cp[p] = cp[p].replace(builddir, newbuilddir) 144 145 def patch_suite(s): 146 for x in s: 147 if isinstance(x, unittest.TestSuite): 148 patch_suite(x) 149 else: 150 patch_test(x) 151 152 patch_suite(suite) 153 154 return (builddir, newbuilddir) 155 156 def prepareSuite(self, suites, processes): 157 if processes: 158 from oeqa.core.utils.concurrencytest import ConcurrentTestSuite 159 160 return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) 161 else: 162 return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir, self.bb_vars) 163 164 def runTests(self, processes=None, machine=None, skips=[]): 165 return super(OESelftestTestContext, self).runTests(processes, skips) 166 167 def listTests(self, display_type, machine=None): 168 return super(OESelftestTestContext, self).listTests(display_type) 169 170class OESelftestTestContextExecutor(OETestContextExecutor): 171 _context_class = OESelftestTestContext 172 _script_executor = 'oe-selftest' 173 174 name = 'oe-selftest' 175 help = 'oe-selftest test component' 176 description = 'Executes selftest tests' 177 178 def register_commands(self, logger, parser): 179 group = parser.add_mutually_exclusive_group(required=True) 180 181 group.add_argument('-a', '--run-all-tests', default=False, 182 action="store_true", dest="run_all_tests", 183 help='Run all (unhidden) tests') 184 group.add_argument('-r', '--run-tests', required=False, action='store', 185 nargs='+', dest="run_tests", default=None, 186 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') 187 188 group.add_argument('-m', '--list-modules', required=False, 189 action="store_true", default=False, 190 help='List all available test modules.') 191 group.add_argument('--list-classes', required=False, 192 action="store_true", default=False, 193 help='List all available test classes.') 194 group.add_argument('-l', '--list-tests', required=False, 195 action="store_true", default=False, 196 help='List all available tests.') 197 198 parser.add_argument('-R', '--skip-tests', required=False, action='store', 199 nargs='+', dest="skips", default=None, 200 help='Skip the tests specified. Format should be <module>[.<class>[.<test_method>]]') 201 202 def check_parallel_support(parameter): 203 if not parameter.isdigit(): 204 import argparse 205 raise argparse.ArgumentTypeError("argument -j/--num-processes: invalid int value: '%s' " % str(parameter)) 206 207 processes = int(parameter) 208 if processes: 209 try: 210 import testtools, subunit 211 except ImportError: 212 print("Failed to import testtools or subunit, the testcases will run serially") 213 processes = None 214 return processes 215 216 parser.add_argument('-j', '--num-processes', dest='processes', action='store', 217 type=check_parallel_support, help="number of processes to execute in parallel with") 218 219 parser.add_argument('-t', '--select-tag', dest="select_tags", 220 action='append', default=None, 221 help='Filter all (unhidden) tests to any that match any of the specified tag(s).') 222 parser.add_argument('-T', '--exclude-tag', dest="exclude_tags", 223 action='append', default=None, 224 help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)') 225 226 parser.add_argument('-K', '--keep-builddir', action='store_true', 227 help='Keep the test build directory even if all tests pass') 228 229 parser.add_argument('-B', '--newbuilddir', help='New build directory to use for tests.') 230 parser.add_argument('-v', '--verbose', action='store_true') 231 parser.set_defaults(func=self.run) 232 233 def _get_cases_paths(self, bbpath): 234 cases_paths = [] 235 for layer in bbpath: 236 cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases') 237 if os.path.isdir(cases_dir): 238 cases_paths.append(cases_dir) 239 return cases_paths 240 241 def _process_args(self, logger, args): 242 args.test_start_time = time.strftime("%Y%m%d%H%M%S") 243 args.test_data_file = None 244 args.CASES_PATHS = None 245 246 bbvars = get_bb_vars() 247 logdir = os.environ.get("BUILDDIR") 248 if 'LOG_DIR' in bbvars: 249 logdir = bbvars['LOG_DIR'] 250 bb.utils.mkdirhier(logdir) 251 args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time) 252 253 super(OESelftestTestContextExecutor, self)._process_args(logger, args) 254 255 if args.list_modules: 256 args.list_tests = 'module' 257 elif args.list_classes: 258 args.list_tests = 'class' 259 elif args.list_tests: 260 args.list_tests = 'name' 261 262 self.tc_kwargs['init']['td'] = bbvars 263 264 builddir = os.environ.get("BUILDDIR") 265 self.tc_kwargs['init']['config_paths'] = {} 266 self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer(bbvars["BBLAYERS"]) 267 self.tc_kwargs['init']['config_paths']['builddir'] = builddir 268 self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf") 269 self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf") 270 self.tc_kwargs['init']['newbuilddir'] = args.newbuilddir 271 self.tc_kwargs['init']['keep_builddir'] = args.keep_builddir 272 273 def tag_filter(tags): 274 if args.exclude_tags: 275 if any(tag in args.exclude_tags for tag in tags): 276 return True 277 if args.select_tags: 278 if not tags or not any(tag in args.select_tags for tag in tags): 279 return True 280 return False 281 282 if args.select_tags or args.exclude_tags: 283 self.tc_kwargs['load']['tags_filter'] = tag_filter 284 285 self.tc_kwargs['run']['skips'] = args.skips 286 self.tc_kwargs['run']['processes'] = args.processes 287 288 def _pre_run(self): 289 def _check_required_env_variables(vars): 290 for var in vars: 291 if not os.environ.get(var): 292 self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var) 293 raise OEQAPreRun 294 295 def _check_presence_meta_selftest(): 296 builddir = os.environ.get("BUILDDIR") 297 if os.getcwd() != builddir: 298 self.tc.logger.info("Changing cwd to %s" % builddir) 299 os.chdir(builddir) 300 301 if not "meta-selftest" in self.tc.td["BBLAYERS"]: 302 self.tc.logger.info("meta-selftest layer not found in BBLAYERS, adding it") 303 meta_selftestdir = os.path.join( 304 self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') 305 if os.path.isdir(meta_selftestdir): 306 runCmd("bitbake-layers add-layer %s" % meta_selftestdir) 307 # reload data is needed because a meta-selftest layer was add 308 self.tc.td = get_bb_vars() 309 self.tc.config_paths['testlayer_path'] = get_test_layer(self.tc.td["BBLAYERS"]) 310 else: 311 self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) 312 raise OEQAPreRun 313 314 def _add_layer_libs(): 315 bbpath = self.tc.td['BBPATH'].split(':') 316 layer_libdirs = [p for p in (os.path.join(l, 'lib') \ 317 for l in bbpath) if os.path.exists(p)] 318 if layer_libdirs: 319 self.tc.logger.info("Adding layer libraries:") 320 for l in layer_libdirs: 321 self.tc.logger.info("\t%s" % l) 322 323 sys.path.extend(layer_libdirs) 324 importlib.reload(oeqa.selftest) 325 326 _check_required_env_variables(["BUILDDIR"]) 327 _check_presence_meta_selftest() 328 329 if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]: 330 self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.") 331 raise OEQAPreRun 332 333 if "rm_work.bbclass" in self.tc.td["BBINCLUDED"]: 334 self.tc.logger.error("You have rm_work enabled which isn't recommended while running oe-selftest. Please disable it before continuing.") 335 raise OEQAPreRun 336 337 if "PRSERV_HOST" in self.tc.td: 338 self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest") 339 raise OEQAPreRun 340 341 if "SANITY_TESTED_DISTROS" in self.tc.td: 342 self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest") 343 raise OEQAPreRun 344 345 _add_layer_libs() 346 347 self.tc.logger.info("Checking base configuration is valid/parsable") 348 349 with bb.tinfoil.Tinfoil(tracking=True) as tinfoil: 350 tinfoil.prepare(quiet=2, config_only=True) 351 d = tinfoil.config_data 352 vars = {} 353 vars['SSTATE_DIR'] = str(d.getVar('SSTATE_DIR')) 354 vars['BBLAYERS'] = str(d.getVar('BBLAYERS')) 355 self.tc.set_variables(vars) 356 357 def get_json_result_dir(self, args): 358 json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') 359 if "OEQA_JSON_RESULT_DIR" in self.tc.td: 360 json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"] 361 362 return json_result_dir 363 364 def get_configuration(self, args): 365 import platform 366 from oeqa.utils.metadata import metadata_from_bb 367 metadata = metadata_from_bb() 368 oeselftest_metadata = get_oeselftest_metadata(args) 369 configuration = {'TEST_TYPE': 'oeselftest', 370 'STARTTIME': args.test_start_time, 371 'MACHINE': self.tc.td["MACHINE"], 372 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 373 'HOST_NAME': metadata['hostname'], 374 'LAYERS': metadata['layers'], 375 'OESELFTEST_METADATA': oeselftest_metadata} 376 return configuration 377 378 def get_result_id(self, configuration): 379 return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME']) 380 381 def _internal_run(self, logger, args): 382 self.module_paths = self._get_cases_paths( 383 self.tc_kwargs['init']['td']['BBPATH'].split(':')) 384 385 self.tc = self._context_class(**self.tc_kwargs['init']) 386 try: 387 self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) 388 except OEQATestNotFound as ex: 389 logger.error(ex) 390 sys.exit(1) 391 392 if args.list_tests: 393 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list']) 394 else: 395 self._pre_run() 396 rc = self.tc.runTests(**self.tc_kwargs['run']) 397 configuration = self.get_configuration(args) 398 rc.logDetails(self.get_json_result_dir(args), 399 configuration, 400 self.get_result_id(configuration)) 401 rc.logSummary(self.name) 402 403 return rc 404 405 def run(self, logger, args): 406 self._process_args(logger, args) 407 408 rc = None 409 try: 410 rc = self._internal_run(logger, args) 411 finally: 412 config_paths = self.tc_kwargs['init']['config_paths'] 413 414 output_link = os.path.join(os.path.dirname(args.output_log), 415 "%s-results.log" % self.name) 416 if os.path.lexists(output_link): 417 os.unlink(output_link) 418 os.symlink(args.output_log, output_link) 419 420 return rc 421 422_executor_class = OESelftestTestContextExecutor 423