1c342db35SBrad Bishop# 292b42cb3SPatrick Williams# Copyright OpenEmbedded Contributors 392b42cb3SPatrick Williams# 4c342db35SBrad Bishop# SPDX-License-Identifier: MIT 5c342db35SBrad Bishop# 6c342db35SBrad Bishop 782c905dcSAndrew Geisslerimport os 8f86d0556SBrad Bishopimport unittest 9f86d0556SBrad Bishopimport pprint 1099467dabSAndrew Geisslerimport datetime 11f86d0556SBrad Bishop 12eb8dc403SDave Cobbleyfrom oeqa.runtime.case import OERuntimeTestCase 13eb8dc403SDave Cobbleyfrom oeqa.core.decorator.depends import OETestDepends 14eb8dc403SDave Cobbleyfrom oeqa.core.decorator.data import skipIfNotFeature 15977dc1acSBrad Bishopfrom oeqa.runtime.decorator.package import OEHasPackage 1699467dabSAndrew Geisslerfrom oeqa.utils.logparser import PtestParser 17eb8dc403SDave Cobbley 18eb8dc403SDave Cobbleyclass PtestRunnerTest(OERuntimeTestCase): 19eb8dc403SDave Cobbley 20eb8dc403SDave Cobbley @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES') 21eb8dc403SDave Cobbley @OETestDepends(['ssh.SSHTest.test_ssh']) 22977dc1acSBrad Bishop @OEHasPackage(['ptest-runner']) 23f86d0556SBrad Bishop @unittest.expectedFailure 2482c905dcSAndrew Geissler def test_ptestrunner_expectfail(self): 2582c905dcSAndrew Geissler if not self.td.get('PTEST_EXPECT_FAILURE'): 2682c905dcSAndrew Geissler self.skipTest('Cannot run ptests with @expectedFailure as ptests are required to pass') 2782c905dcSAndrew Geissler self.do_ptestrunner() 2882c905dcSAndrew Geissler 2982c905dcSAndrew Geissler @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES') 3082c905dcSAndrew Geissler @OETestDepends(['ssh.SSHTest.test_ssh']) 3182c905dcSAndrew Geissler @OEHasPackage(['ptest-runner']) 3282c905dcSAndrew Geissler def test_ptestrunner_expectsuccess(self): 3382c905dcSAndrew Geissler if self.td.get('PTEST_EXPECT_FAILURE'): 3482c905dcSAndrew Geissler self.skipTest('Cannot run ptests without @expectedFailure as ptests are expected to fail') 3582c905dcSAndrew Geissler self.do_ptestrunner() 3682c905dcSAndrew Geissler 3782c905dcSAndrew Geissler def do_ptestrunner(self): 38eb8dc403SDave Cobbley status, output = self.target.run('which ptest-runner', 0) 39eb8dc403SDave Cobbley if status != 0: 40eb8dc403SDave Cobbley self.skipTest("No -ptest packages are installed in the image") 41eb8dc403SDave Cobbley 42eb8dc403SDave Cobbley test_log_dir = self.td.get('TEST_LOG_DIR', '') 43eb8dc403SDave Cobbley # The TEST_LOG_DIR maybe NULL when testimage is added after 44eb8dc403SDave Cobbley # testdata.json is generated. 45eb8dc403SDave Cobbley if not test_log_dir: 46eb8dc403SDave Cobbley test_log_dir = os.path.join(self.td.get('WORKDIR', ''), 'testimage') 47d25ed324SAndrew Geissler # Make the test output path absolute, otherwise the output content will be 48d25ed324SAndrew Geissler # created relative to current directory 49d25ed324SAndrew Geissler if not os.path.isabs(test_log_dir): 50d25ed324SAndrew Geissler test_log_dir = os.path.join(self.td.get('TOPDIR', ''), test_log_dir) 51eb8dc403SDave Cobbley # Don't use self.td.get('DATETIME'), it's from testdata.json, not 52eb8dc403SDave Cobbley # up-to-date, and may cause "File exists" when re-reun. 5399467dabSAndrew Geissler timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S') 54eb8dc403SDave Cobbley ptest_log_dir_link = os.path.join(test_log_dir, 'ptest_log') 5599467dabSAndrew Geissler ptest_log_dir = '%s.%s' % (ptest_log_dir_link, timestamp) 56eb8dc403SDave Cobbley ptest_runner_log = os.path.join(ptest_log_dir, 'ptest-runner.log') 57eb8dc403SDave Cobbley 584b740dc9SAndrew Geissler libdir = self.td.get('libdir', '') 594b740dc9SAndrew Geissler ptest_dirs = [ '/usr/lib' ] 604b740dc9SAndrew Geissler if not libdir in ptest_dirs: 614b740dc9SAndrew Geissler ptest_dirs.append(libdir) 62213cb269SPatrick Williams status, output = self.target.run('ptest-runner -t 450 -d \"{}\"'.format(' '.join(ptest_dirs)), 0) 63eb8dc403SDave Cobbley os.makedirs(ptest_log_dir) 64eb8dc403SDave Cobbley with open(ptest_runner_log, 'w') as f: 65eb8dc403SDave Cobbley f.write(output) 66eb8dc403SDave Cobbley 67eb8dc403SDave Cobbley # status != 0 is OK since some ptest tests may fail 68eb8dc403SDave Cobbley self.assertTrue(status != 127, msg="Cannot execute ptest-runner!") 69eb8dc403SDave Cobbley 70f86d0556SBrad Bishop if not hasattr(self.tc, "extraresults"): 71f86d0556SBrad Bishop self.tc.extraresults = {} 72f86d0556SBrad Bishop extras = self.tc.extraresults 73f86d0556SBrad Bishop extras['ptestresult.rawlogs'] = {'log': output} 74f86d0556SBrad Bishop 75eb8dc403SDave Cobbley # Parse and save results 7699467dabSAndrew Geissler parser = PtestParser() 7799467dabSAndrew Geissler results, sections = parser.parse(ptest_runner_log) 7899467dabSAndrew Geissler parser.results_as_files(ptest_log_dir) 79eb8dc403SDave Cobbley if os.path.exists(ptest_log_dir_link): 80eb8dc403SDave Cobbley # Remove the old link to create a new one 81eb8dc403SDave Cobbley os.remove(ptest_log_dir_link) 82eb8dc403SDave Cobbley os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link) 83eb8dc403SDave Cobbley 8499467dabSAndrew Geissler extras['ptestresult.sections'] = sections 8599467dabSAndrew Geissler 86028142bdSAndrew Geissler zerolength = [] 87f86d0556SBrad Bishop trans = str.maketrans("()", "__") 8899467dabSAndrew Geissler for section in results: 8999467dabSAndrew Geissler for test in results[section]: 9099467dabSAndrew Geissler result = results[section][test] 9199467dabSAndrew Geissler testname = "ptestresult." + (section or "No-section") + "." + "_".join(test.translate(trans).split()) 9299467dabSAndrew Geissler extras[testname] = {'status': result} 93028142bdSAndrew Geissler if not results[section]: 94028142bdSAndrew Geissler zerolength.append(section) 95f86d0556SBrad Bishop 96eb8dc403SDave Cobbley failed_tests = {} 9782c905dcSAndrew Geissler 9882c905dcSAndrew Geissler for section in sections: 99*220dafdbSAndrew Geissler if 'exitcode' in sections[section].keys() or 'timeout' in sections[section].keys(): 10082c905dcSAndrew Geissler failed_tests[section] = sections[section]["log"] 10182c905dcSAndrew Geissler 10299467dabSAndrew Geissler for section in results: 10382c905dcSAndrew Geissler failed_testcases = [ "_".join(test.translate(trans).split()) for test in results[section] if results[section][test] == 'FAILED' ] 104eb8dc403SDave Cobbley if failed_testcases: 105eb8dc403SDave Cobbley failed_tests[section] = failed_testcases 106eb8dc403SDave Cobbley 10799467dabSAndrew Geissler failmsg = "" 10899467dabSAndrew Geissler status, output = self.target.run('dmesg | grep "Killed process"', 0) 10999467dabSAndrew Geissler if output: 11099467dabSAndrew Geissler failmsg = "ERROR: Processes were killed by the OOM Killer:\n%s\n" % output 11199467dabSAndrew Geissler 112f86d0556SBrad Bishop if failed_tests: 113028142bdSAndrew Geissler failmsg = failmsg + "\nFailed ptests:\n%s\n" % pprint.pformat(failed_tests) 114028142bdSAndrew Geissler 115028142bdSAndrew Geissler if zerolength: 116028142bdSAndrew Geissler failmsg = failmsg + "\nptests which had no test results:\n%s" % pprint.pformat(zerolength) 11799467dabSAndrew Geissler 11899467dabSAndrew Geissler if failmsg: 119d1e89497SAndrew Geissler self.logger.warning("There were failing ptests.") 12099467dabSAndrew Geissler self.fail(failmsg) 121