1#!/usr/bin/python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A collection of tests for tools/testing/kunit/kunit.py 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Brendan Higgins <brendanhiggins@google.com> 8 9import unittest 10from unittest import mock 11 12import tempfile, shutil # Handling test_tmpdir 13 14import json 15import os 16 17import kunit_config 18import kunit_parser 19import kunit_kernel 20import kunit_json 21import kunit 22 23test_tmpdir = '' 24 25def setUpModule(): 26 global test_tmpdir 27 test_tmpdir = tempfile.mkdtemp() 28 29def tearDownModule(): 30 shutil.rmtree(test_tmpdir) 31 32def get_absolute_path(path): 33 return os.path.join(os.path.dirname(__file__), path) 34 35class KconfigTest(unittest.TestCase): 36 37 def test_is_subset_of(self): 38 kconfig0 = kunit_config.Kconfig() 39 self.assertTrue(kconfig0.is_subset_of(kconfig0)) 40 41 kconfig1 = kunit_config.Kconfig() 42 kconfig1.add_entry(kunit_config.KconfigEntry('TEST', 'y')) 43 self.assertTrue(kconfig1.is_subset_of(kconfig1)) 44 self.assertTrue(kconfig0.is_subset_of(kconfig1)) 45 self.assertFalse(kconfig1.is_subset_of(kconfig0)) 46 47 def test_read_from_file(self): 48 kconfig = kunit_config.Kconfig() 49 kconfig_path = get_absolute_path( 50 'test_data/test_read_from_file.kconfig') 51 52 kconfig.read_from_file(kconfig_path) 53 54 expected_kconfig = kunit_config.Kconfig() 55 expected_kconfig.add_entry( 56 kunit_config.KconfigEntry('UML', 'y')) 57 expected_kconfig.add_entry( 58 kunit_config.KconfigEntry('MMU', 'y')) 59 expected_kconfig.add_entry( 60 kunit_config.KconfigEntry('TEST', 'y')) 61 expected_kconfig.add_entry( 62 kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) 63 expected_kconfig.add_entry( 64 kunit_config.KconfigEntry('MK8', 'n')) 65 66 self.assertEqual(kconfig.entries(), expected_kconfig.entries()) 67 68 def test_write_to_file(self): 69 kconfig_path = os.path.join(test_tmpdir, '.config') 70 71 expected_kconfig = kunit_config.Kconfig() 72 expected_kconfig.add_entry( 73 kunit_config.KconfigEntry('UML', 'y')) 74 expected_kconfig.add_entry( 75 kunit_config.KconfigEntry('MMU', 'y')) 76 expected_kconfig.add_entry( 77 kunit_config.KconfigEntry('TEST', 'y')) 78 expected_kconfig.add_entry( 79 kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) 80 expected_kconfig.add_entry( 81 kunit_config.KconfigEntry('MK8', 'n')) 82 83 expected_kconfig.write_to_file(kconfig_path) 84 85 actual_kconfig = kunit_config.Kconfig() 86 actual_kconfig.read_from_file(kconfig_path) 87 88 self.assertEqual(actual_kconfig.entries(), 89 expected_kconfig.entries()) 90 91class KUnitParserTest(unittest.TestCase): 92 93 def assertContains(self, needle, haystack): 94 for line in haystack: 95 if needle in line: 96 return 97 raise AssertionError('"' + 98 str(needle) + '" not found in "' + str(haystack) + '"!') 99 100 def test_output_isolated_correctly(self): 101 log_path = get_absolute_path( 102 'test_data/test_output_isolated_correctly.log') 103 file = open(log_path) 104 result = kunit_parser.isolate_kunit_output(file.readlines()) 105 self.assertContains('TAP version 14\n', result) 106 self.assertContains(' # Subtest: example', result) 107 self.assertContains(' 1..2', result) 108 self.assertContains(' ok 1 - example_simple_test', result) 109 self.assertContains(' ok 2 - example_mock_test', result) 110 self.assertContains('ok 1 - example', result) 111 file.close() 112 113 def test_output_with_prefix_isolated_correctly(self): 114 log_path = get_absolute_path( 115 'test_data/test_pound_sign.log') 116 with open(log_path) as file: 117 result = kunit_parser.isolate_kunit_output(file.readlines()) 118 self.assertContains('TAP version 14\n', result) 119 self.assertContains(' # Subtest: kunit-resource-test', result) 120 self.assertContains(' 1..5', result) 121 self.assertContains(' ok 1 - kunit_resource_test_init_resources', result) 122 self.assertContains(' ok 2 - kunit_resource_test_alloc_resource', result) 123 self.assertContains(' ok 3 - kunit_resource_test_destroy_resource', result) 124 self.assertContains(' foo bar #', result) 125 self.assertContains(' ok 4 - kunit_resource_test_cleanup_resources', result) 126 self.assertContains(' ok 5 - kunit_resource_test_proper_free_ordering', result) 127 self.assertContains('ok 1 - kunit-resource-test', result) 128 self.assertContains(' foo bar # non-kunit output', result) 129 self.assertContains(' # Subtest: kunit-try-catch-test', result) 130 self.assertContains(' 1..2', result) 131 self.assertContains(' ok 1 - kunit_test_try_catch_successful_try_no_catch', 132 result) 133 self.assertContains(' ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch', 134 result) 135 self.assertContains('ok 2 - kunit-try-catch-test', result) 136 self.assertContains(' # Subtest: string-stream-test', result) 137 self.assertContains(' 1..3', result) 138 self.assertContains(' ok 1 - string_stream_test_empty_on_creation', result) 139 self.assertContains(' ok 2 - string_stream_test_not_empty_after_add', result) 140 self.assertContains(' ok 3 - string_stream_test_get_string', result) 141 self.assertContains('ok 3 - string-stream-test', result) 142 143 def test_parse_successful_test_log(self): 144 all_passed_log = get_absolute_path( 145 'test_data/test_is_test_passed-all_passed.log') 146 file = open(all_passed_log) 147 result = kunit_parser.parse_run_tests(file.readlines()) 148 self.assertEqual( 149 kunit_parser.TestStatus.SUCCESS, 150 result.status) 151 file.close() 152 153 def test_parse_failed_test_log(self): 154 failed_log = get_absolute_path( 155 'test_data/test_is_test_passed-failure.log') 156 file = open(failed_log) 157 result = kunit_parser.parse_run_tests(file.readlines()) 158 self.assertEqual( 159 kunit_parser.TestStatus.FAILURE, 160 result.status) 161 file.close() 162 163 def test_no_tests(self): 164 empty_log = get_absolute_path( 165 'test_data/test_is_test_passed-no_tests_run.log') 166 file = open(empty_log) 167 result = kunit_parser.parse_run_tests( 168 kunit_parser.isolate_kunit_output(file.readlines())) 169 self.assertEqual(0, len(result.suites)) 170 self.assertEqual( 171 kunit_parser.TestStatus.NO_TESTS, 172 result.status) 173 file.close() 174 175 def test_no_kunit_output(self): 176 crash_log = get_absolute_path( 177 'test_data/test_insufficient_memory.log') 178 file = open(crash_log) 179 print_mock = mock.patch('builtins.print').start() 180 result = kunit_parser.parse_run_tests( 181 kunit_parser.isolate_kunit_output(file.readlines())) 182 print_mock.assert_any_call(StrContains("no kunit output detected")) 183 print_mock.stop() 184 file.close() 185 186 def test_crashed_test(self): 187 crashed_log = get_absolute_path( 188 'test_data/test_is_test_passed-crash.log') 189 file = open(crashed_log) 190 result = kunit_parser.parse_run_tests(file.readlines()) 191 self.assertEqual( 192 kunit_parser.TestStatus.TEST_CRASHED, 193 result.status) 194 file.close() 195 196 def test_ignores_prefix_printk_time(self): 197 prefix_log = get_absolute_path( 198 'test_data/test_config_printk_time.log') 199 with open(prefix_log) as file: 200 result = kunit_parser.parse_run_tests(file.readlines()) 201 self.assertEqual('kunit-resource-test', result.suites[0].name) 202 203 def test_ignores_multiple_prefixes(self): 204 prefix_log = get_absolute_path( 205 'test_data/test_multiple_prefixes.log') 206 with open(prefix_log) as file: 207 result = kunit_parser.parse_run_tests(file.readlines()) 208 self.assertEqual('kunit-resource-test', result.suites[0].name) 209 210 def test_prefix_mixed_kernel_output(self): 211 mixed_prefix_log = get_absolute_path( 212 'test_data/test_interrupted_tap_output.log') 213 with open(mixed_prefix_log) as file: 214 result = kunit_parser.parse_run_tests(file.readlines()) 215 self.assertEqual('kunit-resource-test', result.suites[0].name) 216 217 def test_prefix_poundsign(self): 218 pound_log = get_absolute_path('test_data/test_pound_sign.log') 219 with open(pound_log) as file: 220 result = kunit_parser.parse_run_tests(file.readlines()) 221 self.assertEqual('kunit-resource-test', result.suites[0].name) 222 223 def test_kernel_panic_end(self): 224 panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log') 225 with open(panic_log) as file: 226 result = kunit_parser.parse_run_tests(file.readlines()) 227 self.assertEqual('kunit-resource-test', result.suites[0].name) 228 229 def test_pound_no_prefix(self): 230 pound_log = get_absolute_path('test_data/test_pound_no_prefix.log') 231 with open(pound_log) as file: 232 result = kunit_parser.parse_run_tests(file.readlines()) 233 self.assertEqual('kunit-resource-test', result.suites[0].name) 234 235class KUnitJsonTest(unittest.TestCase): 236 237 def _json_for(self, log_file): 238 with(open(get_absolute_path(log_file))) as file: 239 test_result = kunit_parser.parse_run_tests(file) 240 json_obj = kunit_json.get_json_result( 241 test_result=test_result, 242 def_config='kunit_defconfig', 243 build_dir=None, 244 json_path='stdout') 245 return json.loads(json_obj) 246 247 def test_failed_test_json(self): 248 result = self._json_for( 249 'test_data/test_is_test_passed-failure.log') 250 self.assertEqual( 251 {'name': 'example_simple_test', 'status': 'FAIL'}, 252 result["sub_groups"][1]["test_cases"][0]) 253 254 def test_crashed_test_json(self): 255 result = self._json_for( 256 'test_data/test_is_test_passed-crash.log') 257 self.assertEqual( 258 {'name': 'example_simple_test', 'status': 'ERROR'}, 259 result["sub_groups"][1]["test_cases"][0]) 260 261 def test_no_tests_json(self): 262 result = self._json_for( 263 'test_data/test_is_test_passed-no_tests_run.log') 264 self.assertEqual(0, len(result['sub_groups'])) 265 266class StrContains(str): 267 def __eq__(self, other): 268 return self in other 269 270class KUnitMainTest(unittest.TestCase): 271 def setUp(self): 272 path = get_absolute_path('test_data/test_is_test_passed-all_passed.log') 273 file = open(path) 274 all_passed_log = file.readlines() 275 self.print_patch = mock.patch('builtins.print') 276 self.print_mock = self.print_patch.start() 277 self.linux_source_mock = mock.Mock() 278 self.linux_source_mock.build_reconfig = mock.Mock(return_value=True) 279 self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True) 280 self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log) 281 282 def tearDown(self): 283 self.print_patch.stop() 284 pass 285 286 def test_config_passes_args_pass(self): 287 kunit.main(['config', '--build_dir=.kunit'], self.linux_source_mock) 288 assert self.linux_source_mock.build_reconfig.call_count == 1 289 assert self.linux_source_mock.run_kernel.call_count == 0 290 291 def test_build_passes_args_pass(self): 292 kunit.main(['build'], self.linux_source_mock) 293 assert self.linux_source_mock.build_reconfig.call_count == 0 294 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, '.kunit', None) 295 assert self.linux_source_mock.run_kernel.call_count == 0 296 297 def test_exec_passes_args_pass(self): 298 kunit.main(['exec'], self.linux_source_mock) 299 assert self.linux_source_mock.build_reconfig.call_count == 0 300 assert self.linux_source_mock.run_kernel.call_count == 1 301 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=300) 302 self.print_mock.assert_any_call(StrContains('Testing complete.')) 303 304 def test_run_passes_args_pass(self): 305 kunit.main(['run'], self.linux_source_mock) 306 assert self.linux_source_mock.build_reconfig.call_count == 1 307 assert self.linux_source_mock.run_kernel.call_count == 1 308 self.linux_source_mock.run_kernel.assert_called_once_with( 309 build_dir='.kunit', timeout=300) 310 self.print_mock.assert_any_call(StrContains('Testing complete.')) 311 312 def test_exec_passes_args_fail(self): 313 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 314 with self.assertRaises(SystemExit) as e: 315 kunit.main(['exec'], self.linux_source_mock) 316 assert type(e.exception) == SystemExit 317 assert e.exception.code == 1 318 319 def test_run_passes_args_fail(self): 320 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 321 with self.assertRaises(SystemExit) as e: 322 kunit.main(['run'], self.linux_source_mock) 323 assert type(e.exception) == SystemExit 324 assert e.exception.code == 1 325 assert self.linux_source_mock.build_reconfig.call_count == 1 326 assert self.linux_source_mock.run_kernel.call_count == 1 327 self.print_mock.assert_any_call(StrContains(' 0 tests run')) 328 329 def test_exec_raw_output(self): 330 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 331 kunit.main(['exec', '--raw_output'], self.linux_source_mock) 332 assert self.linux_source_mock.run_kernel.call_count == 1 333 for kall in self.print_mock.call_args_list: 334 assert kall != mock.call(StrContains('Testing complete.')) 335 assert kall != mock.call(StrContains(' 0 tests run')) 336 337 def test_run_raw_output(self): 338 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 339 kunit.main(['run', '--raw_output'], self.linux_source_mock) 340 assert self.linux_source_mock.build_reconfig.call_count == 1 341 assert self.linux_source_mock.run_kernel.call_count == 1 342 for kall in self.print_mock.call_args_list: 343 assert kall != mock.call(StrContains('Testing complete.')) 344 assert kall != mock.call(StrContains(' 0 tests run')) 345 346 def test_exec_timeout(self): 347 timeout = 3453 348 kunit.main(['exec', '--timeout', str(timeout)], self.linux_source_mock) 349 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=timeout) 350 self.print_mock.assert_any_call(StrContains('Testing complete.')) 351 352 def test_run_timeout(self): 353 timeout = 3453 354 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) 355 assert self.linux_source_mock.build_reconfig.call_count == 1 356 self.linux_source_mock.run_kernel.assert_called_once_with( 357 build_dir='.kunit', timeout=timeout) 358 self.print_mock.assert_any_call(StrContains('Testing complete.')) 359 360 def test_run_builddir(self): 361 build_dir = '.kunit' 362 kunit.main(['run', '--build_dir=.kunit'], self.linux_source_mock) 363 assert self.linux_source_mock.build_reconfig.call_count == 1 364 self.linux_source_mock.run_kernel.assert_called_once_with( 365 build_dir=build_dir, timeout=300) 366 self.print_mock.assert_any_call(StrContains('Testing complete.')) 367 368 def test_config_builddir(self): 369 build_dir = '.kunit' 370 kunit.main(['config', '--build_dir', build_dir], self.linux_source_mock) 371 assert self.linux_source_mock.build_reconfig.call_count == 1 372 373 def test_build_builddir(self): 374 build_dir = '.kunit' 375 kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock) 376 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, build_dir, None) 377 378 def test_exec_builddir(self): 379 build_dir = '.kunit' 380 kunit.main(['exec', '--build_dir', build_dir], self.linux_source_mock) 381 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300) 382 self.print_mock.assert_any_call(StrContains('Testing complete.')) 383 384if __name__ == '__main__': 385 unittest.main() 386