1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A collection of tests for tools/testing/kunit/kunit.py 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Brendan Higgins <brendanhiggins@google.com> 8 9import unittest 10from unittest import mock 11 12import tempfile, shutil # Handling test_tmpdir 13 14import json 15import os 16 17import kunit_config 18import kunit_parser 19import kunit_kernel 20import kunit_json 21import kunit 22 23test_tmpdir = '' 24 25def setUpModule(): 26 global test_tmpdir 27 test_tmpdir = tempfile.mkdtemp() 28 29def tearDownModule(): 30 shutil.rmtree(test_tmpdir) 31 32def get_absolute_path(path): 33 return os.path.join(os.path.dirname(__file__), path) 34 35class KconfigTest(unittest.TestCase): 36 37 def test_is_subset_of(self): 38 kconfig0 = kunit_config.Kconfig() 39 self.assertTrue(kconfig0.is_subset_of(kconfig0)) 40 41 kconfig1 = kunit_config.Kconfig() 42 kconfig1.add_entry(kunit_config.KconfigEntry('TEST', 'y')) 43 self.assertTrue(kconfig1.is_subset_of(kconfig1)) 44 self.assertTrue(kconfig0.is_subset_of(kconfig1)) 45 self.assertFalse(kconfig1.is_subset_of(kconfig0)) 46 47 def test_read_from_file(self): 48 kconfig = kunit_config.Kconfig() 49 kconfig_path = get_absolute_path( 50 'test_data/test_read_from_file.kconfig') 51 52 kconfig.read_from_file(kconfig_path) 53 54 expected_kconfig = kunit_config.Kconfig() 55 expected_kconfig.add_entry( 56 kunit_config.KconfigEntry('UML', 'y')) 57 expected_kconfig.add_entry( 58 kunit_config.KconfigEntry('MMU', 'y')) 59 expected_kconfig.add_entry( 60 kunit_config.KconfigEntry('TEST', 'y')) 61 expected_kconfig.add_entry( 62 kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) 63 expected_kconfig.add_entry( 64 kunit_config.KconfigEntry('MK8', 'n')) 65 66 self.assertEqual(kconfig.entries(), expected_kconfig.entries()) 67 68 def test_write_to_file(self): 69 kconfig_path = os.path.join(test_tmpdir, '.config') 70 71 expected_kconfig = kunit_config.Kconfig() 72 expected_kconfig.add_entry( 73 kunit_config.KconfigEntry('UML', 'y')) 74 expected_kconfig.add_entry( 75 kunit_config.KconfigEntry('MMU', 'y')) 76 expected_kconfig.add_entry( 77 kunit_config.KconfigEntry('TEST', 'y')) 78 expected_kconfig.add_entry( 79 kunit_config.KconfigEntry('EXAMPLE_TEST', 'y')) 80 expected_kconfig.add_entry( 81 kunit_config.KconfigEntry('MK8', 'n')) 82 83 expected_kconfig.write_to_file(kconfig_path) 84 85 actual_kconfig = kunit_config.Kconfig() 86 actual_kconfig.read_from_file(kconfig_path) 87 88 self.assertEqual(actual_kconfig.entries(), 89 expected_kconfig.entries()) 90 91class KUnitParserTest(unittest.TestCase): 92 93 def assertContains(self, needle, haystack): 94 for line in haystack: 95 if needle in line: 96 return 97 raise AssertionError('"' + 98 str(needle) + '" not found in "' + str(haystack) + '"!') 99 100 def test_output_isolated_correctly(self): 101 log_path = get_absolute_path( 102 'test_data/test_output_isolated_correctly.log') 103 file = open(log_path) 104 result = kunit_parser.isolate_kunit_output(file.readlines()) 105 self.assertContains('TAP version 14', result) 106 self.assertContains(' # Subtest: example', result) 107 self.assertContains(' 1..2', result) 108 self.assertContains(' ok 1 - example_simple_test', result) 109 self.assertContains(' ok 2 - example_mock_test', result) 110 self.assertContains('ok 1 - example', result) 111 file.close() 112 113 def test_output_with_prefix_isolated_correctly(self): 114 log_path = get_absolute_path( 115 'test_data/test_pound_sign.log') 116 with open(log_path) as file: 117 result = kunit_parser.isolate_kunit_output(file.readlines()) 118 self.assertContains('TAP version 14', result) 119 self.assertContains(' # Subtest: kunit-resource-test', result) 120 self.assertContains(' 1..5', result) 121 self.assertContains(' ok 1 - kunit_resource_test_init_resources', result) 122 self.assertContains(' ok 2 - kunit_resource_test_alloc_resource', result) 123 self.assertContains(' ok 3 - kunit_resource_test_destroy_resource', result) 124 self.assertContains(' foo bar #', result) 125 self.assertContains(' ok 4 - kunit_resource_test_cleanup_resources', result) 126 self.assertContains(' ok 5 - kunit_resource_test_proper_free_ordering', result) 127 self.assertContains('ok 1 - kunit-resource-test', result) 128 self.assertContains(' foo bar # non-kunit output', result) 129 self.assertContains(' # Subtest: kunit-try-catch-test', result) 130 self.assertContains(' 1..2', result) 131 self.assertContains(' ok 1 - kunit_test_try_catch_successful_try_no_catch', 132 result) 133 self.assertContains(' ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch', 134 result) 135 self.assertContains('ok 2 - kunit-try-catch-test', result) 136 self.assertContains(' # Subtest: string-stream-test', result) 137 self.assertContains(' 1..3', result) 138 self.assertContains(' ok 1 - string_stream_test_empty_on_creation', result) 139 self.assertContains(' ok 2 - string_stream_test_not_empty_after_add', result) 140 self.assertContains(' ok 3 - string_stream_test_get_string', result) 141 self.assertContains('ok 3 - string-stream-test', result) 142 143 def test_parse_successful_test_log(self): 144 all_passed_log = get_absolute_path( 145 'test_data/test_is_test_passed-all_passed.log') 146 file = open(all_passed_log) 147 result = kunit_parser.parse_run_tests(file.readlines()) 148 self.assertEqual( 149 kunit_parser.TestStatus.SUCCESS, 150 result.status) 151 file.close() 152 153 def test_parse_failed_test_log(self): 154 failed_log = get_absolute_path( 155 'test_data/test_is_test_passed-failure.log') 156 file = open(failed_log) 157 result = kunit_parser.parse_run_tests(file.readlines()) 158 self.assertEqual( 159 kunit_parser.TestStatus.FAILURE, 160 result.status) 161 file.close() 162 163 def test_no_tests(self): 164 empty_log = get_absolute_path( 165 'test_data/test_is_test_passed-no_tests_run.log') 166 file = open(empty_log) 167 result = kunit_parser.parse_run_tests( 168 kunit_parser.isolate_kunit_output(file.readlines())) 169 self.assertEqual(0, len(result.suites)) 170 self.assertEqual( 171 kunit_parser.TestStatus.NO_TESTS, 172 result.status) 173 file.close() 174 175 def test_no_kunit_output(self): 176 crash_log = get_absolute_path( 177 'test_data/test_insufficient_memory.log') 178 file = open(crash_log) 179 print_mock = mock.patch('builtins.print').start() 180 result = kunit_parser.parse_run_tests( 181 kunit_parser.isolate_kunit_output(file.readlines())) 182 print_mock.assert_any_call(StrContains('no tests run!')) 183 print_mock.stop() 184 file.close() 185 186 def test_crashed_test(self): 187 crashed_log = get_absolute_path( 188 'test_data/test_is_test_passed-crash.log') 189 file = open(crashed_log) 190 result = kunit_parser.parse_run_tests(file.readlines()) 191 self.assertEqual( 192 kunit_parser.TestStatus.TEST_CRASHED, 193 result.status) 194 file.close() 195 196 def test_ignores_prefix_printk_time(self): 197 prefix_log = get_absolute_path( 198 'test_data/test_config_printk_time.log') 199 with open(prefix_log) as file: 200 result = kunit_parser.parse_run_tests(file.readlines()) 201 self.assertEqual( 202 kunit_parser.TestStatus.SUCCESS, 203 result.status) 204 self.assertEqual('kunit-resource-test', result.suites[0].name) 205 206 def test_ignores_multiple_prefixes(self): 207 prefix_log = get_absolute_path( 208 'test_data/test_multiple_prefixes.log') 209 with open(prefix_log) as file: 210 result = kunit_parser.parse_run_tests(file.readlines()) 211 self.assertEqual( 212 kunit_parser.TestStatus.SUCCESS, 213 result.status) 214 self.assertEqual('kunit-resource-test', result.suites[0].name) 215 216 def test_prefix_mixed_kernel_output(self): 217 mixed_prefix_log = get_absolute_path( 218 'test_data/test_interrupted_tap_output.log') 219 with open(mixed_prefix_log) as file: 220 result = kunit_parser.parse_run_tests(file.readlines()) 221 self.assertEqual( 222 kunit_parser.TestStatus.SUCCESS, 223 result.status) 224 self.assertEqual('kunit-resource-test', result.suites[0].name) 225 226 def test_prefix_poundsign(self): 227 pound_log = get_absolute_path('test_data/test_pound_sign.log') 228 with open(pound_log) as file: 229 result = kunit_parser.parse_run_tests(file.readlines()) 230 self.assertEqual( 231 kunit_parser.TestStatus.SUCCESS, 232 result.status) 233 self.assertEqual('kunit-resource-test', result.suites[0].name) 234 235 def test_kernel_panic_end(self): 236 panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log') 237 with open(panic_log) as file: 238 result = kunit_parser.parse_run_tests(file.readlines()) 239 self.assertEqual( 240 kunit_parser.TestStatus.TEST_CRASHED, 241 result.status) 242 self.assertEqual('kunit-resource-test', result.suites[0].name) 243 244 def test_pound_no_prefix(self): 245 pound_log = get_absolute_path('test_data/test_pound_no_prefix.log') 246 with open(pound_log) as file: 247 result = kunit_parser.parse_run_tests(file.readlines()) 248 self.assertEqual( 249 kunit_parser.TestStatus.SUCCESS, 250 result.status) 251 self.assertEqual('kunit-resource-test', result.suites[0].name) 252 253class KUnitJsonTest(unittest.TestCase): 254 255 def _json_for(self, log_file): 256 with(open(get_absolute_path(log_file))) as file: 257 test_result = kunit_parser.parse_run_tests(file) 258 json_obj = kunit_json.get_json_result( 259 test_result=test_result, 260 def_config='kunit_defconfig', 261 build_dir=None, 262 json_path='stdout') 263 return json.loads(json_obj) 264 265 def test_failed_test_json(self): 266 result = self._json_for( 267 'test_data/test_is_test_passed-failure.log') 268 self.assertEqual( 269 {'name': 'example_simple_test', 'status': 'FAIL'}, 270 result["sub_groups"][1]["test_cases"][0]) 271 272 def test_crashed_test_json(self): 273 result = self._json_for( 274 'test_data/test_is_test_passed-crash.log') 275 self.assertEqual( 276 {'name': 'example_simple_test', 'status': 'ERROR'}, 277 result["sub_groups"][1]["test_cases"][0]) 278 279 def test_no_tests_json(self): 280 result = self._json_for( 281 'test_data/test_is_test_passed-no_tests_run.log') 282 self.assertEqual(0, len(result['sub_groups'])) 283 284class StrContains(str): 285 def __eq__(self, other): 286 return self in other 287 288class KUnitMainTest(unittest.TestCase): 289 def setUp(self): 290 path = get_absolute_path('test_data/test_is_test_passed-all_passed.log') 291 file = open(path) 292 all_passed_log = file.readlines() 293 self.print_patch = mock.patch('builtins.print') 294 self.print_mock = self.print_patch.start() 295 self.linux_source_mock = mock.Mock() 296 self.linux_source_mock.build_reconfig = mock.Mock(return_value=True) 297 self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True) 298 self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log) 299 300 def tearDown(self): 301 self.print_patch.stop() 302 pass 303 304 def test_config_passes_args_pass(self): 305 kunit.main(['config', '--build_dir=.kunit'], self.linux_source_mock) 306 assert self.linux_source_mock.build_reconfig.call_count == 1 307 assert self.linux_source_mock.run_kernel.call_count == 0 308 309 def test_build_passes_args_pass(self): 310 kunit.main(['build'], self.linux_source_mock) 311 assert self.linux_source_mock.build_reconfig.call_count == 0 312 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, '.kunit', None) 313 assert self.linux_source_mock.run_kernel.call_count == 0 314 315 def test_exec_passes_args_pass(self): 316 kunit.main(['exec'], self.linux_source_mock) 317 assert self.linux_source_mock.build_reconfig.call_count == 0 318 assert self.linux_source_mock.run_kernel.call_count == 1 319 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=300) 320 self.print_mock.assert_any_call(StrContains('Testing complete.')) 321 322 def test_run_passes_args_pass(self): 323 kunit.main(['run'], self.linux_source_mock) 324 assert self.linux_source_mock.build_reconfig.call_count == 1 325 assert self.linux_source_mock.run_kernel.call_count == 1 326 self.linux_source_mock.run_kernel.assert_called_once_with( 327 build_dir='.kunit', timeout=300) 328 self.print_mock.assert_any_call(StrContains('Testing complete.')) 329 330 def test_exec_passes_args_fail(self): 331 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 332 with self.assertRaises(SystemExit) as e: 333 kunit.main(['exec'], self.linux_source_mock) 334 assert type(e.exception) == SystemExit 335 assert e.exception.code == 1 336 337 def test_run_passes_args_fail(self): 338 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 339 with self.assertRaises(SystemExit) as e: 340 kunit.main(['run'], self.linux_source_mock) 341 assert type(e.exception) == SystemExit 342 assert e.exception.code == 1 343 assert self.linux_source_mock.build_reconfig.call_count == 1 344 assert self.linux_source_mock.run_kernel.call_count == 1 345 self.print_mock.assert_any_call(StrContains(' 0 tests run')) 346 347 def test_exec_raw_output(self): 348 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 349 kunit.main(['exec', '--raw_output'], self.linux_source_mock) 350 assert self.linux_source_mock.run_kernel.call_count == 1 351 for kall in self.print_mock.call_args_list: 352 assert kall != mock.call(StrContains('Testing complete.')) 353 assert kall != mock.call(StrContains(' 0 tests run')) 354 355 def test_run_raw_output(self): 356 self.linux_source_mock.run_kernel = mock.Mock(return_value=[]) 357 kunit.main(['run', '--raw_output'], self.linux_source_mock) 358 assert self.linux_source_mock.build_reconfig.call_count == 1 359 assert self.linux_source_mock.run_kernel.call_count == 1 360 for kall in self.print_mock.call_args_list: 361 assert kall != mock.call(StrContains('Testing complete.')) 362 assert kall != mock.call(StrContains(' 0 tests run')) 363 364 def test_exec_timeout(self): 365 timeout = 3453 366 kunit.main(['exec', '--timeout', str(timeout)], self.linux_source_mock) 367 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=timeout) 368 self.print_mock.assert_any_call(StrContains('Testing complete.')) 369 370 def test_run_timeout(self): 371 timeout = 3453 372 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock) 373 assert self.linux_source_mock.build_reconfig.call_count == 1 374 self.linux_source_mock.run_kernel.assert_called_once_with( 375 build_dir='.kunit', timeout=timeout) 376 self.print_mock.assert_any_call(StrContains('Testing complete.')) 377 378 def test_run_builddir(self): 379 build_dir = '.kunit' 380 kunit.main(['run', '--build_dir=.kunit'], self.linux_source_mock) 381 assert self.linux_source_mock.build_reconfig.call_count == 1 382 self.linux_source_mock.run_kernel.assert_called_once_with( 383 build_dir=build_dir, timeout=300) 384 self.print_mock.assert_any_call(StrContains('Testing complete.')) 385 386 def test_config_builddir(self): 387 build_dir = '.kunit' 388 kunit.main(['config', '--build_dir', build_dir], self.linux_source_mock) 389 assert self.linux_source_mock.build_reconfig.call_count == 1 390 391 def test_build_builddir(self): 392 build_dir = '.kunit' 393 kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock) 394 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, build_dir, None) 395 396 def test_exec_builddir(self): 397 build_dir = '.kunit' 398 kunit.main(['exec', '--build_dir', build_dir], self.linux_source_mock) 399 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300) 400 self.print_mock.assert_any_call(StrContains('Testing complete.')) 401 402if __name__ == '__main__': 403 unittest.main() 404