1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> 9 10import argparse 11import sys 12import os 13import time 14 15assert sys.version_info >= (3, 7), "Python version is too old" 16 17from collections import namedtuple 18from enum import Enum, auto 19from typing import Iterable 20 21import kunit_config 22import kunit_json 23import kunit_kernel 24import kunit_parser 25 26KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 27 28KunitConfigRequest = namedtuple('KunitConfigRequest', 29 ['build_dir', 'make_options']) 30KunitBuildRequest = namedtuple('KunitBuildRequest', 31 ['jobs', 'build_dir', 'alltests', 32 'make_options']) 33KunitExecRequest = namedtuple('KunitExecRequest', 34 ['timeout', 'build_dir', 'alltests', 35 'filter_glob', 'kernel_args']) 36KunitParseRequest = namedtuple('KunitParseRequest', 37 ['raw_output', 'input_data', 'build_dir', 'json']) 38KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 39 'build_dir', 'alltests', 'filter_glob', 40 'kernel_args', 'json', 'make_options']) 41 42KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 43 44class KunitStatus(Enum): 45 SUCCESS = auto() 46 CONFIG_FAILURE = auto() 47 BUILD_FAILURE = auto() 48 TEST_FAILURE = auto() 49 50def get_kernel_root_path() -> str: 51 path = sys.argv[0] if not __file__ else __file__ 52 parts = os.path.realpath(path).split('tools/testing/kunit') 53 if len(parts) != 2: 54 sys.exit(1) 55 return parts[0] 56 57def config_tests(linux: kunit_kernel.LinuxSourceTree, 58 request: KunitConfigRequest) -> KunitResult: 59 kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...') 60 61 config_start = time.time() 62 success = linux.build_reconfig(request.build_dir, request.make_options) 63 config_end = time.time() 64 if not success: 65 return KunitResult(KunitStatus.CONFIG_FAILURE, 66 'could not configure kernel', 67 config_end - config_start) 68 return KunitResult(KunitStatus.SUCCESS, 69 'configured kernel successfully', 70 config_end - config_start) 71 72def build_tests(linux: kunit_kernel.LinuxSourceTree, 73 request: KunitBuildRequest) -> KunitResult: 74 kunit_parser.print_with_timestamp('Building KUnit Kernel ...') 75 76 build_start = time.time() 77 success = linux.build_kernel(request.alltests, 78 request.jobs, 79 request.build_dir, 80 request.make_options) 81 build_end = time.time() 82 if not success: 83 return KunitResult(KunitStatus.BUILD_FAILURE, 84 'could not build kernel', 85 build_end - build_start) 86 if not success: 87 return KunitResult(KunitStatus.BUILD_FAILURE, 88 'could not build kernel', 89 build_end - build_start) 90 return KunitResult(KunitStatus.SUCCESS, 91 'built kernel successfully', 92 build_end - build_start) 93 94def exec_tests(linux: kunit_kernel.LinuxSourceTree, 95 request: KunitExecRequest) -> KunitResult: 96 kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') 97 test_start = time.time() 98 result = linux.run_kernel( 99 args=request.kernel_args, 100 timeout=None if request.alltests else request.timeout, 101 filter_glob=request.filter_glob, 102 build_dir=request.build_dir) 103 104 test_end = time.time() 105 106 return KunitResult(KunitStatus.SUCCESS, 107 result, 108 test_end - test_start) 109 110def parse_tests(request: KunitParseRequest) -> KunitResult: 111 parse_start = time.time() 112 113 test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, 114 [], 115 'Tests not Parsed.') 116 117 if request.raw_output: 118 output: Iterable[str] = request.input_data 119 if request.raw_output == 'all': 120 pass 121 elif request.raw_output == 'kunit': 122 output = kunit_parser.extract_tap_lines(output) 123 else: 124 print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr) 125 for line in output: 126 print(line.rstrip()) 127 128 else: 129 test_result = kunit_parser.parse_run_tests(request.input_data) 130 parse_end = time.time() 131 132 if request.json: 133 json_obj = kunit_json.get_json_result( 134 test_result=test_result, 135 def_config='kunit_defconfig', 136 build_dir=request.build_dir, 137 json_path=request.json) 138 if request.json == 'stdout': 139 print(json_obj) 140 141 if test_result.status != kunit_parser.TestStatus.SUCCESS: 142 return KunitResult(KunitStatus.TEST_FAILURE, test_result, 143 parse_end - parse_start) 144 145 return KunitResult(KunitStatus.SUCCESS, test_result, 146 parse_end - parse_start) 147 148def run_tests(linux: kunit_kernel.LinuxSourceTree, 149 request: KunitRequest) -> KunitResult: 150 run_start = time.time() 151 152 config_request = KunitConfigRequest(request.build_dir, 153 request.make_options) 154 config_result = config_tests(linux, config_request) 155 if config_result.status != KunitStatus.SUCCESS: 156 return config_result 157 158 build_request = KunitBuildRequest(request.jobs, request.build_dir, 159 request.alltests, 160 request.make_options) 161 build_result = build_tests(linux, build_request) 162 if build_result.status != KunitStatus.SUCCESS: 163 return build_result 164 165 exec_request = KunitExecRequest(request.timeout, request.build_dir, 166 request.alltests, request.filter_glob, 167 request.kernel_args) 168 exec_result = exec_tests(linux, exec_request) 169 if exec_result.status != KunitStatus.SUCCESS: 170 return exec_result 171 172 parse_request = KunitParseRequest(request.raw_output, 173 exec_result.result, 174 request.build_dir, 175 request.json) 176 parse_result = parse_tests(parse_request) 177 178 run_end = time.time() 179 180 kunit_parser.print_with_timestamp(( 181 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 182 'building, %.3fs running\n') % ( 183 run_end - run_start, 184 config_result.elapsed_time, 185 build_result.elapsed_time, 186 exec_result.elapsed_time)) 187 return parse_result 188 189def add_common_opts(parser) -> None: 190 parser.add_argument('--build_dir', 191 help='As in the make command, it specifies the build ' 192 'directory.', 193 type=str, default='.kunit', metavar='build_dir') 194 parser.add_argument('--make_options', 195 help='X=Y make option, can be repeated.', 196 action='append') 197 parser.add_argument('--alltests', 198 help='Run all KUnit tests through allyesconfig', 199 action='store_true') 200 parser.add_argument('--kunitconfig', 201 help='Path to Kconfig fragment that enables KUnit tests.' 202 ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 203 'will get automatically appended.', 204 metavar='kunitconfig') 205 206 parser.add_argument('--arch', 207 help=('Specifies the architecture to run tests under. ' 208 'The architecture specified here must match the ' 209 'string passed to the ARCH make param, ' 210 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 211 'architectures run on QEMU.'), 212 type=str, default='um', metavar='arch') 213 214 parser.add_argument('--cross_compile', 215 help=('Sets make\'s CROSS_COMPILE variable; it should ' 216 'be set to a toolchain path prefix (the prefix ' 217 'of gcc and other tools in your toolchain, for ' 218 'example `sparc64-linux-gnu-` if you have the ' 219 'sparc toolchain installed on your system, or ' 220 '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 221 'if you have downloaded the microblaze toolchain ' 222 'from the 0-day website to a directory in your ' 223 'home directory called `toolchains`).'), 224 metavar='cross_compile') 225 226 parser.add_argument('--qemu_config', 227 help=('Takes a path to a path to a file containing ' 228 'a QemuArchParams object.'), 229 type=str, metavar='qemu_config') 230 231def add_build_opts(parser) -> None: 232 parser.add_argument('--jobs', 233 help='As in the make command, "Specifies the number of ' 234 'jobs (commands) to run simultaneously."', 235 type=int, default=8, metavar='jobs') 236 237def add_exec_opts(parser) -> None: 238 parser.add_argument('--timeout', 239 help='maximum number of seconds to allow for all tests ' 240 'to run. This does not include time taken to build the ' 241 'tests.', 242 type=int, 243 default=300, 244 metavar='timeout') 245 parser.add_argument('filter_glob', 246 help='maximum number of seconds to allow for all tests ' 247 'to run. This does not include time taken to build the ' 248 'tests.', 249 type=str, 250 nargs='?', 251 default='', 252 metavar='filter_glob') 253 parser.add_argument('--kernel_args', 254 help='Kernel command-line parameters. Maybe be repeated', 255 action='append') 256 257def add_parse_opts(parser) -> None: 258 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 259 'If set to --raw_output=kunit, filters to just KUnit output.', 260 type=str, nargs='?', const='all', default=None) 261 parser.add_argument('--json', 262 nargs='?', 263 help='Stores test results in a JSON, and either ' 264 'prints to stdout or saves to file if a ' 265 'filename is specified', 266 type=str, const='stdout', default=None) 267 268def main(argv, linux=None): 269 parser = argparse.ArgumentParser( 270 description='Helps writing and running KUnit tests.') 271 subparser = parser.add_subparsers(dest='subcommand') 272 273 # The 'run' command will config, build, exec, and parse in one go. 274 run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 275 add_common_opts(run_parser) 276 add_build_opts(run_parser) 277 add_exec_opts(run_parser) 278 add_parse_opts(run_parser) 279 280 config_parser = subparser.add_parser('config', 281 help='Ensures that .config contains all of ' 282 'the options in .kunitconfig') 283 add_common_opts(config_parser) 284 285 build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 286 add_common_opts(build_parser) 287 add_build_opts(build_parser) 288 289 exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 290 add_common_opts(exec_parser) 291 add_exec_opts(exec_parser) 292 add_parse_opts(exec_parser) 293 294 # The 'parse' option is special, as it doesn't need the kernel source 295 # (therefore there is no need for a build_dir, hence no add_common_opts) 296 # and the '--file' argument is not relevant to 'run', so isn't in 297 # add_parse_opts() 298 parse_parser = subparser.add_parser('parse', 299 help='Parses KUnit results from a file, ' 300 'and parses formatted results.') 301 add_parse_opts(parse_parser) 302 parse_parser.add_argument('file', 303 help='Specifies the file to read results from.', 304 type=str, nargs='?', metavar='input_file') 305 306 cli_args = parser.parse_args(argv) 307 308 if get_kernel_root_path(): 309 os.chdir(get_kernel_root_path()) 310 311 if cli_args.subcommand == 'run': 312 if not os.path.exists(cli_args.build_dir): 313 os.mkdir(cli_args.build_dir) 314 315 if not linux: 316 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 317 kunitconfig_path=cli_args.kunitconfig, 318 arch=cli_args.arch, 319 cross_compile=cli_args.cross_compile, 320 qemu_config_path=cli_args.qemu_config) 321 322 request = KunitRequest(cli_args.raw_output, 323 cli_args.timeout, 324 cli_args.jobs, 325 cli_args.build_dir, 326 cli_args.alltests, 327 cli_args.filter_glob, 328 cli_args.kernel_args, 329 cli_args.json, 330 cli_args.make_options) 331 result = run_tests(linux, request) 332 if result.status != KunitStatus.SUCCESS: 333 sys.exit(1) 334 elif cli_args.subcommand == 'config': 335 if cli_args.build_dir and ( 336 not os.path.exists(cli_args.build_dir)): 337 os.mkdir(cli_args.build_dir) 338 339 if not linux: 340 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 341 kunitconfig_path=cli_args.kunitconfig, 342 arch=cli_args.arch, 343 cross_compile=cli_args.cross_compile, 344 qemu_config_path=cli_args.qemu_config) 345 346 request = KunitConfigRequest(cli_args.build_dir, 347 cli_args.make_options) 348 result = config_tests(linux, request) 349 kunit_parser.print_with_timestamp(( 350 'Elapsed time: %.3fs\n') % ( 351 result.elapsed_time)) 352 if result.status != KunitStatus.SUCCESS: 353 sys.exit(1) 354 elif cli_args.subcommand == 'build': 355 if not linux: 356 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 357 kunitconfig_path=cli_args.kunitconfig, 358 arch=cli_args.arch, 359 cross_compile=cli_args.cross_compile, 360 qemu_config_path=cli_args.qemu_config) 361 362 request = KunitBuildRequest(cli_args.jobs, 363 cli_args.build_dir, 364 cli_args.alltests, 365 cli_args.make_options) 366 result = build_tests(linux, request) 367 kunit_parser.print_with_timestamp(( 368 'Elapsed time: %.3fs\n') % ( 369 result.elapsed_time)) 370 if result.status != KunitStatus.SUCCESS: 371 sys.exit(1) 372 elif cli_args.subcommand == 'exec': 373 if not linux: 374 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 375 kunitconfig_path=cli_args.kunitconfig, 376 arch=cli_args.arch, 377 cross_compile=cli_args.cross_compile, 378 qemu_config_path=cli_args.qemu_config) 379 380 exec_request = KunitExecRequest(cli_args.timeout, 381 cli_args.build_dir, 382 cli_args.alltests, 383 cli_args.filter_glob, 384 cli_args.kernel_args) 385 exec_result = exec_tests(linux, exec_request) 386 parse_request = KunitParseRequest(cli_args.raw_output, 387 exec_result.result, 388 cli_args.build_dir, 389 cli_args.json) 390 result = parse_tests(parse_request) 391 kunit_parser.print_with_timestamp(( 392 'Elapsed time: %.3fs\n') % ( 393 exec_result.elapsed_time)) 394 if result.status != KunitStatus.SUCCESS: 395 sys.exit(1) 396 elif cli_args.subcommand == 'parse': 397 if cli_args.file == None: 398 kunit_output = sys.stdin 399 else: 400 with open(cli_args.file, 'r') as f: 401 kunit_output = f.read().splitlines() 402 request = KunitParseRequest(cli_args.raw_output, 403 kunit_output, 404 None, 405 cli_args.json) 406 result = parse_tests(request) 407 if result.status != KunitStatus.SUCCESS: 408 sys.exit(1) 409 else: 410 parser.print_help() 411 412if __name__ == '__main__': 413 main(sys.argv[1:]) 414