1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of tests under KVM. By default, this series is specified 5# by the relevant CFLIST file, but can be overridden by the --configs 6# command-line argument. 7# 8# Usage: kvm.sh [ options ] 9# 10# Copyright (C) IBM Corporation, 2011 11# 12# Authors: Paul E. McKenney <paulmck@linux.ibm.com> 13 14scriptname=$0 15args="$*" 16 17T=${TMPDIR-/tmp}/kvm.sh.$$ 18trap 'rm -rf $T' 0 19mkdir $T 20 21cd `dirname $scriptname`/../../../../../ 22 23dur=$((30*60)) 24dryrun="" 25KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM 26PATH=${KVM}/bin:$PATH; export PATH 27. functions.sh 28 29TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 30TORTURE_DEFCONFIG=defconfig 31TORTURE_BOOT_IMAGE="" 32TORTURE_INITRD="$KVM/initrd"; export TORTURE_INITRD 33TORTURE_KCONFIG_ARG="" 34TORTURE_KCONFIG_KASAN_ARG="" 35TORTURE_KCONFIG_KCSAN_ARG="" 36TORTURE_KMAKE_ARG="" 37TORTURE_QEMU_MEM=512 38TORTURE_SHUTDOWN_GRACE=180 39TORTURE_SUITE=rcu 40TORTURE_TRUST_MAKE="" 41resdir="" 42configs="" 43cpus=0 44ds=`date +%Y.%m.%d-%H.%M.%S` 45jitter="-1" 46 47usage () { 48 echo "Usage: $scriptname optional arguments:" 49 echo " --bootargs kernel-boot-arguments" 50 echo " --bootimage relative-path-to-kernel-boot-image" 51 echo " --buildonly" 52 echo " --configs \"config-file list w/ repeat factor (3*TINY01)\"" 53 echo " --cpus N" 54 echo " --datestamp string" 55 echo " --defconfig string" 56 echo " --dryrun sched|script" 57 echo " --duration minutes" 58 echo " --interactive" 59 echo " --jitter N [ maxsleep (us) [ maxspin (us) ] ]" 60 echo " --kconfig Kconfig-options" 61 echo " --kmake-arg kernel-make-arguments" 62 echo " --mac nn:nn:nn:nn:nn:nn" 63 echo " --memory megabytes | nnnG" 64 echo " --no-initrd" 65 echo " --qemu-args qemu-arguments" 66 echo " --qemu-cmd qemu-system-..." 67 echo " --results absolute-pathname" 68 echo " --torture rcu" 69 echo " --trust-make" 70 exit 1 71} 72 73while test $# -gt 0 74do 75 case "$1" in 76 --bootargs|--bootarg) 77 checkarg --bootargs "(list of kernel boot arguments)" "$#" "$2" '.*' '^--' 78 TORTURE_BOOTARGS="$2" 79 shift 80 ;; 81 --bootimage) 82 checkarg --bootimage "(relative path to kernel boot image)" "$#" "$2" '[a-zA-Z0-9][a-zA-Z0-9_]*' '^--' 83 TORTURE_BOOT_IMAGE="$2" 84 shift 85 ;; 86 --buildonly) 87 TORTURE_BUILDONLY=1 88 ;; 89 --configs|--config) 90 checkarg --configs "(list of config files)" "$#" "$2" '^[^/]*$' '^--' 91 configs="$2" 92 shift 93 ;; 94 --cpus) 95 checkarg --cpus "(number)" "$#" "$2" '^[0-9]*$' '^--' 96 cpus=$2 97 TORTURE_ALLOTED_CPUS="$2" 98 max_cpus="`identify_qemu_vcpus`" 99 if test "$TORTURE_ALLOTED_CPUS" -gt "$max_cpus" 100 then 101 TORTURE_ALLOTED_CPUS=$max_cpus 102 fi 103 shift 104 ;; 105 --datestamp) 106 checkarg --datestamp "(relative pathname)" "$#" "$2" '^[^/]*$' '^--' 107 ds=$2 108 shift 109 ;; 110 --defconfig) 111 checkarg --defconfig "defconfigtype" "$#" "$2" '^[^/][^/]*$' '^--' 112 TORTURE_DEFCONFIG=$2 113 shift 114 ;; 115 --dryrun) 116 checkarg --dryrun "sched|script" $# "$2" 'sched\|script' '^--' 117 dryrun=$2 118 shift 119 ;; 120 --duration) 121 checkarg --duration "(minutes)" $# "$2" '^[0-9]*$' '^error' 122 dur=$(($2*60)) 123 shift 124 ;; 125 --interactive) 126 TORTURE_QEMU_INTERACTIVE=1; export TORTURE_QEMU_INTERACTIVE 127 ;; 128 --jitter) 129 checkarg --jitter "(# threads [ sleep [ spin ] ])" $# "$2" '^-\{,1\}[0-9]\+\( \+[0-9]\+\)\{,2\} *$' '^error$' 130 jitter="$2" 131 shift 132 ;; 133 --kconfig) 134 checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\)*$' '^error$' 135 TORTURE_KCONFIG_ARG="$2" 136 shift 137 ;; 138 --kasan) 139 TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG 140 ;; 141 --kcsan) 142 TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_KCSAN_INTERRUPT_WATCHER=y"; export TORTURE_KCONFIG_KCSAN_ARG 143 ;; 144 --kmake-arg) 145 checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 146 TORTURE_KMAKE_ARG="$2" 147 shift 148 ;; 149 --mac) 150 checkarg --mac "(MAC address)" $# "$2" '^\([0-9a-fA-F]\{2\}:\)\{5\}[0-9a-fA-F]\{2\}$' error 151 TORTURE_QEMU_MAC=$2 152 shift 153 ;; 154 --memory) 155 checkarg --memory "(memory size)" $# "$2" '^[0-9]\+[MG]\?$' error 156 TORTURE_QEMU_MEM=$2 157 shift 158 ;; 159 --no-initrd) 160 TORTURE_INITRD=""; export TORTURE_INITRD 161 ;; 162 --qemu-args|--qemu-arg) 163 checkarg --qemu-args "(qemu arguments)" $# "$2" '^-' '^error' 164 TORTURE_QEMU_ARG="$2" 165 shift 166 ;; 167 --qemu-cmd) 168 checkarg --qemu-cmd "(qemu-system-...)" $# "$2" 'qemu-system-' '^--' 169 TORTURE_QEMU_CMD="$2" 170 shift 171 ;; 172 --results) 173 checkarg --results "(absolute pathname)" "$#" "$2" '^/' '^error' 174 resdir=$2 175 shift 176 ;; 177 --shutdown-grace) 178 checkarg --shutdown-grace "(seconds)" "$#" "$2" '^[0-9]*$' '^error' 179 TORTURE_SHUTDOWN_GRACE=$2 180 shift 181 ;; 182 --torture) 183 checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\|rcuperf\)$' '^--' 184 TORTURE_SUITE=$2 185 shift 186 if test "$TORTURE_SUITE" = rcuperf 187 then 188 # If you really want jitter for rcuperf, specify 189 # it after specifying rcuperf. (But why?) 190 jitter=0 191 fi 192 ;; 193 --trust-make) 194 TORTURE_TRUST_MAKE="y" 195 ;; 196 *) 197 echo Unknown argument $1 198 usage 199 ;; 200 esac 201 shift 202done 203 204if test -z "$TORTURE_INITRD" || tools/testing/selftests/rcutorture/bin/mkinitrd.sh 205then 206 : 207else 208 echo No initrd and unable to create one, aborting test >&2 209 exit 1 210fi 211 212CONFIGFRAG=${KVM}/configs/${TORTURE_SUITE}; export CONFIGFRAG 213 214defaultconfigs="`tr '\012' ' ' < $CONFIGFRAG/CFLIST`" 215if test -z "$configs" 216then 217 configs=$defaultconfigs 218fi 219 220if test -z "$resdir" 221then 222 resdir=$KVM/res 223fi 224 225# Create a file of test-name/#cpus pairs, sorted by decreasing #cpus. 226configs_derep= 227for CF in $configs 228do 229 case $CF in 230 [0-9]\**|[0-9][0-9]\**|[0-9][0-9][0-9]\**) 231 config_reps=`echo $CF | sed -e 's/\*.*$//'` 232 CF1=`echo $CF | sed -e 's/^[^*]*\*//'` 233 ;; 234 *) 235 config_reps=1 236 CF1=$CF 237 ;; 238 esac 239 for ((cur_rep=0;cur_rep<$config_reps;cur_rep++)) 240 do 241 configs_derep="$configs_derep $CF1" 242 done 243done 244touch $T/cfgcpu 245configs_derep="`echo $configs_derep | sed -e "s/\<CFLIST\>/$defaultconfigs/g"`" 246for CF1 in $configs_derep 247do 248 if test -f "$CONFIGFRAG/$CF1" 249 then 250 cpu_count=`configNR_CPUS.sh $CONFIGFRAG/$CF1` 251 cpu_count=`configfrag_boot_cpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 252 cpu_count=`configfrag_boot_maxcpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 253 echo $CF1 $cpu_count >> $T/cfgcpu 254 else 255 echo "The --configs file $CF1 does not exist, terminating." 256 exit 1 257 fi 258done 259sort -k2nr $T/cfgcpu -T="$T" > $T/cfgcpu.sort 260 261# Use a greedy bin-packing algorithm, sorting the list accordingly. 262awk < $T/cfgcpu.sort > $T/cfgcpu.pack -v ncpus=$cpus ' 263BEGIN { 264 njobs = 0; 265} 266 267{ 268 # Read file of tests and corresponding required numbers of CPUs. 269 cf[njobs] = $1; 270 cpus[njobs] = $2; 271 njobs++; 272} 273 274END { 275 batch = 0; 276 nc = -1; 277 278 # Each pass through the following loop creates on test batch 279 # that can be executed concurrently given ncpus. Note that a 280 # given test that requires more than the available CPUs will run in 281 # their own batch. Such tests just have to make do with what 282 # is available. 283 while (nc != ncpus) { 284 batch++; 285 nc = ncpus; 286 287 # Each pass through the following loop considers one 288 # test for inclusion in the current batch. 289 for (i = 0; i < njobs; i++) { 290 if (done[i]) 291 continue; # Already part of a batch. 292 if (nc >= cpus[i] || nc == ncpus) { 293 294 # This test fits into the current batch. 295 done[i] = batch; 296 nc -= cpus[i]; 297 if (nc <= 0) 298 break; # Too-big test in its own batch. 299 } 300 } 301 } 302 303 # Dump out the tests in batch order. 304 for (b = 1; b <= batch; b++) 305 for (i = 0; i < njobs; i++) 306 if (done[i] == b) 307 print cf[i], cpus[i]; 308}' 309 310# Generate a script to execute the tests in appropriate batches. 311cat << ___EOF___ > $T/script 312CONFIGFRAG="$CONFIGFRAG"; export CONFIGFRAG 313KVM="$KVM"; export KVM 314PATH="$PATH"; export PATH 315TORTURE_ALLOTED_CPUS="$TORTURE_ALLOTED_CPUS"; export TORTURE_ALLOTED_CPUS 316TORTURE_BOOT_IMAGE="$TORTURE_BOOT_IMAGE"; export TORTURE_BOOT_IMAGE 317TORTURE_BUILDONLY="$TORTURE_BUILDONLY"; export TORTURE_BUILDONLY 318TORTURE_DEFCONFIG="$TORTURE_DEFCONFIG"; export TORTURE_DEFCONFIG 319TORTURE_INITRD="$TORTURE_INITRD"; export TORTURE_INITRD 320TORTURE_KCONFIG_ARG="$TORTURE_KCONFIG_ARG"; export TORTURE_KCONFIG_ARG 321TORTURE_KCONFIG_KASAN_ARG="$TORTURE_KCONFIG_KASAN_ARG"; export TORTURE_KCONFIG_KASAN_ARG 322TORTURE_KCONFIG_KCSAN_ARG="$TORTURE_KCONFIG_KCSAN_ARG"; export TORTURE_KCONFIG_KCSAN_ARG 323TORTURE_KMAKE_ARG="$TORTURE_KMAKE_ARG"; export TORTURE_KMAKE_ARG 324TORTURE_QEMU_CMD="$TORTURE_QEMU_CMD"; export TORTURE_QEMU_CMD 325TORTURE_QEMU_INTERACTIVE="$TORTURE_QEMU_INTERACTIVE"; export TORTURE_QEMU_INTERACTIVE 326TORTURE_QEMU_MAC="$TORTURE_QEMU_MAC"; export TORTURE_QEMU_MAC 327TORTURE_QEMU_MEM="$TORTURE_QEMU_MEM"; export TORTURE_QEMU_MEM 328TORTURE_SHUTDOWN_GRACE="$TORTURE_SHUTDOWN_GRACE"; export TORTURE_SHUTDOWN_GRACE 329TORTURE_SUITE="$TORTURE_SUITE"; export TORTURE_SUITE 330TORTURE_TRUST_MAKE="$TORTURE_TRUST_MAKE"; export TORTURE_TRUST_MAKE 331if ! test -e $resdir 332then 333 mkdir -p "$resdir" || : 334fi 335mkdir $resdir/$ds 336echo Results directory: $resdir/$ds 337echo $scriptname $args 338touch $resdir/$ds/log 339echo $scriptname $args >> $resdir/$ds/log 340echo ${TORTURE_SUITE} > $resdir/$ds/TORTURE_SUITE 341pwd > $resdir/$ds/testid.txt 342if test -d .git 343then 344 git status >> $resdir/$ds/testid.txt 345 git rev-parse HEAD >> $resdir/$ds/testid.txt 346 git diff HEAD >> $resdir/$ds/testid.txt 347fi 348___EOF___ 349awk < $T/cfgcpu.pack \ 350 -v TORTURE_BUILDONLY="$TORTURE_BUILDONLY" \ 351 -v CONFIGDIR="$CONFIGFRAG/" \ 352 -v KVM="$KVM" \ 353 -v ncpus=$cpus \ 354 -v jitter="$jitter" \ 355 -v rd=$resdir/$ds/ \ 356 -v dur=$dur \ 357 -v TORTURE_QEMU_ARG="$TORTURE_QEMU_ARG" \ 358 -v TORTURE_BOOTARGS="$TORTURE_BOOTARGS" \ 359'BEGIN { 360 i = 0; 361} 362 363{ 364 cf[i] = $1; 365 cpus[i] = $2; 366 i++; 367} 368 369# Dump out the scripting required to run one test batch. 370function dump(first, pastlast, batchnum) 371{ 372 print "echo ----Start batch " batchnum ": `date` | tee -a " rd "log"; 373 print "needqemurun=" 374 jn=1 375 for (j = first; j < pastlast; j++) { 376 builddir=KVM "/b" j - first + 1 377 cpusr[jn] = cpus[j]; 378 if (cfrep[cf[j]] == "") { 379 cfr[jn] = cf[j]; 380 cfrep[cf[j]] = 1; 381 } else { 382 cfrep[cf[j]]++; 383 cfr[jn] = cf[j] "." cfrep[cf[j]]; 384 } 385 if (cpusr[jn] > ncpus && ncpus != 0) 386 ovf = "-ovf"; 387 else 388 ovf = ""; 389 print "echo ", cfr[jn], cpusr[jn] ovf ": Starting build. `date` | tee -a " rd "log"; 390 print "rm -f " builddir ".*"; 391 print "touch " builddir ".wait"; 392 print "mkdir " rd cfr[jn] " || :"; 393 print "kvm-test-1-run.sh " CONFIGDIR cf[j], builddir, rd cfr[jn], dur " \"" TORTURE_QEMU_ARG "\" \"" TORTURE_BOOTARGS "\" > " rd cfr[jn] "/kvm-test-1-run.sh.out 2>&1 &" 394 print "echo ", cfr[jn], cpusr[jn] ovf ": Waiting for build to complete. `date` | tee -a " rd "log"; 395 print "while test -f " builddir ".wait" 396 print "do" 397 print "\tsleep 1" 398 print "done" 399 print "echo ", cfr[jn], cpusr[jn] ovf ": Build complete. `date` | tee -a " rd "log"; 400 jn++; 401 } 402 for (j = 1; j < jn; j++) { 403 builddir=KVM "/b" j 404 print "rm -f " builddir ".ready" 405 print "if test -f \"" rd cfr[j] "/builtkernel\"" 406 print "then" 407 print "\techo ----", cfr[j], cpusr[j] ovf ": Kernel present. `date` | tee -a " rd "log"; 408 print "\tneedqemurun=1" 409 print "fi" 410 } 411 njitter = 0; 412 split(jitter, ja); 413 if (ja[1] == -1 && ncpus == 0) 414 njitter = 1; 415 else if (ja[1] == -1) 416 njitter = ncpus; 417 else 418 njitter = ja[1]; 419 if (TORTURE_BUILDONLY && njitter != 0) { 420 njitter = 0; 421 print "echo Build-only run, so suppressing jitter | tee -a " rd "log" 422 } 423 if (TORTURE_BUILDONLY) { 424 print "needqemurun=" 425 } 426 print "if test -n \"$needqemurun\"" 427 print "then" 428 print "\techo ---- Starting kernels. `date` | tee -a " rd "log"; 429 for (j = 0; j < njitter; j++) 430 print "\tjitter.sh " j " " dur " " ja[2] " " ja[3] "&" 431 print "\twait" 432 print "\techo ---- All kernel runs complete. `date` | tee -a " rd "log"; 433 print "else" 434 print "\twait" 435 print "\techo ---- No kernel runs. `date` | tee -a " rd "log"; 436 print "fi" 437 for (j = 1; j < jn; j++) { 438 builddir=KVM "/b" j 439 print "echo ----", cfr[j], cpusr[j] ovf ": Build/run results: | tee -a " rd "log"; 440 print "cat " rd cfr[j] "/kvm-test-1-run.sh.out | tee -a " rd "log"; 441 } 442} 443 444END { 445 njobs = i; 446 nc = ncpus; 447 first = 0; 448 batchnum = 1; 449 450 # Each pass through the following loop considers one test. 451 for (i = 0; i < njobs; i++) { 452 if (ncpus == 0) { 453 # Sequential test specified, each test its own batch. 454 dump(i, i + 1, batchnum); 455 first = i; 456 batchnum++; 457 } else if (nc < cpus[i] && i != 0) { 458 # Out of CPUs, dump out a batch. 459 dump(first, i, batchnum); 460 first = i; 461 nc = ncpus; 462 batchnum++; 463 } 464 # Account for the CPUs needed by the current test. 465 nc -= cpus[i]; 466 } 467 # Dump the last batch. 468 if (ncpus != 0) 469 dump(first, i, batchnum); 470}' >> $T/script 471 472cat << ___EOF___ >> $T/script 473echo 474echo 475echo " --- `date` Test summary:" 476echo Results directory: $resdir/$ds 477kcsan-collapse.sh $resdir/$ds 478kvm-recheck.sh $resdir/$ds 479___EOF___ 480 481if test "$dryrun" = script 482then 483 cat $T/script 484 exit 0 485elif test "$dryrun" = sched 486then 487 # Extract the test run schedule from the script. 488 egrep 'Start batch|Starting build\.' $T/script | 489 grep -v ">>" | 490 sed -e 's/:.*$//' -e 's/^echo //' 491 exit 0 492else 493 # Not a dryrun, so run the script. 494 sh $T/script 495fi 496 497# Tracing: trace_event=rcu:rcu_grace_period,rcu:rcu_future_grace_period,rcu:rcu_grace_period_init,rcu:rcu_nocb_wake,rcu:rcu_preempt_task,rcu:rcu_unlock_preempted_task,rcu:rcu_quiescent_state_report,rcu:rcu_fqs,rcu:rcu_callback,rcu:rcu_kfree_callback,rcu:rcu_batch_start,rcu:rcu_invoke_callback,rcu:rcu_invoke_kfree_callback,rcu:rcu_batch_end,rcu:rcu_torture_read,rcu:rcu_barrier 498# Function-graph tracing: ftrace=function_graph ftrace_graph_filter=sched_setaffinity,migration_cpu_stop 499# Also --kconfig "CONFIG_FUNCTION_TRACER=y CONFIG_FUNCTION_GRAPH_TRACER=y" 500