1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of torture tests, intended for overnight or 5# longer timeframes, and also for large systems. 6# 7# Usage: torture.sh [ options ] 8# 9# Copyright (C) 2020 Facebook, Inc. 10# 11# Authors: Paul E. McKenney <paulmck@kernel.org> 12 13scriptname=$0 14args="$*" 15 16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE 17PATH=${RCUTORTURE}/bin:$PATH; export PATH 18. functions.sh 19 20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2)) 22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2)) 23if test "$HALF_ALLOTED_CPUS" -lt 1 24then 25 HALF_ALLOTED_CPUS=1 26fi 27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16)) 28if test "$VERBOSE_BATCH_CPUS" -lt 2 29then 30 VERBOSE_BATCH_CPUS=0 31fi 32 33# Configurations/scenarios. 34configs_rcutorture= 35configs_locktorture= 36configs_scftorture= 37kcsan_kmake_args= 38 39# Default compression, duration, and apportionment. 40compress_concurrency="`identify_qemu_vcpus`" 41duration_base=10 42duration_rcutorture_frac=7 43duration_locktorture_frac=1 44duration_scftorture_frac=2 45 46# "yes" or "no" parameters 47do_allmodconfig=yes 48do_rcutorture=yes 49do_locktorture=yes 50do_scftorture=yes 51do_rcuscale=yes 52do_refscale=yes 53do_kvfree=yes 54do_kasan=yes 55do_kcsan=no 56do_clocksourcewd=yes 57 58# doyesno - Helper function for yes/no arguments 59function doyesno () { 60 if test "$1" = "$2" 61 then 62 echo yes 63 else 64 echo no 65 fi 66} 67 68usage () { 69 echo "Usage: $scriptname optional arguments:" 70 echo " --compress-concurrency concurrency" 71 echo " --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\"" 72 echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\"" 73 echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\"" 74 echo " --do-all" 75 echo " --do-allmodconfig / --do-no-allmodconfig" 76 echo " --do-clocksourcewd / --do-no-clocksourcewd" 77 echo " --do-kasan / --do-no-kasan" 78 echo " --do-kcsan / --do-no-kcsan" 79 echo " --do-kvfree / --do-no-kvfree" 80 echo " --do-locktorture / --do-no-locktorture" 81 echo " --do-none" 82 echo " --do-rcuscale / --do-no-rcuscale" 83 echo " --do-rcutorture / --do-no-rcutorture" 84 echo " --do-refscale / --do-no-refscale" 85 echo " --do-scftorture / --do-no-scftorture" 86 echo " --duration [ <minutes> | <hours>h | <days>d ]" 87 echo " --kcsan-kmake-arg kernel-make-arguments" 88 exit 1 89} 90 91while test $# -gt 0 92do 93 case "$1" in 94 --compress-concurrency) 95 checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error' 96 compress_concurrency=$2 97 shift 98 ;; 99 --config-rcutorture|--configs-rcutorture) 100 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 101 configs_rcutorture="$configs_rcutorture $2" 102 shift 103 ;; 104 --config-locktorture|--configs-locktorture) 105 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 106 configs_locktorture="$configs_locktorture $2" 107 shift 108 ;; 109 --config-scftorture|--configs-scftorture) 110 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 111 configs_scftorture="$configs_scftorture $2" 112 shift 113 ;; 114 --do-all|--doall) 115 do_allmodconfig=yes 116 do_rcutorture=yes 117 do_locktorture=yes 118 do_scftorture=yes 119 do_rcuscale=yes 120 do_refscale=yes 121 do_kvfree=yes 122 do_kasan=yes 123 do_kcsan=yes 124 do_clocksourcewd=yes 125 ;; 126 --do-allmodconfig|--do-no-allmodconfig) 127 do_allmodconfig=`doyesno "$1" --do-allmodconfig` 128 ;; 129 --do-clocksourcewd|--do-no-clocksourcewd) 130 do_clocksourcewd=`doyesno "$1" --do-clocksourcewd` 131 ;; 132 --do-kasan|--do-no-kasan) 133 do_kasan=`doyesno "$1" --do-kasan` 134 ;; 135 --do-kcsan|--do-no-kcsan) 136 do_kcsan=`doyesno "$1" --do-kcsan` 137 ;; 138 --do-kvfree|--do-no-kvfree) 139 do_kvfree=`doyesno "$1" --do-kvfree` 140 ;; 141 --do-locktorture|--do-no-locktorture) 142 do_locktorture=`doyesno "$1" --do-locktorture` 143 ;; 144 --do-none|--donone) 145 do_allmodconfig=no 146 do_rcutorture=no 147 do_locktorture=no 148 do_scftorture=no 149 do_rcuscale=no 150 do_refscale=no 151 do_kvfree=no 152 do_kasan=no 153 do_kcsan=no 154 do_clocksourcewd=no 155 ;; 156 --do-rcuscale|--do-no-rcuscale) 157 do_rcuscale=`doyesno "$1" --do-rcuscale` 158 ;; 159 --do-rcutorture|--do-no-rcutorture) 160 do_rcutorture=`doyesno "$1" --do-rcutorture` 161 ;; 162 --do-refscale|--do-no-refscale) 163 do_refscale=`doyesno "$1" --do-refscale` 164 ;; 165 --do-scftorture|--do-no-scftorture) 166 do_scftorture=`doyesno "$1" --do-scftorture` 167 ;; 168 --duration) 169 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' 170 mult=1 171 if echo "$2" | grep -q 'm$' 172 then 173 mult=1 174 elif echo "$2" | grep -q 'h$' 175 then 176 mult=60 177 elif echo "$2" | grep -q 'd$' 178 then 179 mult=1440 180 fi 181 ts=`echo $2 | sed -e 's/[smhd]$//'` 182 duration_base=$(($ts*mult)) 183 shift 184 ;; 185 --kcsan-kmake-arg|--kcsan-kmake-args) 186 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 187 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`" 188 shift 189 ;; 190 *) 191 echo Unknown argument $1 192 usage 193 ;; 194 esac 195 shift 196done 197 198ds="`date +%Y.%m.%d-%H.%M.%S`-torture" 199startdate="`date`" 200starttime="`get_starttime`" 201 202T=/tmp/torture.sh.$$ 203trap 'rm -rf $T' 0 2 204mkdir $T 205 206echo " --- " $scriptname $args | tee -a $T/log 207echo " --- Results directory: " $ds | tee -a $T/log 208 209# Calculate rcutorture defaults and apportion time 210if test -z "$configs_rcutorture" 211then 212 configs_rcutorture=CFLIST 213fi 214duration_rcutorture=$((duration_base*duration_rcutorture_frac/10)) 215if test "$duration_rcutorture" -eq 0 216then 217 echo " --- Zero time for rcutorture, disabling" | tee -a $T/log 218 do_rcutorture=no 219fi 220 221# Calculate locktorture defaults and apportion time 222if test -z "$configs_locktorture" 223then 224 configs_locktorture=CFLIST 225fi 226duration_locktorture=$((duration_base*duration_locktorture_frac/10)) 227if test "$duration_locktorture" -eq 0 228then 229 echo " --- Zero time for locktorture, disabling" | tee -a $T/log 230 do_locktorture=no 231fi 232 233# Calculate scftorture defaults and apportion time 234if test -z "$configs_scftorture" 235then 236 configs_scftorture=CFLIST 237fi 238duration_scftorture=$((duration_base*duration_scftorture_frac/10)) 239if test "$duration_scftorture" -eq 0 240then 241 echo " --- Zero time for scftorture, disabling" | tee -a $T/log 242 do_scftorture=no 243fi 244 245touch $T/failures 246touch $T/successes 247 248# torture_one - Does a single kvm.sh run. 249# 250# Usage: 251# torture_bootargs="[ kernel boot arguments ]" 252# torture_one flavor [ kvm.sh arguments ] 253# 254# Note that "flavor" is an arbitrary string. Supply --torture if needed. 255# Note that quoting is problematic. So on the command line, pass multiple 256# values with multiple kvm.sh argument instances. 257function torture_one { 258 local cur_bootargs= 259 local boottag= 260 261 echo " --- $curflavor:" Start `date` | tee -a $T/log 262 if test -n "$torture_bootargs" 263 then 264 boottag="--bootargs" 265 cur_bootargs="$torture_bootargs" 266 fi 267 "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1 268 retcode=$? 269 resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`" 270 if test -z "$resdir" 271 then 272 cat $T/$curflavor.out | tee -a $T/log 273 echo retcode=$retcode | tee -a $T/log 274 fi 275 if test "$retcode" == 0 276 then 277 echo "$curflavor($retcode)" $resdir >> $T/successes 278 else 279 echo "$curflavor($retcode)" $resdir >> $T/failures 280 fi 281} 282 283# torture_set - Does a set of tortures with and without KASAN and KCSAN. 284# 285# Usage: 286# torture_bootargs="[ kernel boot arguments ]" 287# torture_set flavor [ kvm.sh arguments ] 288# 289# Note that "flavor" is an arbitrary string that does not affect kvm.sh 290# in any way. So also supply --torture if you need something other than 291# the default. 292function torture_set { 293 local cur_kcsan_kmake_args= 294 local kcsan_kmake_tag= 295 local flavor=$1 296 shift 297 curflavor=$flavor 298 torture_one "$@" 299 if test "$do_kasan" = "yes" 300 then 301 curflavor=${flavor}-kasan 302 torture_one "$@" --kasan 303 fi 304 if test "$do_kcsan" = "yes" 305 then 306 curflavor=${flavor}-kcsan 307 if test -n "$kcsan_kmake_args" 308 then 309 kcsan_kmake_tag="--kmake-args" 310 cur_kcsan_kmake_args="$kcsan_kmake_args" 311 fi 312 torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan 313 fi 314} 315 316# make allmodconfig 317if test "$do_allmodconfig" = "yes" 318then 319 echo " --- allmodconfig:" Start `date` | tee -a $T/log 320 amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig" 321 mkdir -p "$amcdir" 322 echo " --- make clean" > "$amcdir/Make.out" 2>&1 323 make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1 324 echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1 325 make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1 326 echo " --- make " >> "$amcdir/Make.out" 2>&1 327 make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1 328 retcode="$?" 329 echo $retcode > "$amcdir/Make.exitcode" 330 if test "$retcode" == 0 331 then 332 echo "allmodconfig($retcode)" $amcdir >> $T/successes 333 else 334 echo "allmodconfig($retcode)" $amcdir >> $T/failures 335 fi 336fi 337 338# --torture rcu 339if test "$do_rcutorture" = "yes" 340then 341 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000" 342 torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make 343fi 344 345if test "$do_locktorture" = "yes" 346then 347 torture_bootargs="torture.disable_onoff_at_boot" 348 torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make 349fi 350 351if test "$do_scftorture" = "yes" 352then 353 torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot" 354 torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make 355fi 356 357if test "$do_refscale" = yes 358then 359 primlist="`grep '\.name[ ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 360else 361 primlist= 362fi 363for prim in $primlist 364do 365 torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot" 366 torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make 367done 368 369if test "$do_rcuscale" = yes 370then 371 primlist="`grep '\.name[ ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 372else 373 primlist= 374fi 375for prim in $primlist 376do 377 torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot" 378 torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make 379done 380 381if test "$do_kvfree" = "yes" 382then 383 torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" 384 torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make 385fi 386 387if test "$do_clocksourcewd" = "yes" 388then 389 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000" 390 torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make 391 392 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1" 393 torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make 394 395 # In case our work is already done... 396 if test "$do_rcutorture" != "yes" 397 then 398 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000" 399 torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make 400 fi 401fi 402 403echo " --- " $scriptname $args 404echo " --- " Done `date` | tee -a $T/log 405ret=0 406nsuccesses=0 407echo SUCCESSES: | tee -a $T/log 408if test -s "$T/successes" 409then 410 cat "$T/successes" | tee -a $T/log 411 nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`" 412fi 413nfailures=0 414echo FAILURES: | tee -a $T/log 415if test -s "$T/failures" 416then 417 awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/ /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum" 418 nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`" 419 grep "^ Summary: " "$T/failuresum" | 420 grep -v '^ Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan" 421 if test -s "$T/nonkcsan" 422 then 423 nonkcsanbug="yes" 424 fi 425 ret=2 426fi 427if test "$do_kcsan" = "yes" 428then 429 TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum 430fi 431echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log 432echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log 433if test -z "$nonkcsanbug" && test -s "$T/failuresum" 434then 435 echo " All bugs were KCSAN failures." 436fi 437tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" 438if test -n "$tdir" && test $compress_concurrency -gt 0 439then 440 # KASAN vmlinux files can approach 1GB in size, so compress them. 441 echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1 442 find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo 443 ncompresses=0 444 batchno=1 445 if test -s $T/xz-todo 446 then 447 for i in `cat $T/xz-todo` 448 do 449 find $i -name 'vmlinux*' -print 450 done | wc -l | awk '{ print $1 }' > $T/xz-todo-count 451 n2compress="`cat $T/xz-todo-count`" 452 echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 453 for i in `cat $T/xz-todo` 454 do 455 echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1 456 for j in $i/*/vmlinux 457 do 458 xz "$j" >> "$tdir/log-xz" 2>&1 & 459 ncompresses=$((ncompresses+1)) 460 if test $ncompresses -ge $compress_concurrency 461 then 462 echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 463 wait 464 ncompresses=0 465 batchno=$((batchno+1)) 466 fi 467 done 468 done 469 if test $ncompresses -gt 0 470 then 471 echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 472 fi 473 wait 474 echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 475 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log 476 else 477 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1 478 fi 479fi 480if test -n "$tdir" 481then 482 cp $T/log "$tdir" 483fi 484exit $ret 485