1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Run a series of torture tests, intended for overnight or
5# longer timeframes, and also for large systems.
6#
7# Usage: torture.sh [ options ]
8#
9# Copyright (C) 2020 Facebook, Inc.
10#
11# Authors: Paul E. McKenney <paulmck@kernel.org>
12
13scriptname=$0
14args="$*"
15
16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
17PATH=${RCUTORTURE}/bin:$PATH; export PATH
18. functions.sh
19
20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
23if test "$HALF_ALLOTED_CPUS" -lt 1
24then
25	HALF_ALLOTED_CPUS=1
26fi
27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
28if test "$VERBOSE_BATCH_CPUS" -lt 2
29then
30	VERBOSE_BATCH_CPUS=0
31fi
32
33# Configurations/scenarios.
34configs_rcutorture=
35configs_locktorture=
36configs_scftorture=
37kcsan_kmake_args=
38
39# Default compression, duration, and apportionment.
40compress_concurrency="`identify_qemu_vcpus`"
41duration_base=10
42duration_rcutorture_frac=7
43duration_locktorture_frac=1
44duration_scftorture_frac=2
45
46# "yes" or "no" parameters
47do_allmodconfig=yes
48do_rcutorture=yes
49do_locktorture=yes
50do_scftorture=yes
51do_rcuscale=yes
52do_refscale=yes
53do_kvfree=yes
54do_kasan=yes
55do_kcsan=no
56do_clocksourcewd=yes
57do_rt=yes
58do_rcutasksflavors=yes
59do_srcu_lockdep=yes
60
61# doyesno - Helper function for yes/no arguments
62function doyesno () {
63	if test "$1" = "$2"
64	then
65		echo yes
66	else
67		echo no
68	fi
69}
70
71usage () {
72	echo "Usage: $scriptname optional arguments:"
73	echo "       --compress-concurrency concurrency"
74	echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
75	echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
76	echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
77	echo "       --do-all"
78	echo "       --do-allmodconfig / --do-no-allmodconfig / --no-allmodconfig"
79	echo "       --do-clocksourcewd / --do-no-clocksourcewd / --no-clocksourcewd"
80	echo "       --do-kasan / --do-no-kasan / --no-kasan"
81	echo "       --do-kcsan / --do-no-kcsan / --no-kcsan"
82	echo "       --do-kvfree / --do-no-kvfree / --no-kvfree"
83	echo "       --do-locktorture / --do-no-locktorture / --no-locktorture"
84	echo "       --do-none"
85	echo "       --do-rcuscale / --do-no-rcuscale / --no-rcuscale"
86	echo "       --do-rcutasksflavors / --do-no-rcutasksflavors / --no-rcutasksflavors"
87	echo "       --do-rcutorture / --do-no-rcutorture / --no-rcutorture"
88	echo "       --do-refscale / --do-no-refscale / --no-refscale"
89	echo "       --do-rt / --do-no-rt / --no-rt"
90	echo "       --do-scftorture / --do-no-scftorture / --no-scftorture"
91	echo "       --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep"
92	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
93	echo "       --kcsan-kmake-arg kernel-make-arguments"
94	exit 1
95}
96
97while test $# -gt 0
98do
99	case "$1" in
100	--compress-concurrency)
101		checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
102		compress_concurrency=$2
103		shift
104		;;
105	--config-rcutorture|--configs-rcutorture)
106		checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
107		configs_rcutorture="$configs_rcutorture $2"
108		shift
109		;;
110	--config-locktorture|--configs-locktorture)
111		checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
112		configs_locktorture="$configs_locktorture $2"
113		shift
114		;;
115	--config-scftorture|--configs-scftorture)
116		checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
117		configs_scftorture="$configs_scftorture $2"
118		shift
119		;;
120	--do-all|--doall)
121		do_allmodconfig=yes
122		do_rcutasksflavor=yes
123		do_rcutorture=yes
124		do_locktorture=yes
125		do_scftorture=yes
126		do_rcuscale=yes
127		do_refscale=yes
128		do_rt=yes
129		do_kvfree=yes
130		do_kasan=yes
131		do_kcsan=yes
132		do_clocksourcewd=yes
133		do_srcu_lockdep=yes
134		;;
135	--do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig)
136		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
137		;;
138	--do-clocksourcewd|--do-no-clocksourcewd|--no-clocksourcewd)
139		do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
140		;;
141	--do-kasan|--do-no-kasan|--no-kasan)
142		do_kasan=`doyesno "$1" --do-kasan`
143		;;
144	--do-kcsan|--do-no-kcsan|--no-kcsan)
145		do_kcsan=`doyesno "$1" --do-kcsan`
146		;;
147	--do-kvfree|--do-no-kvfree|--no-kvfree)
148		do_kvfree=`doyesno "$1" --do-kvfree`
149		;;
150	--do-locktorture|--do-no-locktorture|--no-locktorture)
151		do_locktorture=`doyesno "$1" --do-locktorture`
152		;;
153	--do-none|--donone)
154		do_allmodconfig=no
155		do_rcutasksflavors=no
156		do_rcutorture=no
157		do_locktorture=no
158		do_scftorture=no
159		do_rcuscale=no
160		do_refscale=no
161		do_rt=no
162		do_kvfree=no
163		do_kasan=no
164		do_kcsan=no
165		do_clocksourcewd=no
166		do_srcu_lockdep=no
167		;;
168	--do-rcuscale|--do-no-rcuscale|--no-rcuscale)
169		do_rcuscale=`doyesno "$1" --do-rcuscale`
170		;;
171	--do-rcutasksflavors|--do-no-rcutasksflavors|--no-rcutasksflavors)
172		do_rcutasksflavors=`doyesno "$1" --do-rcutasksflavors`
173		;;
174	--do-rcutorture|--do-no-rcutorture|--no-rcutorture)
175		do_rcutorture=`doyesno "$1" --do-rcutorture`
176		;;
177	--do-refscale|--do-no-refscale|--no-refscale)
178		do_refscale=`doyesno "$1" --do-refscale`
179		;;
180	--do-rt|--do-no-rt|--no-rt)
181		do_rt=`doyesno "$1" --do-rt`
182		;;
183	--do-scftorture|--do-no-scftorture|--no-scftorture)
184		do_scftorture=`doyesno "$1" --do-scftorture`
185		;;
186	--do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep)
187		do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep`
188		;;
189	--duration)
190		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
191		mult=1
192		if echo "$2" | grep -q 'm$'
193		then
194			mult=1
195		elif echo "$2" | grep -q 'h$'
196		then
197			mult=60
198		elif echo "$2" | grep -q 'd$'
199		then
200			mult=1440
201		fi
202		ts=`echo $2 | sed -e 's/[smhd]$//'`
203		duration_base=$(($ts*mult))
204		shift
205		;;
206	--kcsan-kmake-arg|--kcsan-kmake-args)
207		checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
208		kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
209		shift
210		;;
211	*)
212		echo Unknown argument $1
213		usage
214		;;
215	esac
216	shift
217done
218
219ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
220startdate="`date`"
221starttime="`get_starttime`"
222
223T="`mktemp -d ${TMPDIR-/tmp}/torture.sh.XXXXXX`"
224trap 'rm -rf $T' 0 2
225
226echo " --- " $scriptname $args | tee -a $T/log
227echo " --- Results directory: " $ds | tee -a $T/log
228
229# Calculate rcutorture defaults and apportion time
230if test -z "$configs_rcutorture"
231then
232	configs_rcutorture=CFLIST
233fi
234duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
235if test "$duration_rcutorture" -eq 0
236then
237	echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
238	do_rcutorture=no
239fi
240
241# Calculate locktorture defaults and apportion time
242if test -z "$configs_locktorture"
243then
244	configs_locktorture=CFLIST
245fi
246duration_locktorture=$((duration_base*duration_locktorture_frac/10))
247if test "$duration_locktorture" -eq 0
248then
249	echo " --- Zero time for locktorture, disabling" | tee -a $T/log
250	do_locktorture=no
251fi
252
253# Calculate scftorture defaults and apportion time
254if test -z "$configs_scftorture"
255then
256	configs_scftorture=CFLIST
257fi
258duration_scftorture=$((duration_base*duration_scftorture_frac/10))
259if test "$duration_scftorture" -eq 0
260then
261	echo " --- Zero time for scftorture, disabling" | tee -a $T/log
262	do_scftorture=no
263fi
264
265touch $T/failures
266touch $T/successes
267
268# torture_one - Does a single kvm.sh run.
269#
270# Usage:
271#	torture_bootargs="[ kernel boot arguments ]"
272#	torture_one flavor [ kvm.sh arguments ]
273#
274# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
275# Note that quoting is problematic.  So on the command line, pass multiple
276# values with multiple kvm.sh argument instances.
277function torture_one {
278	local cur_bootargs=
279	local boottag=
280
281	echo " --- $curflavor:" Start `date` | tee -a $T/log
282	if test -n "$torture_bootargs"
283	then
284		boottag="--bootargs"
285		cur_bootargs="$torture_bootargs"
286	fi
287	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
288	retcode=$?
289	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
290	if test -z "$resdir"
291	then
292		cat $T/$curflavor.out | tee -a $T/log
293		echo retcode=$retcode | tee -a $T/log
294	else
295		echo $resdir > $T/last-resdir
296	fi
297	if test "$retcode" == 0
298	then
299		echo "$curflavor($retcode)" $resdir >> $T/successes
300	else
301		echo "$curflavor($retcode)" $resdir >> $T/failures
302	fi
303}
304
305# torture_set - Does a set of tortures with and without KASAN and KCSAN.
306#
307# Usage:
308#	torture_bootargs="[ kernel boot arguments ]"
309#	torture_set flavor [ kvm.sh arguments ]
310#
311# Note that "flavor" is an arbitrary string that does not affect kvm.sh
312# in any way.  So also supply --torture if you need something other than
313# the default.
314function torture_set {
315	local cur_kcsan_kmake_args=
316	local kcsan_kmake_tag=
317	local flavor=$1
318	shift
319	curflavor=$flavor
320	torture_one "$@"
321	mv $T/last-resdir $T/last-resdir-nodebug || :
322	if test "$do_kasan" = "yes"
323	then
324		curflavor=${flavor}-kasan
325		torture_one "$@" --kasan
326		mv $T/last-resdir $T/last-resdir-kasan || :
327	fi
328	if test "$do_kcsan" = "yes"
329	then
330		curflavor=${flavor}-kcsan
331		if test -n "$kcsan_kmake_args"
332		then
333			kcsan_kmake_tag="--kmake-args"
334			cur_kcsan_kmake_args="$kcsan_kmake_args"
335		fi
336		torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
337		mv $T/last-resdir $T/last-resdir-kcsan || :
338	fi
339}
340
341# make allmodconfig
342if test "$do_allmodconfig" = "yes"
343then
344	echo " --- allmodconfig:" Start `date` | tee -a $T/log
345	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
346	mkdir -p "$amcdir"
347	echo " --- make clean" | tee $amcdir/log > "$amcdir/Make.out" 2>&1
348	make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
349	retcode=$?
350	buildphase='"make clean"'
351	if test "$retcode" -eq 0
352	then
353		echo " --- make allmodconfig" | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
354		cp .config $amcdir
355		make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
356		retcode=$?
357		buildphase='"make allmodconfig"'
358	fi
359	if test "$retcode" -eq 0
360	then
361		echo " --- make " | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
362		make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
363		retcode="$?"
364		echo $retcode > "$amcdir/Make.exitcode"
365		buildphase='"make"'
366	fi
367	if test "$retcode" -eq 0
368	then
369		echo "allmodconfig($retcode)" $amcdir >> $T/successes
370		echo Success >> $amcdir/log
371	else
372		echo "allmodconfig($retcode)" $amcdir >> $T/failures
373		echo " --- allmodconfig Test summary:" >> $amcdir/log
374		echo " --- Summary: Exit code $retcode from $buildphase, see Make.out" >> $amcdir/log
375	fi
376fi
377
378# Test building RCU Tasks flavors in isolation, both SMP and !SMP
379if test "$do_rcutasksflavors" = "yes"
380then
381	echo " --- rcutasksflavors:" Start `date` | tee -a $T/log
382	rtfdir="tools/testing/selftests/rcutorture/res/$ds/results-rcutasksflavors"
383	mkdir -p "$rtfdir"
384	cat > $T/rcutasksflavors << __EOF__
385#CHECK#CONFIG_TASKS_RCU=n
386#CHECK#CONFIG_TASKS_RUDE_RCU=n
387#CHECK#CONFIG_TASKS_TRACE_RCU=n
388__EOF__
389	for flavor in CONFIG_TASKS_RCU CONFIG_TASKS_RUDE_RCU CONFIG_TASKS_TRACE_RCU
390	do
391		forceflavor="`echo $flavor | sed -e 's/^CONFIG/CONFIG_FORCE/'`"
392		deselectedflavors="`grep -v $flavor $T/rcutasksflavors | tr '\012' ' ' | tr -s ' ' | sed -e 's/ *$//'`"
393		echo " --- Running RCU Tasks Trace flavor $flavor `date`" >> $rtfdir/log
394		tools/testing/selftests/rcutorture/bin/kvm.sh --datestamp "$ds/results-rcutasksflavors/$flavor" --buildonly --configs "TINY01 TREE04" --kconfig "CONFIG_RCU_EXPERT=y CONFIG_RCU_SCALE_TEST=y $forceflavor=y $deselectedflavors" --trust-make > $T/$flavor.out 2>&1
395		retcode=$?
396		if test "$retcode" -ne 0
397		then
398			break
399		fi
400	done
401	if test "$retcode" -eq 0
402	then
403		echo "rcutasksflavors($retcode)" $rtfdir >> $T/successes
404		echo Success >> $rtfdir/log
405	else
406		echo "rcutasksflavors($retcode)" $rtfdir >> $T/failures
407		echo " --- rcutasksflavors Test summary:" >> $rtfdir/log
408		echo " --- Summary: Exit code $retcode from $flavor, see Make.out" >> $rtfdir/log
409	fi
410fi
411
412# --torture rcu
413if test "$do_rcutorture" = "yes"
414then
415	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
416	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
417fi
418
419if test "$do_locktorture" = "yes"
420then
421	torture_bootargs="torture.disable_onoff_at_boot"
422	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
423fi
424
425if test "$do_scftorture" = "yes"
426then
427	# Scale memory based on the number of CPUs.
428	scfmem=$((2+HALF_ALLOTED_CPUS/16))
429	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1"
430	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory ${scfmem}G --trust-make
431fi
432
433if test "$do_rt" = "yes"
434then
435	# With all post-boot grace periods forced to normal.
436	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_normal=1"
437	torture_set "rcurttorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
438
439	# With all post-boot grace periods forced to expedited.
440	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_expedited=1"
441	torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
442fi
443
444if test "$do_srcu_lockdep" = "yes"
445then
446	echo " --- do-srcu-lockdep:" Start `date` | tee -a $T/log
447	tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh --datestamp "$ds/results-srcu-lockdep" > $T/srcu_lockdep.sh.out 2>&1
448	retcode=$?
449	cp $T/srcu_lockdep.sh.out "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
450	if test "$retcode" -eq 0
451	then
452		echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/successes
453		echo Success >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
454	else
455		echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/failures
456		echo " --- srcu_lockdep Test Summary:" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
457		echo " --- Summary: Exit code $retcode from srcu_lockdep.sh, see ds/results-srcu-lockdep" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log"
458	fi
459fi
460
461if test "$do_refscale" = yes
462then
463	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
464else
465	primlist=
466fi
467firsttime=1
468do_kasan_save="$do_kasan"
469do_kcsan_save="$do_kcsan"
470for prim in $primlist
471do
472	if test -n "$firsttime"
473	then
474		torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
475		torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
476		mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
477		if test -f "$T/last-resdir-kasan"
478		then
479			mv $T/last-resdir-kasan $T/first-resdir-kasan || :
480		fi
481		if test -f "$T/last-resdir-kcsan"
482		then
483			mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
484		fi
485		firsttime=
486		do_kasan=
487		do_kcsan=
488	else
489		torture_bootargs=
490		for i in $T/first-resdir-*
491		do
492			case "$i" in
493			*-nodebug)
494				torture_suffix=
495				;;
496			*-kasan)
497				torture_suffix="-kasan"
498				;;
499			*-kcsan)
500				torture_suffix="-kcsan"
501				;;
502			esac
503			torture_set "refscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "refscale.scale_type=$prim"
504		done
505	fi
506done
507do_kasan="$do_kasan_save"
508do_kcsan="$do_kcsan_save"
509
510if test "$do_rcuscale" = yes
511then
512	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
513else
514	primlist=
515fi
516firsttime=1
517do_kasan_save="$do_kasan"
518do_kcsan_save="$do_kcsan"
519for prim in $primlist
520do
521	if test -n "$firsttime"
522	then
523		torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
524		torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
525		mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
526		if test -f "$T/last-resdir-kasan"
527		then
528			mv $T/last-resdir-kasan $T/first-resdir-kasan || :
529		fi
530		if test -f "$T/last-resdir-kcsan"
531		then
532			mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
533		fi
534		firsttime=
535		do_kasan=
536		do_kcsan=
537	else
538		torture_bootargs=
539		for i in $T/first-resdir-*
540		do
541			case "$i" in
542			*-nodebug)
543				torture_suffix=
544				;;
545			*-kasan)
546				torture_suffix="-kasan"
547				;;
548			*-kcsan)
549				torture_suffix="-kcsan"
550				;;
551			esac
552			torture_set "rcuscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "rcuscale.scale_type=$prim"
553		done
554	fi
555done
556do_kasan="$do_kasan_save"
557do_kcsan="$do_kcsan_save"
558
559if test "$do_kvfree" = "yes"
560then
561	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
562	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
563fi
564
565if test "$do_clocksourcewd" = "yes"
566then
567	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
568	torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
569
570	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1 tsc=watchdog"
571	torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
572
573	# In case our work is already done...
574	if test "$do_rcutorture" != "yes"
575	then
576		torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
577		torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
578	fi
579fi
580
581echo " --- " $scriptname $args
582echo " --- " Done `date` | tee -a $T/log
583ret=0
584nsuccesses=0
585echo SUCCESSES: | tee -a $T/log
586if test -s "$T/successes"
587then
588	cat "$T/successes" | tee -a $T/log
589	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
590fi
591nfailures=0
592echo FAILURES: | tee -a $T/log
593if test -s "$T/failures"
594then
595	awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/  /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum"
596	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
597	grep "^  Summary: " "$T/failuresum" |
598		grep -v '^  Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan"
599	if test -s "$T/nonkcsan"
600	then
601		nonkcsanbug="yes"
602	fi
603	ret=2
604fi
605if test "$do_kcsan" = "yes"
606then
607	TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum
608fi
609echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
610echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
611tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
612find "$tdir" -name 'ConfigFragment.diags' -print > $T/configerrors
613find "$tdir" -name 'Make.out.diags' -print > $T/builderrors
614if test -s "$T/configerrors"
615then
616	echo "  Scenarios with .config errors: `wc -l "$T/configerrors" | awk '{ print $1 }'`"
617	nonkcsanbug="yes"
618fi
619if test -s "$T/builderrors"
620then
621	echo "  Scenarios with build errors: `wc -l "$T/builderrors" | awk '{ print $1 }'`"
622	nonkcsanbug="yes"
623fi
624if test -z "$nonkcsanbug" && test -s "$T/failuresum"
625then
626	echo "  All bugs were KCSAN failures."
627fi
628if test -n "$tdir" && test $compress_concurrency -gt 0
629then
630	# KASAN vmlinux files can approach 1GB in size, so compress them.
631	echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1
632	find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo-all
633	find "$tdir" -type f -name 're-run' -print | sed -e 's,/re-run,,' |
634		grep -e '-k[ac]san$' > $T/xz-todo-copy
635	sort $T/xz-todo-all $T/xz-todo-copy | uniq -u > $T/xz-todo
636	ncompresses=0
637	batchno=1
638	if test -s $T/xz-todo
639	then
640		for i in `cat $T/xz-todo`
641		do
642			find $i -name 'vmlinux*' -print
643		done | wc -l | awk '{ print $1 }' > $T/xz-todo-count
644		n2compress="`cat $T/xz-todo-count`"
645		echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
646		for i in `cat $T/xz-todo`
647		do
648			echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
649			for j in $i/*/vmlinux
650			do
651				xz "$j" >> "$tdir/log-xz" 2>&1 &
652				ncompresses=$((ncompresses+1))
653				if test $ncompresses -ge $compress_concurrency
654				then
655					echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
656					wait
657					ncompresses=0
658					batchno=$((batchno+1))
659				fi
660			done
661		done
662		if test $ncompresses -gt 0
663		then
664			echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
665		fi
666		wait
667		if test -s $T/xz-todo-copy
668		then
669			# The trick here is that we need corresponding
670			# vmlinux files from corresponding scenarios.
671			echo Linking vmlinux.xz files to re-use scenarios `date` | tee -a "$tdir/log-xz" | tee -a $T/log
672			dirstash="`pwd`"
673			for i in `cat $T/xz-todo-copy`
674			do
675				cd $i
676				find . -name vmlinux -print > $T/xz-todo-copy-vmlinux
677				for v in `cat $T/xz-todo-copy-vmlinux`
678				do
679					rm -f "$v"
680					cp -l `cat $i/re-run`/"$i/$v".xz "`dirname "$v"`"
681				done
682				cd "$dirstash"
683			done
684		fi
685		echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
686		echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
687	else
688		echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
689	fi
690fi
691if test -n "$tdir"
692then
693	cp $T/log "$tdir"
694fi
695exit $ret
696