1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Run a series of torture tests, intended for overnight or
5# longer timeframes, and also for large systems.
6#
7# Usage: torture.sh [ options ]
8#
9# Copyright (C) 2020 Facebook, Inc.
10#
11# Authors: Paul E. McKenney <paulmck@kernel.org>
12
13scriptname=$0
14args="$*"
15
16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
17PATH=${RCUTORTURE}/bin:$PATH; export PATH
18. functions.sh
19
20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
23if test "$HALF_ALLOTED_CPUS" -lt 1
24then
25	HALF_ALLOTED_CPUS=1
26fi
27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
28if test "$VERBOSE_BATCH_CPUS" -lt 2
29then
30	VERBOSE_BATCH_CPUS=0
31fi
32
33# Configurations/scenarios.
34configs_rcutorture=
35configs_locktorture=
36configs_scftorture=
37kcsan_kmake_args=
38
39# Default compression, duration, and apportionment.
40compress_concurrency="`identify_qemu_vcpus`"
41duration_base=10
42duration_rcutorture_frac=7
43duration_locktorture_frac=1
44duration_scftorture_frac=2
45
46# "yes" or "no" parameters
47do_allmodconfig=yes
48do_rcutorture=yes
49do_locktorture=yes
50do_scftorture=yes
51do_rcuscale=yes
52do_refscale=yes
53do_kvfree=yes
54do_kasan=yes
55do_kcsan=no
56do_clocksourcewd=yes
57
58# doyesno - Helper function for yes/no arguments
59function doyesno () {
60	if test "$1" = "$2"
61	then
62		echo yes
63	else
64		echo no
65	fi
66}
67
68usage () {
69	echo "Usage: $scriptname optional arguments:"
70	echo "       --compress-concurrency concurrency"
71	echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
72	echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
73	echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
74	echo "       --do-all"
75	echo "       --do-allmodconfig / --do-no-allmodconfig"
76	echo "       --do-clocksourcewd / --do-no-clocksourcewd"
77	echo "       --do-kasan / --do-no-kasan"
78	echo "       --do-kcsan / --do-no-kcsan"
79	echo "       --do-kvfree / --do-no-kvfree"
80	echo "       --do-locktorture / --do-no-locktorture"
81	echo "       --do-none"
82	echo "       --do-rcuscale / --do-no-rcuscale"
83	echo "       --do-rcutorture / --do-no-rcutorture"
84	echo "       --do-refscale / --do-no-refscale"
85	echo "       --do-scftorture / --do-no-scftorture"
86	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
87	echo "       --kcsan-kmake-arg kernel-make-arguments"
88	exit 1
89}
90
91while test $# -gt 0
92do
93	case "$1" in
94	--compress-concurrency)
95		checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
96		compress_concurrency=$2
97		shift
98		;;
99	--config-rcutorture|--configs-rcutorture)
100		checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
101		configs_rcutorture="$configs_rcutorture $2"
102		shift
103		;;
104	--config-locktorture|--configs-locktorture)
105		checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
106		configs_locktorture="$configs_locktorture $2"
107		shift
108		;;
109	--config-scftorture|--configs-scftorture)
110		checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
111		configs_scftorture="$configs_scftorture $2"
112		shift
113		;;
114	--do-all|--doall)
115		do_allmodconfig=yes
116		do_rcutorture=yes
117		do_locktorture=yes
118		do_scftorture=yes
119		do_rcuscale=yes
120		do_refscale=yes
121		do_kvfree=yes
122		do_kasan=yes
123		do_kcsan=yes
124		do_clocksourcewd=yes
125		;;
126	--do-allmodconfig|--do-no-allmodconfig)
127		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
128		;;
129	--do-clocksourcewd|--do-no-clocksourcewd)
130		do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
131		;;
132	--do-kasan|--do-no-kasan)
133		do_kasan=`doyesno "$1" --do-kasan`
134		;;
135	--do-kcsan|--do-no-kcsan)
136		do_kcsan=`doyesno "$1" --do-kcsan`
137		;;
138	--do-kvfree|--do-no-kvfree)
139		do_kvfree=`doyesno "$1" --do-kvfree`
140		;;
141	--do-locktorture|--do-no-locktorture)
142		do_locktorture=`doyesno "$1" --do-locktorture`
143		;;
144	--do-none|--donone)
145		do_allmodconfig=no
146		do_rcutorture=no
147		do_locktorture=no
148		do_scftorture=no
149		do_rcuscale=no
150		do_refscale=no
151		do_kvfree=no
152		do_kasan=no
153		do_kcsan=no
154		do_clocksourcewd=no
155		;;
156	--do-rcuscale|--do-no-rcuscale)
157		do_rcuscale=`doyesno "$1" --do-rcuscale`
158		;;
159	--do-rcutorture|--do-no-rcutorture)
160		do_rcutorture=`doyesno "$1" --do-rcutorture`
161		;;
162	--do-refscale|--do-no-refscale)
163		do_refscale=`doyesno "$1" --do-refscale`
164		;;
165	--do-scftorture|--do-no-scftorture)
166		do_scftorture=`doyesno "$1" --do-scftorture`
167		;;
168	--duration)
169		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
170		mult=1
171		if echo "$2" | grep -q 'm$'
172		then
173			mult=1
174		elif echo "$2" | grep -q 'h$'
175		then
176			mult=60
177		elif echo "$2" | grep -q 'd$'
178		then
179			mult=1440
180		fi
181		ts=`echo $2 | sed -e 's/[smhd]$//'`
182		duration_base=$(($ts*mult))
183		shift
184		;;
185	--kcsan-kmake-arg|--kcsan-kmake-args)
186		checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
187		kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
188		shift
189		;;
190	*)
191		echo Unknown argument $1
192		usage
193		;;
194	esac
195	shift
196done
197
198ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
199startdate="`date`"
200starttime="`get_starttime`"
201
202T=/tmp/torture.sh.$$
203trap 'rm -rf $T' 0 2
204mkdir $T
205
206echo " --- " $scriptname $args | tee -a $T/log
207echo " --- Results directory: " $ds | tee -a $T/log
208
209# Calculate rcutorture defaults and apportion time
210if test -z "$configs_rcutorture"
211then
212	configs_rcutorture=CFLIST
213fi
214duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
215if test "$duration_rcutorture" -eq 0
216then
217	echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
218	do_rcutorture=no
219fi
220
221# Calculate locktorture defaults and apportion time
222if test -z "$configs_locktorture"
223then
224	configs_locktorture=CFLIST
225fi
226duration_locktorture=$((duration_base*duration_locktorture_frac/10))
227if test "$duration_locktorture" -eq 0
228then
229	echo " --- Zero time for locktorture, disabling" | tee -a $T/log
230	do_locktorture=no
231fi
232
233# Calculate scftorture defaults and apportion time
234if test -z "$configs_scftorture"
235then
236	configs_scftorture=CFLIST
237fi
238duration_scftorture=$((duration_base*duration_scftorture_frac/10))
239if test "$duration_scftorture" -eq 0
240then
241	echo " --- Zero time for scftorture, disabling" | tee -a $T/log
242	do_scftorture=no
243fi
244
245touch $T/failures
246touch $T/successes
247
248# torture_one - Does a single kvm.sh run.
249#
250# Usage:
251#	torture_bootargs="[ kernel boot arguments ]"
252#	torture_one flavor [ kvm.sh arguments ]
253#
254# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
255# Note that quoting is problematic.  So on the command line, pass multiple
256# values with multiple kvm.sh argument instances.
257function torture_one {
258	local cur_bootargs=
259	local boottag=
260
261	echo " --- $curflavor:" Start `date` | tee -a $T/log
262	if test -n "$torture_bootargs"
263	then
264		boottag="--bootargs"
265		cur_bootargs="$torture_bootargs"
266	fi
267	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
268	retcode=$?
269	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
270	if test -z "$resdir"
271	then
272		cat $T/$curflavor.out | tee -a $T/log
273		echo retcode=$retcode | tee -a $T/log
274	fi
275	if test "$retcode" == 0
276	then
277		echo "$curflavor($retcode)" $resdir >> $T/successes
278	else
279		echo "$curflavor($retcode)" $resdir >> $T/failures
280	fi
281}
282
283# torture_set - Does a set of tortures with and without KASAN and KCSAN.
284#
285# Usage:
286#	torture_bootargs="[ kernel boot arguments ]"
287#	torture_set flavor [ kvm.sh arguments ]
288#
289# Note that "flavor" is an arbitrary string that does not affect kvm.sh
290# in any way.  So also supply --torture if you need something other than
291# the default.
292function torture_set {
293	local cur_kcsan_kmake_args=
294	local kcsan_kmake_tag=
295	local flavor=$1
296	shift
297	curflavor=$flavor
298	torture_one "$@"
299	if test "$do_kasan" = "yes"
300	then
301		curflavor=${flavor}-kasan
302		torture_one "$@" --kasan
303	fi
304	if test "$do_kcsan" = "yes"
305	then
306		curflavor=${flavor}-kcsan
307		if test -n "$kcsan_kmake_args"
308		then
309			kcsan_kmake_tag="--kmake-args"
310			cur_kcsan_kmake_args="$kcsan_kmake_args"
311		fi
312		torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
313	fi
314}
315
316# make allmodconfig
317if test "$do_allmodconfig" = "yes"
318then
319	echo " --- allmodconfig:" Start `date` | tee -a $T/log
320	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
321	mkdir -p "$amcdir"
322	echo " --- make clean" > "$amcdir/Make.out" 2>&1
323	make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
324	echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1
325	make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
326	echo " --- make " >> "$amcdir/Make.out" 2>&1
327	make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
328	retcode="$?"
329	echo $retcode > "$amcdir/Make.exitcode"
330	if test "$retcode" == 0
331	then
332		echo "allmodconfig($retcode)" $amcdir >> $T/successes
333	else
334		echo "allmodconfig($retcode)" $amcdir >> $T/failures
335	fi
336fi
337
338# --torture rcu
339if test "$do_rcutorture" = "yes"
340then
341	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
342	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
343fi
344
345if test "$do_locktorture" = "yes"
346then
347	torture_bootargs="torture.disable_onoff_at_boot"
348	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
349fi
350
351if test "$do_scftorture" = "yes"
352then
353	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot"
354	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make
355fi
356
357if test "$do_refscale" = yes
358then
359	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
360else
361	primlist=
362fi
363for prim in $primlist
364do
365	torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
366	torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
367done
368
369if test "$do_rcuscale" = yes
370then
371	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
372else
373	primlist=
374fi
375for prim in $primlist
376do
377	torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
378	torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
379done
380
381if test "$do_kvfree" = "yes"
382then
383	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
384	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make
385fi
386
387if test "$do_clocksourcewd" = "yes"
388then
389	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
390	torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
391
392	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1"
393	torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
394
395	# In case our work is already done...
396	if test "$do_rcutorture" != "yes"
397	then
398		torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
399		torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
400	fi
401fi
402
403echo " --- " $scriptname $args
404echo " --- " Done `date` | tee -a $T/log
405ret=0
406nsuccesses=0
407echo SUCCESSES: | tee -a $T/log
408if test -s "$T/successes"
409then
410	cat "$T/successes" | tee -a $T/log
411	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
412fi
413nfailures=0
414echo FAILURES: | tee -a $T/log
415if test -s "$T/failures"
416then
417	awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/  /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum"
418	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
419	grep "^  Summary: " "$T/failuresum" |
420		grep -v '^  Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan"
421	if test -s "$T/nonkcsan"
422	then
423		nonkcsanbug="yes"
424	fi
425	ret=2
426fi
427if test "$do_kcsan" = "yes"
428then
429	TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum
430fi
431echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
432echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
433if test -z "$nonkcsanbug" && test -s "$T/failuresum"
434then
435	echo "  All bugs were KCSAN failures."
436fi
437tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
438if test -n "$tdir" && test $compress_concurrency -gt 0
439then
440	# KASAN vmlinux files can approach 1GB in size, so compress them.
441	echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1
442	find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo
443	ncompresses=0
444	batchno=1
445	if test -s $T/xz-todo
446	then
447		for i in `cat $T/xz-todo`
448		do
449			find $i -name 'vmlinux*' -print
450		done | wc -l | awk '{ print $1 }' > $T/xz-todo-count
451		n2compress="`cat $T/xz-todo-count`"
452		echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
453		for i in `cat $T/xz-todo`
454		do
455			echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
456			for j in $i/*/vmlinux
457			do
458				xz "$j" >> "$tdir/log-xz" 2>&1 &
459				ncompresses=$((ncompresses+1))
460				if test $ncompresses -ge $compress_concurrency
461				then
462					echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
463					wait
464					ncompresses=0
465					batchno=$((batchno+1))
466				fi
467			done
468		done
469		if test $ncompresses -gt 0
470		then
471			echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
472		fi
473		wait
474		echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
475		echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
476	else
477		echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
478	fi
479fi
480if test -n "$tdir"
481then
482	cp $T/log "$tdir"
483fi
484exit $ret
485