1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Run a series of torture tests, intended for overnight or
5# longer timeframes, and also for large systems.
6#
7# Usage: torture.sh [ options ]
8#
9# Copyright (C) 2020 Facebook, Inc.
10#
11# Authors: Paul E. McKenney <paulmck@kernel.org>
12
13scriptname=$0
14args="$*"
15
16KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM
17PATH=${KVM}/bin:$PATH; export PATH
18. functions.sh
19
20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
23if test "$HALF_ALLOTED_CPUS" -lt 1
24then
25	HALF_ALLOTED_CPUS=1
26fi
27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
28if test "$VERBOSE_BATCH_CPUS" -lt 2
29then
30	VERBOSE_BATCH_CPUS=0
31fi
32
33# Configurations/scenarios.
34configs_rcutorture=
35configs_locktorture=
36configs_scftorture=
37kcsan_kmake_args=
38
39# Default compression, duration, and apportionment.
40compress_kasan_vmlinux="`identify_qemu_vcpus`"
41duration_base=10
42duration_rcutorture_frac=7
43duration_locktorture_frac=1
44duration_scftorture_frac=2
45
46# "yes" or "no" parameters
47do_allmodconfig=yes
48do_rcutorture=yes
49do_locktorture=yes
50do_scftorture=yes
51do_rcuscale=yes
52do_refscale=yes
53do_kvfree=yes
54do_kasan=yes
55do_kcsan=no
56
57# doyesno - Helper function for yes/no arguments
58function doyesno () {
59	if test "$1" = "$2"
60	then
61		echo yes
62	else
63		echo no
64	fi
65}
66
67usage () {
68	echo "Usage: $scriptname optional arguments:"
69	echo "       --compress-kasan-vmlinux concurrency"
70	echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
71	echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
72	echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
73	echo "       --doall"
74	echo "       --doallmodconfig / --do-no-allmodconfig"
75	echo "       --do-kasan / --do-no-kasan"
76	echo "       --do-kcsan / --do-no-kcsan"
77	echo "       --do-kvfree / --do-no-kvfree"
78	echo "       --do-locktorture / --do-no-locktorture"
79	echo "       --do-none"
80	echo "       --do-rcuscale / --do-no-rcuscale"
81	echo "       --do-rcutorture / --do-no-rcutorture"
82	echo "       --do-refscale / --do-no-refscale"
83	echo "       --do-scftorture / --do-no-scftorture"
84	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
85	echo "       --kcsan-kmake-arg kernel-make-arguments"
86	exit 1
87}
88
89while test $# -gt 0
90do
91	case "$1" in
92	--compress-kasan-vmlinux)
93		checkarg --compress-kasan-vmlinux "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
94		compress_kasan_vmlinux=$2
95		shift
96		;;
97	--config-rcutorture|--configs-rcutorture)
98		checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
99		configs_rcutorture="$configs_rcutorture $2"
100		shift
101		;;
102	--config-locktorture|--configs-locktorture)
103		checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
104		configs_locktorture="$configs_locktorture $2"
105		shift
106		;;
107	--config-scftorture|--configs-scftorture)
108		checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
109		configs_scftorture="$configs_scftorture $2"
110		shift
111		;;
112	--doall)
113		do_allmodconfig=yes
114		do_rcutorture=yes
115		do_locktorture=yes
116		do_scftorture=yes
117		do_rcuscale=yes
118		do_refscale=yes
119		do_kvfree=yes
120		do_kasan=yes
121		do_kcsan=yes
122		;;
123	--do-allmodconfig|--do-no-allmodconfig)
124		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
125		;;
126	--do-kasan|--do-no-kasan)
127		do_kasan=`doyesno "$1" --do-kasan`
128		;;
129	--do-kcsan|--do-no-kcsan)
130		do_kcsan=`doyesno "$1" --do-kcsan`
131		;;
132	--do-kvfree|--do-no-kvfree)
133		do_kvfree=`doyesno "$1" --do-kvfree`
134		;;
135	--do-locktorture|--do-no-locktorture)
136		do_locktorture=`doyesno "$1" --do-locktorture`
137		;;
138	--do-none)
139		do_allmodconfig=no
140		do_rcutorture=no
141		do_locktorture=no
142		do_scftorture=no
143		do_rcuscale=no
144		do_refscale=no
145		do_kvfree=no
146		do_kasan=no
147		do_kcsan=no
148		;;
149	--do-rcuscale|--do-no-rcuscale)
150		do_rcuscale=`doyesno "$1" --do-rcuscale`
151		;;
152	--do-rcutorture|--do-no-rcutorture)
153		do_rcutorture=`doyesno "$1" --do-rcutorture`
154		;;
155	--do-refscale|--do-no-refscale)
156		do_refscale=`doyesno "$1" --do-refscale`
157		;;
158	--do-scftorture|--do-no-scftorture)
159		do_scftorture=`doyesno "$1" --do-scftorture`
160		;;
161	--duration)
162		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
163		mult=1
164		if echo "$2" | grep -q 'm$'
165		then
166			mult=1
167		elif echo "$2" | grep -q 'h$'
168		then
169			mult=60
170		elif echo "$2" | grep -q 'd$'
171		then
172			mult=1440
173		fi
174		ts=`echo $2 | sed -e 's/[smhd]$//'`
175		duration_base=$(($ts*mult))
176		shift
177		;;
178	--kcsan-kmake-arg|--kcsan-kmake-args)
179		checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
180		kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
181		shift
182		;;
183	*)
184		echo Unknown argument $1
185		usage
186		;;
187	esac
188	shift
189done
190
191ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
192startdate="`date`"
193starttime="`get_starttime`"
194
195T=/tmp/torture.sh.$$
196trap 'rm -rf $T' 0 2
197mkdir $T
198
199echo " --- " $scriptname $args | tee -a $T/log
200echo " --- Results directory: " $ds | tee -a $T/log
201
202# Calculate rcutorture defaults and apportion time
203if test -z "$configs_rcutorture"
204then
205	configs_rcutorture=CFLIST
206fi
207duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
208if test "$duration_rcutorture" -eq 0
209then
210	echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
211	do_rcutorture=no
212fi
213
214# Calculate locktorture defaults and apportion time
215if test -z "$configs_locktorture"
216then
217	configs_locktorture=CFLIST
218fi
219duration_locktorture=$((duration_base*duration_locktorture_frac/10))
220if test "$duration_locktorture" -eq 0
221then
222	echo " --- Zero time for locktorture, disabling" | tee -a $T/log
223	do_locktorture=no
224fi
225
226# Calculate scftorture defaults and apportion time
227if test -z "$configs_scftorture"
228then
229	configs_scftorture=CFLIST
230fi
231duration_scftorture=$((duration_base*duration_scftorture_frac/10))
232if test "$duration_scftorture" -eq 0
233then
234	echo " --- Zero time for scftorture, disabling" | tee -a $T/log
235	do_scftorture=no
236fi
237
238touch $T/failures
239touch $T/successes
240
241# torture_one - Does a single kvm.sh run.
242#
243# Usage:
244#	torture_bootargs="[ kernel boot arguments ]"
245#	torture_one flavor [ kvm.sh arguments ]
246#
247# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
248# Note that quoting is problematic.  So on the command line, pass multiple
249# values with multiple kvm.sh argument instances.
250function torture_one {
251	local cur_bootargs=
252	local boottag=
253
254	echo " --- $curflavor:" Start `date` | tee -a $T/log
255	if test -n "$torture_bootargs"
256	then
257		boottag="--bootargs"
258		cur_bootargs="$torture_bootargs"
259	fi
260	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
261	retcode=$?
262	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
263	if test -z "$resdir"
264	then
265		cat $T/$curflavor.out | tee -a $T/log
266		echo retcode=$retcode | tee -a $T/log
267	fi
268	if test "$retcode" == 0
269	then
270		echo "$curflavor($retcode)" $resdir >> $T/successes
271	else
272		echo "$curflavor($retcode)" $resdir >> $T/failures
273	fi
274}
275
276# torture_set - Does a set of tortures with and without KASAN and KCSAN.
277#
278# Usage:
279#	torture_bootargs="[ kernel boot arguments ]"
280#	torture_set flavor [ kvm.sh arguments ]
281#
282# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
283# Note that quoting is problematic.  So on the command line, pass multiple
284# values with multiple kvm.sh argument instances.
285function torture_set {
286	local cur_kcsan_kmake_args=
287	local kcsan_kmake_tag=
288	local flavor=$1
289	shift
290	curflavor=$flavor
291	torture_one "$@"
292	if test "$do_kasan" = "yes"
293	then
294		curflavor=${flavor}-kasan
295		torture_one "$@" --kasan
296	fi
297	if test "$do_kcsan" = "yes"
298	then
299		curflavor=${flavor}-kcsan
300		if test -n "$kcsan_kmake_args"
301		then
302			kcsan_kmake_tag="--kmake-args"
303			cur_kcsan_kmake_args="$kcsan_kmake_args"
304		fi
305		torture_one $* --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
306	fi
307}
308
309# make allmodconfig
310if test "$do_allmodconfig" = "yes"
311then
312	echo " --- allmodconfig:" Start `date` | tee -a $T/log
313	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
314	mkdir -p "$amcdir"
315	echo " --- make clean" > "$amcdir/Make.out" 2>&1
316	make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
317	echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1
318	make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
319	echo " --- make " >> "$amcdir/Make.out" 2>&1
320	make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
321	retcode="$?"
322	echo $retcode > "$amcdir/Make.exitcode"
323	if test "$retcode" == 0
324	then
325		echo "allmodconfig($retcode)" $amcdir >> $T/successes
326	else
327		echo "allmodconfig($retcode)" $amcdir >> $T/failures
328	fi
329fi
330
331# --torture rcu
332if test "$do_rcutorture" = "yes"
333then
334	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
335	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
336fi
337
338if test "$do_locktorture" = "yes"
339then
340	torture_bootargs="torture.disable_onoff_at_boot"
341	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
342fi
343
344if test "$do_scftorture" = "yes"
345then
346	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot"
347	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
348fi
349
350if test "$do_refscale" = yes
351then
352	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
353else
354	primlist=
355fi
356for prim in $primlist
357do
358	torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
359	torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
360done
361
362if test "$do_rcuscale" = yes
363then
364	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
365else
366	primlist=
367fi
368for prim in $primlist
369do
370	torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
371	torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
372done
373
374if test "$do_kvfree" = "yes"
375then
376	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
377	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make
378fi
379
380echo " --- " $scriptname $args
381echo " --- " Done `date` | tee -a $T/log
382ret=0
383nsuccesses=0
384echo SUCCESSES: | tee -a $T/log
385if test -s "$T/successes"
386then
387	cat "$T/successes" | tee -a $T/log
388	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
389fi
390nfailures=0
391echo FAILURES: | tee -a $T/log
392if test -s "$T/failures"
393then
394	cat "$T/failures" | tee -a $T/log
395	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
396	ret=2
397fi
398echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
399echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
400tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
401if test -n "$tdir" && test $compress_kasan_vmlinux -gt 0
402then
403	# KASAN vmlinux files can approach 1GB in size, so compress them.
404	echo Looking for KASAN files to compress: `date` > "$tdir/log-xz" 2>&1
405	find "$tdir" -type d -name '*-kasan' -print > $T/xz-todo
406	ncompresses=0
407	batchno=1
408	if test -s $T/xz-todo
409	then
410		echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
411		for i in `cat $T/xz-todo`
412		do
413			echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
414			for j in $i/*/vmlinux
415			do
416				xz "$j" >> "$tdir/log-xz" 2>&1 &
417				ncompresses=$((ncompresses+1))
418				if test $ncompresses -ge $compress_kasan_vmlinux
419				then
420					echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
421					wait
422					ncompresses=0
423					batchno=$((batchno+1))
424				fi
425			done
426		done
427		if test $ncompresses -gt 0
428		then
429			echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
430		fi
431		wait
432		echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
433		echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
434	else
435		echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
436	fi
437fi
438if test -n "$tdir"
439then
440	cp $T/log "$tdir"
441fi
442exit $ret
443