1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Run a series of torture tests, intended for overnight or
5# longer timeframes, and also for large systems.
6#
7# Usage: torture.sh [ options ]
8#
9# Copyright (C) 2020 Facebook, Inc.
10#
11# Authors: Paul E. McKenney <paulmck@kernel.org>
12
13scriptname=$0
14args="$*"
15
16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
17PATH=${RCUTORTURE}/bin:$PATH; export PATH
18. functions.sh
19
20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`"
21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2))
22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2))
23if test "$HALF_ALLOTED_CPUS" -lt 1
24then
25	HALF_ALLOTED_CPUS=1
26fi
27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16))
28if test "$VERBOSE_BATCH_CPUS" -lt 2
29then
30	VERBOSE_BATCH_CPUS=0
31fi
32
33# Configurations/scenarios.
34configs_rcutorture=
35configs_locktorture=
36configs_scftorture=
37kcsan_kmake_args=
38
39# Default compression, duration, and apportionment.
40compress_concurrency="`identify_qemu_vcpus`"
41duration_base=10
42duration_rcutorture_frac=7
43duration_locktorture_frac=1
44duration_scftorture_frac=2
45
46# "yes" or "no" parameters
47do_allmodconfig=yes
48do_rcutorture=yes
49do_locktorture=yes
50do_scftorture=yes
51do_rcuscale=yes
52do_refscale=yes
53do_kvfree=yes
54do_kasan=yes
55do_kcsan=no
56do_clocksourcewd=yes
57do_rt=yes
58
59# doyesno - Helper function for yes/no arguments
60function doyesno () {
61	if test "$1" = "$2"
62	then
63		echo yes
64	else
65		echo no
66	fi
67}
68
69usage () {
70	echo "Usage: $scriptname optional arguments:"
71	echo "       --compress-concurrency concurrency"
72	echo "       --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\""
73	echo "       --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\""
74	echo "       --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\""
75	echo "       --do-all"
76	echo "       --do-allmodconfig / --do-no-allmodconfig"
77	echo "       --do-clocksourcewd / --do-no-clocksourcewd"
78	echo "       --do-kasan / --do-no-kasan"
79	echo "       --do-kcsan / --do-no-kcsan"
80	echo "       --do-kvfree / --do-no-kvfree"
81	echo "       --do-locktorture / --do-no-locktorture"
82	echo "       --do-none"
83	echo "       --do-rcuscale / --do-no-rcuscale"
84	echo "       --do-rcutorture / --do-no-rcutorture"
85	echo "       --do-refscale / --do-no-refscale"
86	echo "       --do-rt / --do-no-rt"
87	echo "       --do-scftorture / --do-no-scftorture"
88	echo "       --duration [ <minutes> | <hours>h | <days>d ]"
89	echo "       --kcsan-kmake-arg kernel-make-arguments"
90	exit 1
91}
92
93while test $# -gt 0
94do
95	case "$1" in
96	--compress-concurrency)
97		checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error'
98		compress_concurrency=$2
99		shift
100		;;
101	--config-rcutorture|--configs-rcutorture)
102		checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
103		configs_rcutorture="$configs_rcutorture $2"
104		shift
105		;;
106	--config-locktorture|--configs-locktorture)
107		checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
108		configs_locktorture="$configs_locktorture $2"
109		shift
110		;;
111	--config-scftorture|--configs-scftorture)
112		checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--'
113		configs_scftorture="$configs_scftorture $2"
114		shift
115		;;
116	--do-all|--doall)
117		do_allmodconfig=yes
118		do_rcutorture=yes
119		do_locktorture=yes
120		do_scftorture=yes
121		do_rcuscale=yes
122		do_refscale=yes
123		do_rt=yes
124		do_kvfree=yes
125		do_kasan=yes
126		do_kcsan=yes
127		do_clocksourcewd=yes
128		;;
129	--do-allmodconfig|--do-no-allmodconfig)
130		do_allmodconfig=`doyesno "$1" --do-allmodconfig`
131		;;
132	--do-clocksourcewd|--do-no-clocksourcewd)
133		do_clocksourcewd=`doyesno "$1" --do-clocksourcewd`
134		;;
135	--do-kasan|--do-no-kasan)
136		do_kasan=`doyesno "$1" --do-kasan`
137		;;
138	--do-kcsan|--do-no-kcsan)
139		do_kcsan=`doyesno "$1" --do-kcsan`
140		;;
141	--do-kvfree|--do-no-kvfree)
142		do_kvfree=`doyesno "$1" --do-kvfree`
143		;;
144	--do-locktorture|--do-no-locktorture)
145		do_locktorture=`doyesno "$1" --do-locktorture`
146		;;
147	--do-none|--donone)
148		do_allmodconfig=no
149		do_rcutorture=no
150		do_locktorture=no
151		do_scftorture=no
152		do_rcuscale=no
153		do_refscale=no
154		do_rt=no
155		do_kvfree=no
156		do_kasan=no
157		do_kcsan=no
158		do_clocksourcewd=no
159		;;
160	--do-rcuscale|--do-no-rcuscale)
161		do_rcuscale=`doyesno "$1" --do-rcuscale`
162		;;
163	--do-rcutorture|--do-no-rcutorture)
164		do_rcutorture=`doyesno "$1" --do-rcutorture`
165		;;
166	--do-refscale|--do-no-refscale)
167		do_refscale=`doyesno "$1" --do-refscale`
168		;;
169	--do-rt|--do-no-rt)
170		do_rt=`doyesno "$1" --do-rt`
171		;;
172	--do-scftorture|--do-no-scftorture)
173		do_scftorture=`doyesno "$1" --do-scftorture`
174		;;
175	--duration)
176		checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error'
177		mult=1
178		if echo "$2" | grep -q 'm$'
179		then
180			mult=1
181		elif echo "$2" | grep -q 'h$'
182		then
183			mult=60
184		elif echo "$2" | grep -q 'd$'
185		then
186			mult=1440
187		fi
188		ts=`echo $2 | sed -e 's/[smhd]$//'`
189		duration_base=$(($ts*mult))
190		shift
191		;;
192	--kcsan-kmake-arg|--kcsan-kmake-args)
193		checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$'
194		kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`"
195		shift
196		;;
197	*)
198		echo Unknown argument $1
199		usage
200		;;
201	esac
202	shift
203done
204
205ds="`date +%Y.%m.%d-%H.%M.%S`-torture"
206startdate="`date`"
207starttime="`get_starttime`"
208
209T="`mktemp -d ${TMPDIR-/tmp}/torture.sh.XXXXXX`"
210trap 'rm -rf $T' 0 2
211
212echo " --- " $scriptname $args | tee -a $T/log
213echo " --- Results directory: " $ds | tee -a $T/log
214
215# Calculate rcutorture defaults and apportion time
216if test -z "$configs_rcutorture"
217then
218	configs_rcutorture=CFLIST
219fi
220duration_rcutorture=$((duration_base*duration_rcutorture_frac/10))
221if test "$duration_rcutorture" -eq 0
222then
223	echo " --- Zero time for rcutorture, disabling" | tee -a $T/log
224	do_rcutorture=no
225fi
226
227# Calculate locktorture defaults and apportion time
228if test -z "$configs_locktorture"
229then
230	configs_locktorture=CFLIST
231fi
232duration_locktorture=$((duration_base*duration_locktorture_frac/10))
233if test "$duration_locktorture" -eq 0
234then
235	echo " --- Zero time for locktorture, disabling" | tee -a $T/log
236	do_locktorture=no
237fi
238
239# Calculate scftorture defaults and apportion time
240if test -z "$configs_scftorture"
241then
242	configs_scftorture=CFLIST
243fi
244duration_scftorture=$((duration_base*duration_scftorture_frac/10))
245if test "$duration_scftorture" -eq 0
246then
247	echo " --- Zero time for scftorture, disabling" | tee -a $T/log
248	do_scftorture=no
249fi
250
251touch $T/failures
252touch $T/successes
253
254# torture_one - Does a single kvm.sh run.
255#
256# Usage:
257#	torture_bootargs="[ kernel boot arguments ]"
258#	torture_one flavor [ kvm.sh arguments ]
259#
260# Note that "flavor" is an arbitrary string.  Supply --torture if needed.
261# Note that quoting is problematic.  So on the command line, pass multiple
262# values with multiple kvm.sh argument instances.
263function torture_one {
264	local cur_bootargs=
265	local boottag=
266
267	echo " --- $curflavor:" Start `date` | tee -a $T/log
268	if test -n "$torture_bootargs"
269	then
270		boottag="--bootargs"
271		cur_bootargs="$torture_bootargs"
272	fi
273	"$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1
274	retcode=$?
275	resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`"
276	if test -z "$resdir"
277	then
278		cat $T/$curflavor.out | tee -a $T/log
279		echo retcode=$retcode | tee -a $T/log
280	else
281		echo $resdir > $T/last-resdir
282	fi
283	if test "$retcode" == 0
284	then
285		echo "$curflavor($retcode)" $resdir >> $T/successes
286	else
287		echo "$curflavor($retcode)" $resdir >> $T/failures
288	fi
289}
290
291# torture_set - Does a set of tortures with and without KASAN and KCSAN.
292#
293# Usage:
294#	torture_bootargs="[ kernel boot arguments ]"
295#	torture_set flavor [ kvm.sh arguments ]
296#
297# Note that "flavor" is an arbitrary string that does not affect kvm.sh
298# in any way.  So also supply --torture if you need something other than
299# the default.
300function torture_set {
301	local cur_kcsan_kmake_args=
302	local kcsan_kmake_tag=
303	local flavor=$1
304	shift
305	curflavor=$flavor
306	torture_one "$@"
307	mv $T/last-resdir $T/last-resdir-nodebug || :
308	if test "$do_kasan" = "yes"
309	then
310		curflavor=${flavor}-kasan
311		torture_one "$@" --kasan
312		mv $T/last-resdir $T/last-resdir-kasan || :
313	fi
314	if test "$do_kcsan" = "yes"
315	then
316		curflavor=${flavor}-kcsan
317		if test -n "$kcsan_kmake_args"
318		then
319			kcsan_kmake_tag="--kmake-args"
320			cur_kcsan_kmake_args="$kcsan_kmake_args"
321		fi
322		torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan
323		mv $T/last-resdir $T/last-resdir-kcsan || :
324	fi
325}
326
327# make allmodconfig
328if test "$do_allmodconfig" = "yes"
329then
330	echo " --- allmodconfig:" Start `date` | tee -a $T/log
331	amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig"
332	mkdir -p "$amcdir"
333	echo " --- make clean" | tee $amcdir/log > "$amcdir/Make.out" 2>&1
334	make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1
335	retcode=$?
336	buildphase='"make clean"'
337	if test "$retcode" -eq 0
338	then
339		echo " --- make allmodconfig" | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
340		cp .config $amcdir
341		make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1
342		retcode=$?
343		buildphase='"make allmodconfig"'
344	fi
345	if test "$retcode" -eq 0
346	then
347		echo " --- make " | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1
348		make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1
349		retcode="$?"
350		echo $retcode > "$amcdir/Make.exitcode"
351		buildphase='"make"'
352	fi
353	if test "$retcode" -eq 0
354	then
355		echo "allmodconfig($retcode)" $amcdir >> $T/successes
356		echo Success >> $amcdir/log
357	else
358		echo "allmodconfig($retcode)" $amcdir >> $T/failures
359		echo " --- allmodconfig Test summary:" >> $amcdir/log
360		echo " --- Summary: Exit code $retcode from $buildphase, see Make.out" >> $amcdir/log
361	fi
362fi
363
364# --torture rcu
365if test "$do_rcutorture" = "yes"
366then
367	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000"
368	torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make
369fi
370
371if test "$do_locktorture" = "yes"
372then
373	torture_bootargs="torture.disable_onoff_at_boot"
374	torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make
375fi
376
377if test "$do_scftorture" = "yes"
378then
379	torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1"
380	torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
381fi
382
383if test "$do_rt" = "yes"
384then
385	# With all post-boot grace periods forced to normal.
386	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_normal=1"
387	torture_set "rcurttorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
388
389	# With all post-boot grace periods forced to expedited.
390	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcupdate.rcu_expedited=1"
391	torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make
392fi
393
394if test "$do_refscale" = yes
395then
396	primlist="`grep '\.name[ 	]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
397else
398	primlist=
399fi
400firsttime=1
401do_kasan_save="$do_kasan"
402do_kcsan_save="$do_kcsan"
403for prim in $primlist
404do
405	if test -n "$firsttime"
406	then
407		torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot"
408		torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make
409		mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
410		if test -f "$T/last-resdir-kasan"
411		then
412			mv $T/last-resdir-kasan $T/first-resdir-kasan || :
413		fi
414		if test -f "$T/last-resdir-kcsan"
415		then
416			mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
417		fi
418		firsttime=
419		do_kasan=
420		do_kcsan=
421	else
422		torture_bootargs=
423		for i in $T/first-resdir-*
424		do
425			case "$i" in
426			*-nodebug)
427				torture_suffix=
428				;;
429			*-kasan)
430				torture_suffix="-kasan"
431				;;
432			*-kcsan)
433				torture_suffix="-kcsan"
434				;;
435			esac
436			torture_set "refscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "refscale.scale_type=$prim"
437		done
438	fi
439done
440do_kasan="$do_kasan_save"
441do_kcsan="$do_kcsan_save"
442
443if test "$do_rcuscale" = yes
444then
445	primlist="`grep '\.name[ 	]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`"
446else
447	primlist=
448fi
449firsttime=1
450do_kasan_save="$do_kasan"
451do_kcsan_save="$do_kcsan"
452for prim in $primlist
453do
454	if test -n "$firsttime"
455	then
456		torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot"
457		torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make
458		mv $T/last-resdir-nodebug $T/first-resdir-nodebug || :
459		if test -f "$T/last-resdir-kasan"
460		then
461			mv $T/last-resdir-kasan $T/first-resdir-kasan || :
462		fi
463		if test -f "$T/last-resdir-kcsan"
464		then
465			mv $T/last-resdir-kcsan $T/first-resdir-kcsan || :
466		fi
467		firsttime=
468		do_kasan=
469		do_kcsan=
470	else
471		torture_bootargs=
472		for i in $T/first-resdir-*
473		do
474			case "$i" in
475			*-nodebug)
476				torture_suffix=
477				;;
478			*-kasan)
479				torture_suffix="-kasan"
480				;;
481			*-kcsan)
482				torture_suffix="-kcsan"
483				;;
484			esac
485			torture_set "rcuscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "rcuscale.scale_type=$prim"
486		done
487	fi
488done
489do_kasan="$do_kasan_save"
490do_kcsan="$do_kcsan_save"
491
492if test "$do_kvfree" = "yes"
493then
494	torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot"
495	torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make
496fi
497
498if test "$do_clocksourcewd" = "yes"
499then
500	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
501	torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
502
503	torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 clocksource.max_cswd_read_retries=1 tsc=watchdog"
504	torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make
505
506	# In case our work is already done...
507	if test "$do_rcutorture" != "yes"
508	then
509		torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog"
510		torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make
511	fi
512fi
513
514echo " --- " $scriptname $args
515echo " --- " Done `date` | tee -a $T/log
516ret=0
517nsuccesses=0
518echo SUCCESSES: | tee -a $T/log
519if test -s "$T/successes"
520then
521	cat "$T/successes" | tee -a $T/log
522	nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`"
523fi
524nfailures=0
525echo FAILURES: | tee -a $T/log
526if test -s "$T/failures"
527then
528	awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/  /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum"
529	nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`"
530	grep "^  Summary: " "$T/failuresum" |
531		grep -v '^  Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan"
532	if test -s "$T/nonkcsan"
533	then
534		nonkcsanbug="yes"
535	fi
536	ret=2
537fi
538if test "$do_kcsan" = "yes"
539then
540	TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum
541fi
542echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log
543echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log
544if test -z "$nonkcsanbug" && test -s "$T/failuresum"
545then
546	echo "  All bugs were KCSAN failures."
547fi
548tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`"
549if test -n "$tdir" && test $compress_concurrency -gt 0
550then
551	# KASAN vmlinux files can approach 1GB in size, so compress them.
552	echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1
553	find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo-all
554	find "$tdir" -type f -name 're-run' -print | sed -e 's,/re-run,,' |
555		grep -e '-k[ac]san$' > $T/xz-todo-copy
556	sort $T/xz-todo-all $T/xz-todo-copy | uniq -u > $T/xz-todo
557	ncompresses=0
558	batchno=1
559	if test -s $T/xz-todo
560	then
561		for i in `cat $T/xz-todo`
562		do
563			find $i -name 'vmlinux*' -print
564		done | wc -l | awk '{ print $1 }' > $T/xz-todo-count
565		n2compress="`cat $T/xz-todo-count`"
566		echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
567		for i in `cat $T/xz-todo`
568		do
569			echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1
570			for j in $i/*/vmlinux
571			do
572				xz "$j" >> "$tdir/log-xz" 2>&1 &
573				ncompresses=$((ncompresses+1))
574				if test $ncompresses -ge $compress_concurrency
575				then
576					echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
577					wait
578					ncompresses=0
579					batchno=$((batchno+1))
580				fi
581			done
582		done
583		if test $ncompresses -gt 0
584		then
585			echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log
586		fi
587		wait
588		if test -s $T/xz-todo-copy
589		then
590			# The trick here is that we need corresponding
591			# vmlinux files from corresponding scenarios.
592			echo Linking vmlinux.xz files to re-use scenarios `date` | tee -a "$tdir/log-xz" | tee -a $T/log
593			dirstash="`pwd`"
594			for i in `cat $T/xz-todo-copy`
595			do
596				cd $i
597				find . -name vmlinux -print > $T/xz-todo-copy-vmlinux
598				for v in `cat $T/xz-todo-copy-vmlinux`
599				do
600					rm -f "$v"
601					cp -l `cat $i/re-run`/"$i/$v".xz "`dirname "$v"`"
602				done
603				cd "$dirstash"
604			done
605		fi
606		echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log
607		echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log
608	else
609		echo No compression needed: `date` >> "$tdir/log-xz" 2>&1
610	fi
611fi
612if test -n "$tdir"
613then
614	cp $T/log "$tdir"
615fi
616exit $ret
617