xref: /openbmc/qemu/tests/qemu-iotests/check (revision 9543fdaf)
1#!/bin/bash
2#
3# Copyright (C) 2009 Red Hat, Inc.
4# Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation.
9#
10# This program is distributed in the hope that it would be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18#
19# Control script for QA
20#
21
22status=0
23needwrap=true
24try=0
25n_bad=0
26bad=""
27notrun=""
28interrupt=true
29
30# by default don't output timestamps
31timestamp=${TIMESTAMP:=false}
32
33_init_error()
34{
35    echo "check: $1" >&2
36    exit 1
37}
38
39if [ -L "$0" ]
40then
41    # called from the build tree
42    source_iotests=$(dirname "$(readlink "$0")")
43    if [ -z "$source_iotests" ]
44    then
45        _init_error "failed to obtain source tree name from check symlink"
46    fi
47    source_iotests=$(cd "$source_iotests"; pwd) || _init_error "failed to enter source tree"
48    build_iotests=$PWD
49else
50    # called from the source tree
51    source_iotests=$PWD
52    # this may be an in-tree build (note that in the following code we may not
53    # assume that it truly is and have to test whether the build results
54    # actually exist)
55    build_iotests=$PWD
56fi
57
58build_root="$build_iotests/../.."
59
60# we need common.env
61if ! . "$build_iotests/common.env"
62then
63    _init_error "failed to source common.env (make sure the qemu-iotests are run from tests/qemu-iotests in the build tree)"
64fi
65
66# we need common.config
67if ! . "$source_iotests/common.config"
68then
69    _init_error "failed to source common.config"
70fi
71
72_full_imgfmt_details()
73{
74    if [ -n "$IMGOPTS" ]; then
75        echo "$IMGFMT ($IMGOPTS)"
76    else
77        echo "$IMGFMT"
78    fi
79}
80
81_full_platform_details()
82{
83    os=$(uname -s)
84    host=$(hostname -s)
85    kernel=$(uname -r)
86    platform=$(uname -m)
87    echo "$os/$platform $host $kernel"
88}
89
90# $1 = prog to look for
91set_prog_path()
92{
93    p=$(command -v $1 2> /dev/null)
94    if [ -n "$p" -a -x "$p" ]; then
95        type -p "$p"
96    else
97        return 1
98    fi
99}
100
101if [ -z "$TEST_DIR" ]; then
102        TEST_DIR=$PWD/scratch
103fi
104
105if [ ! -e "$TEST_DIR" ]; then
106        mkdir "$TEST_DIR"
107fi
108
109diff="diff -u"
110verbose=false
111debug=false
112group=false
113xgroup=false
114imgopts=false
115showme=false
116sortme=false
117expunge=true
118have_test_arg=false
119cachemode=false
120
121tmp="${TEST_DIR}"/$$
122rm -f $tmp.list $tmp.tmp $tmp.sed
123
124export IMGFMT=raw
125export IMGFMT_GENERIC=true
126export IMGPROTO=file
127export IMGOPTS=""
128export CACHEMODE="writeback"
129export QEMU_IO_OPTIONS=""
130export QEMU_IO_OPTIONS_NO_FMT=""
131export CACHEMODE_IS_DEFAULT=true
132export QEMU_OPTIONS="-nodefaults -machine accel=qtest"
133export VALGRIND_QEMU=
134export IMGKEYSECRET=
135export IMGOPTSSYNTAX=false
136
137# Save current tty settings, since an aborting qemu call may leave things
138# screwed up
139STTY_RESTORE=
140if test -t 0; then
141    STTY_RESTORE=$(stty -g)
142fi
143
144for r
145do
146
147    if $group
148    then
149        # arg after -g
150        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
151s/ .*//p
152}')
153        if [ -z "$group_list" ]
154        then
155            echo "Group \"$r\" is empty or not defined?"
156            exit 1
157        fi
158        [ ! -s $tmp.list ] && touch $tmp.list
159        for t in $group_list
160        do
161            if grep -s "^$t\$" $tmp.list >/dev/null
162            then
163                :
164            else
165                echo "$t" >>$tmp.list
166            fi
167        done
168        group=false
169        continue
170
171    elif $xgroup
172    then
173        # arg after -x
174        # Populate $tmp.list with all tests
175        awk '/^[0-9]{3,}/ {print $1}' "${source_iotests}/group" > $tmp.list 2>/dev/null
176        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
177s/ .*//p
178}')
179        if [ -z "$group_list" ]
180        then
181            echo "Group \"$r\" is empty or not defined?"
182            exit 1
183        fi
184        numsed=0
185        rm -f $tmp.sed
186        for t in $group_list
187        do
188            if [ $numsed -gt 100 ]
189            then
190                sed -f $tmp.sed <$tmp.list >$tmp.tmp
191                mv $tmp.tmp $tmp.list
192                numsed=0
193                rm -f $tmp.sed
194            fi
195            echo "/^$t\$/d" >>$tmp.sed
196            numsed=$(expr $numsed + 1)
197        done
198        sed -f $tmp.sed <$tmp.list >$tmp.tmp
199        mv $tmp.tmp $tmp.list
200        xgroup=false
201        continue
202
203    elif $imgopts
204    then
205        IMGOPTS="$r"
206        imgopts=false
207        continue
208    elif $cachemode
209    then
210        CACHEMODE="$r"
211        CACHEMODE_IS_DEFAULT=false
212        cachemode=false
213        continue
214    fi
215
216    xpand=true
217    case "$r"
218    in
219
220        -\? | -h | --help)        # usage
221            echo "Usage: $0 [options] [testlist]"'
222
223common options
224    -v                  verbose
225    -d                  debug
226
227image format options
228    -raw                test raw (default)
229    -bochs              test bochs
230    -cloop              test cloop
231    -parallels          test parallels
232    -qcow               test qcow
233    -qcow2              test qcow2
234    -qed                test qed
235    -vdi                test vdi
236    -vpc                test vpc
237    -vhdx               test vhdx
238    -vmdk               test vmdk
239    -luks               test luks
240
241image protocol options
242    -file               test file (default)
243    -rbd                test rbd
244    -sheepdog           test sheepdog
245    -nbd                test nbd
246    -ssh                test ssh
247    -nfs                test nfs
248    -vxhs               test vxhs
249
250other options
251    -xdiff              graphical mode diff
252    -nocache            use O_DIRECT on backing file
253    -misalign           misalign memory allocations
254    -n                  show me, do not run tests
255    -o options          -o options to pass to qemu-img create/convert
256    -T                  output timestamps
257    -c mode             cache mode
258
259testlist options
260    -g group[,group...]        include tests from these groups
261    -x group[,group...]        exclude tests from these groups
262    NNN                        include test NNN
263    NNN-NNN                    include test range (eg. 012-021)
264'
265            exit 0
266            ;;
267
268        -raw)
269            IMGFMT=raw
270            xpand=false
271            ;;
272
273        -bochs)
274            IMGFMT=bochs
275            IMGFMT_GENERIC=false
276            xpand=false
277            ;;
278
279        -cloop)
280            IMGFMT=cloop
281            IMGFMT_GENERIC=false
282            xpand=false
283            ;;
284
285        -parallels)
286            IMGFMT=parallels
287            xpand=false
288            ;;
289
290        -qcow)
291            IMGFMT=qcow
292            xpand=false
293            ;;
294
295        -qcow2)
296            IMGFMT=qcow2
297            xpand=false
298            ;;
299
300        -luks)
301            IMGOPTSSYNTAX=true
302            IMGFMT=luks
303            IMGKEYSECRET=123456
304            xpand=false
305            ;;
306
307        -qed)
308            IMGFMT=qed
309            xpand=false
310            ;;
311
312        -vdi)
313            IMGFMT=vdi
314            xpand=false
315            ;;
316
317        -vmdk)
318            IMGFMT=vmdk
319            xpand=false
320            ;;
321
322        -vpc)
323            IMGFMT=vpc
324            xpand=false
325            ;;
326
327        -vhdx)
328            IMGFMT=vhdx
329            xpand=false
330            ;;
331
332        -file)
333            IMGPROTO=file
334            xpand=false
335            ;;
336
337        -rbd)
338            IMGPROTO=rbd
339            xpand=false
340            ;;
341
342        -sheepdog)
343            IMGPROTO=sheepdog
344            xpand=false
345            ;;
346
347        -nbd)
348            IMGPROTO=nbd
349            xpand=false
350            ;;
351
352        -vxhs)
353            IMGPROTO=vxhs
354            xpand=false
355            ;;
356
357        -ssh)
358            IMGPROTO=ssh
359            xpand=false
360            ;;
361
362        -nfs)
363            IMGPROTO=nfs
364            xpand=false
365            ;;
366
367        -nocache)
368            CACHEMODE="none"
369            CACHEMODE_IS_DEFAULT=false
370            xpand=false
371            ;;
372
373        -misalign)
374            QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --misalign"
375            xpand=false
376            ;;
377
378        -valgrind)
379            VALGRIND_QEMU='y'
380            xpand=false
381            ;;
382
383        -g)        # -g group ... pick from group file
384            group=true
385            xpand=false
386            ;;
387
388        -xdiff)        # graphical diff mode
389            xpand=false
390
391            if [ ! -z "$DISPLAY" ]
392            then
393                command -v xdiff >/dev/null 2>&1 && diff=xdiff
394                command -v gdiff >/dev/null 2>&1 && diff=gdiff
395                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
396                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
397            fi
398            ;;
399
400        -n)        # show me, don't do it
401            showme=true
402            xpand=false
403            ;;
404        -o)
405            imgopts=true
406            xpand=false
407            ;;
408        -c)
409            cachemode=true
410            xpand=false
411            ;;
412        -T)        # turn on timestamp output
413            timestamp=true
414            xpand=false
415            ;;
416
417        -v)
418            verbose=true
419            xpand=false
420            ;;
421        -d)
422            debug=true
423            xpand=false
424            ;;
425        -x)        # -x group ... exclude from group file
426            xgroup=true
427            xpand=false
428            ;;
429        '[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
430            echo "No tests?"
431            status=1
432            exit $status
433            ;;
434
435        [0-9]*-[0-9]*)
436            eval $(echo $r | sed -e 's/^/start=/' -e 's/-/ end=/')
437            ;;
438
439        [0-9]*-)
440            eval $(echo $r | sed -e 's/^/start=/' -e 's/-//')
441            end=$(echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/  *$//' -e 's/.* //')
442            if [ -z "$end" ]
443            then
444                echo "No tests in range \"$r\"?"
445                status=1
446                exit $status
447            fi
448            ;;
449
450        *)
451            start=$r
452            end=$r
453            ;;
454
455    esac
456
457    # get rid of leading 0s as can be interpreted as octal
458    start=$(echo $start | sed 's/^0*//')
459    end=$(echo $end | sed 's/^0*//')
460
461    if $xpand
462    then
463        have_test_arg=true
464        awk </dev/null '
465BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
466        | while read id
467        do
468            if grep -s "^$id " "$source_iotests/group" >/dev/null
469            then
470                # in group file ... OK
471                echo $id >>$tmp.list
472            else
473                if [ -f expunged ] && $expunge && egrep "^$id([         ]|\$)" expunged >/dev/null
474                then
475                    # expunged ... will be reported, but not run, later
476                    echo $id >>$tmp.list
477                else
478                    # oops
479                    if [ "$start" == "$end" -a "$id" == "$end" ]
480                    then
481                        echo "$id - unknown test"
482                        exit 1
483                    else
484                        echo "$id - unknown test, ignored"
485                    fi
486                fi
487            fi
488        done || exit 1
489    fi
490
491done
492
493# Set qemu-io cache mode with $CACHEMODE we have
494QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
495
496QEMU_IO_OPTIONS_NO_FMT="$QEMU_IO_OPTIONS"
497if [ "$IMGOPTSSYNTAX" != "true" ]; then
498    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT"
499fi
500
501# Set default options for qemu-img create -o if they were not specified
502if [ "$IMGFMT" == "qcow2" ] && ! (echo "$IMGOPTS" | grep "compat=" > /dev/null); then
503    IMGOPTS=$(_optstr_add "$IMGOPTS" "compat=1.1")
504fi
505if [ "$IMGFMT" == "luks" ] && ! (echo "$IMGOPTS" | grep "iter-time=" > /dev/null); then
506    IMGOPTS=$(_optstr_add "$IMGOPTS" "iter-time=10")
507fi
508
509if [ -z "$SAMPLE_IMG_DIR" ]; then
510        SAMPLE_IMG_DIR="$source_iotests/sample_images"
511fi
512
513export TEST_DIR
514export SAMPLE_IMG_DIR
515
516if [ -s $tmp.list ]
517then
518    # found some valid test numbers ... this is good
519    :
520else
521    if $have_test_arg
522    then
523        # had test numbers, but none in group file ... do nothing
524        touch $tmp.list
525    else
526        # no test numbers, do everything from group file
527        sed -n -e '/^[0-9][0-9][0-9]*/s/[         ].*//p' <"$source_iotests/group" >$tmp.list
528    fi
529fi
530
531# should be sort -n, but this did not work for Linux when this
532# was ported from IRIX
533#
534list=$(sort $tmp.list)
535rm -f $tmp.list $tmp.tmp $tmp.sed
536
537if [ -z "$QEMU_PROG" ]
538then
539    if [ -x "$build_iotests/qemu" ]; then
540        export QEMU_PROG="$build_iotests/qemu"
541    elif [ -x "$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" ]; then
542        export QEMU_PROG="$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}"
543    else
544        pushd "$build_root" > /dev/null
545        for binary in *-softmmu/qemu-system-*
546        do
547            if [ -x "$binary" ]
548            then
549                export QEMU_PROG="$build_root/$binary"
550                break
551            fi
552        done
553        popd > /dev/null
554        [ "$QEMU_PROG" = "" ] && _init_error "qemu not found"
555    fi
556fi
557export QEMU_PROG="$(type -p "$QEMU_PROG")"
558
559if [ -z "$QEMU_IMG_PROG" ]; then
560    if [ -x "$build_iotests/qemu-img" ]; then
561        export QEMU_IMG_PROG="$build_iotests/qemu-img"
562    elif [ -x "$build_root/qemu-img" ]; then
563        export QEMU_IMG_PROG="$build_root/qemu-img"
564    else
565        _init_error "qemu-img not found"
566    fi
567fi
568export QEMU_IMG_PROG="$(type -p "$QEMU_IMG_PROG")"
569
570if [ -z "$QEMU_IO_PROG" ]; then
571    if [ -x "$build_iotests/qemu-io" ]; then
572        export QEMU_IO_PROG="$build_iotests/qemu-io"
573    elif [ -x "$build_root/qemu-io" ]; then
574        export QEMU_IO_PROG="$build_root/qemu-io"
575    else
576        _init_error "qemu-io not found"
577    fi
578fi
579export QEMU_IO_PROG="$(type -p "$QEMU_IO_PROG")"
580
581if [ -z $QEMU_NBD_PROG ]; then
582    if [ -x "$build_iotests/qemu-nbd" ]; then
583        export QEMU_NBD_PROG="$build_iotests/qemu-nbd"
584    elif [ -x "$build_root/qemu-nbd" ]; then
585        export QEMU_NBD_PROG="$build_root/qemu-nbd"
586    else
587        _init_error "qemu-nbd not found"
588    fi
589fi
590export QEMU_NBD_PROG="$(type -p "$QEMU_NBD_PROG")"
591
592if [ -z "$QEMU_VXHS_PROG" ]; then
593    export QEMU_VXHS_PROG="$(set_prog_path qnio_server)"
594fi
595
596if [ -x "$build_iotests/socket_scm_helper" ]
597then
598    export SOCKET_SCM_HELPER="$build_iotests/socket_scm_helper"
599fi
600
601default_machine=$($QEMU_PROG -machine help | sed -n '/(default)/ s/ .*//p')
602default_alias_machine=$($QEMU_PROG -machine help | \
603   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
604if [[ "$default_alias_machine" ]]; then
605    default_machine="$default_alias_machine"
606fi
607
608export QEMU_DEFAULT_MACHINE="$default_machine"
609
610TIMESTAMP_FILE=check.time-$IMGPROTO-$IMGFMT
611
612_wallclock()
613{
614    date "+%H %M %S" | awk '{ print $1*3600 + $2*60 + $3 }'
615}
616
617_timestamp()
618{
619    now=$(date "+%T")
620    printf %s " [$now]"
621}
622
623_wrapup()
624{
625    if $showme
626    then
627        :
628    elif $needwrap
629    then
630        if [ -f $TIMESTAMP_FILE -a -f $tmp.time ]
631        then
632            cat $TIMESTAMP_FILE $tmp.time \
633            | awk '
634        { t[$1] = $2 }
635END        { if (NR > 0) {
636            for (i in t) print i " " t[i]
637          }
638        }' \
639            | sort -n >$tmp.out
640            mv $tmp.out $TIMESTAMP_FILE
641        fi
642
643        if [ -f $tmp.expunged ]
644        then
645            notrun=$(wc -l <$tmp.expunged | sed -e 's/  *//g')
646            try=$(expr $try - $notrun)
647            list=$(echo "$list" | sed -f $tmp.expunged)
648        fi
649
650        echo "" >>check.log
651        date >>check.log
652        echo $list | fmt | sed -e 's/^/    /' >>check.log
653        $interrupt && echo "Interrupted!" >>check.log
654
655        if [ ! -z "$notrun" ]
656        then
657            echo "Not run:$notrun"
658            echo "Not run:$notrun" >>check.log
659        fi
660        if [ ! -z "$n_bad" -a $n_bad != 0 ]
661        then
662            echo "Failures:$bad"
663            echo "Failed $n_bad of $try tests"
664            echo "Failures:$bad" | fmt >>check.log
665            echo "Failed $n_bad of $try tests" >>check.log
666        else
667            echo "Passed all $try tests"
668            echo "Passed all $try tests" >>check.log
669        fi
670        needwrap=false
671    fi
672
673    if test -n "$STTY_RESTORE"; then
674        stty $STTY_RESTORE
675    fi
676    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
677    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
678    rm -f $tmp.*
679}
680
681trap "_wrapup; exit \$status" 0 1 2 3 15
682
683[ -f $TIMESTAMP_FILE ] || touch $TIMESTAMP_FILE
684
685FULL_IMGFMT_DETAILS=$(_full_imgfmt_details)
686FULL_HOST_DETAILS=$(_full_platform_details)
687
688cat <<EOF
689QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
690QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
691QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
692QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
693IMGFMT        -- $FULL_IMGFMT_DETAILS
694IMGPROTO      -- $IMGPROTO
695PLATFORM      -- $FULL_HOST_DETAILS
696TEST_DIR      -- $TEST_DIR
697SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
698
699EOF
700
701seq="check"
702
703[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG
704
705for seq in $list
706do
707    err=false
708    printf %s "$seq"
709    if [ -n "$TESTS_REMAINING_LOG" ] ; then
710        sed -e "s/$seq//" -e 's/  / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
711        mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
712        sync
713    fi
714
715    if $showme
716    then
717        echo
718        continue
719    elif [ -f expunged ] && $expunge && egrep "^$seq([         ]|\$)" expunged >/dev/null
720    then
721        echo " - expunged"
722        rm -f $seq.out.bad
723        echo "/^$seq\$/d" >>$tmp.expunged
724    elif [ ! -f "$source_iotests/$seq" ]
725    then
726        echo " - no such test?"
727        echo "/^$seq\$/d" >>$tmp.expunged
728    else
729        # really going to try and run this one
730        #
731        rm -f $seq.out.bad
732        lasttime=$(sed -n -e "/^$seq /s/.* //p" <$TIMESTAMP_FILE)
733        if [ "X$lasttime" != X ]; then
734                printf %s " ${lasttime}s ..."
735        else
736                printf "        "        # prettier output with timestamps.
737        fi
738        rm -f core $seq.notrun
739
740        start=$(_wallclock)
741        $timestamp && printf %s "        [$(date "+%T")]"
742
743        if [ "$(head -n 1 "$source_iotests/$seq")" == "#!/usr/bin/env python" ]; then
744            run_command="$PYTHON $seq"
745        else
746            run_command="./$seq"
747        fi
748        export OUTPUT_DIR=$PWD
749        if $debug; then
750            (cd "$source_iotests";
751            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
752                    $run_command -d 2>&1 | tee $tmp.out)
753        else
754            (cd "$source_iotests";
755            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
756                    $run_command >$tmp.out 2>&1)
757        fi
758        sts=$?
759        $timestamp && _timestamp
760        stop=$(_wallclock)
761
762        if [ -f core ]
763        then
764            printf " [dumped core]"
765            mv core $seq.core
766            err=true
767        fi
768
769        if [ -f $seq.notrun ]
770        then
771            $timestamp || printf " [not run] "
772            $timestamp && echo " [not run]" && printf %s "        $seq -- "
773            cat $seq.notrun
774            notrun="$notrun $seq"
775        else
776            if [ $sts -ne 0 ]
777            then
778                printf %s " [failed, exit status $sts]"
779                err=true
780            fi
781
782            reference="$source_iotests/$seq.out"
783            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
784            if [ -f "$reference_machine" ]; then
785                reference="$reference_machine"
786            fi
787
788            reference_format="$source_iotests/$seq.out.$IMGFMT"
789            if [ -f "$reference_format" ]; then
790                reference="$reference_format"
791            fi
792
793            if [ "$CACHEMODE" = "none" ]; then
794                [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
795            fi
796
797            if [ ! -f "$reference" ]
798            then
799                echo " - no qualified output"
800                err=true
801            else
802                if diff -w "$reference" $tmp.out >/dev/null 2>&1
803                then
804                    echo ""
805                    if $err
806                    then
807                        :
808                    else
809                        echo "$seq $(expr $stop - $start)" >>$tmp.time
810                    fi
811                else
812                    echo " - output mismatch (see $seq.out.bad)"
813                    mv $tmp.out $seq.out.bad
814                    $diff -w "$reference" "$PWD"/$seq.out.bad
815                    err=true
816                fi
817            fi
818        fi
819
820    fi
821
822    # come here for each test, except when $showme is true
823    #
824    if $err
825    then
826        bad="$bad $seq"
827        n_bad=$(expr $n_bad + 1)
828        quick=false
829    fi
830    [ -f $seq.notrun ] || try=$(expr $try + 1)
831
832    seq="after_$seq"
833done
834
835interrupt=false
836status=$(expr $n_bad)
837exit
838