xref: /openbmc/qemu/tests/qemu-iotests/check (revision dc5bd18f)
1#!/bin/bash
2#
3# Copyright (C) 2009 Red Hat, Inc.
4# Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation.
9#
10# This program is distributed in the hope that it would be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18#
19# Control script for QA
20#
21
22status=0
23needwrap=true
24try=0
25n_bad=0
26bad=""
27notrun=""
28interrupt=true
29
30# by default don't output timestamps
31timestamp=${TIMESTAMP:=false}
32
33_init_error()
34{
35    echo "check: $1" >&2
36    exit 1
37}
38
39if [ -L "$0" ]
40then
41    # called from the build tree
42    source_iotests=$(dirname "$(readlink "$0")")
43    if [ -z "$source_iotests" ]
44    then
45        _init_error "failed to obtain source tree name from check symlink"
46    fi
47    source_iotests=$(cd "$source_iotests"; pwd) || _init_error "failed to enter source tree"
48    build_iotests=$PWD
49else
50    # called from the source tree
51    source_iotests=$PWD
52    # this may be an in-tree build (note that in the following code we may not
53    # assume that it truly is and have to test whether the build results
54    # actually exist)
55    build_iotests=$PWD
56fi
57
58build_root="$build_iotests/../.."
59
60# we need common.env
61if ! . "$build_iotests/common.env"
62then
63    _init_error "failed to source common.env (make sure the qemu-iotests are run from tests/qemu-iotests in the build tree)"
64fi
65
66# we need common.config
67if ! . "$source_iotests/common.config"
68then
69    _init_error "failed to source common.config"
70fi
71
72_full_imgfmt_details()
73{
74    if [ -n "$IMGOPTS" ]; then
75        echo "$IMGFMT ($IMGOPTS)"
76    else
77        echo "$IMGFMT"
78    fi
79}
80
81_full_platform_details()
82{
83    os=`uname -s`
84    host=`hostname -s`
85    kernel=`uname -r`
86    platform=`uname -m`
87    echo "$os/$platform $host $kernel"
88}
89
90# $1 = prog to look for
91set_prog_path()
92{
93    p=`command -v $1 2> /dev/null`
94    if [ -n "$p" -a -x "$p" ]; then
95        realpath -- "$(type -p "$p")"
96    else
97        return 1
98    fi
99}
100
101if [ -z "$TEST_DIR" ]; then
102        TEST_DIR=`pwd`/scratch
103fi
104
105if [ ! -e "$TEST_DIR" ]; then
106        mkdir "$TEST_DIR"
107fi
108
109diff="diff -u"
110verbose=false
111debug=false
112group=false
113xgroup=false
114imgopts=false
115showme=false
116sortme=false
117expunge=true
118have_test_arg=false
119cachemode=false
120
121tmp="${TEST_DIR}"/$$
122rm -f $tmp.list $tmp.tmp $tmp.sed
123
124export IMGFMT=raw
125export IMGFMT_GENERIC=true
126export IMGPROTO=file
127export IMGOPTS=""
128export CACHEMODE="writeback"
129export QEMU_IO_OPTIONS=""
130export QEMU_IO_OPTIONS_NO_FMT=""
131export CACHEMODE_IS_DEFAULT=true
132export QEMU_OPTIONS="-nodefaults -machine accel=qtest"
133export VALGRIND_QEMU=
134export IMGKEYSECRET=
135export IMGOPTSSYNTAX=false
136
137# Save current tty settings, since an aborting qemu call may leave things
138# screwed up
139STTY_RESTORE=
140if test -t 0; then
141    STTY_RESTORE=$(stty -g)
142fi
143
144for r
145do
146
147    if $group
148    then
149        # arg after -g
150        group_list=`sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
151s/ .*//p
152}'`
153        if [ -z "$group_list" ]
154        then
155            echo "Group \"$r\" is empty or not defined?"
156            exit 1
157        fi
158        [ ! -s $tmp.list ] && touch $tmp.list
159        for t in $group_list
160        do
161            if grep -s "^$t\$" $tmp.list >/dev/null
162            then
163                :
164            else
165                echo "$t" >>$tmp.list
166            fi
167        done
168        group=false
169        continue
170
171    elif $xgroup
172    then
173        # arg after -x
174        # Populate $tmp.list with all tests
175        awk '/^[0-9]{3,}/ {print $1}' "${source_iotests}/group" > $tmp.list 2>/dev/null
176        group_list=`sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
177s/ .*//p
178}'`
179        if [ -z "$group_list" ]
180        then
181            echo "Group \"$r\" is empty or not defined?"
182            exit 1
183        fi
184        numsed=0
185        rm -f $tmp.sed
186        for t in $group_list
187        do
188            if [ $numsed -gt 100 ]
189            then
190                sed -f $tmp.sed <$tmp.list >$tmp.tmp
191                mv $tmp.tmp $tmp.list
192                numsed=0
193                rm -f $tmp.sed
194            fi
195            echo "/^$t\$/d" >>$tmp.sed
196            numsed=`expr $numsed + 1`
197        done
198        sed -f $tmp.sed <$tmp.list >$tmp.tmp
199        mv $tmp.tmp $tmp.list
200        xgroup=false
201        continue
202
203    elif $imgopts
204    then
205        IMGOPTS="$r"
206        imgopts=false
207        continue
208    elif $cachemode
209    then
210        CACHEMODE="$r"
211        CACHEMODE_IS_DEFAULT=false
212        cachemode=false
213        continue
214    fi
215
216    xpand=true
217    case "$r"
218    in
219
220        -\? | -h | --help)        # usage
221            echo "Usage: $0 [options] [testlist]"'
222
223common options
224    -v                  verbose
225    -d                  debug
226
227image format options
228    -raw                test raw (default)
229    -bochs              test bochs
230    -cloop              test cloop
231    -parallels          test parallels
232    -qcow               test qcow
233    -qcow2              test qcow2
234    -qed                test qed
235    -vdi                test vdi
236    -vpc                test vpc
237    -vhdx               test vhdx
238    -vmdk               test vmdk
239    -luks               test luks
240
241image protocol options
242    -file               test file (default)
243    -rbd                test rbd
244    -sheepdog           test sheepdog
245    -nbd                test nbd
246    -ssh                test ssh
247    -nfs                test nfs
248    -vxhs               test vxhs
249
250other options
251    -xdiff              graphical mode diff
252    -nocache            use O_DIRECT on backing file
253    -misalign           misalign memory allocations
254    -n                  show me, do not run tests
255    -o options          -o options to pass to qemu-img create/convert
256    -T                  output timestamps
257    -c mode             cache mode
258
259testlist options
260    -g group[,group...]        include tests from these groups
261    -x group[,group...]        exclude tests from these groups
262    NNN                        include test NNN
263    NNN-NNN                    include test range (eg. 012-021)
264'
265            exit 0
266            ;;
267
268        -raw)
269            IMGFMT=raw
270            xpand=false
271            ;;
272
273        -bochs)
274            IMGFMT=bochs
275            IMGFMT_GENERIC=false
276            xpand=false
277            ;;
278
279        -cloop)
280            IMGFMT=cloop
281            IMGFMT_GENERIC=false
282            xpand=false
283            ;;
284
285        -parallels)
286            IMGFMT=parallels
287            IMGFMT_GENERIC=false
288            xpand=false
289            ;;
290
291        -qcow)
292            IMGFMT=qcow
293            xpand=false
294            ;;
295
296        -qcow2)
297            IMGFMT=qcow2
298            xpand=false
299            ;;
300
301        -luks)
302            IMGOPTSSYNTAX=true
303            IMGFMT=luks
304            IMGKEYSECRET=123456
305            xpand=false
306            ;;
307
308        -qed)
309            IMGFMT=qed
310            xpand=false
311            ;;
312
313        -vdi)
314            IMGFMT=vdi
315            xpand=false
316            ;;
317
318        -vmdk)
319            IMGFMT=vmdk
320            xpand=false
321            ;;
322
323        -vpc)
324            IMGFMT=vpc
325            xpand=false
326            ;;
327
328        -vhdx)
329            IMGFMT=vhdx
330            xpand=false
331            ;;
332
333        -file)
334            IMGPROTO=file
335            xpand=false
336            ;;
337
338        -rbd)
339            IMGPROTO=rbd
340            xpand=false
341            ;;
342
343        -sheepdog)
344            IMGPROTO=sheepdog
345            xpand=false
346            ;;
347
348        -nbd)
349            IMGPROTO=nbd
350            xpand=false
351            ;;
352
353        -vxhs)
354            IMGPROTO=vxhs
355            xpand=false
356            ;;
357
358        -ssh)
359            IMGPROTO=ssh
360            xpand=false
361            ;;
362
363        -nfs)
364            IMGPROTO=nfs
365            xpand=false
366            ;;
367
368        -nocache)
369            CACHEMODE="none"
370            CACHEMODE_IS_DEFAULT=false
371            xpand=false
372            ;;
373
374        -misalign)
375            QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --misalign"
376            xpand=false
377            ;;
378
379        -valgrind)
380            VALGRIND_QEMU='y'
381            xpand=false
382            ;;
383
384        -g)        # -g group ... pick from group file
385            group=true
386            xpand=false
387            ;;
388
389        -xdiff)        # graphical diff mode
390            xpand=false
391
392            if [ ! -z "$DISPLAY" ]
393            then
394                command -v xdiff >/dev/null 2>&1 && diff=xdiff
395                command -v gdiff >/dev/null 2>&1 && diff=gdiff
396                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
397                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
398            fi
399            ;;
400
401        -n)        # show me, don't do it
402            showme=true
403            xpand=false
404            ;;
405        -o)
406            imgopts=true
407            xpand=false
408            ;;
409        -c)
410            cachemode=true
411            xpand=false
412            ;;
413        -T)        # turn on timestamp output
414            timestamp=true
415            xpand=false
416            ;;
417
418        -v)
419            verbose=true
420            xpand=false
421            ;;
422        -d)
423            debug=true
424            xpand=false
425            ;;
426        -x)        # -x group ... exclude from group file
427            xgroup=true
428            xpand=false
429            ;;
430        '[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
431            echo "No tests?"
432            status=1
433            exit $status
434            ;;
435
436        [0-9]*-[0-9]*)
437            eval `echo $r | sed -e 's/^/start=/' -e 's/-/ end=/'`
438            ;;
439
440        [0-9]*-)
441            eval `echo $r | sed -e 's/^/start=/' -e 's/-//'`
442            end=`echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/  *$//' -e 's/.* //'`
443            if [ -z "$end" ]
444            then
445                echo "No tests in range \"$r\"?"
446                status=1
447                exit $status
448            fi
449            ;;
450
451        *)
452            start=$r
453            end=$r
454            ;;
455
456    esac
457
458    # get rid of leading 0s as can be interpreted as octal
459    start=`echo $start | sed 's/^0*//'`
460    end=`echo $end | sed 's/^0*//'`
461
462    if $xpand
463    then
464        have_test_arg=true
465        awk </dev/null '
466BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
467        | while read id
468        do
469            if grep -s "^$id " "$source_iotests/group" >/dev/null
470            then
471                # in group file ... OK
472                echo $id >>$tmp.list
473            else
474                if [ -f expunged ] && $expunge && egrep "^$id([         ]|\$)" expunged >/dev/null
475                then
476                    # expunged ... will be reported, but not run, later
477                    echo $id >>$tmp.list
478                else
479                    # oops
480                    if [ "$start" == "$end" -a "$id" == "$end" ]
481                    then
482                        echo "$id - unknown test"
483                        exit 1
484                    else
485                        echo "$id - unknown test, ignored"
486                    fi
487                fi
488            fi
489        done || exit 1
490    fi
491
492done
493
494# Set qemu-io cache mode with $CACHEMODE we have
495QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
496
497QEMU_IO_OPTIONS_NO_FMT="$QEMU_IO_OPTIONS"
498if [ "$IMGOPTSSYNTAX" != "true" ]; then
499    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT"
500fi
501
502# Set default options for qemu-img create -o if they were not specified
503if [ "$IMGFMT" == "qcow2" ] && ! (echo "$IMGOPTS" | grep "compat=" > /dev/null); then
504    IMGOPTS=$(_optstr_add "$IMGOPTS" "compat=1.1")
505fi
506if [ "$IMGFMT" == "luks" ] && ! (echo "$IMGOPTS" | grep "iter-time=" > /dev/null); then
507    IMGOPTS=$(_optstr_add "$IMGOPTS" "iter-time=10")
508fi
509
510if [ -z "$SAMPLE_IMG_DIR" ]; then
511        SAMPLE_IMG_DIR="$source_iotests/sample_images"
512fi
513
514export TEST_DIR
515export SAMPLE_IMG_DIR
516
517if [ -s $tmp.list ]
518then
519    # found some valid test numbers ... this is good
520    :
521else
522    if $have_test_arg
523    then
524        # had test numbers, but none in group file ... do nothing
525        touch $tmp.list
526    else
527        # no test numbers, do everything from group file
528        sed -n -e '/^[0-9][0-9][0-9]*/s/[         ].*//p' <"$source_iotests/group" >$tmp.list
529    fi
530fi
531
532# should be sort -n, but this did not work for Linux when this
533# was ported from IRIX
534#
535list=`sort $tmp.list`
536rm -f $tmp.list $tmp.tmp $tmp.sed
537
538if [ -z "$QEMU_PROG" ]
539then
540    if [ -x "$build_iotests/qemu" ]; then
541        export QEMU_PROG="$build_iotests/qemu"
542    elif [ -x "$build_root/$arch-softmmu/qemu-system-$arch" ]; then
543        export QEMU_PROG="$build_root/$arch-softmmu/qemu-system-$arch"
544    else
545        pushd "$build_root" > /dev/null
546        for binary in *-softmmu/qemu-system-*
547        do
548            if [ -x "$binary" ]
549            then
550                export QEMU_PROG="$build_root/$binary"
551                break
552            fi
553        done
554        popd > /dev/null
555        [ "$QEMU_PROG" = "" ] && _init_error "qemu not found"
556    fi
557fi
558export QEMU_PROG=$(realpath -- "$(type -p "$QEMU_PROG")")
559
560if [ -z "$QEMU_IMG_PROG" ]; then
561    if [ -x "$build_iotests/qemu-img" ]; then
562        export QEMU_IMG_PROG="$build_iotests/qemu-img"
563    elif [ -x "$build_root/qemu-img" ]; then
564        export QEMU_IMG_PROG="$build_root/qemu-img"
565    else
566        _init_error "qemu-img not found"
567    fi
568fi
569export QEMU_IMG_PROG=$(realpath -- "$(type -p "$QEMU_IMG_PROG")")
570
571if [ -z "$QEMU_IO_PROG" ]; then
572    if [ -x "$build_iotests/qemu-io" ]; then
573        export QEMU_IO_PROG="$build_iotests/qemu-io"
574    elif [ -x "$build_root/qemu-io" ]; then
575        export QEMU_IO_PROG="$build_root/qemu-io"
576    else
577        _init_error "qemu-io not found"
578    fi
579fi
580export QEMU_IO_PROG=$(realpath -- "$(type -p "$QEMU_IO_PROG")")
581
582if [ -z $QEMU_NBD_PROG ]; then
583    if [ -x "$build_iotests/qemu-nbd" ]; then
584        export QEMU_NBD_PROG="$build_iotests/qemu-nbd"
585    elif [ -x "$build_root/qemu-nbd" ]; then
586        export QEMU_NBD_PROG="$build_root/qemu-nbd"
587    else
588        _init_error "qemu-nbd not found"
589    fi
590fi
591export QEMU_NBD_PROG=$(realpath -- "$(type -p "$QEMU_NBD_PROG")")
592
593if [ -z "$QEMU_VXHS_PROG" ]; then
594  export QEMU_VXHS_PROG="`set_prog_path qnio_server`"
595fi
596
597if [ -x "$build_iotests/socket_scm_helper" ]
598then
599    export SOCKET_SCM_HELPER="$build_iotests/socket_scm_helper"
600fi
601
602default_machine=$($QEMU_PROG -machine help | sed -n '/(default)/ s/ .*//p')
603default_alias_machine=$($QEMU_PROG -machine help | \
604   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
605if [[ "$default_alias_machine" ]]; then
606    default_machine="$default_alias_machine"
607fi
608
609export QEMU_DEFAULT_MACHINE="$default_machine"
610
611TIMESTAMP_FILE=check.time-$IMGPROTO-$IMGFMT
612
613_wallclock()
614{
615    date "+%H %M %S" | awk '{ print $1*3600 + $2*60 + $3 }'
616}
617
618_timestamp()
619{
620    now=`date "+%T"`
621    printf %s " [$now]"
622}
623
624_wrapup()
625{
626    if $showme
627    then
628        :
629    elif $needwrap
630    then
631        if [ -f $TIMESTAMP_FILE -a -f $tmp.time ]
632        then
633            cat $TIMESTAMP_FILE $tmp.time \
634            | awk '
635        { t[$1] = $2 }
636END        { if (NR > 0) {
637            for (i in t) print i " " t[i]
638          }
639        }' \
640            | sort -n >$tmp.out
641            mv $tmp.out $TIMESTAMP_FILE
642        fi
643
644        if [ -f $tmp.expunged ]
645        then
646            notrun=`wc -l <$tmp.expunged | sed -e 's/  *//g'`
647            try=`expr $try - $notrun`
648            list=`echo "$list" | sed -f $tmp.expunged`
649        fi
650
651        echo "" >>check.log
652        date >>check.log
653        echo $list | fmt | sed -e 's/^/    /' >>check.log
654        $interrupt && echo "Interrupted!" >>check.log
655
656        if [ ! -z "$notrun" ]
657        then
658            echo "Not run:$notrun"
659            echo "Not run:$notrun" >>check.log
660        fi
661        if [ ! -z "$n_bad" -a $n_bad != 0 ]
662        then
663            echo "Failures:$bad"
664            echo "Failed $n_bad of $try tests"
665            echo "Failures:$bad" | fmt >>check.log
666            echo "Failed $n_bad of $try tests" >>check.log
667        else
668            echo "Passed all $try tests"
669            echo "Passed all $try tests" >>check.log
670        fi
671        needwrap=false
672    fi
673
674    if test -n "$STTY_RESTORE"; then
675        stty $STTY_RESTORE
676    fi
677    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
678    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
679    rm -f $tmp.*
680}
681
682trap "_wrapup; exit \$status" 0 1 2 3 15
683
684[ -f $TIMESTAMP_FILE ] || touch $TIMESTAMP_FILE
685
686FULL_IMGFMT_DETAILS=`_full_imgfmt_details`
687FULL_HOST_DETAILS=`_full_platform_details`
688
689cat <<EOF
690QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
691QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
692QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
693QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
694IMGFMT        -- $FULL_IMGFMT_DETAILS
695IMGPROTO      -- $IMGPROTO
696PLATFORM      -- $FULL_HOST_DETAILS
697TEST_DIR      -- $TEST_DIR
698SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
699
700EOF
701
702seq="check"
703
704[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG
705
706for seq in $list
707do
708    err=false
709    printf %s "$seq"
710    if [ -n "$TESTS_REMAINING_LOG" ] ; then
711        sed -e "s/$seq//" -e 's/  / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
712        mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
713        sync
714    fi
715
716    if $showme
717    then
718        echo
719        continue
720    elif [ -f expunged ] && $expunge && egrep "^$seq([         ]|\$)" expunged >/dev/null
721    then
722        echo " - expunged"
723        rm -f $seq.out.bad
724        echo "/^$seq\$/d" >>$tmp.expunged
725    elif [ ! -f "$source_iotests/$seq" ]
726    then
727        echo " - no such test?"
728        echo "/^$seq\$/d" >>$tmp.expunged
729    else
730        # really going to try and run this one
731        #
732        rm -f $seq.out.bad
733        lasttime=`sed -n -e "/^$seq /s/.* //p" <$TIMESTAMP_FILE`
734        if [ "X$lasttime" != X ]; then
735                printf %s " ${lasttime}s ..."
736        else
737                printf "        "        # prettier output with timestamps.
738        fi
739        rm -f core $seq.notrun
740
741        start=`_wallclock`
742        $timestamp && printf %s "        [$(date "+%T")]"
743
744        if [ "$(head -n 1 "$source_iotests/$seq")" == "#!/usr/bin/env python" ]; then
745            run_command="$PYTHON $seq"
746        else
747            run_command="./$seq"
748        fi
749        export OUTPUT_DIR=$PWD
750        if $debug; then
751            (cd "$source_iotests";
752            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
753                    $run_command -d 2>&1 | tee $tmp.out)
754        else
755            (cd "$source_iotests";
756            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
757                    $run_command >$tmp.out 2>&1)
758        fi
759        sts=$?
760        $timestamp && _timestamp
761        stop=`_wallclock`
762
763        if [ -f core ]
764        then
765            printf " [dumped core]"
766            mv core $seq.core
767            err=true
768        fi
769
770        if [ -f $seq.notrun ]
771        then
772            $timestamp || printf " [not run] "
773            $timestamp && echo " [not run]" && printf %s "        $seq -- "
774            cat $seq.notrun
775            notrun="$notrun $seq"
776        else
777            if [ $sts -ne 0 ]
778            then
779                printf %s " [failed, exit status $sts]"
780                err=true
781            fi
782
783            reference="$source_iotests/$seq.out"
784            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
785            if [ -f "$reference_machine" ]; then
786                reference="$reference_machine"
787            fi
788
789            reference_format="$source_iotests/$seq.out.$IMGFMT"
790            if [ -f "$reference_format" ]; then
791                reference="$reference_format"
792            fi
793
794            if [ "$CACHEMODE" = "none" ]; then
795                [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
796            fi
797
798            if [ ! -f "$reference" ]
799            then
800                echo " - no qualified output"
801                err=true
802            else
803                if diff -w "$reference" $tmp.out >/dev/null 2>&1
804                then
805                    echo ""
806                    if $err
807                    then
808                        :
809                    else
810                        echo "$seq `expr $stop - $start`" >>$tmp.time
811                    fi
812                else
813                    echo " - output mismatch (see $seq.out.bad)"
814                    mv $tmp.out $seq.out.bad
815                    $diff -w "$reference" $(realpath $seq.out.bad)
816                    err=true
817                fi
818            fi
819        fi
820
821    fi
822
823    # come here for each test, except when $showme is true
824    #
825    if $err
826    then
827        bad="$bad $seq"
828        n_bad=`expr $n_bad + 1`
829        quick=false
830    fi
831    [ -f $seq.notrun ] || try=`expr $try + 1`
832
833    seq="after_$seq"
834done
835
836interrupt=false
837status=`expr $n_bad`
838exit
839