xref: /openbmc/qemu/tests/qemu-iotests/check (revision 38dc1294)
1#!/usr/bin/env bash
2#
3# Copyright (C) 2009 Red Hat, Inc.
4# Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation.
9#
10# This program is distributed in the hope that it would be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18#
19# Control script for QA
20#
21
22status=0
23needwrap=true
24try=0
25n_bad=0
26bad=""
27notrun=""
28casenotrun=""
29interrupt=true
30
31# by default don't output timestamps
32timestamp=${TIMESTAMP:=false}
33
34_init_error()
35{
36    echo "check: $1" >&2
37    exit 1
38}
39
40if [ -L "$0" ]
41then
42    # called from the build tree
43    source_iotests=$(dirname "$(readlink "$0")")
44    if [ -z "$source_iotests" ]
45    then
46        _init_error "failed to obtain source tree name from check symlink"
47    fi
48    source_iotests=$(cd "$source_iotests"; pwd) || _init_error "failed to enter source tree"
49    build_iotests=$PWD
50else
51    # called from the source tree
52    source_iotests=$PWD
53    # this may be an in-tree build (note that in the following code we may not
54    # assume that it truly is and have to test whether the build results
55    # actually exist)
56    build_iotests=$PWD
57fi
58
59build_root="$build_iotests/../.."
60
61# we need common.env
62if ! . "$build_iotests/common.env"
63then
64    _init_error "failed to source common.env (make sure the qemu-iotests are run from tests/qemu-iotests in the build tree)"
65fi
66
67# we need common.config
68if ! . "$source_iotests/common.config"
69then
70    _init_error "failed to source common.config"
71fi
72
73_full_imgfmt_details()
74{
75    if [ -n "$IMGOPTS" ]; then
76        echo "$IMGFMT ($IMGOPTS)"
77    else
78        echo "$IMGFMT"
79    fi
80}
81
82_full_platform_details()
83{
84    os=$(uname -s)
85    host=$(hostname -s)
86    kernel=$(uname -r)
87    platform=$(uname -m)
88    echo "$os/$platform $host $kernel"
89}
90
91# $1 = prog to look for
92set_prog_path()
93{
94    p=$(command -v $1 2> /dev/null)
95    if [ -n "$p" -a -x "$p" ]; then
96        type -p "$p"
97    else
98        return 1
99    fi
100}
101
102if [ -z "$TEST_DIR" ]; then
103        TEST_DIR=$PWD/scratch
104fi
105
106if [ ! -e "$TEST_DIR" ]; then
107        mkdir "$TEST_DIR"
108fi
109
110diff="diff -u"
111verbose=false
112debug=false
113group=false
114xgroup=false
115imgopts=false
116showme=false
117sortme=false
118expunge=true
119have_test_arg=false
120cachemode=false
121
122tmp="${TEST_DIR}"/$$
123rm -f $tmp.list $tmp.tmp $tmp.sed
124
125export IMGFMT=raw
126export IMGFMT_GENERIC=true
127export IMGPROTO=file
128export IMGOPTS=""
129export CACHEMODE="writeback"
130export QEMU_IO_OPTIONS=""
131export QEMU_IO_OPTIONS_NO_FMT=""
132export CACHEMODE_IS_DEFAULT=true
133export VALGRIND_QEMU=
134export IMGKEYSECRET=
135export IMGOPTSSYNTAX=false
136
137# Save current tty settings, since an aborting qemu call may leave things
138# screwed up
139STTY_RESTORE=
140if test -t 0; then
141    STTY_RESTORE=$(stty -g)
142fi
143
144for r
145do
146
147    if $group
148    then
149        # arg after -g
150        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
151s/ .*//p
152}')
153        if [ -z "$group_list" ]
154        then
155            echo "Group \"$r\" is empty or not defined?"
156            exit 1
157        fi
158        [ ! -s $tmp.list ] && touch $tmp.list
159        for t in $group_list
160        do
161            if grep -s "^$t\$" $tmp.list >/dev/null
162            then
163                :
164            else
165                echo "$t" >>$tmp.list
166            fi
167        done
168        group=false
169        continue
170
171    elif $xgroup
172    then
173        # arg after -x
174        # Populate $tmp.list with all tests
175        awk '/^[0-9]{3,}/ {print $1}' "${source_iotests}/group" > $tmp.list 2>/dev/null
176        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
177s/ .*//p
178}')
179        if [ -z "$group_list" ]
180        then
181            echo "Group \"$r\" is empty or not defined?"
182            exit 1
183        fi
184        numsed=0
185        rm -f $tmp.sed
186        for t in $group_list
187        do
188            if [ $numsed -gt 100 ]
189            then
190                sed -f $tmp.sed <$tmp.list >$tmp.tmp
191                mv $tmp.tmp $tmp.list
192                numsed=0
193                rm -f $tmp.sed
194            fi
195            echo "/^$t\$/d" >>$tmp.sed
196            numsed=$(expr $numsed + 1)
197        done
198        sed -f $tmp.sed <$tmp.list >$tmp.tmp
199        mv $tmp.tmp $tmp.list
200        xgroup=false
201        continue
202
203    elif $imgopts
204    then
205        IMGOPTS="$r"
206        imgopts=false
207        continue
208    elif $cachemode
209    then
210        CACHEMODE="$r"
211        CACHEMODE_IS_DEFAULT=false
212        cachemode=false
213        continue
214    fi
215
216    xpand=true
217    case "$r"
218    in
219
220        -\? | -h | --help)        # usage
221            echo "Usage: $0 [options] [testlist]"'
222
223common options
224    -v                  verbose
225    -d                  debug
226
227image format options
228    -raw                test raw (default)
229    -bochs              test bochs
230    -cloop              test cloop
231    -parallels          test parallels
232    -qcow               test qcow
233    -qcow2              test qcow2
234    -qed                test qed
235    -vdi                test vdi
236    -vpc                test vpc
237    -vhdx               test vhdx
238    -vmdk               test vmdk
239    -luks               test luks
240    -dmg                test dmg
241
242image protocol options
243    -file               test file (default)
244    -rbd                test rbd
245    -sheepdog           test sheepdog
246    -nbd                test nbd
247    -ssh                test ssh
248    -nfs                test nfs
249    -vxhs               test vxhs
250
251other options
252    -xdiff              graphical mode diff
253    -nocache            use O_DIRECT on backing file
254    -misalign           misalign memory allocations
255    -n                  show me, do not run tests
256    -o options          -o options to pass to qemu-img create/convert
257    -T                  output timestamps
258    -c mode             cache mode
259
260testlist options
261    -g group[,group...]        include tests from these groups
262    -x group[,group...]        exclude tests from these groups
263    NNN                        include test NNN
264    NNN-NNN                    include test range (eg. 012-021)
265'
266            exit 0
267            ;;
268
269        -raw)
270            IMGFMT=raw
271            xpand=false
272            ;;
273
274        -bochs)
275            IMGFMT=bochs
276            IMGFMT_GENERIC=false
277            xpand=false
278            ;;
279
280        -cloop)
281            IMGFMT=cloop
282            IMGFMT_GENERIC=false
283            xpand=false
284            ;;
285
286        -parallels)
287            IMGFMT=parallels
288            xpand=false
289            ;;
290
291        -qcow)
292            IMGFMT=qcow
293            xpand=false
294            ;;
295
296        -qcow2)
297            IMGFMT=qcow2
298            xpand=false
299            ;;
300
301        -luks)
302            IMGOPTSSYNTAX=true
303            IMGFMT=luks
304            IMGKEYSECRET=123456
305            xpand=false
306            ;;
307
308        -dmg)
309            IMGFMT=dmg
310            IMGFMT_GENERIC=false
311            xpand=false
312            ;;
313
314        -qed)
315            IMGFMT=qed
316            xpand=false
317            ;;
318
319        -vdi)
320            IMGFMT=vdi
321            xpand=false
322            ;;
323
324        -vmdk)
325            IMGFMT=vmdk
326            xpand=false
327            ;;
328
329        -vpc)
330            IMGFMT=vpc
331            xpand=false
332            ;;
333
334        -vhdx)
335            IMGFMT=vhdx
336            xpand=false
337            ;;
338
339        -file)
340            IMGPROTO=file
341            xpand=false
342            ;;
343
344        -rbd)
345            IMGPROTO=rbd
346            xpand=false
347            ;;
348
349        -sheepdog)
350            IMGPROTO=sheepdog
351            xpand=false
352            ;;
353
354        -nbd)
355            IMGPROTO=nbd
356            xpand=false
357            ;;
358
359        -vxhs)
360            IMGPROTO=vxhs
361            xpand=false
362            ;;
363
364        -ssh)
365            IMGPROTO=ssh
366            xpand=false
367            ;;
368
369        -nfs)
370            IMGPROTO=nfs
371            xpand=false
372            ;;
373
374        -nocache)
375            CACHEMODE="none"
376            CACHEMODE_IS_DEFAULT=false
377            xpand=false
378            ;;
379
380        -misalign)
381            QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --misalign"
382            xpand=false
383            ;;
384
385        -valgrind)
386            VALGRIND_QEMU='y'
387            xpand=false
388            ;;
389
390        -g)        # -g group ... pick from group file
391            group=true
392            xpand=false
393            ;;
394
395        -xdiff)        # graphical diff mode
396            xpand=false
397
398            if [ ! -z "$DISPLAY" ]
399            then
400                command -v xdiff >/dev/null 2>&1 && diff=xdiff
401                command -v gdiff >/dev/null 2>&1 && diff=gdiff
402                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
403                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
404            fi
405            ;;
406
407        -n)        # show me, don't do it
408            showme=true
409            xpand=false
410            ;;
411        -o)
412            imgopts=true
413            xpand=false
414            ;;
415        -c)
416            cachemode=true
417            xpand=false
418            ;;
419        -T)        # turn on timestamp output
420            timestamp=true
421            xpand=false
422            ;;
423
424        -v)
425            verbose=true
426            xpand=false
427            ;;
428        -d)
429            debug=true
430            xpand=false
431            ;;
432        -x)        # -x group ... exclude from group file
433            xgroup=true
434            xpand=false
435            ;;
436        '[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
437            echo "No tests?"
438            status=1
439            exit $status
440            ;;
441
442        [0-9]*-[0-9]*)
443            eval $(echo $r | sed -e 's/^/start=/' -e 's/-/ end=/')
444            ;;
445
446        [0-9]*-)
447            eval $(echo $r | sed -e 's/^/start=/' -e 's/-//')
448            end=$(echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/  *$//' -e 's/.* //')
449            if [ -z "$end" ]
450            then
451                echo "No tests in range \"$r\"?"
452                status=1
453                exit $status
454            fi
455            ;;
456
457        *)
458            start=$r
459            end=$r
460            ;;
461
462    esac
463
464    # get rid of leading 0s as can be interpreted as octal
465    start=$(echo $start | sed 's/^0*//')
466    end=$(echo $end | sed 's/^0*//')
467
468    if $xpand
469    then
470        have_test_arg=true
471        awk </dev/null '
472BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
473        | while read id
474        do
475            if grep -s "^$id " "$source_iotests/group" >/dev/null
476            then
477                # in group file ... OK
478                echo $id >>$tmp.list
479            else
480                if [ -f expunged ] && $expunge && egrep "^$id([         ]|\$)" expunged >/dev/null
481                then
482                    # expunged ... will be reported, but not run, later
483                    echo $id >>$tmp.list
484                else
485                    # oops
486                    if [ "$start" == "$end" -a "$id" == "$end" ]
487                    then
488                        echo "$id - unknown test"
489                        exit 1
490                    else
491                        echo "$id - unknown test, ignored"
492                    fi
493                fi
494            fi
495        done || exit 1
496    fi
497
498done
499
500# Set qemu-io cache mode with $CACHEMODE we have
501QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
502
503QEMU_IO_OPTIONS_NO_FMT="$QEMU_IO_OPTIONS"
504if [ "$IMGOPTSSYNTAX" != "true" ]; then
505    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT"
506fi
507
508# Set default options for qemu-img create -o if they were not specified
509if [ "$IMGFMT" == "qcow2" ] && ! (echo "$IMGOPTS" | grep "compat=" > /dev/null); then
510    IMGOPTS=$(_optstr_add "$IMGOPTS" "compat=1.1")
511fi
512if [ "$IMGFMT" == "luks" ] && ! (echo "$IMGOPTS" | grep "iter-time=" > /dev/null); then
513    IMGOPTS=$(_optstr_add "$IMGOPTS" "iter-time=10")
514fi
515
516if [ -z "$SAMPLE_IMG_DIR" ]; then
517        SAMPLE_IMG_DIR="$source_iotests/sample_images"
518fi
519
520export TEST_DIR
521export SAMPLE_IMG_DIR
522
523if [ -s $tmp.list ]
524then
525    # found some valid test numbers ... this is good
526    :
527else
528    if $have_test_arg
529    then
530        # had test numbers, but none in group file ... do nothing
531        touch $tmp.list
532    else
533        # no test numbers, do everything from group file
534        sed -n -e '/^[0-9][0-9][0-9]*/s/[         ].*//p' <"$source_iotests/group" >$tmp.list
535    fi
536fi
537
538# should be sort -n, but this did not work for Linux when this
539# was ported from IRIX
540#
541list=$(sort $tmp.list)
542rm -f $tmp.list $tmp.tmp $tmp.sed
543
544if [ -z "$QEMU_PROG" ]
545then
546    if [ -x "$build_iotests/qemu" ]; then
547        export QEMU_PROG="$build_iotests/qemu"
548    elif [ -x "$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" ]; then
549        export QEMU_PROG="$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}"
550    else
551        pushd "$build_root" > /dev/null
552        for binary in *-softmmu/qemu-system-*
553        do
554            if [ -x "$binary" ]
555            then
556                export QEMU_PROG="$build_root/$binary"
557                break
558            fi
559        done
560        popd > /dev/null
561        [ "$QEMU_PROG" = "" ] && _init_error "qemu not found"
562    fi
563fi
564export QEMU_PROG="$(type -p "$QEMU_PROG")"
565
566case "$QEMU_PROG" in
567    *qemu-system-arm|*qemu-system-aarch64)
568        export QEMU_OPTIONS="-nodefaults -machine virt,accel=qtest"
569        ;;
570    *qemu-system-tricore)
571        export QEMU_OPTIONS="-nodefaults -machine tricore_testboard,accel=qtest"
572        ;;
573    *)
574        export QEMU_OPTIONS="-nodefaults -machine accel=qtest"
575        ;;
576esac
577
578if [ -z "$QEMU_IMG_PROG" ]; then
579    if [ -x "$build_iotests/qemu-img" ]; then
580        export QEMU_IMG_PROG="$build_iotests/qemu-img"
581    elif [ -x "$build_root/qemu-img" ]; then
582        export QEMU_IMG_PROG="$build_root/qemu-img"
583    else
584        _init_error "qemu-img not found"
585    fi
586fi
587export QEMU_IMG_PROG="$(type -p "$QEMU_IMG_PROG")"
588
589if [ -z "$QEMU_IO_PROG" ]; then
590    if [ -x "$build_iotests/qemu-io" ]; then
591        export QEMU_IO_PROG="$build_iotests/qemu-io"
592    elif [ -x "$build_root/qemu-io" ]; then
593        export QEMU_IO_PROG="$build_root/qemu-io"
594    else
595        _init_error "qemu-io not found"
596    fi
597fi
598export QEMU_IO_PROG="$(type -p "$QEMU_IO_PROG")"
599
600if [ -z $QEMU_NBD_PROG ]; then
601    if [ -x "$build_iotests/qemu-nbd" ]; then
602        export QEMU_NBD_PROG="$build_iotests/qemu-nbd"
603    elif [ -x "$build_root/qemu-nbd" ]; then
604        export QEMU_NBD_PROG="$build_root/qemu-nbd"
605    else
606        _init_error "qemu-nbd not found"
607    fi
608fi
609export QEMU_NBD_PROG="$(type -p "$QEMU_NBD_PROG")"
610
611if [ -z "$QEMU_VXHS_PROG" ]; then
612    export QEMU_VXHS_PROG="$(set_prog_path qnio_server)"
613fi
614
615if [ -x "$build_iotests/socket_scm_helper" ]
616then
617    export SOCKET_SCM_HELPER="$build_iotests/socket_scm_helper"
618fi
619
620default_machine=$($QEMU_PROG -machine help | sed -n '/(default)/ s/ .*//p')
621default_alias_machine=$($QEMU_PROG -machine help | \
622   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
623if [[ "$default_alias_machine" ]]; then
624    default_machine="$default_alias_machine"
625fi
626
627export QEMU_DEFAULT_MACHINE="$default_machine"
628
629TIMESTAMP_FILE=check.time-$IMGPROTO-$IMGFMT
630
631_wallclock()
632{
633    date "+%H %M %S" | awk '{ print $1*3600 + $2*60 + $3 }'
634}
635
636_timestamp()
637{
638    now=$(date "+%T")
639    printf %s " [$now]"
640}
641
642_wrapup()
643{
644    if $showme
645    then
646        :
647    elif $needwrap
648    then
649        if [ -f $TIMESTAMP_FILE -a -f $tmp.time ]
650        then
651            cat $TIMESTAMP_FILE $tmp.time \
652            | awk '
653        { t[$1] = $2 }
654END        { if (NR > 0) {
655            for (i in t) print i " " t[i]
656          }
657        }' \
658            | sort -n >$tmp.out
659            mv $tmp.out $TIMESTAMP_FILE
660        fi
661
662        if [ -f $tmp.expunged ]
663        then
664            notrun=$(wc -l <$tmp.expunged | sed -e 's/  *//g')
665            try=$(expr $try - $notrun)
666            list=$(echo "$list" | sed -f $tmp.expunged)
667        fi
668
669        echo "" >>check.log
670        date >>check.log
671        echo $list | fmt | sed -e 's/^/    /' >>check.log
672        $interrupt && echo "Interrupted!" >>check.log
673
674        if [ ! -z "$notrun" ]
675        then
676            echo "Not run:$notrun"
677            echo "Not run:$notrun" >>check.log
678        fi
679        if [ ! -z "$casenotrun" ]
680        then
681            echo "Some cases not run in:$casenotrun"
682            echo "Some cases not run in:$casenotrun" >>check.log
683        fi
684        if [ ! -z "$n_bad" -a $n_bad != 0 ]
685        then
686            echo "Failures:$bad"
687            echo "Failed $n_bad of $try tests"
688            echo "Failures:$bad" | fmt >>check.log
689            echo "Failed $n_bad of $try tests" >>check.log
690        else
691            echo "Passed all $try tests"
692            echo "Passed all $try tests" >>check.log
693        fi
694        needwrap=false
695    fi
696
697    if test -n "$STTY_RESTORE"; then
698        stty $STTY_RESTORE
699    fi
700    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
701    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
702    rm -f $tmp.*
703}
704
705trap "_wrapup; exit \$status" 0 1 2 3 15
706
707[ -f $TIMESTAMP_FILE ] || touch $TIMESTAMP_FILE
708
709FULL_IMGFMT_DETAILS=$(_full_imgfmt_details)
710FULL_HOST_DETAILS=$(_full_platform_details)
711
712cat <<EOF
713QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
714QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
715QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
716QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
717IMGFMT        -- $FULL_IMGFMT_DETAILS
718IMGPROTO      -- $IMGPROTO
719PLATFORM      -- $FULL_HOST_DETAILS
720TEST_DIR      -- $TEST_DIR
721SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
722
723EOF
724
725seq="check"
726
727[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG
728
729for seq in $list
730do
731    err=false
732    printf %s "$seq"
733    if [ -n "$TESTS_REMAINING_LOG" ] ; then
734        sed -e "s/$seq//" -e 's/  / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
735        mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
736        sync
737    fi
738
739    if $showme
740    then
741        echo
742        continue
743    elif [ -f expunged ] && $expunge && egrep "^$seq([         ]|\$)" expunged >/dev/null
744    then
745        echo " - expunged"
746        rm -f $seq.out.bad
747        echo "/^$seq\$/d" >>$tmp.expunged
748    elif [ ! -f "$source_iotests/$seq" ]
749    then
750        echo " - no such test?"
751        echo "/^$seq\$/d" >>$tmp.expunged
752    else
753        # really going to try and run this one
754        #
755        rm -f $seq.out.bad
756        lasttime=$(sed -n -e "/^$seq /s/.* //p" <$TIMESTAMP_FILE)
757        if [ "X$lasttime" != X ]; then
758                printf %s " ${lasttime}s ..."
759        else
760                printf "        "        # prettier output with timestamps.
761        fi
762        rm -f core $seq.notrun
763        rm -f $seq.casenotrun
764
765        start=$(_wallclock)
766        $timestamp && printf %s "        [$(date "+%T")]"
767
768        if [ "$(head -n 1 "$source_iotests/$seq")" == "#!/usr/bin/env python" ]; then
769            run_command="$PYTHON $seq"
770        else
771            run_command="./$seq"
772        fi
773        export OUTPUT_DIR=$PWD
774        if $debug; then
775            (cd "$source_iotests";
776            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
777                    $run_command -d 2>&1 | tee $tmp.out)
778        else
779            (cd "$source_iotests";
780            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
781                    $run_command >$tmp.out 2>&1)
782        fi
783        sts=$?
784        $timestamp && _timestamp
785        stop=$(_wallclock)
786
787        if [ -f core ]
788        then
789            printf " [dumped core]"
790            mv core $seq.core
791            err=true
792        fi
793
794        if [ -f $seq.notrun ]
795        then
796            $timestamp || printf " [not run] "
797            $timestamp && echo " [not run]" && printf %s "        $seq -- "
798            cat $seq.notrun
799            notrun="$notrun $seq"
800        else
801            if [ $sts -ne 0 ]
802            then
803                printf %s " [failed, exit status $sts]"
804                err=true
805            fi
806
807            reference="$source_iotests/$seq.out"
808            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
809            if [ -f "$reference_machine" ]; then
810                reference="$reference_machine"
811            fi
812
813            reference_format="$source_iotests/$seq.out.$IMGFMT"
814            if [ -f "$reference_format" ]; then
815                reference="$reference_format"
816            fi
817
818            if [ "$CACHEMODE" = "none" ]; then
819                [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
820            fi
821
822            if [ ! -f "$reference" ]
823            then
824                echo " - no qualified output"
825                err=true
826            else
827                if diff -w "$reference" $tmp.out >/dev/null 2>&1
828                then
829                    echo ""
830                    if $err
831                    then
832                        :
833                    else
834                        echo "$seq $(expr $stop - $start)" >>$tmp.time
835                    fi
836                else
837                    echo " - output mismatch (see $seq.out.bad)"
838                    mv $tmp.out $seq.out.bad
839                    $diff -w "$reference" "$PWD"/$seq.out.bad
840                    err=true
841                fi
842            fi
843        fi
844        if [ -f $seq.casenotrun ]
845        then
846            cat $seq.casenotrun
847            casenotrun="$casenotrun $seq"
848        fi
849    fi
850
851    # come here for each test, except when $showme is true
852    #
853    if $err
854    then
855        bad="$bad $seq"
856        n_bad=$(expr $n_bad + 1)
857        quick=false
858    fi
859    [ -f $seq.notrun ] || try=$(expr $try + 1)
860
861    seq="after_$seq"
862done
863
864interrupt=false
865status=$(expr $n_bad)
866exit
867