xref: /openbmc/qemu/tests/qemu-iotests/check (revision 88cd34ee)
1#!/usr/bin/env bash
2#
3# Copyright (C) 2009 Red Hat, Inc.
4# Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation.
9#
10# This program is distributed in the hope that it would be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18#
19# Control script for QA
20#
21
22status=0
23needwrap=true
24try=0
25n_bad=0
26bad=""
27notrun=""
28casenotrun=""
29interrupt=true
30makecheck=false
31
32_init_error()
33{
34    echo "check: $1" >&2
35    exit 1
36}
37
38if [ -L "$0" ]
39then
40    # called from the build tree
41    source_iotests=$(dirname "$(readlink "$0")")
42    if [ -z "$source_iotests" ]
43    then
44        _init_error "failed to obtain source tree name from check symlink"
45    fi
46    source_iotests=$(cd "$source_iotests"; pwd) || _init_error "failed to enter source tree"
47    build_iotests=$PWD
48else
49    # called from the source tree
50    source_iotests=$PWD
51    # this may be an in-tree build (note that in the following code we may not
52    # assume that it truly is and have to test whether the build results
53    # actually exist)
54    build_iotests=$PWD
55fi
56
57build_root="$build_iotests/../.."
58
59# we need common.env
60if ! . "$build_iotests/common.env"
61then
62    _init_error "failed to source common.env (make sure the qemu-iotests are run from tests/qemu-iotests in the build tree)"
63fi
64
65# we need common.config
66if ! . "$source_iotests/common.config"
67then
68    _init_error "failed to source common.config"
69fi
70
71_full_imgfmt_details()
72{
73    if [ -n "$IMGOPTS" ]; then
74        echo "$IMGFMT ($IMGOPTS)"
75    else
76        echo "$IMGFMT"
77    fi
78}
79
80_full_platform_details()
81{
82    os=$(uname -s)
83    host=$(hostname -s)
84    kernel=$(uname -r)
85    platform=$(uname -m)
86    echo "$os/$platform $host $kernel"
87}
88
89_full_env_details()
90{
91    cat <<EOF
92QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
93QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
94QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
95QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
96IMGFMT        -- $FULL_IMGFMT_DETAILS
97IMGPROTO      -- $IMGPROTO
98PLATFORM      -- $FULL_HOST_DETAILS
99TEST_DIR      -- $TEST_DIR
100SOCK_DIR      -- $SOCK_DIR
101SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
102
103EOF
104}
105
106# $1 = prog to look for
107set_prog_path()
108{
109    p=$(command -v $1 2> /dev/null)
110    if [ -n "$p" -a -x "$p" ]; then
111        type -p "$p"
112    else
113        return 1
114    fi
115}
116
117if [ -z "$TEST_DIR" ]; then
118        TEST_DIR=$PWD/scratch
119fi
120mkdir -p "$TEST_DIR" || _init_error 'Failed to create TEST_DIR'
121
122tmp_sock_dir=false
123if [ -z "$SOCK_DIR" ]; then
124    SOCK_DIR=$(mktemp -d)
125    tmp_sock_dir=true
126fi
127mkdir -p "$SOCK_DIR" || _init_error 'Failed to create SOCK_DIR'
128
129diff="diff -u"
130verbose=false
131debug=false
132group=false
133xgroup=false
134imgopts=false
135showme=false
136sortme=false
137expunge=true
138have_test_arg=false
139cachemode=false
140aiomode=false
141
142tmp="${TEST_DIR}"/$$
143rm -f $tmp.list $tmp.tmp $tmp.sed
144
145export IMGFMT=raw
146export IMGFMT_GENERIC=true
147export IMGPROTO=file
148export IMGOPTS=""
149export CACHEMODE="writeback"
150export AIOMODE="threads"
151export QEMU_IO_OPTIONS=""
152export QEMU_IO_OPTIONS_NO_FMT=""
153export CACHEMODE_IS_DEFAULT=true
154export VALGRIND_QEMU=
155export IMGKEYSECRET=
156export IMGOPTSSYNTAX=false
157
158# Save current tty settings, since an aborting qemu call may leave things
159# screwed up
160STTY_RESTORE=
161if test -t 0; then
162    STTY_RESTORE=$(stty -g)
163fi
164
165for r
166do
167
168    if $group
169    then
170        # arg after -g
171        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
172s/ .*//p
173}')
174        if [ -z "$group_list" ]
175        then
176            echo "Group \"$r\" is empty or not defined?"
177            exit 1
178        fi
179        [ ! -s $tmp.list ] && touch $tmp.list
180        for t in $group_list
181        do
182            if grep -s "^$t\$" $tmp.list >/dev/null
183            then
184                :
185            else
186                echo "$t" >>$tmp.list
187            fi
188        done
189        group=false
190        continue
191
192    elif $xgroup
193    then
194        # arg after -x
195        # Populate $tmp.list with all tests
196        awk '/^[0-9]{3,}/ {print $1}' "${source_iotests}/group" > $tmp.list 2>/dev/null
197        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
198s/ .*//p
199}')
200        if [ -z "$group_list" ]
201        then
202            echo "Group \"$r\" is empty or not defined?"
203            exit 1
204        fi
205        numsed=0
206        rm -f $tmp.sed
207        for t in $group_list
208        do
209            if [ $numsed -gt 100 ]
210            then
211                sed -f $tmp.sed <$tmp.list >$tmp.tmp
212                mv $tmp.tmp $tmp.list
213                numsed=0
214                rm -f $tmp.sed
215            fi
216            echo "/^$t\$/d" >>$tmp.sed
217            numsed=$(expr $numsed + 1)
218        done
219        sed -f $tmp.sed <$tmp.list >$tmp.tmp
220        mv $tmp.tmp $tmp.list
221        xgroup=false
222        continue
223
224    elif $imgopts
225    then
226        IMGOPTS="$r"
227        imgopts=false
228        continue
229    elif $cachemode
230    then
231        CACHEMODE="$r"
232        CACHEMODE_IS_DEFAULT=false
233        cachemode=false
234        continue
235    elif $aiomode
236    then
237        AIOMODE="$r"
238        aiomode=false
239        continue
240    fi
241
242    xpand=true
243    case "$r"
244    in
245
246        -\? | -h | --help)        # usage
247            echo "Usage: $0 [options] [testlist]"'
248
249common options
250    -v                  verbose
251    -d                  debug
252
253image format options
254    -raw                test raw (default)
255    -bochs              test bochs
256    -cloop              test cloop
257    -parallels          test parallels
258    -qcow               test qcow
259    -qcow2              test qcow2
260    -qed                test qed
261    -vdi                test vdi
262    -vpc                test vpc
263    -vhdx               test vhdx
264    -vmdk               test vmdk
265    -luks               test luks
266    -dmg                test dmg
267
268image protocol options
269    -file               test file (default)
270    -rbd                test rbd
271    -sheepdog           test sheepdog
272    -nbd                test nbd
273    -ssh                test ssh
274    -nfs                test nfs
275    -vxhs               test vxhs
276
277other options
278    -xdiff              graphical mode diff
279    -nocache            use O_DIRECT on backing file
280    -misalign           misalign memory allocations
281    -n                  show me, do not run tests
282    -o options          -o options to pass to qemu-img create/convert
283    -c mode             cache mode
284    -i mode             AIO mode
285    -makecheck          pretty print output for make check
286
287testlist options
288    -g group[,group...]        include tests from these groups
289    -x group[,group...]        exclude tests from these groups
290    NNN                        include test NNN
291    NNN-NNN                    include test range (eg. 012-021)
292'
293            exit 0
294            ;;
295
296        -raw)
297            IMGFMT=raw
298            xpand=false
299            ;;
300
301        -bochs)
302            IMGFMT=bochs
303            IMGFMT_GENERIC=false
304            xpand=false
305            ;;
306
307        -cloop)
308            IMGFMT=cloop
309            IMGFMT_GENERIC=false
310            xpand=false
311            ;;
312
313        -parallels)
314            IMGFMT=parallels
315            xpand=false
316            ;;
317
318        -qcow)
319            IMGFMT=qcow
320            xpand=false
321            ;;
322
323        -qcow2)
324            IMGFMT=qcow2
325            xpand=false
326            ;;
327
328        -luks)
329            IMGOPTSSYNTAX=true
330            IMGFMT=luks
331            IMGKEYSECRET=123456
332            xpand=false
333            ;;
334
335        -dmg)
336            IMGFMT=dmg
337            IMGFMT_GENERIC=false
338            xpand=false
339            ;;
340
341        -qed)
342            IMGFMT=qed
343            xpand=false
344            ;;
345
346        -vdi)
347            IMGFMT=vdi
348            xpand=false
349            ;;
350
351        -vmdk)
352            IMGFMT=vmdk
353            xpand=false
354            ;;
355
356        -vpc)
357            IMGFMT=vpc
358            xpand=false
359            ;;
360
361        -vhdx)
362            IMGFMT=vhdx
363            xpand=false
364            ;;
365
366        -file)
367            IMGPROTO=file
368            xpand=false
369            ;;
370
371        -rbd)
372            IMGPROTO=rbd
373            xpand=false
374            ;;
375
376        -sheepdog)
377            IMGPROTO=sheepdog
378            xpand=false
379            ;;
380
381        -nbd)
382            IMGPROTO=nbd
383            xpand=false
384            ;;
385
386        -vxhs)
387            IMGPROTO=vxhs
388            xpand=false
389            ;;
390
391        -ssh)
392            IMGPROTO=ssh
393            xpand=false
394            ;;
395
396        -nfs)
397            IMGPROTO=nfs
398            xpand=false
399            ;;
400
401        -nocache)
402            CACHEMODE="none"
403            CACHEMODE_IS_DEFAULT=false
404            xpand=false
405            ;;
406
407        -misalign)
408            QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --misalign"
409            xpand=false
410            ;;
411
412        -valgrind)
413            VALGRIND_QEMU='y'
414            xpand=false
415            ;;
416
417        -g)        # -g group ... pick from group file
418            group=true
419            xpand=false
420            ;;
421
422        -xdiff)        # graphical diff mode
423            xpand=false
424
425            if [ ! -z "$DISPLAY" ]
426            then
427                command -v xdiff >/dev/null 2>&1 && diff=xdiff
428                command -v gdiff >/dev/null 2>&1 && diff=gdiff
429                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
430                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
431            fi
432            ;;
433        -makecheck)   # makecheck friendly output
434            makecheck=true
435            xpand=false
436            ;;
437        -n)        # show me, don't do it
438            showme=true
439            xpand=false
440            ;;
441        -o)
442            imgopts=true
443            xpand=false
444            ;;
445        -c)
446            cachemode=true
447            xpand=false
448            ;;
449        -i)
450            aiomode=true
451            xpand=false
452            ;;
453        -T)        # deprecated timestamp option
454            xpand=false
455            ;;
456        -v)
457            verbose=true
458            xpand=false
459            ;;
460        -d)
461            debug=true
462            xpand=false
463            ;;
464        -x)        # -x group ... exclude from group file
465            xgroup=true
466            xpand=false
467            ;;
468        '[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
469            echo "No tests?"
470            status=1
471            exit $status
472            ;;
473
474        [0-9]*-[0-9]*)
475            eval $(echo $r | sed -e 's/^/start=/' -e 's/-/ end=/')
476            ;;
477
478        [0-9]*-)
479            eval $(echo $r | sed -e 's/^/start=/' -e 's/-//')
480            end=$(echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/  *$//' -e 's/.* //')
481            if [ -z "$end" ]
482            then
483                echo "No tests in range \"$r\"?"
484                status=1
485                exit $status
486            fi
487            ;;
488
489        *)
490            start=$r
491            end=$r
492            ;;
493
494    esac
495
496    # get rid of leading 0s as can be interpreted as octal
497    start=$(echo $start | sed 's/^0*//')
498    end=$(echo $end | sed 's/^0*//')
499
500    if $xpand
501    then
502        have_test_arg=true
503        awk </dev/null '
504BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
505        | while read id
506        do
507            if grep -s "^$id\( \|\$\)" "$source_iotests/group" >/dev/null
508            then
509                # in group file ... OK
510                echo $id >>$tmp.list
511            else
512                if [ -f expunged ] && $expunge && egrep "^$id([         ]|\$)" expunged >/dev/null
513                then
514                    # expunged ... will be reported, but not run, later
515                    echo $id >>$tmp.list
516                else
517                    # oops
518                    if [ "$start" == "$end" -a "$id" == "$end" ]
519                    then
520                        echo "$id - unknown test"
521                        exit 1
522                    else
523                        echo "$id - unknown test, ignored"
524                    fi
525                fi
526            fi
527        done || exit 1
528    fi
529
530done
531
532# Set qemu-io cache mode with $CACHEMODE we have
533QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
534# Set qemu-io aio mode with $AIOMODE we have
535QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --aio $AIOMODE"
536
537QEMU_IO_OPTIONS_NO_FMT="$QEMU_IO_OPTIONS"
538if [ "$IMGOPTSSYNTAX" != "true" ]; then
539    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT"
540fi
541
542# Set default options for qemu-img create -o if they were not specified
543if [ "$IMGFMT" == "qcow2" ] && ! (echo "$IMGOPTS" | grep "compat=" > /dev/null); then
544    IMGOPTS=$(_optstr_add "$IMGOPTS" "compat=1.1")
545fi
546if [ "$IMGFMT" == "luks" ] && ! (echo "$IMGOPTS" | grep "iter-time=" > /dev/null); then
547    IMGOPTS=$(_optstr_add "$IMGOPTS" "iter-time=10")
548fi
549
550if [ -z "$SAMPLE_IMG_DIR" ]; then
551        SAMPLE_IMG_DIR="$source_iotests/sample_images"
552fi
553
554export TEST_DIR
555export SOCK_DIR
556export SAMPLE_IMG_DIR
557
558if [ -s $tmp.list ]
559then
560    # found some valid test numbers ... this is good
561    :
562else
563    if $have_test_arg
564    then
565        # had test numbers, but none in group file ... do nothing
566        touch $tmp.list
567    else
568        # no test numbers, do everything from group file
569        sed -n -e '/^[0-9][0-9][0-9]*/s/^\([0-9]*\).*/\1/p' <"$source_iotests/group" >$tmp.list
570    fi
571fi
572
573# should be sort -n, but this did not work for Linux when this
574# was ported from IRIX
575#
576list=$(sort $tmp.list)
577rm -f $tmp.list $tmp.tmp $tmp.sed
578
579if [ -z "$QEMU_PROG" ]
580then
581    if [ -x "$build_iotests/qemu" ]; then
582        export QEMU_PROG="$build_iotests/qemu"
583    elif [ -x "$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" ]; then
584        export QEMU_PROG="$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}"
585    else
586        pushd "$build_root" > /dev/null
587        for binary in *-softmmu/qemu-system-*
588        do
589            if [ -x "$binary" ]
590            then
591                export QEMU_PROG="$build_root/$binary"
592                break
593            fi
594        done
595        popd > /dev/null
596        [ "$QEMU_PROG" = "" ] && _init_error "qemu not found"
597    fi
598fi
599export QEMU_PROG="$(type -p "$QEMU_PROG")"
600
601case "$QEMU_PROG" in
602    *qemu-system-arm|*qemu-system-aarch64)
603        export QEMU_OPTIONS="-nodefaults -display none -machine virt -accel qtest"
604        ;;
605    *qemu-system-tricore)
606        export QEMU_OPTIONS="-nodefaults -display none -machine tricore_testboard -accel qtest"
607        ;;
608    *)
609        export QEMU_OPTIONS="-nodefaults -display none -accel qtest"
610        ;;
611esac
612
613if [ -z "$QEMU_IMG_PROG" ]; then
614    if [ -x "$build_iotests/qemu-img" ]; then
615        export QEMU_IMG_PROG="$build_iotests/qemu-img"
616    elif [ -x "$build_root/qemu-img" ]; then
617        export QEMU_IMG_PROG="$build_root/qemu-img"
618    else
619        _init_error "qemu-img not found"
620    fi
621fi
622export QEMU_IMG_PROG="$(type -p "$QEMU_IMG_PROG")"
623
624if [ -z "$QEMU_IO_PROG" ]; then
625    if [ -x "$build_iotests/qemu-io" ]; then
626        export QEMU_IO_PROG="$build_iotests/qemu-io"
627    elif [ -x "$build_root/qemu-io" ]; then
628        export QEMU_IO_PROG="$build_root/qemu-io"
629    else
630        _init_error "qemu-io not found"
631    fi
632fi
633export QEMU_IO_PROG="$(type -p "$QEMU_IO_PROG")"
634
635if [ -z $QEMU_NBD_PROG ]; then
636    if [ -x "$build_iotests/qemu-nbd" ]; then
637        export QEMU_NBD_PROG="$build_iotests/qemu-nbd"
638    elif [ -x "$build_root/qemu-nbd" ]; then
639        export QEMU_NBD_PROG="$build_root/qemu-nbd"
640    else
641        _init_error "qemu-nbd not found"
642    fi
643fi
644export QEMU_NBD_PROG="$(type -p "$QEMU_NBD_PROG")"
645
646if [ -z "$QEMU_VXHS_PROG" ]; then
647    export QEMU_VXHS_PROG="$(set_prog_path qnio_server)"
648fi
649
650if [ -x "$build_iotests/socket_scm_helper" ]
651then
652    export SOCKET_SCM_HELPER="$build_iotests/socket_scm_helper"
653fi
654
655python_usable=false
656if $PYTHON -c 'import sys; sys.exit(0 if sys.version_info >= (3,6) else 1)'
657then
658    # Our python framework also requires virtio-blk
659    if "$QEMU_PROG" -M none -device help | grep -q virtio-blk >/dev/null 2>&1
660    then
661        python_usable=true
662    else
663        python_unusable_because="Missing virtio-blk in QEMU binary"
664    fi
665else
666    python_unusable_because="Unsupported Python version"
667fi
668
669default_machine=$($QEMU_PROG -machine help | sed -n '/(default)/ s/ .*//p')
670default_alias_machine=$($QEMU_PROG -machine help | \
671   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
672if [[ "$default_alias_machine" ]]; then
673    default_machine="$default_alias_machine"
674fi
675
676export QEMU_DEFAULT_MACHINE="$default_machine"
677
678TIMESTAMP_FILE=check.time-$IMGPROTO-$IMGFMT
679
680_wallclock()
681{
682    date "+%H %M %S" | awk '{ print $1*3600 + $2*60 + $3 }'
683}
684
685_wrapup()
686{
687    if $showme
688    then
689        :
690    elif $needwrap
691    then
692        if [ -f $TIMESTAMP_FILE -a -f $tmp.time ]
693        then
694            cat $TIMESTAMP_FILE $tmp.time \
695            | awk '
696        { t[$1] = $2 }
697END        { if (NR > 0) {
698            for (i in t) print i " " t[i]
699          }
700        }' \
701            | sort -n >$tmp.out
702            mv $tmp.out $TIMESTAMP_FILE
703        fi
704
705        if [ -f $tmp.expunged ]
706        then
707            notrun=$(wc -l <$tmp.expunged | sed -e 's/  *//g')
708            try=$(expr $try - $notrun)
709            list=$(echo "$list" | sed -f $tmp.expunged)
710        fi
711
712        echo "" >>check.log
713        date >>check.log
714        echo $list | fmt | sed -e 's/^/    /' >>check.log
715        $interrupt && echo "Interrupted!" >>check.log
716
717        if [ ! -z "$notrun" ]
718        then
719            echo "Not run:$notrun"
720            echo "Not run:$notrun" >>check.log
721        fi
722        if [ ! -z "$casenotrun" ]
723        then
724            echo "Some cases not run in:$casenotrun"
725            echo "Some cases not run in:$casenotrun" >>check.log
726        fi
727        if [ ! -z "$n_bad" -a $n_bad != 0 ]
728        then
729            echo "Failures:$bad"
730            echo "Failed $n_bad of $try iotests"
731            echo "Failures:$bad" | fmt >>check.log
732            echo "Failed $n_bad of $try iotests" >>check.log
733        else
734            echo "Passed all $try iotests"
735            echo "Passed all $try iotests" >>check.log
736        fi
737        needwrap=false
738    fi
739
740    if test -n "$STTY_RESTORE"; then
741        stty $STTY_RESTORE
742    fi
743    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
744    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
745    rm -f $tmp.*
746
747    if $tmp_sock_dir
748    then
749        rm -rf "$SOCK_DIR"
750    fi
751}
752
753trap "_wrapup; exit \$status" 0 1 2 3 15
754
755# Report the test start and results. For makecheck we want to pretty
756# print the whole report at the end of the execution.
757# args: $seq, $starttime, $lasttime
758_report_test_start()
759{
760    if ! $makecheck; then
761        if [ -n "$3" ]; then
762            local lasttime=" (last: $3s)"
763        fi
764        printf "%-8s %-10s [%s]            %4s%-14s\r" "$1" "..." "$2" "..." "$lasttime"
765    fi
766}
767# args:$seq $status $starttime $lasttime $thistime $details
768_report_test_result()
769{
770    local status lasttime thistime
771    if $makecheck; then
772        if [ -n "$2" ] && [ "$2" != "pass" ]; then
773            status=" [$2]"
774        fi
775        printf "  TEST    iotest-$IMGFMT: %s%s\n" "$1" "$status"
776        return
777    fi
778
779    if [ -n "$4" ]; then
780        lasttime=" (last: $4s)"
781    fi
782    if [ -n "$5" ]; then
783        thistime=" $5s"
784    fi
785    case "$2" in
786        "pass")     status=$(printf "\e[32m%-10s\e[0m" "$2") ;;
787        "fail")     status=$(printf "\e[1m\e[31m%-10s\e[0m" "$2") ;;
788        "not run")  status=$(printf "\e[33m%-10s\e[0m" "$2") ;;
789        *)          status=$(printf "%-10s" "$2") ;;
790    esac
791
792    printf "%-8s %s [%s] [%s] %4s%-14s %s\n" "$1" "$status" "$3" "$(date '+%T')" "$thistime" "$lasttime" "$6"
793}
794
795[ -f $TIMESTAMP_FILE ] || touch $TIMESTAMP_FILE
796
797FULL_IMGFMT_DETAILS=$(_full_imgfmt_details)
798FULL_HOST_DETAILS=$(_full_platform_details)
799
800if ! $makecheck; then
801    _full_env_details
802fi
803
804seq="check"
805
806[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG
807
808for seq in $list
809do
810    err=false       # error flag
811    printdiff=false # show diff to reference output?
812    status=""       # test result summary
813    results=""      # test result details
814    thistime=""     # time the test took
815
816    if [ -n "$TESTS_REMAINING_LOG" ] ; then
817        sed -e "s/$seq//" -e 's/  / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
818        mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
819        sync
820    fi
821
822    lasttime=$(sed -n -e "/^$seq /s/.* //p" <$TIMESTAMP_FILE)
823    starttime=$(date "+%T")
824    _report_test_start $seq $starttime $lasttime
825
826    if $showme
827    then
828        status="not run"
829    elif [ -f expunged ] && $expunge && egrep "^$seq([         ]|\$)" expunged >/dev/null
830    then
831        status="not run"
832        results="expunged"
833        rm -f $seq.out.bad
834        echo "/^$seq\$/d" >>$tmp.expunged
835    elif [ ! -f "$source_iotests/$seq" ]
836    then
837        status="not run"
838        results="no such test?"
839        echo "/^$seq\$/d" >>$tmp.expunged
840    else
841        # really going to try and run this one
842        #
843        rm -f $seq.out.bad
844        rm -f core $seq.notrun
845        rm -f $seq.casenotrun
846
847        start=$(_wallclock)
848
849        if [ "$(head -n 1 "$source_iotests/$seq")" == "#!/usr/bin/env python3" ]; then
850            if $python_usable; then
851                run_command="$PYTHON $seq"
852            else
853                run_command="false"
854                echo "$python_unusable_because" > $seq.notrun
855            fi
856        else
857            run_command="./$seq"
858        fi
859        export OUTPUT_DIR=$PWD
860        if $debug; then
861            (cd "$source_iotests";
862            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
863                    $run_command -d 2>&1 | tee $tmp.out)
864        else
865            (cd "$source_iotests";
866            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
867                    $run_command >$tmp.out 2>&1)
868        fi
869        sts=$?
870        stop=$(_wallclock)
871
872        if [ -f core ]
873        then
874            mv core $seq.core
875            status="fail"
876            results="[dumped core] $seq.core"
877            err=true
878        fi
879
880        if [ -f $seq.notrun ]
881        then
882            # overwrites timestamp output
883            status="not run"
884            results="$(cat $seq.notrun)"
885        else
886            if [ $sts -ne 0 ]
887            then
888                status="fail"
889                results=$(printf %s "[failed, exit status $sts]")
890                err=true
891            fi
892
893            reference="$source_iotests/$seq.out"
894            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
895            if [ -f "$reference_machine" ]; then
896                reference="$reference_machine"
897            fi
898
899            reference_format="$source_iotests/$seq.out.$IMGFMT"
900            if [ -f "$reference_format" ]; then
901                reference="$reference_format"
902            fi
903
904            if [ "$CACHEMODE" = "none" ]; then
905                [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
906            fi
907
908            if [ ! -f "$reference" ]
909            then
910                status="fail"
911                results="no qualified output"
912                err=true
913            else
914                if diff -w "$reference" $tmp.out >/dev/null 2>&1
915                then
916                    if ! $err; then
917                        status="pass"
918                        thistime=$(expr $stop - $start)
919                        echo "$seq $thistime" >>$tmp.time
920                    fi
921                else
922                    mv $tmp.out $seq.out.bad
923                    status="fail"
924                    results="output mismatch (see $seq.out.bad)"
925                    printdiff=true
926                    err=true
927                fi
928            fi
929        fi
930        if [ -f $seq.casenotrun ]
931        then
932            cat $seq.casenotrun
933            casenotrun="$casenotrun $seq"
934        fi
935    fi
936
937    # come here for each test, except when $showme is true
938    #
939    _report_test_result $seq "$status" "$starttime" "$lasttime" "$thistime" "$results"
940    case "$status" in
941        "pass")
942            try=$(expr $try + 1)
943            ;;
944        "fail")
945            try=$(expr $try + 1)
946            if $makecheck; then
947                _full_env_details
948            fi
949            if $printdiff; then
950                $diff -w "$reference" "$PWD"/$seq.out.bad
951            fi
952            bad="$bad $seq"
953            n_bad=$(expr $n_bad + 1)
954            quick=false
955            ;;
956        "not run")
957            notrun="$notrun $seq"
958            ;;
959    esac
960
961    seq="after_$seq"
962done
963
964interrupt=false
965status=$(expr $n_bad)
966exit
967