xref: /openbmc/qemu/tests/qemu-iotests/check (revision 70ff5b07fcdd378180ad2d5cc0b0d5e67e7ef325)

#!/usr/bin/env bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Control script for QA
#

status=0
needwrap=true
try=0
n_bad=0
bad=""
notrun=""
casenotrun=""
interrupt=true
makecheck=false

_init_error()
{
    echo "check: $1" >&2
    exit 1
}

if [ -L "$0" ]
then
    # called from the build tree
    source_iotests=$(dirname "$(readlink "$0")")
    if [ -z "$source_iotests" ]
    then
        _init_error "failed to obtain source tree name from check symlink"
    fi
    source_iotests=$(cd "$source_iotests"; pwd) || _init_error "failed to enter source tree"
    build_iotests=$PWD
else
    # called from the source tree
    source_iotests=$PWD
    # this may be an in-tree build (note that in the following code we may not
    # assume that it truly is and have to test whether the build results
    # actually exist)
    build_iotests=$PWD
fi

build_root="$build_iotests/../.."

# we need common.env
if ! . "$build_iotests/common.env"
then
    _init_error "failed to source common.env (make sure the qemu-iotests are run from tests/qemu-iotests in the build tree)"
fi

# we need common.config
if ! . "$source_iotests/common.config"
then
    _init_error "failed to source common.config"
fi

_full_imgfmt_details()
{
    if [ -n "$IMGOPTS" ]; then
        echo "$IMGFMT ($IMGOPTS)"
    else
        echo "$IMGFMT"
    fi
}

_full_platform_details()
{
    os=$(uname -s)
    host=$(hostname -s)
    kernel=$(uname -r)
    platform=$(uname -m)
    echo "$os/$platform $host $kernel"
}

_full_env_details()
{
    cat <<EOF
QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
IMGFMT        -- $FULL_IMGFMT_DETAILS
IMGPROTO      -- $IMGPROTO
PLATFORM      -- $FULL_HOST_DETAILS
TEST_DIR      -- $TEST_DIR
SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER

EOF
}

# $1 = prog to look for
set_prog_path()
{
    p=$(command -v $1 2> /dev/null)
    if [ -n "$p" -a -x "$p" ]; then
        type -p "$p"
    else
        return 1
    fi
}

if [ -z "$TEST_DIR" ]; then
        TEST_DIR=$PWD/scratch
fi

if [ ! -e "$TEST_DIR" ]; then
        mkdir "$TEST_DIR"
fi

diff="diff -u"
verbose=false
debug=false
group=false
xgroup=false
imgopts=false
showme=false
sortme=false
expunge=true
have_test_arg=false
cachemode=false

tmp="${TEST_DIR}"/$$
rm -f $tmp.list $tmp.tmp $tmp.sed

export IMGFMT=raw
export IMGFMT_GENERIC=true
export IMGPROTO=file
export IMGOPTS=""
export CACHEMODE="writeback"
export QEMU_IO_OPTIONS=""
export QEMU_IO_OPTIONS_NO_FMT=""
export CACHEMODE_IS_DEFAULT=true
export VALGRIND_QEMU=
export IMGKEYSECRET=
export IMGOPTSSYNTAX=false

# Save current tty settings, since an aborting qemu call may leave things
# screwed up
STTY_RESTORE=
if test -t 0; then
    STTY_RESTORE=$(stty -g)
fi

for r
do

    if $group
    then
        # arg after -g
        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
s/ .*//p
}')
        if [ -z "$group_list" ]
        then
            echo "Group \"$r\" is empty or not defined?"
            exit 1
        fi
        [ ! -s $tmp.list ] && touch $tmp.list
        for t in $group_list
        do
            if grep -s "^$t\$" $tmp.list >/dev/null
            then
                :
            else
                echo "$t" >>$tmp.list
            fi
        done
        group=false
        continue

    elif $xgroup
    then
        # arg after -x
        # Populate $tmp.list with all tests
        awk '/^[0-9]{3,}/ {print $1}' "${source_iotests}/group" > $tmp.list 2>/dev/null
        group_list=$(sed -n <"$source_iotests/group" -e 's/$/ /' -e "/^[0-9][0-9][0-9].* $r /"'{
s/ .*//p
}')
        if [ -z "$group_list" ]
        then
            echo "Group \"$r\" is empty or not defined?"
            exit 1
        fi
        numsed=0
        rm -f $tmp.sed
        for t in $group_list
        do
            if [ $numsed -gt 100 ]
            then
                sed -f $tmp.sed <$tmp.list >$tmp.tmp
                mv $tmp.tmp $tmp.list
                numsed=0
                rm -f $tmp.sed
            fi
            echo "/^$t\$/d" >>$tmp.sed
            numsed=$(expr $numsed + 1)
        done
        sed -f $tmp.sed <$tmp.list >$tmp.tmp
        mv $tmp.tmp $tmp.list
        xgroup=false
        continue

    elif $imgopts
    then
        IMGOPTS="$r"
        imgopts=false
        continue
    elif $cachemode
    then
        CACHEMODE="$r"
        CACHEMODE_IS_DEFAULT=false
        cachemode=false
        continue
    fi

    xpand=true
    case "$r"
    in

        -\? | -h | --help)        # usage
            echo "Usage: $0 [options] [testlist]"'

common options
    -v                  verbose
    -d                  debug

image format options
    -raw                test raw (default)
    -bochs              test bochs
    -cloop              test cloop
    -parallels          test parallels
    -qcow               test qcow
    -qcow2              test qcow2
    -qed                test qed
    -vdi                test vdi
    -vpc                test vpc
    -vhdx               test vhdx
    -vmdk               test vmdk
    -luks               test luks
    -dmg                test dmg

image protocol options
    -file               test file (default)
    -rbd                test rbd
    -sheepdog           test sheepdog
    -nbd                test nbd
    -ssh                test ssh
    -nfs                test nfs
    -vxhs               test vxhs

other options
    -xdiff              graphical mode diff
    -nocache            use O_DIRECT on backing file
    -misalign           misalign memory allocations
    -n                  show me, do not run tests
    -o options          -o options to pass to qemu-img create/convert
    -c mode             cache mode
    -makecheck          pretty print output for make check

testlist options
    -g group[,group...]        include tests from these groups
    -x group[,group...]        exclude tests from these groups
    NNN                        include test NNN
    NNN-NNN                    include test range (eg. 012-021)
'
            exit 0
            ;;

        -raw)
            IMGFMT=raw
            xpand=false
            ;;

        -bochs)
            IMGFMT=bochs
            IMGFMT_GENERIC=false
            xpand=false
            ;;

        -cloop)
            IMGFMT=cloop
            IMGFMT_GENERIC=false
            xpand=false
            ;;

        -parallels)
            IMGFMT=parallels
            xpand=false
            ;;

        -qcow)
            IMGFMT=qcow
            xpand=false
            ;;

        -qcow2)
            IMGFMT=qcow2
            xpand=false
            ;;

        -luks)
            IMGOPTSSYNTAX=true
            IMGFMT=luks
            IMGKEYSECRET=123456
            xpand=false
            ;;

        -dmg)
            IMGFMT=dmg
            IMGFMT_GENERIC=false
            xpand=false
            ;;

        -qed)
            IMGFMT=qed
            xpand=false
            ;;

        -vdi)
            IMGFMT=vdi
            xpand=false
            ;;

        -vmdk)
            IMGFMT=vmdk
            xpand=false
            ;;

        -vpc)
            IMGFMT=vpc
            xpand=false
            ;;

        -vhdx)
            IMGFMT=vhdx
            xpand=false
            ;;

        -file)
            IMGPROTO=file
            xpand=false
            ;;

        -rbd)
            IMGPROTO=rbd
            xpand=false
            ;;

        -sheepdog)
            IMGPROTO=sheepdog
            xpand=false
            ;;

        -nbd)
            IMGPROTO=nbd
            xpand=false
            ;;

        -vxhs)
            IMGPROTO=vxhs
            xpand=false
            ;;

        -ssh)
            IMGPROTO=ssh
            xpand=false
            ;;

        -nfs)
            IMGPROTO=nfs
            xpand=false
            ;;

        -nocache)
            CACHEMODE="none"
            CACHEMODE_IS_DEFAULT=false
            xpand=false
            ;;

        -misalign)
            QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --misalign"
            xpand=false
            ;;

        -valgrind)
            VALGRIND_QEMU='y'
            xpand=false
            ;;

        -g)        # -g group ... pick from group file
            group=true
            xpand=false
            ;;

        -xdiff)        # graphical diff mode
            xpand=false

            if [ ! -z "$DISPLAY" ]
            then
                command -v xdiff >/dev/null 2>&1 && diff=xdiff
                command -v gdiff >/dev/null 2>&1 && diff=gdiff
                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
            fi
            ;;
        -makecheck)   # makecheck friendly output
            makecheck=true
            xpand=false
            ;;
        -n)        # show me, don't do it
            showme=true
            xpand=false
            ;;
        -o)
            imgopts=true
            xpand=false
            ;;
        -c)
            cachemode=true
            xpand=false
            ;;
        -T)        # deprecated timestamp option
            xpand=false
            ;;

        -v)
            verbose=true
            xpand=false
            ;;
        -d)
            debug=true
            xpand=false
            ;;
        -x)        # -x group ... exclude from group file
            xgroup=true
            xpand=false
            ;;
        '[0-9][0-9][0-9] [0-9][0-9][0-9][0-9]')
            echo "No tests?"
            status=1
            exit $status
            ;;

        [0-9]*-[0-9]*)
            eval $(echo $r | sed -e 's/^/start=/' -e 's/-/ end=/')
            ;;

        [0-9]*-)
            eval $(echo $r | sed -e 's/^/start=/' -e 's/-//')
            end=$(echo [0-9][0-9][0-9] [0-9][0-9][0-9][0-9] | sed -e 's/\[0-9]//g' -e 's/  *$//' -e 's/.* //')
            if [ -z "$end" ]
            then
                echo "No tests in range \"$r\"?"
                status=1
                exit $status
            fi
            ;;

        *)
            start=$r
            end=$r
            ;;

    esac

    # get rid of leading 0s as can be interpreted as octal
    start=$(echo $start | sed 's/^0*//')
    end=$(echo $end | sed 's/^0*//')

    if $xpand
    then
        have_test_arg=true
        awk </dev/null '
BEGIN        { for (t='$start'; t<='$end'; t++) printf "%03d\n",t }' \
        | while read id
        do
            if grep -s "^$id " "$source_iotests/group" >/dev/null
            then
                # in group file ... OK
                echo $id >>$tmp.list
            else
                if [ -f expunged ] && $expunge && egrep "^$id([         ]|\$)" expunged >/dev/null
                then
                    # expunged ... will be reported, but not run, later
                    echo $id >>$tmp.list
                else
                    # oops
                    if [ "$start" == "$end" -a "$id" == "$end" ]
                    then
                        echo "$id - unknown test"
                        exit 1
                    else
                        echo "$id - unknown test, ignored"
                    fi
                fi
            fi
        done || exit 1
    fi

done

# Set qemu-io cache mode with $CACHEMODE we have
QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"

QEMU_IO_OPTIONS_NO_FMT="$QEMU_IO_OPTIONS"
if [ "$IMGOPTSSYNTAX" != "true" ]; then
    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT"
fi

# Set default options for qemu-img create -o if they were not specified
if [ "$IMGFMT" == "qcow2" ] && ! (echo "$IMGOPTS" | grep "compat=" > /dev/null); then
    IMGOPTS=$(_optstr_add "$IMGOPTS" "compat=1.1")
fi
if [ "$IMGFMT" == "luks" ] && ! (echo "$IMGOPTS" | grep "iter-time=" > /dev/null); then
    IMGOPTS=$(_optstr_add "$IMGOPTS" "iter-time=10")
fi

if [ -z "$SAMPLE_IMG_DIR" ]; then
        SAMPLE_IMG_DIR="$source_iotests/sample_images"
fi

export TEST_DIR
export SAMPLE_IMG_DIR

if [ -s $tmp.list ]
then
    # found some valid test numbers ... this is good
    :
else
    if $have_test_arg
    then
        # had test numbers, but none in group file ... do nothing
        touch $tmp.list
    else
        # no test numbers, do everything from group file
        sed -n -e '/^[0-9][0-9][0-9]*/s/[         ].*//p' <"$source_iotests/group" >$tmp.list
    fi
fi

# should be sort -n, but this did not work for Linux when this
# was ported from IRIX
#
list=$(sort $tmp.list)
rm -f $tmp.list $tmp.tmp $tmp.sed

if [ -z "$QEMU_PROG" ]
then
    if [ -x "$build_iotests/qemu" ]; then
        export QEMU_PROG="$build_iotests/qemu"
    elif [ -x "$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}" ]; then
        export QEMU_PROG="$build_root/${qemu_arch}-softmmu/qemu-system-${qemu_arch}"
    else
        pushd "$build_root" > /dev/null
        for binary in *-softmmu/qemu-system-*
        do
            if [ -x "$binary" ]
            then
                export QEMU_PROG="$build_root/$binary"
                break
            fi
        done
        popd > /dev/null
        [ "$QEMU_PROG" = "" ] && _init_error "qemu not found"
    fi
fi
export QEMU_PROG="$(type -p "$QEMU_PROG")"

case "$QEMU_PROG" in
    *qemu-system-arm|*qemu-system-aarch64)
        export QEMU_OPTIONS="-nodefaults -machine virt,accel=qtest"
        ;;
    *qemu-system-tricore)
        export QEMU_OPTIONS="-nodefaults -machine tricore_testboard,accel=qtest"
        ;;
    *)
        export QEMU_OPTIONS="-nodefaults -machine accel=qtest"
        ;;
esac

if [ -z "$QEMU_IMG_PROG" ]; then
    if [ -x "$build_iotests/qemu-img" ]; then
        export QEMU_IMG_PROG="$build_iotests/qemu-img"
    elif [ -x "$build_root/qemu-img" ]; then
        export QEMU_IMG_PROG="$build_root/qemu-img"
    else
        _init_error "qemu-img not found"
    fi
fi
export QEMU_IMG_PROG="$(type -p "$QEMU_IMG_PROG")"

if [ -z "$QEMU_IO_PROG" ]; then
    if [ -x "$build_iotests/qemu-io" ]; then
        export QEMU_IO_PROG="$build_iotests/qemu-io"
    elif [ -x "$build_root/qemu-io" ]; then
        export QEMU_IO_PROG="$build_root/qemu-io"
    else
        _init_error "qemu-io not found"
    fi
fi
export QEMU_IO_PROG="$(type -p "$QEMU_IO_PROG")"

if [ -z $QEMU_NBD_PROG ]; then
    if [ -x "$build_iotests/qemu-nbd" ]; then
        export QEMU_NBD_PROG="$build_iotests/qemu-nbd"
    elif [ -x "$build_root/qemu-nbd" ]; then
        export QEMU_NBD_PROG="$build_root/qemu-nbd"
    else
        _init_error "qemu-nbd not found"
    fi
fi
export QEMU_NBD_PROG="$(type -p "$QEMU_NBD_PROG")"

if [ -z "$QEMU_VXHS_PROG" ]; then
    export QEMU_VXHS_PROG="$(set_prog_path qnio_server)"
fi

if [ -x "$build_iotests/socket_scm_helper" ]
then
    export SOCKET_SCM_HELPER="$build_iotests/socket_scm_helper"
fi

default_machine=$($QEMU_PROG -machine help | sed -n '/(default)/ s/ .*//p')
default_alias_machine=$($QEMU_PROG -machine help | \
   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
if [[ "$default_alias_machine" ]]; then
    default_machine="$default_alias_machine"
fi

export QEMU_DEFAULT_MACHINE="$default_machine"

TIMESTAMP_FILE=check.time-$IMGPROTO-$IMGFMT

_wallclock()
{
    date "+%H %M %S" | awk '{ print $1*3600 + $2*60 + $3 }'
}

_wrapup()
{
    if $showme
    then
        :
    elif $needwrap
    then
        if [ -f $TIMESTAMP_FILE -a -f $tmp.time ]
        then
            cat $TIMESTAMP_FILE $tmp.time \
            | awk '
        { t[$1] = $2 }
END        { if (NR > 0) {
            for (i in t) print i " " t[i]
          }
        }' \
            | sort -n >$tmp.out
            mv $tmp.out $TIMESTAMP_FILE
        fi

        if [ -f $tmp.expunged ]
        then
            notrun=$(wc -l <$tmp.expunged | sed -e 's/  *//g')
            try=$(expr $try - $notrun)
            list=$(echo "$list" | sed -f $tmp.expunged)
        fi

        echo "" >>check.log
        date >>check.log
        echo $list | fmt | sed -e 's/^/    /' >>check.log
        $interrupt && echo "Interrupted!" >>check.log

        if [ ! -z "$notrun" ]
        then
            echo "Not run:$notrun"
            echo "Not run:$notrun" >>check.log
        fi
        if [ ! -z "$casenotrun" ]
        then
            echo "Some cases not run in:$casenotrun"
            echo "Some cases not run in:$casenotrun" >>check.log
        fi
        if [ ! -z "$n_bad" -a $n_bad != 0 ]
        then
            echo "Failures:$bad"
            echo "Failed $n_bad of $try tests"
            echo "Failures:$bad" | fmt >>check.log
            echo "Failed $n_bad of $try tests" >>check.log
        else
            echo "Passed all $try tests"
            echo "Passed all $try tests" >>check.log
        fi
        needwrap=false
    fi

    if test -n "$STTY_RESTORE"; then
        stty $STTY_RESTORE
    fi
    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
    rm -f $tmp.*
}

trap "_wrapup; exit \$status" 0 1 2 3 15

# Report the test start and results. For makecheck we want to pretty
# print the whole report at the end of the execution.
# args: $seq, $starttime, $lasttime
_report_test_start()
{
    if ! $makecheck; then
        if [ -n "$3" ]; then
            local lasttime=" (last: $3s)"
        fi
        printf "%-8s %-10s [%s]            %4s%-14s\r" "$1" "..." "$2" "..." "$lasttime"
    fi
}
# args:$seq $status $starttime $lasttime $thistime $details
_report_test_result()
{
    local status lasttime thistime
    if $makecheck; then
        if [ -n "$2" ] && [ "$2" != "pass" ]; then
            status=" [$2]"
        fi
        printf "  TEST    iotest-$IMGFMT: %s%s\n" "$1" "$status"
        return
    fi

    if [ -n "$4" ]; then
        lasttime=" (last: $4s)"
    fi
    if [ -n "$5" ]; then
        thistime=" $5s"
    fi
    case "$2" in
        "pass")     status=$(printf "\e[32m%-10s\e[0m" "$2") ;;
        "fail")     status=$(printf "\e[1m\e[31m%-10s\e[0m" "$2") ;;
        "not run")  status=$(printf "\e[33m%-10s\e[0m" "$2") ;;
        *)          status=$(printf "%-10s" "$2") ;;
    esac

    printf "%-8s %s [%s] [%s] %4s%-14s %s\n" "$1" "$status" "$3" "$(date '+%T')" "$thistime" "$lasttime" "$6"
}

[ -f $TIMESTAMP_FILE ] || touch $TIMESTAMP_FILE

FULL_IMGFMT_DETAILS=$(_full_imgfmt_details)
FULL_HOST_DETAILS=$(_full_platform_details)

if ! $makecheck; then
    _full_env_details
fi

seq="check"

[ -n "$TESTS_REMAINING_LOG" ] && echo $list > $TESTS_REMAINING_LOG

for seq in $list
do
    err=false       # error flag
    printdiff=false # show diff to reference output?
    status=""       # test result summary
    results=""      # test result details

    if [ -n "$TESTS_REMAINING_LOG" ] ; then
        sed -e "s/$seq//" -e 's/  / /' -e 's/^ *//' $TESTS_REMAINING_LOG > $TESTS_REMAINING_LOG.tmp
        mv $TESTS_REMAINING_LOG.tmp $TESTS_REMAINING_LOG
        sync
    fi

    lasttime=$(sed -n -e "/^$seq /s/.* //p" <$TIMESTAMP_FILE)
    starttime=$(date "+%T")
    _report_test_start $seq $starttime $lasttime

    if $showme
    then
        status="not run"
    elif [ -f expunged ] && $expunge && egrep "^$seq([         ]|\$)" expunged >/dev/null
    then
        status="not run"
        results="expunged"
        rm -f $seq.out.bad
        echo "/^$seq\$/d" >>$tmp.expunged
    elif [ ! -f "$source_iotests/$seq" ]
    then
        status="not run"
        results="no such test?"
        echo "/^$seq\$/d" >>$tmp.expunged
    else
        # really going to try and run this one
        #
        rm -f $seq.out.bad
        rm -f core $seq.notrun
        rm -f $seq.casenotrun

        start=$(_wallclock)

        if [ "$(head -n 1 "$source_iotests/$seq")" == "#!/usr/bin/env python" ]; then
            run_command="$PYTHON $seq"
        else
            run_command="./$seq"
        fi
        export OUTPUT_DIR=$PWD
        if $debug; then
            (cd "$source_iotests";
            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
                    $run_command -d 2>&1 | tee $tmp.out)
        else
            (cd "$source_iotests";
            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
                    $run_command >$tmp.out 2>&1)
        fi
        sts=$?
        stop=$(_wallclock)

        if [ -f core ]
        then
            mv core $seq.core
            status="fail"
            results="[dumped core] $seq.core"
            err=true
        fi

        if [ -f $seq.notrun ]
        then
            # overwrites timestamp output
            status="not run"
            results="$(cat $seq.notrun)"
        else
            if [ $sts -ne 0 ]
            then
                status="fail"
                results=$(printf %s "[failed, exit status $sts]")
                err=true
            fi

            reference="$source_iotests/$seq.out"
            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
            if [ -f "$reference_machine" ]; then
                reference="$reference_machine"
            fi

            reference_format="$source_iotests/$seq.out.$IMGFMT"
            if [ -f "$reference_format" ]; then
                reference="$reference_format"
            fi

            if [ "$CACHEMODE" = "none" ]; then
                [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
            fi

            if [ ! -f "$reference" ]
            then
                status="fail"
                reason="no qualified output"
                err=true
            else
                if diff -w "$reference" $tmp.out >/dev/null 2>&1
                then
                    if ! $err; then
                        status="pass"
                        thistime=$(expr $stop - $start)
                        echo "$seq $thistime" >>$tmp.time
                    fi
                else
                    mv $tmp.out $seq.out.bad
                    $diff -w "$reference" "$PWD"/$seq.out.bad
                    status="fail"
                    results="output mismatch (see $seq.out.bad)"
                    printdiff=true
                    err=true
                fi
            fi
        fi
        if [ -f $seq.casenotrun ]
        then
            cat $seq.casenotrun
            casenotrun="$casenotrun $seq"
        fi
    fi

    # come here for each test, except when $showme is true
    #
    _report_test_result $seq "$status" "$starttime" "$lasttime" "$thistime" "$results"
    case "$status" in
        "pass")
            try=$(expr $try + 1)
            ;;
        "fail")
            try=$(expr $try + 1)
            if $makecheck; then
                _full_env_details
            fi
            if $printdiff; then
                $diff -w "$reference" "$PWD"/$seq.out.bad
            fi
            bad="$bad $seq"
            n_bad=$(expr $n_bad + 1)
            quick=false
            ;;
        "not run")
            notrun="$notrun $seq"
            ;;
    esac

    seq="after_$seq"
done

interrupt=false
status=$(expr $n_bad)
exit