xref: /openbmc/u-boot/test/fs/fs-test.sh (revision e246b728)
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0+
3#
4# (C) Copyright 2014 Suriyan Ramasami
5
6# Invoke this test script from U-Boot base directory as ./test/fs/fs-test.sh
7# It currently tests the fs/sb and native commands for ext4 and fat partitions
8# Expected results are as follows:
9# EXT4 tests:
10# fs-test.sb.ext4	Summary: PASS: 24 FAIL: 0
11# fs-test.nonfs.ext4	Summary: PASS: 24 FAIL: 0
12# fs-test.fs.ext4	Summary: PASS: 24 FAIL: 0
13# FAT16 tests:
14# fs-test.sb.fat16	Summary: PASS: 24 FAIL: 0
15# fs-test.nonfs.fat16	Summary: PASS: 24 FAIL: 0
16# fs-test.fs.fat16	Summary: PASS: 24 FAIL: 0
17# FAT32 tests:
18# fs-test.sb.fat32	Summary: PASS: 24 FAIL: 0
19# fs-test.nonfs.fat32	Summary: PASS: 24 FAIL: 0
20# fs-test.fs.fat32	Summary: PASS: 24 FAIL: 0
21# --------------------------------------------
22# Total Summary: TOTAL PASS: 216 TOTAL FAIL: 0
23# --------------------------------------------
24
25# pre-requisite binaries list.
26PREREQ_BINS="md5sum mkfs mount umount dd fallocate mkdir"
27
28# All generated output files from this test will be in $OUT_DIR
29# Hence everything is sandboxed.
30OUT_DIR="sandbox/test/fs"
31
32# Location of generated sandbox u-boot
33UBOOT="./sandbox/u-boot"
34
35# Our mount directory will be in the sandbox
36MOUNT_DIR="${OUT_DIR}/mnt"
37
38# The file system image we create will have the $IMG prefix.
39IMG="${OUT_DIR}/3GB"
40
41# $SMALL_FILE is the name of the 1MB file in the file system image
42SMALL_FILE="1MB.file"
43
44# $BIG_FILE is the name of the 2.5GB file in the file system image
45BIG_FILE="2.5GB.file"
46
47# $MD5_FILE will have the expected md5s when we do the test
48# They shall have a suffix which represents their file system (ext4/fat16/...)
49MD5_FILE="${OUT_DIR}/md5s.list"
50
51# $OUT shall be the prefix of the test output. Their suffix will be .out
52OUT="${OUT_DIR}/fs-test"
53
54# Full Path of the 1 MB file that shall be created in the fs image.
55MB1="${MOUNT_DIR}/${SMALL_FILE}"
56GB2p5="${MOUNT_DIR}/${BIG_FILE}"
57
58# ************************
59# * Functions start here *
60# ************************
61
62# Check if the prereq binaries exist, or exit
63function check_prereq() {
64	for prereq in $PREREQ_BINS; do
65		if [ ! -x "`which $prereq`" ]; then
66			echo "Missing $prereq binary. Exiting!"
67			exit
68		fi
69	done
70
71	# We use /dev/urandom to create files. Check if it exists.
72	if [ ! -c /dev/urandom ]; then
73		echo "Missing character special /dev/urandom. Exiting!"
74		exit
75	fi
76}
77
78# If 1st param is "clean", then clean out the generated files and exit
79function check_clean() {
80	if [ "$1" = "clean" ]; then
81		rm -rf "$OUT_DIR"
82		echo "Cleaned up generated files. Exiting"
83		exit
84	fi
85}
86
87# Generate sandbox U-Boot - gleaned from /test/dm/test-dm.sh
88function compile_sandbox() {
89	unset CROSS_COMPILE
90	NUM_CPUS=$(cat /proc/cpuinfo |grep -c processor)
91	make O=sandbox sandbox_config
92	make O=sandbox -s -j${NUM_CPUS}
93
94	# Check if U-Boot exists
95	if [ ! -x "$UBOOT" ]; then
96		echo "$UBOOT does not exist or is not executable"
97		echo "Build error?"
98		echo "Please run this script as ./test/fs/`basename $0`"
99		exit
100	fi
101}
102
103# Clean out all generated files other than the file system images
104# We save time by not deleting and recreating the file system images
105function prepare_env() {
106	rm -f ${MD5_FILE}.* ${OUT}.*
107	mkdir -p ${OUT_DIR}
108}
109
110# 1st parameter is the name of the image file to be created
111# 2nd parameter is the filesystem - fat16 ext4 etc
112# -F cant be used with fat as it means something else.
113function create_image() {
114	# Create image if not already present - saves time, while debugging
115	case "$2" in
116		fat16)
117		MKFS_OPTION="-F 16"
118		FS_TYPE="fat"
119		;;
120		fat32)
121		MKFS_OPTION="-F 32"
122		FS_TYPE="fat"
123		;;
124		ext4)
125		MKFS_OPTION="-F"
126		FS_TYPE="ext4"
127		;;
128	esac
129
130	if [ ! -f "$1" ]; then
131		fallocate -l 3G "$1" &> /dev/null
132		if [ $? -ne 0 ]; then
133			echo fallocate failed - using dd instead
134			dd if=/dev/zero of=$1 bs=1024 count=$((3 * 1024 * 1024))
135			if [ $? -ne 0 ]; then
136				echo Could not create empty disk image
137				exit $?
138			fi
139		fi
140		mkfs -t "$FS_TYPE" $MKFS_OPTION "$1" &> /dev/null
141		if [ $? -ne 0 -a "$FS_TYPE" = "fat" ]; then
142			# If we fail and we did fat, try vfat.
143			mkfs -t vfat $MKFS_OPTION "$1" &> /dev/null
144		fi
145		if [ $? -ne 0 ]; then
146			echo Could not create filesystem
147			exit $?
148		fi
149	fi
150}
151
152# 1st parameter is image file
153# 2nd parameter is file system type - fat16/ext4/...
154# 3rd parameter is name of small file
155# 4th parameter is name of big file
156# 5th parameter is fs/nonfs/sb - to dictate generic fs commands or
157# otherwise or sb hostfs
158# 6th parameter is the directory path for the files. Its "" for generic
159# fs and ext4/fat and full patch for sb hostfs
160# UBOOT is set in env
161function test_image() {
162	addr="0x01000008"
163	length="0x00100000"
164
165	case "$2" in
166		fat*)
167		FPATH=""
168		PREFIX="fat"
169		WRITE="write"
170		;;
171
172		ext4)
173		# ext4 needs absolute path
174		FPATH="/"
175		PREFIX="ext4"
176		WRITE="write"
177		;;
178
179		*)
180		echo "Unhandled filesystem $2. Exiting!"
181		exit
182		;;
183	esac
184
185	case "$5" in
186		fs)
187		PREFIX=""
188		WRITE="save"
189		SUFFIX=" 0:0"
190		;;
191
192		nonfs)
193		SUFFIX=" 0:0"
194		;;
195
196		sb)
197		PREFIX="host "
198		WRITE="save"
199		SUFFIX="fs -"
200		;;
201
202		*)
203		echo "Unhandled mode $5. Exiting!"
204		exit
205		;;
206
207	esac
208
209	# sb always uses full path to mointpoint, irrespective of filesystem
210	if [ "$5" = "sb" ]; then
211		FPATH=${6}/
212	fi
213
214	FILE_WRITE=${3}.w
215	FILE_SMALL=$3
216	FILE_BIG=$4
217
218	# In u-boot commands, <interface> stands for host or hostfs
219	# hostfs maps to the host fs.
220	# host maps to the "host bind" that we do
221
222	$UBOOT << EOF
223sb=$5
224setenv bind 'if test "\$sb" != sb; then host bind 0 "$1"; fi'
225run bind
226# Test Case 1 - ls
227${PREFIX}ls host${SUFFIX} $6
228# In addition, test with a nonexistent directory to see if we crash.
229${PREFIX}ls host${SUFFIX} invalid_d
230#
231# We want ${PREFIX}size host 0:0 $3 for host commands and
232# host size hostfs - $3 for hostfs commands.
233# 1MB is 0x0010 0000
234# Test Case 2a - size of small file
235${PREFIX}size host${SUFFIX} ${FPATH}$FILE_SMALL
236printenv filesize
237setenv filesize
238# Test Case 2b - size of small file via a path using '..'
239${PREFIX}size host${SUFFIX} ${FPATH}SUBDIR/../$FILE_SMALL
240printenv filesize
241setenv filesize
242
243# 2.5GB (1024*1024*2500) is 0x9C40 0000
244# Test Case 3 - size of big file
245${PREFIX}size host${SUFFIX} ${FPATH}$FILE_BIG
246printenv filesize
247setenv filesize
248
249# Notes about load operation
250# If I use 0x01000000 I get DMA misaligned error message
251# Last two parameters are size and offset.
252
253# Test Case 4a - Read full 1MB of small file
254${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_SMALL
255printenv filesize
256# Test Case 4b - Read full 1MB of small file
257md5sum $addr \$filesize
258setenv filesize
259
260# Test Case 5a - First 1MB of big file
261${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG $length 0x0
262printenv filesize
263# Test Case 5b - First 1MB of big file
264md5sum $addr \$filesize
265setenv filesize
266
267# fails for ext as no offset support
268# Test Case 6a - Last 1MB of big file
269${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG $length 0x9C300000
270printenv filesize
271# Test Case 6b - Last 1MB of big file
272md5sum $addr \$filesize
273setenv filesize
274
275# fails for ext as no offset support
276# Test Case 7a - One from the last 1MB chunk of 2GB
277${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG $length 0x7FF00000
278printenv filesize
279# Test Case 7b - One from the last 1MB chunk of 2GB
280md5sum $addr \$filesize
281setenv filesize
282
283# fails for ext as no offset support
284# Test Case 8a - One from the start 1MB chunk from 2GB
285${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG $length 0x80000000
286printenv filesize
287# Test Case 8b - One from the start 1MB chunk from 2GB
288md5sum $addr \$filesize
289setenv filesize
290
291# fails for ext as no offset support
292# Test Case 9a - One 1MB chunk crossing the 2GB boundary
293${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG $length 0x7FF80000
294printenv filesize
295# Test Case 9b - One 1MB chunk crossing the 2GB boundary
296md5sum $addr \$filesize
297setenv filesize
298
299# Generic failure case
300# Test Case 10 - 2MB chunk from the last 1MB of big file
301${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_BIG 0x00200000 0x9C300000
302printenv filesize
303#
304
305# Read 1MB from small file
306${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_SMALL
307# Write it back to test the writes
308# Test Case 11a - Check that the write succeeded
309${PREFIX}${WRITE} host${SUFFIX} $addr ${FPATH}$FILE_WRITE \$filesize
310mw.b $addr 00 100
311${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_WRITE
312# Test Case 11b - Check md5 of written to is same as the one read from
313md5sum $addr \$filesize
314setenv filesize
315#
316
317# Next test case checks writing a file whose dirent
318# is the first in the block, which is always true for "."
319# The write should fail, but the lookup should work
320# Test Case 12 - Check directory traversal
321${PREFIX}${WRITE} host${SUFFIX} $addr ${FPATH}. 0x10
322
323# Read 1MB from small file
324${PREFIX}load host${SUFFIX} $addr ${FPATH}$FILE_SMALL
325# Write it via "same directory", i.e. "." dirent
326# Test Case 13a - Check directory traversal
327${PREFIX}${WRITE} host${SUFFIX} $addr ${FPATH}./${FILE_WRITE}2 \$filesize
328mw.b $addr 00 100
329${PREFIX}load host${SUFFIX} $addr ${FPATH}./${FILE_WRITE}2
330# Test Case 13b - Check md5 of written to is same as the one read from
331md5sum $addr \$filesize
332setenv filesize
333mw.b $addr 00 100
334${PREFIX}load host${SUFFIX} $addr ${FPATH}${FILE_WRITE}2
335# Test Case 13c - Check md5 of written to is same as the one read from
336md5sum $addr \$filesize
337setenv filesize
338#
339reset
340
341EOF
342}
343
344# 1st argument is the name of the image file.
345# 2nd argument is the file where we generate the md5s of the files
346# generated with the appropriate start and length that we use to test.
347# It creates the necessary files in the image to test.
348# $GB2p5 is the path of the big file (2.5 GB)
349# $MB1 is the path of the small file (1 MB)
350# $MOUNT_DIR is the path we can use to mount the image file.
351function create_files() {
352	# Mount the image so we can populate it.
353	mkdir -p "$MOUNT_DIR"
354	sudo mount -o loop,rw "$1" "$MOUNT_DIR"
355
356	# Create a subdirectory.
357	sudo mkdir -p "$MOUNT_DIR/SUBDIR"
358
359	# Create big file in this image.
360	# Note that we work only on the start 1MB, couple MBs in the 2GB range
361	# and the last 1 MB of the huge 2.5GB file.
362	# So, just put random values only in those areas.
363	if [ ! -f "${GB2p5}" ]; then
364		sudo dd if=/dev/urandom of="${GB2p5}" bs=1M count=1 \
365			&> /dev/null
366		sudo dd if=/dev/urandom of="${GB2p5}" bs=1M count=2 seek=2047 \
367			&> /dev/null
368		sudo dd if=/dev/urandom of="${GB2p5}" bs=1M count=1 seek=2499 \
369			&> /dev/null
370	fi
371
372	# Create a small file in this image.
373	if [ ! -f "${MB1}" ]; then
374		sudo dd if=/dev/urandom of="${MB1}" bs=1M count=1 \
375			&> /dev/null
376	fi
377
378	# Delete the small file copies which possibly are written as part of a
379	# previous test.
380	sudo rm -f "${MB1}.w"
381	sudo rm -f "${MB1}.w2"
382
383	# Generate the md5sums of reads that we will test against small file
384	dd if="${MB1}" bs=1M skip=0 count=1 2> /dev/null | md5sum > "$2"
385
386	# Generate the md5sums of reads that we will test against big file
387	# One from beginning of file.
388	dd if="${GB2p5}" bs=1M skip=0 count=1 \
389		2> /dev/null | md5sum >> "$2"
390
391	# One from end of file.
392	dd if="${GB2p5}" bs=1M skip=2499 count=1 \
393		2> /dev/null | md5sum >> "$2"
394
395	# One from the last 1MB chunk of 2GB
396	dd if="${GB2p5}" bs=1M skip=2047 count=1 \
397		2> /dev/null | md5sum >> "$2"
398
399	# One from the start 1MB chunk from 2GB
400	dd if="${GB2p5}" bs=1M skip=2048 count=1 \
401		2> /dev/null | md5sum >> "$2"
402
403	# One 1MB chunk crossing the 2GB boundary
404	dd if="${GB2p5}" bs=512K skip=4095 count=2 \
405		2> /dev/null | md5sum >> "$2"
406
407	sync
408	sudo umount "$MOUNT_DIR"
409	rmdir "$MOUNT_DIR"
410}
411
412# 1st parameter is the text to print
413# if $? is 0 its a pass, else a fail
414# As a side effect it shall update env variable PASS and FAIL
415function pass_fail() {
416	if [ $? -eq 0 ]; then
417		echo pass - "$1"
418		PASS=$((PASS + 1))
419	else
420		echo FAIL - "$1"
421		FAIL=$((FAIL + 1))
422	fi
423}
424
425# 1st parameter is the string which leads to an md5 generation
426# 2nd parameter is the file we grep, for that string
427# 3rd parameter is the name of the file which has md5s in it
428# 4th parameter is the line # in the md5 file that we match it against
429# This function checks if the md5 of the file in the sandbox matches
430# that calculated while generating the file
431# 5th parameter is the string to print with the result
432check_md5() {
433	# md5sum in u-boot has output of form:
434	# md5 for 01000008 ... 01100007 ==> <md5>
435	# the 7th field is the actual md5
436	md5_src=`grep -A2 "$1" "$2" | grep "md5 for" | tr -d '\r'`
437	md5_src=($md5_src)
438	md5_src=${md5_src[6]}
439
440	# The md5 list, each line is of the form:
441	# - <md5>
442	# the 2nd field is the actual md5
443	md5_dst=`sed -n $4p $3`
444	md5_dst=($md5_dst)
445	md5_dst=${md5_dst[0]}
446
447	# For a pass they should match.
448	[ "$md5_src" = "$md5_dst" ]
449	pass_fail "$5"
450}
451
452# 1st parameter is the name of the output file to check
453# 2nd parameter is the name of the file containing the md5 expected
454# 3rd parameter is the name of the small file
455# 4th parameter is the name of the big file
456# 5th paramter is the name of the written file
457# This function checks the output file for correct results.
458function check_results() {
459	echo "** Start $1"
460
461	PASS=0
462	FAIL=0
463
464	# Check if the ls is showing correct results for 2.5 gb file
465	grep -A7 "Test Case 1 " "$1" | egrep -iq "2621440000 *$4"
466	pass_fail "TC1: ls of $4"
467
468	# Check if the ls is showing correct results for 1 mb file
469	grep -A7 "Test Case 1 " "$1" | egrep -iq "1048576 *$3"
470	pass_fail "TC1: ls of $3"
471
472	# Check size command on 1MB.file
473	egrep -A3 "Test Case 2a " "$1" | grep -q "filesize=100000"
474	pass_fail "TC2: size of $3"
475	# Check size command on 1MB.file via a path using '..'
476	egrep -A3 "Test Case 2b " "$1" | grep -q "filesize=100000"
477	pass_fail "TC2: size of $3 via a path using '..'"
478
479	# Check size command on 2.5GB.file
480	egrep -A3 "Test Case 3 " "$1" | grep -q "filesize=9c400000"
481	pass_fail "TC3: size of $4"
482
483	# Check read full mb of 1MB.file
484	grep -A4 "Test Case 4a " "$1" | grep -q "filesize=100000"
485	pass_fail "TC4: load of $3 size"
486	check_md5 "Test Case 4b " "$1" "$2" 1 "TC4: load from $3"
487
488	# Check first mb of 2.5GB.file
489	grep -A4 "Test Case 5a " "$1" | grep -q "filesize=100000"
490	pass_fail "TC5: load of 1st MB from $4 size"
491	check_md5 "Test Case 5b " "$1" "$2" 2 "TC5: load of 1st MB from $4"
492
493	# Check last mb of 2.5GB.file
494	grep -A4 "Test Case 6a " "$1" | grep -q "filesize=100000"
495	pass_fail "TC6: load of last MB from $4 size"
496	check_md5 "Test Case 6b " "$1" "$2" 3 "TC6: load of last MB from $4"
497
498	# Check last 1mb chunk of 2gb from 2.5GB file
499	grep -A4 "Test Case 7a " "$1" | grep -q "filesize=100000"
500	pass_fail "TC7: load of last 1mb chunk of 2GB from $4 size"
501	check_md5 "Test Case 7b " "$1" "$2" 4 \
502		"TC7: load of last 1mb chunk of 2GB from $4"
503
504	# Check first 1mb chunk after 2gb from 2.5GB file
505	grep -A4 "Test Case 8a " "$1" | grep -q "filesize=100000"
506	pass_fail "TC8: load 1st MB chunk after 2GB from $4 size"
507	check_md5 "Test Case 8b " "$1" "$2" 5 \
508		"TC8: load 1st MB chunk after 2GB from $4"
509
510	# Check 1mb chunk crossing the 2gb boundary from 2.5GB file
511	grep -A4 "Test Case 9a " "$1" | grep -q "filesize=100000"
512	pass_fail "TC9: load 1MB chunk crossing 2GB boundary from $4 size"
513	check_md5 "Test Case 9b " "$1" "$2" 6 \
514		"TC9: load 1MB chunk crossing 2GB boundary from $4"
515
516	# Check 2mb chunk from the last 1MB of 2.5GB file loads 1MB
517	grep -A5 "Test Case 10 " "$1" | grep -q "filesize=100000"
518	pass_fail "TC10: load 2MB from the last 1MB of $4 loads 1MB"
519
520	# Check 1mb chunk write
521	grep -A2 "Test Case 11a " "$1" | grep -q '1048576 bytes written'
522	pass_fail "TC11: 1MB write to $3.w - write succeeded"
523	check_md5 "Test Case 11b " "$1" "$2" 1 \
524		"TC11: 1MB write to $3.w - content verified"
525
526	# Check lookup of 'dot' directory
527	grep -A4 "Test Case 12 " "$1" | grep -q 'Unable to write'
528	pass_fail "TC12: 1MB write to . - write denied"
529
530	# Check directory traversal
531	grep -A2 "Test Case 13a " "$1" | grep -q '1048576 bytes written'
532	pass_fail "TC13: 1MB write to ./$3.w2 - write succeeded"
533	check_md5 "Test Case 13b " "$1" "$2" 1 \
534		"TC13: 1MB read from ./$3.w2 - content verified"
535	check_md5 "Test Case 13c " "$1" "$2" 1 \
536		"TC13: 1MB read from $3.w2 - content verified"
537
538	echo "** End $1"
539}
540
541# Takes in one parameter which is "fs" or "nonfs", which then dictates
542# if a fs test (size/load/save) or a nonfs test (fatread/extread) needs to
543# be performed.
544function test_fs_nonfs() {
545	echo "Creating files in $fs image if not already present."
546	create_files $IMAGE $MD5_FILE_FS
547
548	OUT_FILE="${OUT}.$1.${fs}.out"
549	test_image $IMAGE $fs $SMALL_FILE $BIG_FILE $1 "" \
550		> ${OUT_FILE} 2>&1
551	# strip out noise from fs code
552	grep -v -e "File System is consistent\|update journal finished" \
553		-e "reading .*\.file\|writing .*\.file.w" \
554		< ${OUT_FILE} > ${OUT_FILE}_clean
555	check_results ${OUT_FILE}_clean $MD5_FILE_FS $SMALL_FILE \
556		$BIG_FILE
557	TOTAL_FAIL=$((TOTAL_FAIL + FAIL))
558	TOTAL_PASS=$((TOTAL_PASS + PASS))
559	echo "Summary: PASS: $PASS FAIL: $FAIL"
560	echo "--------------------------------------------"
561}
562
563# ********************
564# * End of functions *
565# ********************
566
567check_clean "$1"
568check_prereq
569compile_sandbox
570prepare_env
571
572# Track TOTAL_FAIL and TOTAL_PASS
573TOTAL_FAIL=0
574TOTAL_PASS=0
575
576# In each loop, for a given file system image, we test both the
577# fs command, like load/size/write, the file system specific command
578# like: ext4load/ext4size/ext4write and the host load/ls/save commands.
579for fs in ext4 fat16 fat32; do
580
581	echo "Creating $fs image if not already present."
582	IMAGE=${IMG}.${fs}.img
583	MD5_FILE_FS="${MD5_FILE}.${fs}"
584	create_image $IMAGE $fs
585
586	# host commands test
587	echo "Creating files in $fs image if not already present."
588	create_files $IMAGE $MD5_FILE_FS
589
590	# Lets mount the image and test host hostfs commands
591	mkdir -p "$MOUNT_DIR"
592	case "$fs" in
593		fat*)
594		uid="uid=`id -u`"
595		;;
596		*)
597		uid=""
598		;;
599	esac
600	sudo mount -o loop,rw,$uid "$IMAGE" "$MOUNT_DIR"
601	sudo chmod 777 "$MOUNT_DIR"
602
603	OUT_FILE="${OUT}.sb.${fs}.out"
604	test_image $IMAGE $fs $SMALL_FILE $BIG_FILE sb `pwd`/$MOUNT_DIR \
605		> ${OUT_FILE} 2>&1
606	sudo umount "$MOUNT_DIR"
607	rmdir "$MOUNT_DIR"
608
609	check_results $OUT_FILE $MD5_FILE_FS $SMALL_FILE $BIG_FILE
610	TOTAL_FAIL=$((TOTAL_FAIL + FAIL))
611	TOTAL_PASS=$((TOTAL_PASS + PASS))
612	echo "Summary: PASS: $PASS FAIL: $FAIL"
613	echo "--------------------------------------------"
614
615	test_fs_nonfs nonfs
616	test_fs_nonfs fs
617done
618
619echo "Total Summary: TOTAL PASS: $TOTAL_PASS TOTAL FAIL: $TOTAL_FAIL"
620echo "--------------------------------------------"
621if [ $TOTAL_FAIL -eq 0 ]; then
622	echo "PASSED"
623	exit 0
624else
625	echo "FAILED"
626	exit 1
627fi
628