1#!/bin/bash
2# Copyright (c) 2016 Microsemi. All Rights Reserved.
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License as
6# published by the Free Software Foundation; either version 2 of
7# the License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it would be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# Author: Logan Gunthorpe <logang@deltatee.com>
15
16REMOTE_HOST=
17LIST_DEVS=FALSE
18
19DEBUGFS=${DEBUGFS-/sys/kernel/debug}
20
21PERF_RUN_ORDER=32
22MAX_MW_SIZE=0
23RUN_DMA_TESTS=
24DONT_CLEANUP=
25MW_SIZE=65536
26
27function show_help()
28{
29	echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
30	echo "Run tests on a pair of NTB endpoints."
31	echo
32	echo "If the NTB device loops back to the same host then,"
33	echo "just specifying the two PCI ids on the command line is"
34	echo "sufficient. Otherwise, if the NTB link spans two hosts"
35	echo "use the -r option to specify the hostname for the remote"
36	echo "device. SSH will then be used to test the remote side."
37	echo "An SSH key between the root users of the host would then"
38	echo "be highly recommended."
39	echo
40	echo "Options:"
41	echo "  -C              don't cleanup ntb modules on exit"
42	echo "  -h              show this help message"
43	echo "  -l              list available local and remote PCI ids"
44	echo "  -r REMOTE_HOST  specify the remote's hostname to connect"
45	echo "                  to for the test (using ssh)"
46	echo "  -m MW_SIZE      memory window size for ntb_tool"
47	echo "                  (default: $MW_SIZE)"
48	echo "  -d              run dma tests for ntb_perf"
49	echo "  -p ORDER        total data order for ntb_perf"
50	echo "                  (default: $PERF_RUN_ORDER)"
51	echo "  -w MAX_MW_SIZE  maxmium memory window size for ntb_perf"
52	echo
53}
54
55function parse_args()
56{
57	OPTIND=0
58	while getopts "b:Cdhlm:r:p:w:" opt; do
59		case "$opt" in
60		C)  DONT_CLEANUP=1 ;;
61		d)  RUN_DMA_TESTS=1 ;;
62		h)  show_help; exit 0 ;;
63		l)  LIST_DEVS=TRUE ;;
64		m)  MW_SIZE=${OPTARG} ;;
65		r)  REMOTE_HOST=${OPTARG} ;;
66		p)  PERF_RUN_ORDER=${OPTARG} ;;
67		w)  MAX_MW_SIZE=${OPTARG} ;;
68		\?)
69		    echo "Invalid option: -$OPTARG" >&2
70		    exit 1
71		    ;;
72		esac
73	done
74}
75
76parse_args "$@"
77shift $((OPTIND-1))
78LOCAL_DEV=$1
79shift
80parse_args "$@"
81shift $((OPTIND-1))
82REMOTE_DEV=$1
83shift
84parse_args "$@"
85
86set -e
87
88function _modprobe()
89{
90	modprobe "$@"
91
92	if [[ "$REMOTE_HOST" != "" ]]; then
93		ssh "$REMOTE_HOST" modprobe "$@"
94	fi
95}
96
97function split_remote()
98{
99	VPATH=$1
100	REMOTE=
101
102	if [[ "$VPATH" == *":/"* ]]; then
103		REMOTE=${VPATH%%:*}
104		VPATH=${VPATH#*:}
105	fi
106}
107
108function read_file()
109{
110	split_remote $1
111	if [[ "$REMOTE" != "" ]]; then
112		ssh "$REMOTE" cat "$VPATH"
113	else
114		cat "$VPATH"
115	fi
116}
117
118function write_file()
119{
120	split_remote $2
121	VALUE=$1
122
123	if [[ "$REMOTE" != "" ]]; then
124		ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
125	else
126		echo "$VALUE" > "$VPATH"
127	fi
128}
129
130function check_file()
131{
132	split_remote $1
133
134	if [[ "$REMOTE" != "" ]]; then
135		ssh "$REMOTE" "[[ -e ${VPATH} ]]"
136	else
137		[[ -e ${VPATH} ]]
138	fi
139}
140
141function subdirname()
142{
143	echo $(basename $(dirname $1)) 2> /dev/null
144}
145
146function find_pidx()
147{
148	PORT=$1
149	PPATH=$2
150
151	for ((i = 0; i < 64; i++)); do
152		PEER_DIR="$PPATH/peer$i"
153
154		check_file ${PEER_DIR} || break
155
156		PEER_PORT=$(read_file "${PEER_DIR}/port")
157		if [[ ${PORT} -eq $PEER_PORT ]]; then
158			echo $i
159			return 0
160		fi
161	done
162
163	return 1
164}
165
166function port_test()
167{
168	LOC=$1
169	REM=$2
170
171	echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
172
173	LOCAL_PORT=$(read_file "$LOC/port")
174	REMOTE_PORT=$(read_file "$REM/port")
175
176	LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
177	REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
178
179	echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
180	echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
181
182	echo "  Passed"
183}
184
185function link_test()
186{
187	LOC=$1
188	REM=$2
189	EXP=0
190
191	echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
192
193	if ! write_file "N" "$LOC/../link" 2> /dev/null; then
194		echo "  Unsupported"
195		return
196	fi
197
198	write_file "N" "$LOC/link_event"
199
200	if [[ $(read_file "$REM/link") != "N" ]]; then
201		echo "Expected link to be down in $REM/link" >&2
202		exit -1
203	fi
204
205	write_file "Y" "$LOC/../link"
206
207	echo "  Passed"
208}
209
210function doorbell_test()
211{
212	LOC=$1
213	REM=$2
214	EXP=0
215
216	echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
217
218	DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
219
220	write_file "c $DB_VALID_MASK" "$REM/db"
221
222	for ((i = 0; i < 64; i++)); do
223		DB=$(read_file "$REM/db")
224		if [[ "$DB" -ne "$EXP" ]]; then
225			echo "Doorbell doesn't match expected value $EXP " \
226			     "in $REM/db" >&2
227			exit -1
228		fi
229
230		let "MASK = (1 << $i) & $DB_VALID_MASK" || true
231		let "EXP = $EXP | $MASK" || true
232
233		write_file "s $MASK" "$LOC/peer_db"
234	done
235
236	write_file "c $DB_VALID_MASK" "$REM/db_mask"
237	write_file $DB_VALID_MASK "$REM/db_event"
238	write_file "s $DB_VALID_MASK" "$REM/db_mask"
239
240	write_file "c $DB_VALID_MASK" "$REM/db"
241
242	echo "  Passed"
243}
244
245function get_files_count()
246{
247	NAME=$1
248	LOC=$2
249
250	split_remote $LOC
251
252	if [[ "$REMOTE" == "" ]]; then
253		echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l)
254	else
255		echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
256		       wc -l" 2> /dev/null)
257	fi
258}
259
260function scratchpad_test()
261{
262	LOC=$1
263	REM=$2
264
265	echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
266
267	CNT=$(get_files_count "spad" "$LOC")
268
269	if [[ $CNT -eq 0 ]]; then
270		echo "  Unsupported"
271		return
272	fi
273
274	for ((i = 0; i < $CNT; i++)); do
275		VAL=$RANDOM
276		write_file "$VAL" "$LOC/spad$i"
277		RVAL=$(read_file "$REM/../spad$i")
278
279		if [[ "$VAL" -ne "$RVAL" ]]; then
280			echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
281			exit -1
282		fi
283	done
284
285	echo "  Passed"
286}
287
288function message_test()
289{
290	LOC=$1
291	REM=$2
292
293	echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
294
295	CNT=$(get_files_count "msg" "$LOC")
296
297	if [[ $CNT -eq 0 ]]; then
298		echo "  Unsupported"
299		return
300	fi
301
302	MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
303	MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
304
305	write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
306	write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
307
308	for ((i = 0; i < $CNT; i++)); do
309		VAL=$RANDOM
310		write_file "$VAL" "$LOC/msg$i"
311		RVAL=$(read_file "$REM/../msg$i")
312
313		if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
314			echo "Message $i value $RVAL doesn't match $VAL" >&2
315			exit -1
316		fi
317	done
318
319	echo "  Passed"
320}
321
322function get_number()
323{
324	KEY=$1
325
326	sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
327}
328
329function mw_alloc()
330{
331	IDX=$1
332	LOC=$2
333	REM=$3
334
335	write_file $MW_SIZE "$LOC/mw_trans$IDX"
336
337	INB_MW=$(read_file "$LOC/mw_trans$IDX")
338	MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
339	MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
340
341	write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
342
343	if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
344		echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
345	fi
346}
347
348function write_mw()
349{
350	split_remote $2
351
352	if [[ "$REMOTE" != "" ]]; then
353		ssh "$REMOTE" \
354			dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
355	else
356		dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
357	fi
358}
359
360function mw_check()
361{
362	IDX=$1
363	LOC=$2
364	REM=$3
365
366	write_mw "$LOC/mw$IDX"
367
368	split_remote "$LOC/mw$IDX"
369	if [[ "$REMOTE" == "" ]]; then
370		A=$VPATH
371	else
372		A=/tmp/ntb_test.$$.A
373		ssh "$REMOTE" cat "$VPATH" > "$A"
374	fi
375
376	split_remote "$REM/peer_mw$IDX"
377	if [[ "$REMOTE" == "" ]]; then
378		B=$VPATH
379	else
380		B=/tmp/ntb_test.$$.B
381		ssh "$REMOTE" cat "$VPATH" > "$B"
382	fi
383
384	cmp -n $MW_ALIGNED_SIZE "$A" "$B"
385	if [[ $? != 0 ]]; then
386		echo "Memory window $MW did not match!" >&2
387	fi
388
389	if [[ "$A" == "/tmp/*" ]]; then
390		rm "$A"
391	fi
392
393	if [[ "$B" == "/tmp/*" ]]; then
394		rm "$B"
395	fi
396}
397
398function mw_free()
399{
400	IDX=$1
401	LOC=$2
402	REM=$3
403
404	write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
405
406	write_file 0 "$LOC/mw_trans$IDX"
407}
408
409function mw_test()
410{
411	LOC=$1
412	REM=$2
413
414	CNT=$(get_files_count "mw_trans" "$LOC")
415
416	for ((i = 0; i < $CNT; i++)); do
417		echo "Running mw$i tests on: $(subdirname $LOC) / " \
418		     "$(subdirname $REM)"
419
420		mw_alloc $i $LOC $REM
421
422		mw_check $i $LOC $REM
423
424		mw_free $i $LOC  $REM
425
426		echo "  Passed"
427	done
428
429}
430
431function pingpong_test()
432{
433	LOC=$1
434	REM=$2
435
436	echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
437
438	LOC_START=$(read_file "$LOC/count")
439	REM_START=$(read_file "$REM/count")
440
441	sleep 7
442
443	LOC_END=$(read_file "$LOC/count")
444	REM_END=$(read_file "$REM/count")
445
446	if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
447		echo "Ping pong counter not incrementing!" >&2
448		exit 1
449	fi
450
451	echo "  Passed"
452}
453
454function perf_test()
455{
456	USE_DMA=$1
457
458	if [[ $USE_DMA == "1" ]]; then
459		WITH="with"
460	else
461		WITH="without"
462	fi
463
464	_modprobe ntb_perf total_order=$PERF_RUN_ORDER \
465		max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
466
467	echo "Running local perf test $WITH DMA"
468	write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
469	echo -n "  "
470	read_file "$LOCAL_PERF/run"
471	echo "  Passed"
472
473	echo "Running remote perf test $WITH DMA"
474	write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
475	echo -n "  "
476	read_file "$REMOTE_PERF/run"
477	echo "  Passed"
478
479	_modprobe -r ntb_perf
480}
481
482function ntb_tool_tests()
483{
484	LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
485	REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
486
487	echo "Starting ntb_tool tests..."
488
489	_modprobe ntb_tool
490
491	port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
492
493	LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
494	REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
495
496	link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
497	link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
498
499	#Ensure the link is up on both sides before continuing
500	write_file "Y" "$LOCAL_PEER_TOOL/link_event"
501	write_file "Y" "$REMOTE_PEER_TOOL/link_event"
502
503	doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
504	doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
505
506	scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
507	scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
508
509	message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
510	message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
511
512	mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
513	mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
514
515	_modprobe -r ntb_tool
516}
517
518function ntb_pingpong_tests()
519{
520	LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
521	REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
522
523	echo "Starting ntb_pingpong tests..."
524
525	_modprobe ntb_pingpong
526
527	pingpong_test $LOCAL_PP $REMOTE_PP
528
529	_modprobe -r ntb_pingpong
530}
531
532function ntb_perf_tests()
533{
534	LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
535	REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
536
537	echo "Starting ntb_perf tests..."
538
539	perf_test 0
540
541	if [[ $RUN_DMA_TESTS ]]; then
542		perf_test 1
543	fi
544}
545
546function cleanup()
547{
548	set +e
549	_modprobe -r ntb_tool 2> /dev/null
550	_modprobe -r ntb_perf 2> /dev/null
551	_modprobe -r ntb_pingpong 2> /dev/null
552	_modprobe -r ntb_transport 2> /dev/null
553	set -e
554}
555
556cleanup
557
558if ! [[ $$DONT_CLEANUP ]]; then
559	trap cleanup EXIT
560fi
561
562if [ "$(id -u)" != "0" ]; then
563	echo "This script must be run as root" 1>&2
564	exit 1
565fi
566
567if [[ "$LIST_DEVS" == TRUE ]]; then
568	echo "Local Devices:"
569	ls -1 /sys/bus/ntb/devices
570	echo
571
572	if [[ "$REMOTE_HOST" != "" ]]; then
573		echo "Remote Devices:"
574		ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
575	fi
576
577	exit 0
578fi
579
580if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
581	show_help
582	exit 1
583fi
584
585ntb_tool_tests
586echo
587ntb_pingpong_tests
588echo
589ntb_perf_tests
590echo
591