1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0-or-later
3# Copyright (c) 2016 Microsemi. All Rights Reserved.
4#
5# Author: Logan Gunthorpe <logang@deltatee.com>
6
7REMOTE_HOST=
8LIST_DEVS=FALSE
9
10DEBUGFS=${DEBUGFS-/sys/kernel/debug}
11
12PERF_RUN_ORDER=32
13MAX_MW_SIZE=0
14RUN_DMA_TESTS=
15DONT_CLEANUP=
16MW_SIZE=65536
17
18function show_help()
19{
20	echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
21	echo "Run tests on a pair of NTB endpoints."
22	echo
23	echo "If the NTB device loops back to the same host then,"
24	echo "just specifying the two PCI ids on the command line is"
25	echo "sufficient. Otherwise, if the NTB link spans two hosts"
26	echo "use the -r option to specify the hostname for the remote"
27	echo "device. SSH will then be used to test the remote side."
28	echo "An SSH key between the root users of the host would then"
29	echo "be highly recommended."
30	echo
31	echo "Options:"
32	echo "  -C              don't cleanup ntb modules on exit"
33	echo "  -h              show this help message"
34	echo "  -l              list available local and remote PCI ids"
35	echo "  -r REMOTE_HOST  specify the remote's hostname to connect"
36	echo "                  to for the test (using ssh)"
37	echo "  -m MW_SIZE      memory window size for ntb_tool"
38	echo "                  (default: $MW_SIZE)"
39	echo "  -d              run dma tests for ntb_perf"
40	echo "  -p ORDER        total data order for ntb_perf"
41	echo "                  (default: $PERF_RUN_ORDER)"
42	echo "  -w MAX_MW_SIZE  maxmium memory window size for ntb_perf"
43	echo
44}
45
46function parse_args()
47{
48	OPTIND=0
49	while getopts "b:Cdhlm:r:p:w:" opt; do
50		case "$opt" in
51		C)  DONT_CLEANUP=1 ;;
52		d)  RUN_DMA_TESTS=1 ;;
53		h)  show_help; exit 0 ;;
54		l)  LIST_DEVS=TRUE ;;
55		m)  MW_SIZE=${OPTARG} ;;
56		r)  REMOTE_HOST=${OPTARG} ;;
57		p)  PERF_RUN_ORDER=${OPTARG} ;;
58		w)  MAX_MW_SIZE=${OPTARG} ;;
59		\?)
60		    echo "Invalid option: -$OPTARG" >&2
61		    exit 1
62		    ;;
63		esac
64	done
65}
66
67parse_args "$@"
68shift $((OPTIND-1))
69LOCAL_DEV=$1
70shift
71parse_args "$@"
72shift $((OPTIND-1))
73REMOTE_DEV=$1
74shift
75parse_args "$@"
76
77set -e
78
79function _modprobe()
80{
81	modprobe "$@"
82
83	if [[ "$REMOTE_HOST" != "" ]]; then
84		ssh "$REMOTE_HOST" modprobe "$@"
85	fi
86}
87
88function split_remote()
89{
90	VPATH=$1
91	REMOTE=
92
93	if [[ "$VPATH" == *":/"* ]]; then
94		REMOTE=${VPATH%%:*}
95		VPATH=${VPATH#*:}
96	fi
97}
98
99function read_file()
100{
101	split_remote $1
102	if [[ "$REMOTE" != "" ]]; then
103		ssh "$REMOTE" cat "$VPATH"
104	else
105		cat "$VPATH"
106	fi
107}
108
109function write_file()
110{
111	split_remote $2
112	VALUE=$1
113
114	if [[ "$REMOTE" != "" ]]; then
115		ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
116	else
117		echo "$VALUE" > "$VPATH"
118	fi
119}
120
121function check_file()
122{
123	split_remote $1
124
125	if [[ "$REMOTE" != "" ]]; then
126		ssh "$REMOTE" "[[ -e ${VPATH} ]]"
127	else
128		[[ -e ${VPATH} ]]
129	fi
130}
131
132function subdirname()
133{
134	echo $(basename $(dirname $1)) 2> /dev/null
135}
136
137function find_pidx()
138{
139	PORT=$1
140	PPATH=$2
141
142	for ((i = 0; i < 64; i++)); do
143		PEER_DIR="$PPATH/peer$i"
144
145		check_file ${PEER_DIR} || break
146
147		PEER_PORT=$(read_file "${PEER_DIR}/port")
148		if [[ ${PORT} -eq $PEER_PORT ]]; then
149			echo $i
150			return 0
151		fi
152	done
153
154	return 1
155}
156
157function port_test()
158{
159	LOC=$1
160	REM=$2
161
162	echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
163
164	LOCAL_PORT=$(read_file "$LOC/port")
165	REMOTE_PORT=$(read_file "$REM/port")
166
167	LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
168	REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
169
170	echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
171	echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
172
173	echo "  Passed"
174}
175
176function link_test()
177{
178	LOC=$1
179	REM=$2
180	EXP=0
181
182	echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
183
184	if ! write_file "N" "$LOC/../link" 2> /dev/null; then
185		echo "  Unsupported"
186		return
187	fi
188
189	write_file "N" "$LOC/link_event"
190
191	if [[ $(read_file "$REM/link") != "N" ]]; then
192		echo "Expected link to be down in $REM/link" >&2
193		exit -1
194	fi
195
196	write_file "Y" "$LOC/../link"
197
198	echo "  Passed"
199}
200
201function doorbell_test()
202{
203	LOC=$1
204	REM=$2
205	EXP=0
206
207	echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
208
209	DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
210
211	write_file "c $DB_VALID_MASK" "$REM/db"
212
213	for ((i = 0; i < 64; i++)); do
214		DB=$(read_file "$REM/db")
215		if [[ "$DB" -ne "$EXP" ]]; then
216			echo "Doorbell doesn't match expected value $EXP " \
217			     "in $REM/db" >&2
218			exit -1
219		fi
220
221		let "MASK = (1 << $i) & $DB_VALID_MASK" || true
222		let "EXP = $EXP | $MASK" || true
223
224		write_file "s $MASK" "$LOC/peer_db"
225	done
226
227	write_file "c $DB_VALID_MASK" "$REM/db_mask"
228	write_file $DB_VALID_MASK "$REM/db_event"
229	write_file "s $DB_VALID_MASK" "$REM/db_mask"
230
231	write_file "c $DB_VALID_MASK" "$REM/db"
232
233	echo "  Passed"
234}
235
236function get_files_count()
237{
238	NAME=$1
239	LOC=$2
240
241	split_remote $LOC
242
243	if [[ "$REMOTE" == "" ]]; then
244		echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l)
245	else
246		echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
247		       wc -l" 2> /dev/null)
248	fi
249}
250
251function scratchpad_test()
252{
253	LOC=$1
254	REM=$2
255
256	echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
257
258	CNT=$(get_files_count "spad" "$LOC")
259
260	if [[ $CNT -eq 0 ]]; then
261		echo "  Unsupported"
262		return
263	fi
264
265	for ((i = 0; i < $CNT; i++)); do
266		VAL=$RANDOM
267		write_file "$VAL" "$LOC/spad$i"
268		RVAL=$(read_file "$REM/../spad$i")
269
270		if [[ "$VAL" -ne "$RVAL" ]]; then
271			echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
272			exit -1
273		fi
274	done
275
276	echo "  Passed"
277}
278
279function message_test()
280{
281	LOC=$1
282	REM=$2
283
284	echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
285
286	CNT=$(get_files_count "msg" "$LOC")
287
288	if [[ $CNT -eq 0 ]]; then
289		echo "  Unsupported"
290		return
291	fi
292
293	MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
294	MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
295
296	write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
297	write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
298
299	for ((i = 0; i < $CNT; i++)); do
300		VAL=$RANDOM
301		write_file "$VAL" "$LOC/msg$i"
302		RVAL=$(read_file "$REM/../msg$i")
303
304		if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
305			echo "Message $i value $RVAL doesn't match $VAL" >&2
306			exit -1
307		fi
308	done
309
310	echo "  Passed"
311}
312
313function get_number()
314{
315	KEY=$1
316
317	sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
318}
319
320function mw_alloc()
321{
322	IDX=$1
323	LOC=$2
324	REM=$3
325
326	write_file $MW_SIZE "$LOC/mw_trans$IDX"
327
328	INB_MW=$(read_file "$LOC/mw_trans$IDX")
329	MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
330	MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
331
332	write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
333
334	if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
335		echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
336	fi
337}
338
339function write_mw()
340{
341	split_remote $2
342
343	if [[ "$REMOTE" != "" ]]; then
344		ssh "$REMOTE" \
345			dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
346	else
347		dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
348	fi
349}
350
351function mw_check()
352{
353	IDX=$1
354	LOC=$2
355	REM=$3
356
357	write_mw "$LOC/mw$IDX"
358
359	split_remote "$LOC/mw$IDX"
360	if [[ "$REMOTE" == "" ]]; then
361		A=$VPATH
362	else
363		A=/tmp/ntb_test.$$.A
364		ssh "$REMOTE" cat "$VPATH" > "$A"
365	fi
366
367	split_remote "$REM/peer_mw$IDX"
368	if [[ "$REMOTE" == "" ]]; then
369		B=$VPATH
370	else
371		B=/tmp/ntb_test.$$.B
372		ssh "$REMOTE" cat "$VPATH" > "$B"
373	fi
374
375	cmp -n $MW_ALIGNED_SIZE "$A" "$B"
376	if [[ $? != 0 ]]; then
377		echo "Memory window $MW did not match!" >&2
378	fi
379
380	if [[ "$A" == "/tmp/*" ]]; then
381		rm "$A"
382	fi
383
384	if [[ "$B" == "/tmp/*" ]]; then
385		rm "$B"
386	fi
387}
388
389function mw_free()
390{
391	IDX=$1
392	LOC=$2
393	REM=$3
394
395	write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
396
397	write_file 0 "$LOC/mw_trans$IDX"
398}
399
400function mw_test()
401{
402	LOC=$1
403	REM=$2
404
405	CNT=$(get_files_count "mw_trans" "$LOC")
406
407	for ((i = 0; i < $CNT; i++)); do
408		echo "Running mw$i tests on: $(subdirname $LOC) / " \
409		     "$(subdirname $REM)"
410
411		mw_alloc $i $LOC $REM
412
413		mw_check $i $LOC $REM
414
415		mw_free $i $LOC  $REM
416
417		echo "  Passed"
418	done
419
420}
421
422function pingpong_test()
423{
424	LOC=$1
425	REM=$2
426
427	echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
428
429	LOC_START=$(read_file "$LOC/count")
430	REM_START=$(read_file "$REM/count")
431
432	sleep 7
433
434	LOC_END=$(read_file "$LOC/count")
435	REM_END=$(read_file "$REM/count")
436
437	if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
438		echo "Ping pong counter not incrementing!" >&2
439		exit 1
440	fi
441
442	echo "  Passed"
443}
444
445function perf_test()
446{
447	USE_DMA=$1
448
449	if [[ $USE_DMA == "1" ]]; then
450		WITH="with"
451	else
452		WITH="without"
453	fi
454
455	_modprobe ntb_perf total_order=$PERF_RUN_ORDER \
456		max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
457
458	echo "Running local perf test $WITH DMA"
459	write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
460	echo -n "  "
461	read_file "$LOCAL_PERF/run"
462	echo "  Passed"
463
464	echo "Running remote perf test $WITH DMA"
465	write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
466	echo -n "  "
467	read_file "$REMOTE_PERF/run"
468	echo "  Passed"
469
470	_modprobe -r ntb_perf
471}
472
473function ntb_tool_tests()
474{
475	LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
476	REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
477
478	echo "Starting ntb_tool tests..."
479
480	_modprobe ntb_tool
481
482	port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
483
484	LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
485	REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
486
487	link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
488	link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
489
490	#Ensure the link is up on both sides before continuing
491	write_file "Y" "$LOCAL_PEER_TOOL/link_event"
492	write_file "Y" "$REMOTE_PEER_TOOL/link_event"
493
494	doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
495	doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
496
497	scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
498	scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
499
500	message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
501	message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
502
503	mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
504	mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
505
506	_modprobe -r ntb_tool
507}
508
509function ntb_pingpong_tests()
510{
511	LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
512	REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
513
514	echo "Starting ntb_pingpong tests..."
515
516	_modprobe ntb_pingpong
517
518	pingpong_test $LOCAL_PP $REMOTE_PP
519
520	_modprobe -r ntb_pingpong
521}
522
523function ntb_perf_tests()
524{
525	LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
526	REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
527
528	echo "Starting ntb_perf tests..."
529
530	perf_test 0
531
532	if [[ $RUN_DMA_TESTS ]]; then
533		perf_test 1
534	fi
535}
536
537function cleanup()
538{
539	set +e
540	_modprobe -r ntb_tool 2> /dev/null
541	_modprobe -r ntb_perf 2> /dev/null
542	_modprobe -r ntb_pingpong 2> /dev/null
543	_modprobe -r ntb_transport 2> /dev/null
544	set -e
545}
546
547cleanup
548
549if ! [[ $$DONT_CLEANUP ]]; then
550	trap cleanup EXIT
551fi
552
553if [ "$(id -u)" != "0" ]; then
554	echo "This script must be run as root" 1>&2
555	exit 1
556fi
557
558if [[ "$LIST_DEVS" == TRUE ]]; then
559	echo "Local Devices:"
560	ls -1 /sys/bus/ntb/devices
561	echo
562
563	if [[ "$REMOTE_HOST" != "" ]]; then
564		echo "Remote Devices:"
565		ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
566	fi
567
568	exit 0
569fi
570
571if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
572	show_help
573	exit 1
574fi
575
576ntb_tool_tests
577echo
578ntb_pingpong_tests
579echo
580ntb_perf_tests
581echo
582