1#!/bin/bash
2# Copyright (c) 2016 Microsemi. All Rights Reserved.
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License as
6# published by the Free Software Foundation; either version 2 of
7# the License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it would be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# Author: Logan Gunthorpe <logang@deltatee.com>
15
16REMOTE_HOST=
17LIST_DEVS=FALSE
18
19DEBUGFS=${DEBUGFS-/sys/kernel/debug}
20
21DB_BITMASK=0x7FFF
22PERF_RUN_ORDER=32
23MAX_MW_SIZE=0
24RUN_DMA_TESTS=
25DONT_CLEANUP=
26MW_SIZE=65536
27
28function show_help()
29{
30	echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
31	echo "Run tests on a pair of NTB endpoints."
32	echo
33	echo "If the NTB device loops back to the same host then,"
34	echo "just specifying the two PCI ids on the command line is"
35	echo "sufficient. Otherwise, if the NTB link spans two hosts"
36	echo "use the -r option to specify the hostname for the remote"
37	echo "device. SSH will then be used to test the remote side."
38	echo "An SSH key between the root users of the host would then"
39	echo "be highly recommended."
40	echo
41	echo "Options:"
42	echo "  -b BITMASK      doorbell clear bitmask for ntb_tool"
43	echo "  -C              don't cleanup ntb modules on exit"
44	echo "  -d              run dma tests"
45	echo "  -h              show this help message"
46	echo "  -l              list available local and remote PCI ids"
47	echo "  -r REMOTE_HOST  specify the remote's hostname to connect"
48        echo "                  to for the test (using ssh)"
49	echo "  -p NUM          ntb_perf run order (default: $PERF_RUN_ORDER)"
50	echo "  -w max_mw_size  maxmium memory window size"
51	echo
52}
53
54function parse_args()
55{
56	OPTIND=0
57	while getopts "b:Cdhlm:r:p:w:" opt; do
58		case "$opt" in
59		b)  DB_BITMASK=${OPTARG} ;;
60		C)  DONT_CLEANUP=1 ;;
61		d)  RUN_DMA_TESTS=1 ;;
62		h)  show_help; exit 0 ;;
63		l)  LIST_DEVS=TRUE ;;
64		m)  MW_SIZE=${OPTARG} ;;
65		r)  REMOTE_HOST=${OPTARG} ;;
66		p)  PERF_RUN_ORDER=${OPTARG} ;;
67		w)  MAX_MW_SIZE=${OPTARG} ;;
68		\?)
69		    echo "Invalid option: -$OPTARG" >&2
70		    exit 1
71		    ;;
72		esac
73	done
74}
75
76parse_args "$@"
77shift $((OPTIND-1))
78LOCAL_DEV=$1
79shift
80parse_args "$@"
81shift $((OPTIND-1))
82REMOTE_DEV=$1
83shift
84parse_args "$@"
85
86set -e
87
88function _modprobe()
89{
90        modprobe "$@"
91
92	if [[ "$REMOTE_HOST" != "" ]]; then
93		ssh "$REMOTE_HOST" modprobe "$@"
94	fi
95}
96
97function split_remote()
98{
99	VPATH=$1
100	REMOTE=
101
102	if [[ "$VPATH" == *":/"* ]]; then
103		REMOTE=${VPATH%%:*}
104		VPATH=${VPATH#*:}
105	fi
106}
107
108function read_file()
109{
110	split_remote $1
111	if [[ "$REMOTE" != "" ]]; then
112		ssh "$REMOTE" cat "$VPATH"
113	else
114		cat "$VPATH"
115	fi
116}
117
118function write_file()
119{
120	split_remote $2
121	VALUE=$1
122
123	if [[ "$REMOTE" != "" ]]; then
124		ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
125	else
126		echo "$VALUE" > "$VPATH"
127	fi
128}
129
130function link_test()
131{
132	LOC=$1
133	REM=$2
134	EXP=0
135
136	echo "Running link tests on: $(basename $LOC) / $(basename $REM)"
137
138	if ! write_file "N" "$LOC/link" 2> /dev/null; then
139		echo "  Unsupported"
140		return
141	fi
142
143	write_file "N" "$LOC/link_event"
144
145	if [[ $(read_file "$REM/link") != "N" ]]; then
146		echo "Expected remote link to be down in $REM/link" >&2
147		exit -1
148	fi
149
150	write_file "Y" "$LOC/link"
151	write_file "Y" "$LOC/link_event"
152
153	echo "  Passed"
154}
155
156function doorbell_test()
157{
158	LOC=$1
159	REM=$2
160	EXP=0
161
162	echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
163
164	write_file "c $DB_BITMASK" "$REM/db"
165
166	for ((i=1; i <= 8; i++)); do
167		let DB=$(read_file "$REM/db") || true
168		if [[ "$DB" != "$EXP" ]]; then
169			echo "Doorbell doesn't match expected value $EXP " \
170			     "in $REM/db" >&2
171			exit -1
172		fi
173
174		let "MASK=1 << ($i-1)" || true
175		let "EXP=$EXP | $MASK" || true
176		write_file "s $MASK" "$LOC/peer_db"
177	done
178
179	echo "  Passed"
180}
181
182function read_spad()
183{
184       VPATH=$1
185       IDX=$2
186
187       ROW=($(read_file "$VPATH" | grep -e "^$IDX"))
188       let VAL=${ROW[1]} || true
189       echo $VAL
190}
191
192function scratchpad_test()
193{
194	LOC=$1
195	REM=$2
196	CNT=$(read_file "$LOC/spad" | wc -l)
197
198	echo "Running spad tests on: $(basename $LOC) / $(basename $REM)"
199
200	for ((i = 0; i < $CNT; i++)); do
201		VAL=$RANDOM
202		write_file "$i $VAL" "$LOC/peer_spad"
203		RVAL=$(read_spad "$REM/spad" $i)
204
205		if [[ "$VAL" != "$RVAL" ]]; then
206			echo "Scratchpad doesn't match expected value $VAL " \
207			     "in $REM/spad, got $RVAL" >&2
208			exit -1
209		fi
210
211	done
212
213	echo "  Passed"
214}
215
216function write_mw()
217{
218	split_remote $2
219
220	if [[ "$REMOTE" != "" ]]; then
221		ssh "$REMOTE" \
222			dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
223	else
224		dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
225	fi
226}
227
228function mw_test()
229{
230	IDX=$1
231	LOC=$2
232	REM=$3
233
234	echo "Running $IDX tests on: $(basename $LOC) / $(basename $REM)"
235
236	write_mw "$LOC/$IDX"
237
238	split_remote "$LOC/$IDX"
239	if [[ "$REMOTE" == "" ]]; then
240		A=$VPATH
241	else
242		A=/tmp/ntb_test.$$.A
243		ssh "$REMOTE" cat "$VPATH" > "$A"
244	fi
245
246	split_remote "$REM/peer_$IDX"
247	if [[ "$REMOTE" == "" ]]; then
248		B=$VPATH
249	else
250		B=/tmp/ntb_test.$$.B
251		ssh "$REMOTE" cat "$VPATH" > "$B"
252	fi
253
254	cmp -n $MW_SIZE "$A" "$B"
255	if [[ $? != 0 ]]; then
256		echo "Memory window $MW did not match!" >&2
257	fi
258
259	if [[ "$A" == "/tmp/*" ]]; then
260		rm "$A"
261	fi
262
263	if [[ "$B" == "/tmp/*" ]]; then
264		rm "$B"
265	fi
266
267	echo "  Passed"
268}
269
270function pingpong_test()
271{
272	LOC=$1
273	REM=$2
274
275	echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
276
277	LOC_START=$(read_file $LOC/count)
278	REM_START=$(read_file $REM/count)
279
280	sleep 7
281
282	LOC_END=$(read_file $LOC/count)
283	REM_END=$(read_file $REM/count)
284
285	if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
286		echo "Ping pong counter not incrementing!" >&2
287		exit 1
288	fi
289
290	echo "  Passed"
291}
292
293function perf_test()
294{
295	USE_DMA=$1
296
297	if [[ $USE_DMA == "1" ]]; then
298		WITH="with"
299	else
300		WITH="without"
301	fi
302
303	_modprobe ntb_perf run_order=$PERF_RUN_ORDER \
304		max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
305
306	echo "Running local perf test $WITH DMA"
307	write_file "" $LOCAL_PERF/run
308	echo -n "  "
309	read_file $LOCAL_PERF/run
310	echo "  Passed"
311
312	echo "Running remote perf test $WITH DMA"
313	write_file "" $REMOTE_PERF/run
314	echo -n "  "
315	read_file $REMOTE_PERF/run
316	echo "  Passed"
317
318	_modprobe -r ntb_perf
319}
320
321function ntb_tool_tests()
322{
323	LOCAL_TOOL=$DEBUGFS/ntb_tool/$LOCAL_DEV
324	REMOTE_TOOL=$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV
325
326	echo "Starting ntb_tool tests..."
327
328	_modprobe ntb_tool
329
330	write_file Y $LOCAL_TOOL/link_event
331	write_file Y $REMOTE_TOOL/link_event
332
333	link_test $LOCAL_TOOL $REMOTE_TOOL
334	link_test $REMOTE_TOOL $LOCAL_TOOL
335
336	#Ensure the link is up on both sides before continuing
337	write_file Y $LOCAL_TOOL/link_event
338	write_file Y $REMOTE_TOOL/link_event
339
340	for PEER_TRANS in $(ls $LOCAL_TOOL/peer_trans*); do
341		PT=$(basename $PEER_TRANS)
342		write_file $MW_SIZE $LOCAL_TOOL/$PT
343		write_file $MW_SIZE $REMOTE_TOOL/$PT
344	done
345
346	doorbell_test $LOCAL_TOOL $REMOTE_TOOL
347	doorbell_test $REMOTE_TOOL $LOCAL_TOOL
348	scratchpad_test $LOCAL_TOOL $REMOTE_TOOL
349	scratchpad_test $REMOTE_TOOL $LOCAL_TOOL
350
351	for MW in $(ls $LOCAL_TOOL/mw*); do
352		MW=$(basename $MW)
353
354		mw_test $MW $LOCAL_TOOL $REMOTE_TOOL
355		mw_test $MW $REMOTE_TOOL $LOCAL_TOOL
356	done
357
358	_modprobe -r ntb_tool
359}
360
361function ntb_pingpong_tests()
362{
363	LOCAL_PP=$DEBUGFS/ntb_pingpong/$LOCAL_DEV
364	REMOTE_PP=$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV
365
366	echo "Starting ntb_pingpong tests..."
367
368	_modprobe ntb_pingpong
369
370	pingpong_test $LOCAL_PP $REMOTE_PP
371
372	_modprobe -r ntb_pingpong
373}
374
375function ntb_perf_tests()
376{
377	LOCAL_PERF=$DEBUGFS/ntb_perf/$LOCAL_DEV
378	REMOTE_PERF=$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV
379
380	echo "Starting ntb_perf tests..."
381
382	perf_test 0
383
384	if [[ $RUN_DMA_TESTS ]]; then
385		perf_test 1
386	fi
387}
388
389function cleanup()
390{
391	set +e
392	_modprobe -r ntb_tool 2> /dev/null
393	_modprobe -r ntb_perf 2> /dev/null
394	_modprobe -r ntb_pingpong 2> /dev/null
395	_modprobe -r ntb_transport 2> /dev/null
396	set -e
397}
398
399cleanup
400
401if ! [[ $$DONT_CLEANUP ]]; then
402	trap cleanup EXIT
403fi
404
405if [ "$(id -u)" != "0" ]; then
406	echo "This script must be run as root" 1>&2
407	exit 1
408fi
409
410if [[ "$LIST_DEVS" == TRUE ]]; then
411	echo "Local Devices:"
412	ls -1 /sys/bus/ntb/devices
413	echo
414
415	if [[ "$REMOTE_HOST" != "" ]]; then
416		echo "Remote Devices:"
417		ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
418	fi
419
420	exit 0
421fi
422
423if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
424	show_help
425	exit 1
426fi
427
428ntb_tool_tests
429echo
430ntb_pingpong_tests
431echo
432ntb_perf_tests
433echo
434