1#!/bin/bash
2# Copyright (c) 2016 Microsemi. All Rights Reserved.
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License as
6# published by the Free Software Foundation; either version 2 of
7# the License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it would be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12# GNU General Public License for more details.
13#
14# Author: Logan Gunthorpe <logang@deltatee.com>
15
16REMOTE_HOST=
17LIST_DEVS=FALSE
18
19DEBUGFS=${DEBUGFS-/sys/kernel/debug}
20
21PERF_RUN_ORDER=32
22MAX_MW_SIZE=0
23RUN_DMA_TESTS=
24DONT_CLEANUP=
25MW_SIZE=65536
26
27function show_help()
28{
29	echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
30	echo "Run tests on a pair of NTB endpoints."
31	echo
32	echo "If the NTB device loops back to the same host then,"
33	echo "just specifying the two PCI ids on the command line is"
34	echo "sufficient. Otherwise, if the NTB link spans two hosts"
35	echo "use the -r option to specify the hostname for the remote"
36	echo "device. SSH will then be used to test the remote side."
37	echo "An SSH key between the root users of the host would then"
38	echo "be highly recommended."
39	echo
40	echo "Options:"
41	echo "  -C              don't cleanup ntb modules on exit"
42	echo "  -d              run dma tests"
43	echo "  -h              show this help message"
44	echo "  -l              list available local and remote PCI ids"
45	echo "  -r REMOTE_HOST  specify the remote's hostname to connect"
46        echo "                  to for the test (using ssh)"
47	echo "  -p NUM          ntb_perf run order (default: $PERF_RUN_ORDER)"
48	echo "  -w max_mw_size  maxmium memory window size"
49	echo
50}
51
52function parse_args()
53{
54	OPTIND=0
55	while getopts "Cdhlm:r:p:w:" opt; do
56		case "$opt" in
57		C)  DONT_CLEANUP=1 ;;
58		d)  RUN_DMA_TESTS=1 ;;
59		h)  show_help; exit 0 ;;
60		l)  LIST_DEVS=TRUE ;;
61		m)  MW_SIZE=${OPTARG} ;;
62		r)  REMOTE_HOST=${OPTARG} ;;
63		p)  PERF_RUN_ORDER=${OPTARG} ;;
64		w)  MAX_MW_SIZE=${OPTARG} ;;
65		\?)
66		    echo "Invalid option: -$OPTARG" >&2
67		    exit 1
68		    ;;
69		esac
70	done
71}
72
73parse_args "$@"
74shift $((OPTIND-1))
75LOCAL_DEV=$1
76shift
77parse_args "$@"
78shift $((OPTIND-1))
79REMOTE_DEV=$1
80shift
81parse_args "$@"
82
83set -e
84
85function _modprobe()
86{
87        modprobe "$@"
88}
89
90function split_remote()
91{
92	VPATH=$1
93	REMOTE=
94
95	if [[ "$VPATH" == *":/"* ]]; then
96		REMOTE=${VPATH%%:*}
97		VPATH=${VPATH#*:}
98	fi
99}
100
101function read_file()
102{
103	split_remote $1
104	if [[ "$REMOTE" != "" ]]; then
105		ssh "$REMOTE" cat "$VPATH"
106	else
107		cat "$VPATH"
108	fi
109}
110
111function write_file()
112{
113	split_remote $2
114	VALUE=$1
115
116	if [[ "$REMOTE" != "" ]]; then
117		ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
118	else
119		echo "$VALUE" > "$VPATH"
120	fi
121}
122
123function link_test()
124{
125	LOC=$1
126	REM=$2
127	EXP=0
128
129	echo "Running link tests on: $(basename $LOC) / $(basename $REM)"
130
131	if ! write_file "N" "$LOC/link" 2> /dev/null; then
132		echo "  Unsupported"
133		return
134	fi
135
136	write_file "N" "$LOC/link_event"
137
138	if [[ $(read_file "$REM/link") != "N" ]]; then
139		echo "Expected remote link to be down in $REM/link" >&2
140		exit -1
141	fi
142
143	write_file "Y" "$LOC/link"
144	write_file "Y" "$LOC/link_event"
145
146	echo "  Passed"
147}
148
149function doorbell_test()
150{
151	LOC=$1
152	REM=$2
153	EXP=0
154
155	echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
156
157	write_file "c 0xFFFFFFFF" "$REM/db"
158
159	for ((i=1; i <= 8; i++)); do
160		let DB=$(read_file "$REM/db") || true
161		if [[ "$DB" != "$EXP" ]]; then
162			echo "Doorbell doesn't match expected value $EXP " \
163			     "in $REM/db" >&2
164			exit -1
165		fi
166
167		let "MASK=1 << ($i-1)" || true
168		let "EXP=$EXP | $MASK" || true
169		write_file "s $MASK" "$LOC/peer_db"
170	done
171
172	echo "  Passed"
173}
174
175function read_spad()
176{
177       VPATH=$1
178       IDX=$2
179
180       ROW=($(read_file "$VPATH" | grep -e "^$IDX"))
181       let VAL=${ROW[1]} || true
182       echo $VAL
183}
184
185function scratchpad_test()
186{
187	LOC=$1
188	REM=$2
189	CNT=$(read_file "$LOC/spad" | wc -l)
190
191	echo "Running spad tests on: $(basename $LOC) / $(basename $REM)"
192
193	for ((i = 0; i < $CNT; i++)); do
194		VAL=$RANDOM
195		write_file "$i $VAL" "$LOC/peer_spad"
196		RVAL=$(read_spad "$REM/spad" $i)
197
198		if [[ "$VAL" != "$RVAL" ]]; then
199			echo "Scratchpad doesn't match expected value $VAL " \
200			     "in $REM/spad, got $RVAL" >&2
201			exit -1
202		fi
203
204	done
205
206	echo "  Passed"
207}
208
209function write_mw()
210{
211	split_remote $2
212
213	if [[ "$REMOTE" != "" ]]; then
214		ssh "$REMOTE" \
215			dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
216	else
217		dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
218	fi
219}
220
221function mw_test()
222{
223	IDX=$1
224	LOC=$2
225	REM=$3
226
227	echo "Running $IDX tests on: $(basename $LOC) / $(basename $REM)"
228
229	write_mw "$LOC/$IDX"
230
231	split_remote "$LOC/$IDX"
232	if [[ "$REMOTE" == "" ]]; then
233		A=$VPATH
234	else
235		A=/tmp/ntb_test.$$.A
236		ssh "$REMOTE" cat "$VPATH" > "$A"
237	fi
238
239	split_remote "$REM/peer_$IDX"
240	if [[ "$REMOTE" == "" ]]; then
241		B=$VPATH
242	else
243		B=/tmp/ntb_test.$$.B
244		ssh "$REMOTE" cat "$VPATH" > "$B"
245	fi
246
247	cmp -n $MW_SIZE "$A" "$B"
248	if [[ $? != 0 ]]; then
249		echo "Memory window $MW did not match!" >&2
250	fi
251
252	if [[ "$A" == "/tmp/*" ]]; then
253		rm "$A"
254	fi
255
256	if [[ "$B" == "/tmp/*" ]]; then
257		rm "$B"
258	fi
259
260	echo "  Passed"
261}
262
263function pingpong_test()
264{
265	LOC=$1
266	REM=$2
267
268	echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
269
270	LOC_START=$(read_file $LOC/count)
271	REM_START=$(read_file $REM/count)
272
273	sleep 7
274
275	LOC_END=$(read_file $LOC/count)
276	REM_END=$(read_file $REM/count)
277
278	if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
279		echo "Ping pong counter not incrementing!" >&2
280		exit 1
281	fi
282
283	echo "  Passed"
284}
285
286function perf_test()
287{
288	USE_DMA=$1
289
290	if [[ $USE_DMA == "1" ]]; then
291		WITH="with"
292	else
293		WITH="without"
294	fi
295
296	_modprobe ntb_perf run_order=$PERF_RUN_ORDER \
297		max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
298
299	echo "Running local perf test $WITH DMA"
300	write_file "" $LOCAL_PERF/run
301	echo -n "  "
302	read_file $LOCAL_PERF/run
303	echo "  Passed"
304
305	echo "Running remote perf test $WITH DMA"
306	write_file "" $REMOTE_PERF/run
307	echo -n "  "
308	read_file $LOCAL_PERF/run
309	echo "  Passed"
310
311	_modprobe -r ntb_perf
312}
313
314function ntb_tool_tests()
315{
316	LOCAL_TOOL=$DEBUGFS/ntb_tool/$LOCAL_DEV
317	REMOTE_TOOL=$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV
318
319	echo "Starting ntb_tool tests..."
320
321	_modprobe ntb_tool
322
323	write_file Y $LOCAL_TOOL/link_event
324	write_file Y $REMOTE_TOOL/link_event
325
326	link_test $LOCAL_TOOL $REMOTE_TOOL
327	link_test $REMOTE_TOOL $LOCAL_TOOL
328
329	for PEER_TRANS in $(ls $LOCAL_TOOL/peer_trans*); do
330		PT=$(basename $PEER_TRANS)
331		write_file $MW_SIZE $LOCAL_TOOL/$PT
332		write_file $MW_SIZE $REMOTE_TOOL/$PT
333	done
334
335	doorbell_test $LOCAL_TOOL $REMOTE_TOOL
336	doorbell_test $REMOTE_TOOL $LOCAL_TOOL
337	scratchpad_test $LOCAL_TOOL $REMOTE_TOOL
338	scratchpad_test $REMOTE_TOOL $LOCAL_TOOL
339
340	for MW in $(ls $LOCAL_TOOL/mw*); do
341		MW=$(basename $MW)
342
343		mw_test $MW $LOCAL_TOOL $REMOTE_TOOL
344		mw_test $MW $REMOTE_TOOL $LOCAL_TOOL
345	done
346
347	_modprobe -r ntb_tool
348}
349
350function ntb_pingpong_tests()
351{
352	LOCAL_PP=$DEBUGFS/ntb_pingpong/$LOCAL_DEV
353	REMOTE_PP=$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV
354
355	echo "Starting ntb_pingpong tests..."
356
357	_modprobe ntb_pingpong
358
359	pingpong_test $LOCAL_PP $REMOTE_PP
360
361	_modprobe -r ntb_pingpong
362}
363
364function ntb_perf_tests()
365{
366	LOCAL_PERF=$DEBUGFS/ntb_perf/$LOCAL_DEV
367	REMOTE_PERF=$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV
368
369	echo "Starting ntb_perf tests..."
370
371	perf_test 0
372
373	if [[ $RUN_DMA_TESTS ]]; then
374		perf_test 1
375	fi
376}
377
378function cleanup()
379{
380	set +e
381	_modprobe -r ntb_tool 2> /dev/null
382	_modprobe -r ntb_perf 2> /dev/null
383	_modprobe -r ntb_pingpong 2> /dev/null
384	_modprobe -r ntb_transport 2> /dev/null
385	set -e
386}
387
388cleanup
389
390if ! [[ $$DONT_CLEANUP ]]; then
391	trap cleanup EXIT
392fi
393
394if [ "$(id -u)" != "0" ]; then
395	echo "This script must be run as root" 1>&2
396	exit 1
397fi
398
399if [[ "$LIST_DEVS" == TRUE ]]; then
400	echo "Local Devices:"
401	ls -1 /sys/bus/ntb/devices
402	echo
403
404	if [[ "$REMOTE_HOST" != "" ]]; then
405		echo "Remote Devices:"
406		ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
407	fi
408
409	exit 0
410fi
411
412if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
413	show_help
414	exit 1
415fi
416
417ntb_tool_tests
418echo
419ntb_pingpong_tests
420echo
421ntb_perf_tests
422echo
423