xref: /openbmc/linux/include/trace/events/rpcrdma.h (revision a23c76e9)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (c) 2017, 2018 Oracle.  All rights reserved.
4  *
5  * Trace point definitions for the "rpcrdma" subsystem.
6  */
7 #undef TRACE_SYSTEM
8 #define TRACE_SYSTEM rpcrdma
9 
10 #if !defined(_TRACE_RPCRDMA_H) || defined(TRACE_HEADER_MULTI_READ)
11 #define _TRACE_RPCRDMA_H
12 
13 #include <linux/scatterlist.h>
14 #include <linux/sunrpc/rpc_rdma_cid.h>
15 #include <linux/tracepoint.h>
16 #include <rdma/ib_cm.h>
17 
18 #include <trace/misc/rdma.h>
19 #include <trace/misc/sunrpc.h>
20 
21 /**
22  ** Event classes
23  **/
24 
25 DECLARE_EVENT_CLASS(rpcrdma_completion_class,
26 	TP_PROTO(
27 		const struct ib_wc *wc,
28 		const struct rpc_rdma_cid *cid
29 	),
30 
31 	TP_ARGS(wc, cid),
32 
33 	TP_STRUCT__entry(
34 		__field(u32, cq_id)
35 		__field(int, completion_id)
36 		__field(unsigned long, status)
37 		__field(unsigned int, vendor_err)
38 	),
39 
40 	TP_fast_assign(
41 		__entry->cq_id = cid->ci_queue_id;
42 		__entry->completion_id = cid->ci_completion_id;
43 		__entry->status = wc->status;
44 		if (wc->status)
45 			__entry->vendor_err = wc->vendor_err;
46 		else
47 			__entry->vendor_err = 0;
48 	),
49 
50 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
51 		__entry->cq_id, __entry->completion_id,
52 		rdma_show_wc_status(__entry->status),
53 		__entry->status, __entry->vendor_err
54 	)
55 );
56 
57 #define DEFINE_COMPLETION_EVENT(name)					\
58 		DEFINE_EVENT(rpcrdma_completion_class, name,		\
59 				TP_PROTO(				\
60 					const struct ib_wc *wc,		\
61 					const struct rpc_rdma_cid *cid	\
62 				),					\
63 				TP_ARGS(wc, cid))
64 
65 DECLARE_EVENT_CLASS(rpcrdma_send_completion_class,
66 	TP_PROTO(
67 		const struct ib_wc *wc,
68 		const struct rpc_rdma_cid *cid
69 	),
70 
71 	TP_ARGS(wc, cid),
72 
73 	TP_STRUCT__entry(
74 		__field(u32, cq_id)
75 		__field(int, completion_id)
76 	),
77 
78 	TP_fast_assign(
79 		__entry->cq_id = cid->ci_queue_id;
80 		__entry->completion_id = cid->ci_completion_id;
81 	),
82 
83 	TP_printk("cq.id=%u cid=%d",
84 		__entry->cq_id, __entry->completion_id
85 	)
86 );
87 
88 #define DEFINE_SEND_COMPLETION_EVENT(name)				\
89 		DEFINE_EVENT(rpcrdma_send_completion_class, name,	\
90 				TP_PROTO(				\
91 					const struct ib_wc *wc,		\
92 					const struct rpc_rdma_cid *cid	\
93 				),					\
94 				TP_ARGS(wc, cid))
95 
96 DECLARE_EVENT_CLASS(rpcrdma_send_flush_class,
97 	TP_PROTO(
98 		const struct ib_wc *wc,
99 		const struct rpc_rdma_cid *cid
100 	),
101 
102 	TP_ARGS(wc, cid),
103 
104 	TP_STRUCT__entry(
105 		__field(u32, cq_id)
106 		__field(int, completion_id)
107 		__field(unsigned long, status)
108 		__field(unsigned int, vendor_err)
109 	),
110 
111 	TP_fast_assign(
112 		__entry->cq_id = cid->ci_queue_id;
113 		__entry->completion_id = cid->ci_completion_id;
114 		__entry->status = wc->status;
115 		__entry->vendor_err = wc->vendor_err;
116 	),
117 
118 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
119 		__entry->cq_id, __entry->completion_id,
120 		rdma_show_wc_status(__entry->status),
121 		__entry->status, __entry->vendor_err
122 	)
123 );
124 
125 #define DEFINE_SEND_FLUSH_EVENT(name)					\
126 		DEFINE_EVENT(rpcrdma_send_flush_class, name,		\
127 				TP_PROTO(				\
128 					const struct ib_wc *wc,		\
129 					const struct rpc_rdma_cid *cid	\
130 				),					\
131 				TP_ARGS(wc, cid))
132 
133 DECLARE_EVENT_CLASS(rpcrdma_mr_completion_class,
134 	TP_PROTO(
135 		const struct ib_wc *wc,
136 		const struct rpc_rdma_cid *cid
137 	),
138 
139 	TP_ARGS(wc, cid),
140 
141 	TP_STRUCT__entry(
142 		__field(u32, cq_id)
143 		__field(int, completion_id)
144 		__field(unsigned long, status)
145 		__field(unsigned int, vendor_err)
146 	),
147 
148 	TP_fast_assign(
149 		__entry->cq_id = cid->ci_queue_id;
150 		__entry->completion_id = cid->ci_completion_id;
151 		__entry->status = wc->status;
152 		if (wc->status)
153 			__entry->vendor_err = wc->vendor_err;
154 		else
155 			__entry->vendor_err = 0;
156 	),
157 
158 	TP_printk("cq.id=%u mr.id=%d status=%s (%lu/0x%x)",
159 		__entry->cq_id, __entry->completion_id,
160 		rdma_show_wc_status(__entry->status),
161 		__entry->status, __entry->vendor_err
162 	)
163 );
164 
165 #define DEFINE_MR_COMPLETION_EVENT(name)				\
166 		DEFINE_EVENT(rpcrdma_mr_completion_class, name,		\
167 				TP_PROTO(				\
168 					const struct ib_wc *wc,		\
169 					const struct rpc_rdma_cid *cid	\
170 				),					\
171 				TP_ARGS(wc, cid))
172 
173 DECLARE_EVENT_CLASS(rpcrdma_receive_completion_class,
174 	TP_PROTO(
175 		const struct ib_wc *wc,
176 		const struct rpc_rdma_cid *cid
177 	),
178 
179 	TP_ARGS(wc, cid),
180 
181 	TP_STRUCT__entry(
182 		__field(u32, cq_id)
183 		__field(int, completion_id)
184 		__field(u32, received)
185 		__field(unsigned long, status)
186 		__field(unsigned int, vendor_err)
187 	),
188 
189 	TP_fast_assign(
190 		__entry->cq_id = cid->ci_queue_id;
191 		__entry->completion_id = cid->ci_completion_id;
192 		__entry->status = wc->status;
193 		if (wc->status) {
194 			__entry->received = 0;
195 			__entry->vendor_err = wc->vendor_err;
196 		} else {
197 			__entry->received = wc->byte_len;
198 			__entry->vendor_err = 0;
199 		}
200 	),
201 
202 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x) received=%u",
203 		__entry->cq_id, __entry->completion_id,
204 		rdma_show_wc_status(__entry->status),
205 		__entry->status, __entry->vendor_err,
206 		__entry->received
207 	)
208 );
209 
210 #define DEFINE_RECEIVE_COMPLETION_EVENT(name)				\
211 		DEFINE_EVENT(rpcrdma_receive_completion_class, name,	\
212 				TP_PROTO(				\
213 					const struct ib_wc *wc,		\
214 					const struct rpc_rdma_cid *cid	\
215 				),					\
216 				TP_ARGS(wc, cid))
217 
218 DECLARE_EVENT_CLASS(rpcrdma_receive_success_class,
219 	TP_PROTO(
220 		const struct ib_wc *wc,
221 		const struct rpc_rdma_cid *cid
222 	),
223 
224 	TP_ARGS(wc, cid),
225 
226 	TP_STRUCT__entry(
227 		__field(u32, cq_id)
228 		__field(int, completion_id)
229 		__field(u32, received)
230 	),
231 
232 	TP_fast_assign(
233 		__entry->cq_id = cid->ci_queue_id;
234 		__entry->completion_id = cid->ci_completion_id;
235 		__entry->received = wc->byte_len;
236 	),
237 
238 	TP_printk("cq.id=%u cid=%d received=%u",
239 		__entry->cq_id, __entry->completion_id,
240 		__entry->received
241 	)
242 );
243 
244 #define DEFINE_RECEIVE_SUCCESS_EVENT(name)				\
245 		DEFINE_EVENT(rpcrdma_receive_success_class, name,	\
246 				TP_PROTO(				\
247 					const struct ib_wc *wc,		\
248 					const struct rpc_rdma_cid *cid	\
249 				),					\
250 				TP_ARGS(wc, cid))
251 
252 DECLARE_EVENT_CLASS(rpcrdma_receive_flush_class,
253 	TP_PROTO(
254 		const struct ib_wc *wc,
255 		const struct rpc_rdma_cid *cid
256 	),
257 
258 	TP_ARGS(wc, cid),
259 
260 	TP_STRUCT__entry(
261 		__field(u32, cq_id)
262 		__field(int, completion_id)
263 		__field(unsigned long, status)
264 		__field(unsigned int, vendor_err)
265 	),
266 
267 	TP_fast_assign(
268 		__entry->cq_id = cid->ci_queue_id;
269 		__entry->completion_id = cid->ci_completion_id;
270 		__entry->status = wc->status;
271 		__entry->vendor_err = wc->vendor_err;
272 	),
273 
274 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
275 		__entry->cq_id, __entry->completion_id,
276 		rdma_show_wc_status(__entry->status),
277 		__entry->status, __entry->vendor_err
278 	)
279 );
280 
281 #define DEFINE_RECEIVE_FLUSH_EVENT(name)				\
282 		DEFINE_EVENT(rpcrdma_receive_flush_class, name,		\
283 				TP_PROTO(				\
284 					const struct ib_wc *wc,		\
285 					const struct rpc_rdma_cid *cid	\
286 				),					\
287 				TP_ARGS(wc, cid))
288 
289 DECLARE_EVENT_CLASS(xprtrdma_reply_class,
290 	TP_PROTO(
291 		const struct rpcrdma_rep *rep
292 	),
293 
294 	TP_ARGS(rep),
295 
296 	TP_STRUCT__entry(
297 		__field(u32, xid)
298 		__field(u32, version)
299 		__field(u32, proc)
300 		__string(addr, rpcrdma_addrstr(rep->rr_rxprt))
301 		__string(port, rpcrdma_portstr(rep->rr_rxprt))
302 	),
303 
304 	TP_fast_assign(
305 		__entry->xid = be32_to_cpu(rep->rr_xid);
306 		__entry->version = be32_to_cpu(rep->rr_vers);
307 		__entry->proc = be32_to_cpu(rep->rr_proc);
308 		__assign_str(addr, rpcrdma_addrstr(rep->rr_rxprt));
309 		__assign_str(port, rpcrdma_portstr(rep->rr_rxprt));
310 	),
311 
312 	TP_printk("peer=[%s]:%s xid=0x%08x version=%u proc=%u",
313 		__get_str(addr), __get_str(port),
314 		__entry->xid, __entry->version, __entry->proc
315 	)
316 );
317 
318 #define DEFINE_REPLY_EVENT(name)					\
319 		DEFINE_EVENT(xprtrdma_reply_class,			\
320 				xprtrdma_reply_##name##_err,		\
321 				TP_PROTO(				\
322 					const struct rpcrdma_rep *rep	\
323 				),					\
324 				TP_ARGS(rep))
325 
326 DECLARE_EVENT_CLASS(xprtrdma_rxprt,
327 	TP_PROTO(
328 		const struct rpcrdma_xprt *r_xprt
329 	),
330 
331 	TP_ARGS(r_xprt),
332 
333 	TP_STRUCT__entry(
334 		__string(addr, rpcrdma_addrstr(r_xprt))
335 		__string(port, rpcrdma_portstr(r_xprt))
336 	),
337 
338 	TP_fast_assign(
339 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
340 		__assign_str(port, rpcrdma_portstr(r_xprt));
341 	),
342 
343 	TP_printk("peer=[%s]:%s",
344 		__get_str(addr), __get_str(port)
345 	)
346 );
347 
348 #define DEFINE_RXPRT_EVENT(name)					\
349 		DEFINE_EVENT(xprtrdma_rxprt, name,			\
350 				TP_PROTO(				\
351 					const struct rpcrdma_xprt *r_xprt \
352 				),					\
353 				TP_ARGS(r_xprt))
354 
355 DECLARE_EVENT_CLASS(xprtrdma_connect_class,
356 	TP_PROTO(
357 		const struct rpcrdma_xprt *r_xprt,
358 		int rc
359 	),
360 
361 	TP_ARGS(r_xprt, rc),
362 
363 	TP_STRUCT__entry(
364 		__field(int, rc)
365 		__field(int, connect_status)
366 		__string(addr, rpcrdma_addrstr(r_xprt))
367 		__string(port, rpcrdma_portstr(r_xprt))
368 	),
369 
370 	TP_fast_assign(
371 		__entry->rc = rc;
372 		__entry->connect_status = r_xprt->rx_ep->re_connect_status;
373 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
374 		__assign_str(port, rpcrdma_portstr(r_xprt));
375 	),
376 
377 	TP_printk("peer=[%s]:%s rc=%d connection status=%d",
378 		__get_str(addr), __get_str(port),
379 		__entry->rc, __entry->connect_status
380 	)
381 );
382 
383 #define DEFINE_CONN_EVENT(name)						\
384 		DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,	\
385 				TP_PROTO(				\
386 					const struct rpcrdma_xprt *r_xprt, \
387 					int rc				\
388 				),					\
389 				TP_ARGS(r_xprt, rc))
390 
391 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
392 	TP_PROTO(
393 		const struct rpc_task *task,
394 		unsigned int pos,
395 		struct rpcrdma_mr *mr,
396 		int nsegs
397 	),
398 
399 	TP_ARGS(task, pos, mr, nsegs),
400 
401 	TP_STRUCT__entry(
402 		__field(unsigned int, task_id)
403 		__field(unsigned int, client_id)
404 		__field(unsigned int, pos)
405 		__field(int, nents)
406 		__field(u32, handle)
407 		__field(u32, length)
408 		__field(u64, offset)
409 		__field(int, nsegs)
410 	),
411 
412 	TP_fast_assign(
413 		__entry->task_id = task->tk_pid;
414 		__entry->client_id = task->tk_client->cl_clid;
415 		__entry->pos = pos;
416 		__entry->nents = mr->mr_nents;
417 		__entry->handle = mr->mr_handle;
418 		__entry->length = mr->mr_length;
419 		__entry->offset = mr->mr_offset;
420 		__entry->nsegs = nsegs;
421 	),
422 
423 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
424 		  " pos=%u %u@0x%016llx:0x%08x (%s)",
425 		__entry->task_id, __entry->client_id,
426 		__entry->pos, __entry->length,
427 		(unsigned long long)__entry->offset, __entry->handle,
428 		__entry->nents < __entry->nsegs ? "more" : "last"
429 	)
430 );
431 
432 #define DEFINE_RDCH_EVENT(name)						\
433 		DEFINE_EVENT(xprtrdma_rdch_event, xprtrdma_chunk_##name,\
434 				TP_PROTO(				\
435 					const struct rpc_task *task,	\
436 					unsigned int pos,		\
437 					struct rpcrdma_mr *mr,		\
438 					int nsegs			\
439 				),					\
440 				TP_ARGS(task, pos, mr, nsegs))
441 
442 DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
443 	TP_PROTO(
444 		const struct rpc_task *task,
445 		struct rpcrdma_mr *mr,
446 		int nsegs
447 	),
448 
449 	TP_ARGS(task, mr, nsegs),
450 
451 	TP_STRUCT__entry(
452 		__field(unsigned int, task_id)
453 		__field(unsigned int, client_id)
454 		__field(int, nents)
455 		__field(u32, handle)
456 		__field(u32, length)
457 		__field(u64, offset)
458 		__field(int, nsegs)
459 	),
460 
461 	TP_fast_assign(
462 		__entry->task_id = task->tk_pid;
463 		__entry->client_id = task->tk_client->cl_clid;
464 		__entry->nents = mr->mr_nents;
465 		__entry->handle = mr->mr_handle;
466 		__entry->length = mr->mr_length;
467 		__entry->offset = mr->mr_offset;
468 		__entry->nsegs = nsegs;
469 	),
470 
471 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
472 		  " %u@0x%016llx:0x%08x (%s)",
473 		__entry->task_id, __entry->client_id,
474 		__entry->length, (unsigned long long)__entry->offset,
475 		__entry->handle,
476 		__entry->nents < __entry->nsegs ? "more" : "last"
477 	)
478 );
479 
480 #define DEFINE_WRCH_EVENT(name)						\
481 		DEFINE_EVENT(xprtrdma_wrch_event, xprtrdma_chunk_##name,\
482 				TP_PROTO(				\
483 					const struct rpc_task *task,	\
484 					struct rpcrdma_mr *mr,		\
485 					int nsegs			\
486 				),					\
487 				TP_ARGS(task, mr, nsegs))
488 
489 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
490 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
491 TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
492 TRACE_DEFINE_ENUM(DMA_NONE);
493 
494 #define xprtrdma_show_direction(x)					\
495 		__print_symbolic(x,					\
496 				{ DMA_BIDIRECTIONAL, "BIDIR" },		\
497 				{ DMA_TO_DEVICE, "TO_DEVICE" },		\
498 				{ DMA_FROM_DEVICE, "FROM_DEVICE" },	\
499 				{ DMA_NONE, "NONE" })
500 
501 DECLARE_EVENT_CLASS(xprtrdma_mr_class,
502 	TP_PROTO(
503 		const struct rpcrdma_mr *mr
504 	),
505 
506 	TP_ARGS(mr),
507 
508 	TP_STRUCT__entry(
509 		__field(unsigned int, task_id)
510 		__field(unsigned int, client_id)
511 		__field(u32, mr_id)
512 		__field(int, nents)
513 		__field(u32, handle)
514 		__field(u32, length)
515 		__field(u64, offset)
516 		__field(u32, dir)
517 	),
518 
519 	TP_fast_assign(
520 		const struct rpcrdma_req *req = mr->mr_req;
521 
522 		if (req) {
523 			const struct rpc_task *task = req->rl_slot.rq_task;
524 
525 			__entry->task_id = task->tk_pid;
526 			__entry->client_id = task->tk_client->cl_clid;
527 		} else {
528 			__entry->task_id = 0;
529 			__entry->client_id = -1;
530 		}
531 		__entry->mr_id  = mr->mr_ibmr->res.id;
532 		__entry->nents  = mr->mr_nents;
533 		__entry->handle = mr->mr_handle;
534 		__entry->length = mr->mr_length;
535 		__entry->offset = mr->mr_offset;
536 		__entry->dir    = mr->mr_dir;
537 	),
538 
539 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
540 		  " mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
541 		__entry->task_id, __entry->client_id,
542 		__entry->mr_id, __entry->nents, __entry->length,
543 		(unsigned long long)__entry->offset, __entry->handle,
544 		xprtrdma_show_direction(__entry->dir)
545 	)
546 );
547 
548 #define DEFINE_MR_EVENT(name)						\
549 		DEFINE_EVENT(xprtrdma_mr_class,				\
550 				xprtrdma_mr_##name,			\
551 				TP_PROTO(				\
552 					const struct rpcrdma_mr *mr	\
553 				),					\
554 				TP_ARGS(mr))
555 
556 DECLARE_EVENT_CLASS(xprtrdma_anonymous_mr_class,
557 	TP_PROTO(
558 		const struct rpcrdma_mr *mr
559 	),
560 
561 	TP_ARGS(mr),
562 
563 	TP_STRUCT__entry(
564 		__field(u32, mr_id)
565 		__field(int, nents)
566 		__field(u32, handle)
567 		__field(u32, length)
568 		__field(u64, offset)
569 		__field(u32, dir)
570 	),
571 
572 	TP_fast_assign(
573 		__entry->mr_id  = mr->mr_ibmr->res.id;
574 		__entry->nents  = mr->mr_nents;
575 		__entry->handle = mr->mr_handle;
576 		__entry->length = mr->mr_length;
577 		__entry->offset = mr->mr_offset;
578 		__entry->dir    = mr->mr_dir;
579 	),
580 
581 	TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
582 		__entry->mr_id, __entry->nents, __entry->length,
583 		(unsigned long long)__entry->offset, __entry->handle,
584 		xprtrdma_show_direction(__entry->dir)
585 	)
586 );
587 
588 #define DEFINE_ANON_MR_EVENT(name)					\
589 		DEFINE_EVENT(xprtrdma_anonymous_mr_class,		\
590 				xprtrdma_mr_##name,			\
591 				TP_PROTO(				\
592 					const struct rpcrdma_mr *mr	\
593 				),					\
594 				TP_ARGS(mr))
595 
596 DECLARE_EVENT_CLASS(xprtrdma_callback_class,
597 	TP_PROTO(
598 		const struct rpcrdma_xprt *r_xprt,
599 		const struct rpc_rqst *rqst
600 	),
601 
602 	TP_ARGS(r_xprt, rqst),
603 
604 	TP_STRUCT__entry(
605 		__field(u32, xid)
606 		__string(addr, rpcrdma_addrstr(r_xprt))
607 		__string(port, rpcrdma_portstr(r_xprt))
608 	),
609 
610 	TP_fast_assign(
611 		__entry->xid = be32_to_cpu(rqst->rq_xid);
612 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
613 		__assign_str(port, rpcrdma_portstr(r_xprt));
614 	),
615 
616 	TP_printk("peer=[%s]:%s xid=0x%08x",
617 		__get_str(addr), __get_str(port), __entry->xid
618 	)
619 );
620 
621 #define DEFINE_CALLBACK_EVENT(name)					\
622 		DEFINE_EVENT(xprtrdma_callback_class,			\
623 				xprtrdma_cb_##name,			\
624 				TP_PROTO(				\
625 					const struct rpcrdma_xprt *r_xprt, \
626 					const struct rpc_rqst *rqst	\
627 				),					\
628 				TP_ARGS(r_xprt, rqst))
629 
630 /**
631  ** Connection events
632  **/
633 
634 TRACE_EVENT(xprtrdma_inline_thresh,
635 	TP_PROTO(
636 		const struct rpcrdma_ep *ep
637 	),
638 
639 	TP_ARGS(ep),
640 
641 	TP_STRUCT__entry(
642 		__field(unsigned int, inline_send)
643 		__field(unsigned int, inline_recv)
644 		__field(unsigned int, max_send)
645 		__field(unsigned int, max_recv)
646 		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
647 		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
648 	),
649 
650 	TP_fast_assign(
651 		const struct rdma_cm_id *id = ep->re_id;
652 
653 		__entry->inline_send = ep->re_inline_send;
654 		__entry->inline_recv = ep->re_inline_recv;
655 		__entry->max_send = ep->re_max_inline_send;
656 		__entry->max_recv = ep->re_max_inline_recv;
657 		memcpy(__entry->srcaddr, &id->route.addr.src_addr,
658 		       sizeof(struct sockaddr_in6));
659 		memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
660 		       sizeof(struct sockaddr_in6));
661 	),
662 
663 	TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u",
664 		__entry->srcaddr, __entry->dstaddr,
665 		__entry->inline_send, __entry->inline_recv,
666 		__entry->max_send, __entry->max_recv
667 	)
668 );
669 
670 DEFINE_CONN_EVENT(connect);
671 DEFINE_CONN_EVENT(disconnect);
672 
673 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
674 
675 TRACE_EVENT(xprtrdma_op_connect,
676 	TP_PROTO(
677 		const struct rpcrdma_xprt *r_xprt,
678 		unsigned long delay
679 	),
680 
681 	TP_ARGS(r_xprt, delay),
682 
683 	TP_STRUCT__entry(
684 		__field(unsigned long, delay)
685 		__string(addr, rpcrdma_addrstr(r_xprt))
686 		__string(port, rpcrdma_portstr(r_xprt))
687 	),
688 
689 	TP_fast_assign(
690 		__entry->delay = delay;
691 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
692 		__assign_str(port, rpcrdma_portstr(r_xprt));
693 	),
694 
695 	TP_printk("peer=[%s]:%s delay=%lu",
696 		__get_str(addr), __get_str(port), __entry->delay
697 	)
698 );
699 
700 
701 TRACE_EVENT(xprtrdma_op_set_cto,
702 	TP_PROTO(
703 		const struct rpcrdma_xprt *r_xprt,
704 		unsigned long connect,
705 		unsigned long reconnect
706 	),
707 
708 	TP_ARGS(r_xprt, connect, reconnect),
709 
710 	TP_STRUCT__entry(
711 		__field(unsigned long, connect)
712 		__field(unsigned long, reconnect)
713 		__string(addr, rpcrdma_addrstr(r_xprt))
714 		__string(port, rpcrdma_portstr(r_xprt))
715 	),
716 
717 	TP_fast_assign(
718 		__entry->connect = connect;
719 		__entry->reconnect = reconnect;
720 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
721 		__assign_str(port, rpcrdma_portstr(r_xprt));
722 	),
723 
724 	TP_printk("peer=[%s]:%s connect=%lu reconnect=%lu",
725 		__get_str(addr), __get_str(port),
726 		__entry->connect / HZ, __entry->reconnect / HZ
727 	)
728 );
729 
730 /**
731  ** Call events
732  **/
733 
734 TRACE_EVENT(xprtrdma_createmrs,
735 	TP_PROTO(
736 		const struct rpcrdma_xprt *r_xprt,
737 		unsigned int count
738 	),
739 
740 	TP_ARGS(r_xprt, count),
741 
742 	TP_STRUCT__entry(
743 		__string(addr, rpcrdma_addrstr(r_xprt))
744 		__string(port, rpcrdma_portstr(r_xprt))
745 		__field(unsigned int, count)
746 	),
747 
748 	TP_fast_assign(
749 		__entry->count = count;
750 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
751 		__assign_str(port, rpcrdma_portstr(r_xprt));
752 	),
753 
754 	TP_printk("peer=[%s]:%s created %u MRs",
755 		__get_str(addr), __get_str(port), __entry->count
756 	)
757 );
758 
759 TRACE_EVENT(xprtrdma_nomrs_err,
760 	TP_PROTO(
761 		const struct rpcrdma_xprt *r_xprt,
762 		const struct rpcrdma_req *req
763 	),
764 
765 	TP_ARGS(r_xprt, req),
766 
767 	TP_STRUCT__entry(
768 		__field(unsigned int, task_id)
769 		__field(unsigned int, client_id)
770 		__string(addr, rpcrdma_addrstr(r_xprt))
771 		__string(port, rpcrdma_portstr(r_xprt))
772 	),
773 
774 	TP_fast_assign(
775 		const struct rpc_rqst *rqst = &req->rl_slot;
776 
777 		__entry->task_id = rqst->rq_task->tk_pid;
778 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
779 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
780 		__assign_str(port, rpcrdma_portstr(r_xprt));
781 	),
782 
783 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " peer=[%s]:%s",
784 		__entry->task_id, __entry->client_id,
785 		__get_str(addr), __get_str(port)
786 	)
787 );
788 
789 DEFINE_RDCH_EVENT(read);
790 DEFINE_WRCH_EVENT(write);
791 DEFINE_WRCH_EVENT(reply);
792 DEFINE_WRCH_EVENT(wp);
793 
794 TRACE_DEFINE_ENUM(rpcrdma_noch);
795 TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
796 TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
797 TRACE_DEFINE_ENUM(rpcrdma_readch);
798 TRACE_DEFINE_ENUM(rpcrdma_areadch);
799 TRACE_DEFINE_ENUM(rpcrdma_writech);
800 TRACE_DEFINE_ENUM(rpcrdma_replych);
801 
802 #define xprtrdma_show_chunktype(x)					\
803 		__print_symbolic(x,					\
804 				{ rpcrdma_noch, "inline" },		\
805 				{ rpcrdma_noch_pullup, "pullup" },	\
806 				{ rpcrdma_noch_mapped, "mapped" },	\
807 				{ rpcrdma_readch, "read list" },	\
808 				{ rpcrdma_areadch, "*read list" },	\
809 				{ rpcrdma_writech, "write list" },	\
810 				{ rpcrdma_replych, "reply chunk" })
811 
812 TRACE_EVENT(xprtrdma_marshal,
813 	TP_PROTO(
814 		const struct rpcrdma_req *req,
815 		unsigned int rtype,
816 		unsigned int wtype
817 	),
818 
819 	TP_ARGS(req, rtype, wtype),
820 
821 	TP_STRUCT__entry(
822 		__field(unsigned int, task_id)
823 		__field(unsigned int, client_id)
824 		__field(u32, xid)
825 		__field(unsigned int, hdrlen)
826 		__field(unsigned int, headlen)
827 		__field(unsigned int, pagelen)
828 		__field(unsigned int, taillen)
829 		__field(unsigned int, rtype)
830 		__field(unsigned int, wtype)
831 	),
832 
833 	TP_fast_assign(
834 		const struct rpc_rqst *rqst = &req->rl_slot;
835 
836 		__entry->task_id = rqst->rq_task->tk_pid;
837 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
838 		__entry->xid = be32_to_cpu(rqst->rq_xid);
839 		__entry->hdrlen = req->rl_hdrbuf.len;
840 		__entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
841 		__entry->pagelen = rqst->rq_snd_buf.page_len;
842 		__entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
843 		__entry->rtype = rtype;
844 		__entry->wtype = wtype;
845 	),
846 
847 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
848 		  " xid=0x%08x hdr=%u xdr=%u/%u/%u %s/%s",
849 		__entry->task_id, __entry->client_id, __entry->xid,
850 		__entry->hdrlen,
851 		__entry->headlen, __entry->pagelen, __entry->taillen,
852 		xprtrdma_show_chunktype(__entry->rtype),
853 		xprtrdma_show_chunktype(__entry->wtype)
854 	)
855 );
856 
857 TRACE_EVENT(xprtrdma_marshal_failed,
858 	TP_PROTO(const struct rpc_rqst *rqst,
859 		 int ret
860 	),
861 
862 	TP_ARGS(rqst, ret),
863 
864 	TP_STRUCT__entry(
865 		__field(unsigned int, task_id)
866 		__field(unsigned int, client_id)
867 		__field(u32, xid)
868 		__field(int, ret)
869 	),
870 
871 	TP_fast_assign(
872 		__entry->task_id = rqst->rq_task->tk_pid;
873 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
874 		__entry->xid = be32_to_cpu(rqst->rq_xid);
875 		__entry->ret = ret;
876 	),
877 
878 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
879 		__entry->task_id, __entry->client_id, __entry->xid,
880 		__entry->ret
881 	)
882 );
883 
884 TRACE_EVENT(xprtrdma_prepsend_failed,
885 	TP_PROTO(const struct rpc_rqst *rqst,
886 		 int ret
887 	),
888 
889 	TP_ARGS(rqst, ret),
890 
891 	TP_STRUCT__entry(
892 		__field(unsigned int, task_id)
893 		__field(unsigned int, client_id)
894 		__field(u32, xid)
895 		__field(int, ret)
896 	),
897 
898 	TP_fast_assign(
899 		__entry->task_id = rqst->rq_task->tk_pid;
900 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
901 		__entry->xid = be32_to_cpu(rqst->rq_xid);
902 		__entry->ret = ret;
903 	),
904 
905 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
906 		__entry->task_id, __entry->client_id, __entry->xid,
907 		__entry->ret
908 	)
909 );
910 
911 TRACE_EVENT(xprtrdma_post_send,
912 	TP_PROTO(
913 		const struct rpcrdma_req *req
914 	),
915 
916 	TP_ARGS(req),
917 
918 	TP_STRUCT__entry(
919 		__field(u32, cq_id)
920 		__field(int, completion_id)
921 		__field(unsigned int, task_id)
922 		__field(unsigned int, client_id)
923 		__field(int, num_sge)
924 		__field(int, signaled)
925 	),
926 
927 	TP_fast_assign(
928 		const struct rpc_rqst *rqst = &req->rl_slot;
929 		const struct rpcrdma_sendctx *sc = req->rl_sendctx;
930 
931 		__entry->cq_id = sc->sc_cid.ci_queue_id;
932 		__entry->completion_id = sc->sc_cid.ci_completion_id;
933 		__entry->task_id = rqst->rq_task->tk_pid;
934 		__entry->client_id = rqst->rq_task->tk_client ?
935 				     rqst->rq_task->tk_client->cl_clid : -1;
936 		__entry->num_sge = req->rl_wr.num_sge;
937 		__entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
938 	),
939 
940 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u cid=%d (%d SGE%s) %s",
941 		__entry->task_id, __entry->client_id,
942 		__entry->cq_id, __entry->completion_id,
943 		__entry->num_sge, (__entry->num_sge == 1 ? "" : "s"),
944 		(__entry->signaled ? "signaled" : "")
945 	)
946 );
947 
948 TRACE_EVENT(xprtrdma_post_send_err,
949 	TP_PROTO(
950 		const struct rpcrdma_xprt *r_xprt,
951 		const struct rpcrdma_req *req,
952 		int rc
953 	),
954 
955 	TP_ARGS(r_xprt, req, rc),
956 
957 	TP_STRUCT__entry(
958 		__field(u32, cq_id)
959 		__field(unsigned int, task_id)
960 		__field(unsigned int, client_id)
961 		__field(int, rc)
962 	),
963 
964 	TP_fast_assign(
965 		const struct rpc_rqst *rqst = &req->rl_slot;
966 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
967 
968 		__entry->cq_id = ep ? ep->re_attr.recv_cq->res.id : 0;
969 		__entry->task_id = rqst->rq_task->tk_pid;
970 		__entry->client_id = rqst->rq_task->tk_client ?
971 				     rqst->rq_task->tk_client->cl_clid : -1;
972 		__entry->rc = rc;
973 	),
974 
975 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u rc=%d",
976 		__entry->task_id, __entry->client_id,
977 		__entry->cq_id, __entry->rc
978 	)
979 );
980 
981 TRACE_EVENT(xprtrdma_post_recv,
982 	TP_PROTO(
983 		const struct rpcrdma_rep *rep
984 	),
985 
986 	TP_ARGS(rep),
987 
988 	TP_STRUCT__entry(
989 		__field(u32, cq_id)
990 		__field(int, completion_id)
991 	),
992 
993 	TP_fast_assign(
994 		__entry->cq_id = rep->rr_cid.ci_queue_id;
995 		__entry->completion_id = rep->rr_cid.ci_completion_id;
996 	),
997 
998 	TP_printk("cq.id=%d cid=%d",
999 		__entry->cq_id, __entry->completion_id
1000 	)
1001 );
1002 
1003 TRACE_EVENT(xprtrdma_post_recvs,
1004 	TP_PROTO(
1005 		const struct rpcrdma_xprt *r_xprt,
1006 		unsigned int count
1007 	),
1008 
1009 	TP_ARGS(r_xprt, count),
1010 
1011 	TP_STRUCT__entry(
1012 		__field(u32, cq_id)
1013 		__field(unsigned int, count)
1014 		__field(int, posted)
1015 		__string(addr, rpcrdma_addrstr(r_xprt))
1016 		__string(port, rpcrdma_portstr(r_xprt))
1017 	),
1018 
1019 	TP_fast_assign(
1020 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1021 
1022 		__entry->cq_id = ep->re_attr.recv_cq->res.id;
1023 		__entry->count = count;
1024 		__entry->posted = ep->re_receive_count;
1025 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
1026 		__assign_str(port, rpcrdma_portstr(r_xprt));
1027 	),
1028 
1029 	TP_printk("peer=[%s]:%s cq.id=%d %u new recvs, %d active",
1030 		__get_str(addr), __get_str(port), __entry->cq_id,
1031 		__entry->count, __entry->posted
1032 	)
1033 );
1034 
1035 TRACE_EVENT(xprtrdma_post_recvs_err,
1036 	TP_PROTO(
1037 		const struct rpcrdma_xprt *r_xprt,
1038 		int status
1039 	),
1040 
1041 	TP_ARGS(r_xprt, status),
1042 
1043 	TP_STRUCT__entry(
1044 		__field(u32, cq_id)
1045 		__field(int, status)
1046 		__string(addr, rpcrdma_addrstr(r_xprt))
1047 		__string(port, rpcrdma_portstr(r_xprt))
1048 	),
1049 
1050 	TP_fast_assign(
1051 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1052 
1053 		__entry->cq_id = ep->re_attr.recv_cq->res.id;
1054 		__entry->status = status;
1055 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
1056 		__assign_str(port, rpcrdma_portstr(r_xprt));
1057 	),
1058 
1059 	TP_printk("peer=[%s]:%s cq.id=%d rc=%d",
1060 		__get_str(addr), __get_str(port), __entry->cq_id,
1061 		__entry->status
1062 	)
1063 );
1064 
1065 TRACE_EVENT(xprtrdma_post_linv_err,
1066 	TP_PROTO(
1067 		const struct rpcrdma_req *req,
1068 		int status
1069 	),
1070 
1071 	TP_ARGS(req, status),
1072 
1073 	TP_STRUCT__entry(
1074 		__field(unsigned int, task_id)
1075 		__field(unsigned int, client_id)
1076 		__field(int, status)
1077 	),
1078 
1079 	TP_fast_assign(
1080 		const struct rpc_task *task = req->rl_slot.rq_task;
1081 
1082 		__entry->task_id = task->tk_pid;
1083 		__entry->client_id = task->tk_client->cl_clid;
1084 		__entry->status = status;
1085 	),
1086 
1087 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " status=%d",
1088 		__entry->task_id, __entry->client_id, __entry->status
1089 	)
1090 );
1091 
1092 /**
1093  ** Completion events
1094  **/
1095 
1096 DEFINE_RECEIVE_COMPLETION_EVENT(xprtrdma_wc_receive);
1097 
1098 DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);
1099 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_fastreg);
1100 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li);
1101 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_wake);
1102 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_done);
1103 
1104 TRACE_EVENT(xprtrdma_frwr_alloc,
1105 	TP_PROTO(
1106 		const struct rpcrdma_mr *mr,
1107 		int rc
1108 	),
1109 
1110 	TP_ARGS(mr, rc),
1111 
1112 	TP_STRUCT__entry(
1113 		__field(u32, mr_id)
1114 		__field(int, rc)
1115 	),
1116 
1117 	TP_fast_assign(
1118 		__entry->mr_id = mr->mr_ibmr->res.id;
1119 		__entry->rc = rc;
1120 	),
1121 
1122 	TP_printk("mr.id=%u: rc=%d",
1123 		__entry->mr_id, __entry->rc
1124 	)
1125 );
1126 
1127 TRACE_EVENT(xprtrdma_frwr_dereg,
1128 	TP_PROTO(
1129 		const struct rpcrdma_mr *mr,
1130 		int rc
1131 	),
1132 
1133 	TP_ARGS(mr, rc),
1134 
1135 	TP_STRUCT__entry(
1136 		__field(u32, mr_id)
1137 		__field(int, nents)
1138 		__field(u32, handle)
1139 		__field(u32, length)
1140 		__field(u64, offset)
1141 		__field(u32, dir)
1142 		__field(int, rc)
1143 	),
1144 
1145 	TP_fast_assign(
1146 		__entry->mr_id  = mr->mr_ibmr->res.id;
1147 		__entry->nents  = mr->mr_nents;
1148 		__entry->handle = mr->mr_handle;
1149 		__entry->length = mr->mr_length;
1150 		__entry->offset = mr->mr_offset;
1151 		__entry->dir    = mr->mr_dir;
1152 		__entry->rc	= rc;
1153 	),
1154 
1155 	TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d",
1156 		__entry->mr_id, __entry->nents, __entry->length,
1157 		(unsigned long long)__entry->offset, __entry->handle,
1158 		xprtrdma_show_direction(__entry->dir),
1159 		__entry->rc
1160 	)
1161 );
1162 
1163 TRACE_EVENT(xprtrdma_frwr_sgerr,
1164 	TP_PROTO(
1165 		const struct rpcrdma_mr *mr,
1166 		int sg_nents
1167 	),
1168 
1169 	TP_ARGS(mr, sg_nents),
1170 
1171 	TP_STRUCT__entry(
1172 		__field(u32, mr_id)
1173 		__field(u64, addr)
1174 		__field(u32, dir)
1175 		__field(int, nents)
1176 	),
1177 
1178 	TP_fast_assign(
1179 		__entry->mr_id = mr->mr_ibmr->res.id;
1180 		__entry->addr = mr->mr_sg->dma_address;
1181 		__entry->dir = mr->mr_dir;
1182 		__entry->nents = sg_nents;
1183 	),
1184 
1185 	TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d",
1186 		__entry->mr_id, __entry->addr,
1187 		xprtrdma_show_direction(__entry->dir),
1188 		__entry->nents
1189 	)
1190 );
1191 
1192 TRACE_EVENT(xprtrdma_frwr_maperr,
1193 	TP_PROTO(
1194 		const struct rpcrdma_mr *mr,
1195 		int num_mapped
1196 	),
1197 
1198 	TP_ARGS(mr, num_mapped),
1199 
1200 	TP_STRUCT__entry(
1201 		__field(u32, mr_id)
1202 		__field(u64, addr)
1203 		__field(u32, dir)
1204 		__field(int, num_mapped)
1205 		__field(int, nents)
1206 	),
1207 
1208 	TP_fast_assign(
1209 		__entry->mr_id = mr->mr_ibmr->res.id;
1210 		__entry->addr = mr->mr_sg->dma_address;
1211 		__entry->dir = mr->mr_dir;
1212 		__entry->num_mapped = num_mapped;
1213 		__entry->nents = mr->mr_nents;
1214 	),
1215 
1216 	TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d",
1217 		__entry->mr_id, __entry->addr,
1218 		xprtrdma_show_direction(__entry->dir),
1219 		__entry->num_mapped, __entry->nents
1220 	)
1221 );
1222 
1223 DEFINE_MR_EVENT(fastreg);
1224 DEFINE_MR_EVENT(localinv);
1225 DEFINE_MR_EVENT(reminv);
1226 DEFINE_MR_EVENT(map);
1227 
1228 DEFINE_ANON_MR_EVENT(unmap);
1229 
1230 TRACE_EVENT(xprtrdma_dma_maperr,
1231 	TP_PROTO(
1232 		u64 addr
1233 	),
1234 
1235 	TP_ARGS(addr),
1236 
1237 	TP_STRUCT__entry(
1238 		__field(u64, addr)
1239 	),
1240 
1241 	TP_fast_assign(
1242 		__entry->addr = addr;
1243 	),
1244 
1245 	TP_printk("dma addr=0x%llx\n", __entry->addr)
1246 );
1247 
1248 /**
1249  ** Reply events
1250  **/
1251 
1252 TRACE_EVENT(xprtrdma_reply,
1253 	TP_PROTO(
1254 		const struct rpc_task *task,
1255 		const struct rpcrdma_rep *rep,
1256 		unsigned int credits
1257 	),
1258 
1259 	TP_ARGS(task, rep, credits),
1260 
1261 	TP_STRUCT__entry(
1262 		__field(unsigned int, task_id)
1263 		__field(unsigned int, client_id)
1264 		__field(u32, xid)
1265 		__field(unsigned int, credits)
1266 	),
1267 
1268 	TP_fast_assign(
1269 		__entry->task_id = task->tk_pid;
1270 		__entry->client_id = task->tk_client->cl_clid;
1271 		__entry->xid = be32_to_cpu(rep->rr_xid);
1272 		__entry->credits = credits;
1273 	),
1274 
1275 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x credits=%u",
1276 		__entry->task_id, __entry->client_id, __entry->xid,
1277 		__entry->credits
1278 	)
1279 );
1280 
1281 DEFINE_REPLY_EVENT(vers);
1282 DEFINE_REPLY_EVENT(rqst);
1283 DEFINE_REPLY_EVENT(short);
1284 DEFINE_REPLY_EVENT(hdr);
1285 
1286 TRACE_EVENT(xprtrdma_err_vers,
1287 	TP_PROTO(
1288 		const struct rpc_rqst *rqst,
1289 		__be32 *min,
1290 		__be32 *max
1291 	),
1292 
1293 	TP_ARGS(rqst, min, max),
1294 
1295 	TP_STRUCT__entry(
1296 		__field(unsigned int, task_id)
1297 		__field(unsigned int, client_id)
1298 		__field(u32, xid)
1299 		__field(u32, min)
1300 		__field(u32, max)
1301 	),
1302 
1303 	TP_fast_assign(
1304 		__entry->task_id = rqst->rq_task->tk_pid;
1305 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1306 		__entry->xid = be32_to_cpu(rqst->rq_xid);
1307 		__entry->min = be32_to_cpup(min);
1308 		__entry->max = be32_to_cpup(max);
1309 	),
1310 
1311 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x versions=[%u, %u]",
1312 		__entry->task_id, __entry->client_id, __entry->xid,
1313 		__entry->min, __entry->max
1314 	)
1315 );
1316 
1317 TRACE_EVENT(xprtrdma_err_chunk,
1318 	TP_PROTO(
1319 		const struct rpc_rqst *rqst
1320 	),
1321 
1322 	TP_ARGS(rqst),
1323 
1324 	TP_STRUCT__entry(
1325 		__field(unsigned int, task_id)
1326 		__field(unsigned int, client_id)
1327 		__field(u32, xid)
1328 	),
1329 
1330 	TP_fast_assign(
1331 		__entry->task_id = rqst->rq_task->tk_pid;
1332 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1333 		__entry->xid = be32_to_cpu(rqst->rq_xid);
1334 	),
1335 
1336 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x",
1337 		__entry->task_id, __entry->client_id, __entry->xid
1338 	)
1339 );
1340 
1341 TRACE_EVENT(xprtrdma_err_unrecognized,
1342 	TP_PROTO(
1343 		const struct rpc_rqst *rqst,
1344 		__be32 *procedure
1345 	),
1346 
1347 	TP_ARGS(rqst, procedure),
1348 
1349 	TP_STRUCT__entry(
1350 		__field(unsigned int, task_id)
1351 		__field(unsigned int, client_id)
1352 		__field(u32, xid)
1353 		__field(u32, procedure)
1354 	),
1355 
1356 	TP_fast_assign(
1357 		__entry->task_id = rqst->rq_task->tk_pid;
1358 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1359 		__entry->procedure = be32_to_cpup(procedure);
1360 	),
1361 
1362 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x procedure=%u",
1363 		__entry->task_id, __entry->client_id, __entry->xid,
1364 		__entry->procedure
1365 	)
1366 );
1367 
1368 TRACE_EVENT(xprtrdma_fixup,
1369 	TP_PROTO(
1370 		const struct rpc_rqst *rqst,
1371 		unsigned long fixup
1372 	),
1373 
1374 	TP_ARGS(rqst, fixup),
1375 
1376 	TP_STRUCT__entry(
1377 		__field(unsigned int, task_id)
1378 		__field(unsigned int, client_id)
1379 		__field(unsigned long, fixup)
1380 		__field(size_t, headlen)
1381 		__field(unsigned int, pagelen)
1382 		__field(size_t, taillen)
1383 	),
1384 
1385 	TP_fast_assign(
1386 		__entry->task_id = rqst->rq_task->tk_pid;
1387 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1388 		__entry->fixup = fixup;
1389 		__entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
1390 		__entry->pagelen = rqst->rq_rcv_buf.page_len;
1391 		__entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1392 	),
1393 
1394 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " fixup=%lu xdr=%zu/%u/%zu",
1395 		__entry->task_id, __entry->client_id, __entry->fixup,
1396 		__entry->headlen, __entry->pagelen, __entry->taillen
1397 	)
1398 );
1399 
1400 TRACE_EVENT(xprtrdma_decode_seg,
1401 	TP_PROTO(
1402 		u32 handle,
1403 		u32 length,
1404 		u64 offset
1405 	),
1406 
1407 	TP_ARGS(handle, length, offset),
1408 
1409 	TP_STRUCT__entry(
1410 		__field(u32, handle)
1411 		__field(u32, length)
1412 		__field(u64, offset)
1413 	),
1414 
1415 	TP_fast_assign(
1416 		__entry->handle = handle;
1417 		__entry->length = length;
1418 		__entry->offset = offset;
1419 	),
1420 
1421 	TP_printk("%u@0x%016llx:0x%08x",
1422 		__entry->length, (unsigned long long)__entry->offset,
1423 		__entry->handle
1424 	)
1425 );
1426 
1427 TRACE_EVENT(xprtrdma_mrs_zap,
1428 	TP_PROTO(
1429 		const struct rpc_task *task
1430 	),
1431 
1432 	TP_ARGS(task),
1433 
1434 	TP_STRUCT__entry(
1435 		__field(unsigned int, task_id)
1436 		__field(unsigned int, client_id)
1437 	),
1438 
1439 	TP_fast_assign(
1440 		__entry->task_id = task->tk_pid;
1441 		__entry->client_id = task->tk_client->cl_clid;
1442 	),
1443 
1444 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER,
1445 		__entry->task_id, __entry->client_id
1446 	)
1447 );
1448 
1449 /**
1450  ** Callback events
1451  **/
1452 
1453 TRACE_EVENT(xprtrdma_cb_setup,
1454 	TP_PROTO(
1455 		const struct rpcrdma_xprt *r_xprt,
1456 		unsigned int reqs
1457 	),
1458 
1459 	TP_ARGS(r_xprt, reqs),
1460 
1461 	TP_STRUCT__entry(
1462 		__field(unsigned int, reqs)
1463 		__string(addr, rpcrdma_addrstr(r_xprt))
1464 		__string(port, rpcrdma_portstr(r_xprt))
1465 	),
1466 
1467 	TP_fast_assign(
1468 		__entry->reqs = reqs;
1469 		__assign_str(addr, rpcrdma_addrstr(r_xprt));
1470 		__assign_str(port, rpcrdma_portstr(r_xprt));
1471 	),
1472 
1473 	TP_printk("peer=[%s]:%s %u reqs",
1474 		__get_str(addr), __get_str(port), __entry->reqs
1475 	)
1476 );
1477 
1478 DEFINE_CALLBACK_EVENT(call);
1479 DEFINE_CALLBACK_EVENT(reply);
1480 
1481 /**
1482  ** Server-side RPC/RDMA events
1483  **/
1484 
1485 DECLARE_EVENT_CLASS(svcrdma_accept_class,
1486 	TP_PROTO(
1487 		const struct svcxprt_rdma *rdma,
1488 		long status
1489 	),
1490 
1491 	TP_ARGS(rdma, status),
1492 
1493 	TP_STRUCT__entry(
1494 		__field(long, status)
1495 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1496 	),
1497 
1498 	TP_fast_assign(
1499 		__entry->status = status;
1500 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1501 	),
1502 
1503 	TP_printk("addr=%s status=%ld",
1504 		__get_str(addr), __entry->status
1505 	)
1506 );
1507 
1508 #define DEFINE_ACCEPT_EVENT(name) \
1509 		DEFINE_EVENT(svcrdma_accept_class, svcrdma_##name##_err, \
1510 				TP_PROTO( \
1511 					const struct svcxprt_rdma *rdma, \
1512 					long status \
1513 				), \
1514 				TP_ARGS(rdma, status))
1515 
1516 DEFINE_ACCEPT_EVENT(pd);
1517 DEFINE_ACCEPT_EVENT(qp);
1518 DEFINE_ACCEPT_EVENT(fabric);
1519 DEFINE_ACCEPT_EVENT(initdepth);
1520 DEFINE_ACCEPT_EVENT(accept);
1521 
1522 TRACE_DEFINE_ENUM(RDMA_MSG);
1523 TRACE_DEFINE_ENUM(RDMA_NOMSG);
1524 TRACE_DEFINE_ENUM(RDMA_MSGP);
1525 TRACE_DEFINE_ENUM(RDMA_DONE);
1526 TRACE_DEFINE_ENUM(RDMA_ERROR);
1527 
1528 #define show_rpcrdma_proc(x)						\
1529 		__print_symbolic(x,					\
1530 				{ RDMA_MSG, "RDMA_MSG" },		\
1531 				{ RDMA_NOMSG, "RDMA_NOMSG" },		\
1532 				{ RDMA_MSGP, "RDMA_MSGP" },		\
1533 				{ RDMA_DONE, "RDMA_DONE" },		\
1534 				{ RDMA_ERROR, "RDMA_ERROR" })
1535 
1536 TRACE_EVENT(svcrdma_decode_rqst,
1537 	TP_PROTO(
1538 		const struct svc_rdma_recv_ctxt *ctxt,
1539 		__be32 *p,
1540 		unsigned int hdrlen
1541 	),
1542 
1543 	TP_ARGS(ctxt, p, hdrlen),
1544 
1545 	TP_STRUCT__entry(
1546 		__field(u32, cq_id)
1547 		__field(int, completion_id)
1548 		__field(u32, xid)
1549 		__field(u32, vers)
1550 		__field(u32, proc)
1551 		__field(u32, credits)
1552 		__field(unsigned int, hdrlen)
1553 	),
1554 
1555 	TP_fast_assign(
1556 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1557 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1558 		__entry->xid = be32_to_cpup(p++);
1559 		__entry->vers = be32_to_cpup(p++);
1560 		__entry->credits = be32_to_cpup(p++);
1561 		__entry->proc = be32_to_cpup(p);
1562 		__entry->hdrlen = hdrlen;
1563 	),
1564 
1565 	TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%s hdrlen=%u",
1566 		__entry->cq_id, __entry->completion_id,
1567 		__entry->xid, __entry->vers, __entry->credits,
1568 		show_rpcrdma_proc(__entry->proc), __entry->hdrlen)
1569 );
1570 
1571 TRACE_EVENT(svcrdma_decode_short_err,
1572 	TP_PROTO(
1573 		const struct svc_rdma_recv_ctxt *ctxt,
1574 		unsigned int hdrlen
1575 	),
1576 
1577 	TP_ARGS(ctxt, hdrlen),
1578 
1579 	TP_STRUCT__entry(
1580 		__field(u32, cq_id)
1581 		__field(int, completion_id)
1582 		__field(unsigned int, hdrlen)
1583 	),
1584 
1585 	TP_fast_assign(
1586 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1587 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1588 		__entry->hdrlen = hdrlen;
1589 	),
1590 
1591 	TP_printk("cq.id=%u cid=%d hdrlen=%u",
1592 		__entry->cq_id, __entry->completion_id,
1593 		__entry->hdrlen)
1594 );
1595 
1596 DECLARE_EVENT_CLASS(svcrdma_badreq_event,
1597 	TP_PROTO(
1598 		const struct svc_rdma_recv_ctxt *ctxt,
1599 		__be32 *p
1600 	),
1601 
1602 	TP_ARGS(ctxt, p),
1603 
1604 	TP_STRUCT__entry(
1605 		__field(u32, cq_id)
1606 		__field(int, completion_id)
1607 		__field(u32, xid)
1608 		__field(u32, vers)
1609 		__field(u32, proc)
1610 		__field(u32, credits)
1611 	),
1612 
1613 	TP_fast_assign(
1614 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1615 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1616 		__entry->xid = be32_to_cpup(p++);
1617 		__entry->vers = be32_to_cpup(p++);
1618 		__entry->credits = be32_to_cpup(p++);
1619 		__entry->proc = be32_to_cpup(p);
1620 	),
1621 
1622 	TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%u",
1623 		__entry->cq_id, __entry->completion_id,
1624 		__entry->xid, __entry->vers, __entry->credits, __entry->proc)
1625 );
1626 
1627 #define DEFINE_BADREQ_EVENT(name)					\
1628 		DEFINE_EVENT(svcrdma_badreq_event,			\
1629 			     svcrdma_decode_##name##_err,		\
1630 				TP_PROTO(				\
1631 					const struct svc_rdma_recv_ctxt *ctxt,	\
1632 					__be32 *p			\
1633 				),					\
1634 				TP_ARGS(ctxt, p))
1635 
1636 DEFINE_BADREQ_EVENT(badvers);
1637 DEFINE_BADREQ_EVENT(drop);
1638 DEFINE_BADREQ_EVENT(badproc);
1639 DEFINE_BADREQ_EVENT(parse);
1640 
1641 TRACE_EVENT(svcrdma_encode_wseg,
1642 	TP_PROTO(
1643 		const struct svc_rdma_send_ctxt *ctxt,
1644 		u32 segno,
1645 		u32 handle,
1646 		u32 length,
1647 		u64 offset
1648 	),
1649 
1650 	TP_ARGS(ctxt, segno, handle, length, offset),
1651 
1652 	TP_STRUCT__entry(
1653 		__field(u32, cq_id)
1654 		__field(int, completion_id)
1655 		__field(u32, segno)
1656 		__field(u32, handle)
1657 		__field(u32, length)
1658 		__field(u64, offset)
1659 	),
1660 
1661 	TP_fast_assign(
1662 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
1663 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
1664 		__entry->segno = segno;
1665 		__entry->handle = handle;
1666 		__entry->length = length;
1667 		__entry->offset = offset;
1668 	),
1669 
1670 	TP_printk("cq_id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1671 		__entry->cq_id, __entry->completion_id,
1672 		__entry->segno, __entry->length,
1673 		(unsigned long long)__entry->offset, __entry->handle
1674 	)
1675 );
1676 
1677 TRACE_EVENT(svcrdma_decode_rseg,
1678 	TP_PROTO(
1679 		const struct rpc_rdma_cid *cid,
1680 		const struct svc_rdma_chunk *chunk,
1681 		const struct svc_rdma_segment *segment
1682 	),
1683 
1684 	TP_ARGS(cid, chunk, segment),
1685 
1686 	TP_STRUCT__entry(
1687 		__field(u32, cq_id)
1688 		__field(int, completion_id)
1689 		__field(u32, segno)
1690 		__field(u32, position)
1691 		__field(u32, handle)
1692 		__field(u32, length)
1693 		__field(u64, offset)
1694 	),
1695 
1696 	TP_fast_assign(
1697 		__entry->cq_id = cid->ci_queue_id;
1698 		__entry->completion_id = cid->ci_completion_id;
1699 		__entry->segno = chunk->ch_segcount;
1700 		__entry->position = chunk->ch_position;
1701 		__entry->handle = segment->rs_handle;
1702 		__entry->length = segment->rs_length;
1703 		__entry->offset = segment->rs_offset;
1704 	),
1705 
1706 	TP_printk("cq_id=%u cid=%d segno=%u position=%u %u@0x%016llx:0x%08x",
1707 		__entry->cq_id, __entry->completion_id,
1708 		__entry->segno, __entry->position, __entry->length,
1709 		(unsigned long long)__entry->offset, __entry->handle
1710 	)
1711 );
1712 
1713 TRACE_EVENT(svcrdma_decode_wseg,
1714 	TP_PROTO(
1715 		const struct rpc_rdma_cid *cid,
1716 		const struct svc_rdma_chunk *chunk,
1717 		u32 segno
1718 	),
1719 
1720 	TP_ARGS(cid, chunk, segno),
1721 
1722 	TP_STRUCT__entry(
1723 		__field(u32, cq_id)
1724 		__field(int, completion_id)
1725 		__field(u32, segno)
1726 		__field(u32, handle)
1727 		__field(u32, length)
1728 		__field(u64, offset)
1729 	),
1730 
1731 	TP_fast_assign(
1732 		const struct svc_rdma_segment *segment =
1733 			&chunk->ch_segments[segno];
1734 
1735 		__entry->cq_id = cid->ci_queue_id;
1736 		__entry->completion_id = cid->ci_completion_id;
1737 		__entry->segno = segno;
1738 		__entry->handle = segment->rs_handle;
1739 		__entry->length = segment->rs_length;
1740 		__entry->offset = segment->rs_offset;
1741 	),
1742 
1743 	TP_printk("cq_id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1744 		__entry->cq_id, __entry->completion_id,
1745 		__entry->segno, __entry->length,
1746 		(unsigned long long)__entry->offset, __entry->handle
1747 	)
1748 );
1749 
1750 DECLARE_EVENT_CLASS(svcrdma_error_event,
1751 	TP_PROTO(
1752 		__be32 xid
1753 	),
1754 
1755 	TP_ARGS(xid),
1756 
1757 	TP_STRUCT__entry(
1758 		__field(u32, xid)
1759 	),
1760 
1761 	TP_fast_assign(
1762 		__entry->xid = be32_to_cpu(xid);
1763 	),
1764 
1765 	TP_printk("xid=0x%08x",
1766 		__entry->xid
1767 	)
1768 );
1769 
1770 #define DEFINE_ERROR_EVENT(name)					\
1771 		DEFINE_EVENT(svcrdma_error_event, svcrdma_err_##name,	\
1772 				TP_PROTO(				\
1773 					__be32 xid			\
1774 				),					\
1775 				TP_ARGS(xid))
1776 
1777 DEFINE_ERROR_EVENT(vers);
1778 DEFINE_ERROR_EVENT(chunk);
1779 
1780 /**
1781  ** Server-side RDMA API events
1782  **/
1783 
1784 DECLARE_EVENT_CLASS(svcrdma_dma_map_class,
1785 	TP_PROTO(
1786 		const struct svcxprt_rdma *rdma,
1787 		u64 dma_addr,
1788 		u32 length
1789 	),
1790 
1791 	TP_ARGS(rdma, dma_addr, length),
1792 
1793 	TP_STRUCT__entry(
1794 		__field(u64, dma_addr)
1795 		__field(u32, length)
1796 		__string(device, rdma->sc_cm_id->device->name)
1797 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1798 	),
1799 
1800 	TP_fast_assign(
1801 		__entry->dma_addr = dma_addr;
1802 		__entry->length = length;
1803 		__assign_str(device, rdma->sc_cm_id->device->name);
1804 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1805 	),
1806 
1807 	TP_printk("addr=%s device=%s dma_addr=%llu length=%u",
1808 		__get_str(addr), __get_str(device),
1809 		__entry->dma_addr, __entry->length
1810 	)
1811 );
1812 
1813 #define DEFINE_SVC_DMA_EVENT(name)					\
1814 		DEFINE_EVENT(svcrdma_dma_map_class, svcrdma_##name,	\
1815 				TP_PROTO(				\
1816 					const struct svcxprt_rdma *rdma,\
1817 					u64 dma_addr,			\
1818 					u32 length			\
1819 				),					\
1820 				TP_ARGS(rdma, dma_addr, length))
1821 
1822 DEFINE_SVC_DMA_EVENT(dma_map_page);
1823 DEFINE_SVC_DMA_EVENT(dma_map_err);
1824 DEFINE_SVC_DMA_EVENT(dma_unmap_page);
1825 
1826 TRACE_EVENT(svcrdma_dma_map_rw_err,
1827 	TP_PROTO(
1828 		const struct svcxprt_rdma *rdma,
1829 		unsigned int nents,
1830 		int status
1831 	),
1832 
1833 	TP_ARGS(rdma, nents, status),
1834 
1835 	TP_STRUCT__entry(
1836 		__field(int, status)
1837 		__field(unsigned int, nents)
1838 		__string(device, rdma->sc_cm_id->device->name)
1839 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1840 	),
1841 
1842 	TP_fast_assign(
1843 		__entry->status = status;
1844 		__entry->nents = nents;
1845 		__assign_str(device, rdma->sc_cm_id->device->name);
1846 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1847 	),
1848 
1849 	TP_printk("addr=%s device=%s nents=%u status=%d",
1850 		__get_str(addr), __get_str(device), __entry->nents,
1851 		__entry->status
1852 	)
1853 );
1854 
1855 TRACE_EVENT(svcrdma_no_rwctx_err,
1856 	TP_PROTO(
1857 		const struct svcxprt_rdma *rdma,
1858 		unsigned int num_sges
1859 	),
1860 
1861 	TP_ARGS(rdma, num_sges),
1862 
1863 	TP_STRUCT__entry(
1864 		__field(unsigned int, num_sges)
1865 		__string(device, rdma->sc_cm_id->device->name)
1866 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1867 	),
1868 
1869 	TP_fast_assign(
1870 		__entry->num_sges = num_sges;
1871 		__assign_str(device, rdma->sc_cm_id->device->name);
1872 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1873 	),
1874 
1875 	TP_printk("addr=%s device=%s num_sges=%d",
1876 		__get_str(addr), __get_str(device), __entry->num_sges
1877 	)
1878 );
1879 
1880 TRACE_EVENT(svcrdma_page_overrun_err,
1881 	TP_PROTO(
1882 		const struct svcxprt_rdma *rdma,
1883 		const struct svc_rqst *rqst,
1884 		unsigned int pageno
1885 	),
1886 
1887 	TP_ARGS(rdma, rqst, pageno),
1888 
1889 	TP_STRUCT__entry(
1890 		__field(unsigned int, pageno)
1891 		__field(u32, xid)
1892 		__string(device, rdma->sc_cm_id->device->name)
1893 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1894 	),
1895 
1896 	TP_fast_assign(
1897 		__entry->pageno = pageno;
1898 		__entry->xid = __be32_to_cpu(rqst->rq_xid);
1899 		__assign_str(device, rdma->sc_cm_id->device->name);
1900 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1901 	),
1902 
1903 	TP_printk("addr=%s device=%s xid=0x%08x pageno=%u", __get_str(addr),
1904 		__get_str(device), __entry->xid, __entry->pageno
1905 	)
1906 );
1907 
1908 TRACE_EVENT(svcrdma_small_wrch_err,
1909 	TP_PROTO(
1910 		const struct svcxprt_rdma *rdma,
1911 		unsigned int remaining,
1912 		unsigned int seg_no,
1913 		unsigned int num_segs
1914 	),
1915 
1916 	TP_ARGS(rdma, remaining, seg_no, num_segs),
1917 
1918 	TP_STRUCT__entry(
1919 		__field(unsigned int, remaining)
1920 		__field(unsigned int, seg_no)
1921 		__field(unsigned int, num_segs)
1922 		__string(device, rdma->sc_cm_id->device->name)
1923 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1924 	),
1925 
1926 	TP_fast_assign(
1927 		__entry->remaining = remaining;
1928 		__entry->seg_no = seg_no;
1929 		__entry->num_segs = num_segs;
1930 		__assign_str(device, rdma->sc_cm_id->device->name);
1931 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1932 	),
1933 
1934 	TP_printk("addr=%s device=%s remaining=%u seg_no=%u num_segs=%u",
1935 		__get_str(addr), __get_str(device), __entry->remaining,
1936 		__entry->seg_no, __entry->num_segs
1937 	)
1938 );
1939 
1940 TRACE_EVENT(svcrdma_send_pullup,
1941 	TP_PROTO(
1942 		const struct svc_rdma_send_ctxt *ctxt,
1943 		unsigned int msglen
1944 	),
1945 
1946 	TP_ARGS(ctxt, msglen),
1947 
1948 	TP_STRUCT__entry(
1949 		__field(u32, cq_id)
1950 		__field(int, completion_id)
1951 		__field(unsigned int, hdrlen)
1952 		__field(unsigned int, msglen)
1953 	),
1954 
1955 	TP_fast_assign(
1956 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
1957 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
1958 		__entry->hdrlen = ctxt->sc_hdrbuf.len,
1959 		__entry->msglen = msglen;
1960 	),
1961 
1962 	TP_printk("cq_id=%u cid=%d hdr=%u msg=%u (total %u)",
1963 		__entry->cq_id, __entry->completion_id,
1964 		__entry->hdrlen, __entry->msglen,
1965 		__entry->hdrlen + __entry->msglen)
1966 );
1967 
1968 TRACE_EVENT(svcrdma_send_err,
1969 	TP_PROTO(
1970 		const struct svc_rqst *rqst,
1971 		int status
1972 	),
1973 
1974 	TP_ARGS(rqst, status),
1975 
1976 	TP_STRUCT__entry(
1977 		__field(int, status)
1978 		__field(u32, xid)
1979 		__string(addr, rqst->rq_xprt->xpt_remotebuf)
1980 	),
1981 
1982 	TP_fast_assign(
1983 		__entry->status = status;
1984 		__entry->xid = __be32_to_cpu(rqst->rq_xid);
1985 		__assign_str(addr, rqst->rq_xprt->xpt_remotebuf);
1986 	),
1987 
1988 	TP_printk("addr=%s xid=0x%08x status=%d", __get_str(addr),
1989 		__entry->xid, __entry->status
1990 	)
1991 );
1992 
1993 TRACE_EVENT(svcrdma_post_send,
1994 	TP_PROTO(
1995 		const struct svc_rdma_send_ctxt *ctxt
1996 	),
1997 
1998 	TP_ARGS(ctxt),
1999 
2000 	TP_STRUCT__entry(
2001 		__field(u32, cq_id)
2002 		__field(int, completion_id)
2003 		__field(unsigned int, num_sge)
2004 		__field(u32, inv_rkey)
2005 	),
2006 
2007 	TP_fast_assign(
2008 		const struct ib_send_wr *wr = &ctxt->sc_send_wr;
2009 
2010 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
2011 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
2012 		__entry->num_sge = wr->num_sge;
2013 		__entry->inv_rkey = (wr->opcode == IB_WR_SEND_WITH_INV) ?
2014 					wr->ex.invalidate_rkey : 0;
2015 	),
2016 
2017 	TP_printk("cq_id=%u cid=%d num_sge=%u inv_rkey=0x%08x",
2018 		__entry->cq_id, __entry->completion_id,
2019 		__entry->num_sge, __entry->inv_rkey
2020 	)
2021 );
2022 
2023 DEFINE_SEND_COMPLETION_EVENT(svcrdma_wc_send);
2024 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_flush);
2025 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_err);
2026 
2027 TRACE_EVENT(svcrdma_post_recv,
2028 	TP_PROTO(
2029 		const struct svc_rdma_recv_ctxt *ctxt
2030 	),
2031 
2032 	TP_ARGS(ctxt),
2033 
2034 	TP_STRUCT__entry(
2035 		__field(u32, cq_id)
2036 		__field(int, completion_id)
2037 	),
2038 
2039 	TP_fast_assign(
2040 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
2041 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
2042 	),
2043 
2044 	TP_printk("cq.id=%d cid=%d",
2045 		__entry->cq_id, __entry->completion_id
2046 	)
2047 );
2048 
2049 DEFINE_RECEIVE_SUCCESS_EVENT(svcrdma_wc_recv);
2050 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_flush);
2051 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_err);
2052 
2053 TRACE_EVENT(svcrdma_rq_post_err,
2054 	TP_PROTO(
2055 		const struct svcxprt_rdma *rdma,
2056 		int status
2057 	),
2058 
2059 	TP_ARGS(rdma, status),
2060 
2061 	TP_STRUCT__entry(
2062 		__field(int, status)
2063 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
2064 	),
2065 
2066 	TP_fast_assign(
2067 		__entry->status = status;
2068 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
2069 	),
2070 
2071 	TP_printk("addr=%s status=%d",
2072 		__get_str(addr), __entry->status
2073 	)
2074 );
2075 
2076 DECLARE_EVENT_CLASS(svcrdma_post_chunk_class,
2077 	TP_PROTO(
2078 		const struct rpc_rdma_cid *cid,
2079 		int sqecount
2080 	),
2081 
2082 	TP_ARGS(cid, sqecount),
2083 
2084 	TP_STRUCT__entry(
2085 		__field(u32, cq_id)
2086 		__field(int, completion_id)
2087 		__field(int, sqecount)
2088 	),
2089 
2090 	TP_fast_assign(
2091 		__entry->cq_id = cid->ci_queue_id;
2092 		__entry->completion_id = cid->ci_completion_id;
2093 		__entry->sqecount = sqecount;
2094 	),
2095 
2096 	TP_printk("cq.id=%u cid=%d sqecount=%d",
2097 		__entry->cq_id, __entry->completion_id,
2098 		__entry->sqecount
2099 	)
2100 );
2101 
2102 #define DEFINE_POST_CHUNK_EVENT(name)					\
2103 		DEFINE_EVENT(svcrdma_post_chunk_class,			\
2104 				svcrdma_post_##name##_chunk,		\
2105 				TP_PROTO(				\
2106 					const struct rpc_rdma_cid *cid,	\
2107 					int sqecount			\
2108 				),					\
2109 				TP_ARGS(cid, sqecount))
2110 
2111 DEFINE_POST_CHUNK_EVENT(read);
2112 DEFINE_POST_CHUNK_EVENT(write);
2113 DEFINE_POST_CHUNK_EVENT(reply);
2114 
2115 DEFINE_EVENT(svcrdma_post_chunk_class, svcrdma_cc_release,
2116 	TP_PROTO(
2117 		const struct rpc_rdma_cid *cid,
2118 		int sqecount
2119 	),
2120 	TP_ARGS(cid, sqecount)
2121 );
2122 
2123 TRACE_EVENT(svcrdma_wc_read,
2124 	TP_PROTO(
2125 		const struct ib_wc *wc,
2126 		const struct rpc_rdma_cid *cid,
2127 		unsigned int totalbytes,
2128 		const ktime_t posttime
2129 	),
2130 
2131 	TP_ARGS(wc, cid, totalbytes, posttime),
2132 
2133 	TP_STRUCT__entry(
2134 		__field(u32, cq_id)
2135 		__field(int, completion_id)
2136 		__field(s64, read_latency)
2137 		__field(unsigned int, totalbytes)
2138 	),
2139 
2140 	TP_fast_assign(
2141 		__entry->cq_id = cid->ci_queue_id;
2142 		__entry->completion_id = cid->ci_completion_id;
2143 		__entry->totalbytes = totalbytes;
2144 		__entry->read_latency = ktime_us_delta(ktime_get(), posttime);
2145 	),
2146 
2147 	TP_printk("cq.id=%u cid=%d totalbytes=%u latency-us=%lld",
2148 		__entry->cq_id, __entry->completion_id,
2149 		__entry->totalbytes, __entry->read_latency
2150 	)
2151 );
2152 
2153 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_flush);
2154 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_err);
2155 
2156 DEFINE_SEND_COMPLETION_EVENT(svcrdma_wc_write);
2157 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_flush);
2158 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_err);
2159 
2160 TRACE_EVENT(svcrdma_qp_error,
2161 	TP_PROTO(
2162 		const struct ib_event *event,
2163 		const struct sockaddr *sap
2164 	),
2165 
2166 	TP_ARGS(event, sap),
2167 
2168 	TP_STRUCT__entry(
2169 		__field(unsigned int, event)
2170 		__string(device, event->device->name)
2171 		__array(__u8, addr, INET6_ADDRSTRLEN + 10)
2172 	),
2173 
2174 	TP_fast_assign(
2175 		__entry->event = event->event;
2176 		__assign_str(device, event->device->name);
2177 		snprintf(__entry->addr, sizeof(__entry->addr) - 1,
2178 			 "%pISpc", sap);
2179 	),
2180 
2181 	TP_printk("addr=%s dev=%s event=%s (%u)",
2182 		__entry->addr, __get_str(device),
2183 		rdma_show_ib_event(__entry->event), __entry->event
2184 	)
2185 );
2186 
2187 DECLARE_EVENT_CLASS(svcrdma_sendqueue_event,
2188 	TP_PROTO(
2189 		const struct svcxprt_rdma *rdma
2190 	),
2191 
2192 	TP_ARGS(rdma),
2193 
2194 	TP_STRUCT__entry(
2195 		__field(int, avail)
2196 		__field(int, depth)
2197 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
2198 	),
2199 
2200 	TP_fast_assign(
2201 		__entry->avail = atomic_read(&rdma->sc_sq_avail);
2202 		__entry->depth = rdma->sc_sq_depth;
2203 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
2204 	),
2205 
2206 	TP_printk("addr=%s sc_sq_avail=%d/%d",
2207 		__get_str(addr), __entry->avail, __entry->depth
2208 	)
2209 );
2210 
2211 #define DEFINE_SQ_EVENT(name)						\
2212 		DEFINE_EVENT(svcrdma_sendqueue_event, svcrdma_sq_##name,\
2213 				TP_PROTO(				\
2214 					const struct svcxprt_rdma *rdma \
2215 				),					\
2216 				TP_ARGS(rdma))
2217 
2218 DEFINE_SQ_EVENT(full);
2219 DEFINE_SQ_EVENT(retry);
2220 
2221 TRACE_EVENT(svcrdma_sq_post_err,
2222 	TP_PROTO(
2223 		const struct svcxprt_rdma *rdma,
2224 		int status
2225 	),
2226 
2227 	TP_ARGS(rdma, status),
2228 
2229 	TP_STRUCT__entry(
2230 		__field(int, avail)
2231 		__field(int, depth)
2232 		__field(int, status)
2233 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
2234 	),
2235 
2236 	TP_fast_assign(
2237 		__entry->avail = atomic_read(&rdma->sc_sq_avail);
2238 		__entry->depth = rdma->sc_sq_depth;
2239 		__entry->status = status;
2240 		__assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
2241 	),
2242 
2243 	TP_printk("addr=%s sc_sq_avail=%d/%d status=%d",
2244 		__get_str(addr), __entry->avail, __entry->depth,
2245 		__entry->status
2246 	)
2247 );
2248 
2249 #endif /* _TRACE_RPCRDMA_H */
2250 
2251 #include <trace/define_trace.h>
2252