1 /*
2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
4  * Copyright (c) 2006 Intel Corporation.  All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/err.h>
38 #include <linux/random.h>
39 #include <linux/spinlock.h>
40 #include <linux/slab.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/kref.h>
43 #include <linux/xarray.h>
44 #include <linux/workqueue.h>
45 #include <uapi/linux/if_ether.h>
46 #include <rdma/ib_pack.h>
47 #include <rdma/ib_cache.h>
48 #include <rdma/rdma_netlink.h>
49 #include <net/netlink.h>
50 #include <uapi/rdma/ib_user_sa.h>
51 #include <rdma/ib_marshall.h>
52 #include <rdma/ib_addr.h>
53 #include <rdma/opa_addr.h>
54 #include "sa.h"
55 #include "core_priv.h"
56 
57 #define IB_SA_LOCAL_SVC_TIMEOUT_MIN		100
58 #define IB_SA_LOCAL_SVC_TIMEOUT_DEFAULT		2000
59 #define IB_SA_LOCAL_SVC_TIMEOUT_MAX		200000
60 #define IB_SA_CPI_MAX_RETRY_CNT			3
61 #define IB_SA_CPI_RETRY_WAIT			1000 /*msecs */
62 static int sa_local_svc_timeout_ms = IB_SA_LOCAL_SVC_TIMEOUT_DEFAULT;
63 
64 struct ib_sa_sm_ah {
65 	struct ib_ah        *ah;
66 	struct kref          ref;
67 	u16		     pkey_index;
68 	u8		     src_path_mask;
69 };
70 
71 enum rdma_class_port_info_type {
72 	RDMA_CLASS_PORT_INFO_IB,
73 	RDMA_CLASS_PORT_INFO_OPA
74 };
75 
76 struct rdma_class_port_info {
77 	enum rdma_class_port_info_type type;
78 	union {
79 		struct ib_class_port_info ib;
80 		struct opa_class_port_info opa;
81 	};
82 };
83 
84 struct ib_sa_classport_cache {
85 	bool valid;
86 	int retry_cnt;
87 	struct rdma_class_port_info data;
88 };
89 
90 struct ib_sa_port {
91 	struct ib_mad_agent *agent;
92 	struct ib_sa_sm_ah  *sm_ah;
93 	struct work_struct   update_task;
94 	struct ib_sa_classport_cache classport_info;
95 	struct delayed_work ib_cpi_work;
96 	spinlock_t                   classport_lock; /* protects class port info set */
97 	spinlock_t           ah_lock;
98 	u32		     port_num;
99 };
100 
101 struct ib_sa_device {
102 	int                     start_port, end_port;
103 	struct ib_event_handler event_handler;
104 	struct ib_sa_port port[];
105 };
106 
107 struct ib_sa_query {
108 	void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
109 	void (*release)(struct ib_sa_query *);
110 	struct ib_sa_client    *client;
111 	struct ib_sa_port      *port;
112 	struct ib_mad_send_buf *mad_buf;
113 	struct ib_sa_sm_ah     *sm_ah;
114 	int			id;
115 	u32			flags;
116 	struct list_head	list; /* Local svc request list */
117 	u32			seq; /* Local svc request sequence number */
118 	unsigned long		timeout; /* Local svc timeout */
119 	u8			path_use; /* How will the pathrecord be used */
120 };
121 
122 #define IB_SA_ENABLE_LOCAL_SERVICE	0x00000001
123 #define IB_SA_CANCEL			0x00000002
124 #define IB_SA_QUERY_OPA			0x00000004
125 
126 struct ib_sa_service_query {
127 	void (*callback)(int, struct ib_sa_service_rec *, void *);
128 	void *context;
129 	struct ib_sa_query sa_query;
130 };
131 
132 struct ib_sa_path_query {
133 	void (*callback)(int, struct sa_path_rec *, void *);
134 	void *context;
135 	struct ib_sa_query sa_query;
136 	struct sa_path_rec *conv_pr;
137 };
138 
139 struct ib_sa_guidinfo_query {
140 	void (*callback)(int, struct ib_sa_guidinfo_rec *, void *);
141 	void *context;
142 	struct ib_sa_query sa_query;
143 };
144 
145 struct ib_sa_classport_info_query {
146 	void (*callback)(void *);
147 	void *context;
148 	struct ib_sa_query sa_query;
149 };
150 
151 struct ib_sa_mcmember_query {
152 	void (*callback)(int, struct ib_sa_mcmember_rec *, void *);
153 	void *context;
154 	struct ib_sa_query sa_query;
155 };
156 
157 static LIST_HEAD(ib_nl_request_list);
158 static DEFINE_SPINLOCK(ib_nl_request_lock);
159 static atomic_t ib_nl_sa_request_seq;
160 static struct workqueue_struct *ib_nl_wq;
161 static struct delayed_work ib_nl_timed_work;
162 static const struct nla_policy ib_nl_policy[LS_NLA_TYPE_MAX] = {
163 	[LS_NLA_TYPE_PATH_RECORD]	= {.type = NLA_BINARY,
164 		.len = sizeof(struct ib_path_rec_data)},
165 	[LS_NLA_TYPE_TIMEOUT]		= {.type = NLA_U32},
166 	[LS_NLA_TYPE_SERVICE_ID]	= {.type = NLA_U64},
167 	[LS_NLA_TYPE_DGID]		= {.type = NLA_BINARY,
168 		.len = sizeof(struct rdma_nla_ls_gid)},
169 	[LS_NLA_TYPE_SGID]		= {.type = NLA_BINARY,
170 		.len = sizeof(struct rdma_nla_ls_gid)},
171 	[LS_NLA_TYPE_TCLASS]		= {.type = NLA_U8},
172 	[LS_NLA_TYPE_PKEY]		= {.type = NLA_U16},
173 	[LS_NLA_TYPE_QOS_CLASS]		= {.type = NLA_U16},
174 };
175 
176 
177 static int ib_sa_add_one(struct ib_device *device);
178 static void ib_sa_remove_one(struct ib_device *device, void *client_data);
179 
180 static struct ib_client sa_client = {
181 	.name   = "sa",
182 	.add    = ib_sa_add_one,
183 	.remove = ib_sa_remove_one
184 };
185 
186 static DEFINE_XARRAY_FLAGS(queries, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
187 
188 static DEFINE_SPINLOCK(tid_lock);
189 static u32 tid;
190 
191 #define PATH_REC_FIELD(field) \
192 	.struct_offset_bytes = offsetof(struct sa_path_rec, field),	\
193 	.struct_size_bytes   = sizeof_field(struct sa_path_rec, field),	\
194 	.field_name          = "sa_path_rec:" #field
195 
196 static const struct ib_field path_rec_table[] = {
197 	{ PATH_REC_FIELD(service_id),
198 	  .offset_words = 0,
199 	  .offset_bits  = 0,
200 	  .size_bits    = 64 },
201 	{ PATH_REC_FIELD(dgid),
202 	  .offset_words = 2,
203 	  .offset_bits  = 0,
204 	  .size_bits    = 128 },
205 	{ PATH_REC_FIELD(sgid),
206 	  .offset_words = 6,
207 	  .offset_bits  = 0,
208 	  .size_bits    = 128 },
209 	{ PATH_REC_FIELD(ib.dlid),
210 	  .offset_words = 10,
211 	  .offset_bits  = 0,
212 	  .size_bits    = 16 },
213 	{ PATH_REC_FIELD(ib.slid),
214 	  .offset_words = 10,
215 	  .offset_bits  = 16,
216 	  .size_bits    = 16 },
217 	{ PATH_REC_FIELD(ib.raw_traffic),
218 	  .offset_words = 11,
219 	  .offset_bits  = 0,
220 	  .size_bits    = 1 },
221 	{ RESERVED,
222 	  .offset_words = 11,
223 	  .offset_bits  = 1,
224 	  .size_bits    = 3 },
225 	{ PATH_REC_FIELD(flow_label),
226 	  .offset_words = 11,
227 	  .offset_bits  = 4,
228 	  .size_bits    = 20 },
229 	{ PATH_REC_FIELD(hop_limit),
230 	  .offset_words = 11,
231 	  .offset_bits  = 24,
232 	  .size_bits    = 8 },
233 	{ PATH_REC_FIELD(traffic_class),
234 	  .offset_words = 12,
235 	  .offset_bits  = 0,
236 	  .size_bits    = 8 },
237 	{ PATH_REC_FIELD(reversible),
238 	  .offset_words = 12,
239 	  .offset_bits  = 8,
240 	  .size_bits    = 1 },
241 	{ PATH_REC_FIELD(numb_path),
242 	  .offset_words = 12,
243 	  .offset_bits  = 9,
244 	  .size_bits    = 7 },
245 	{ PATH_REC_FIELD(pkey),
246 	  .offset_words = 12,
247 	  .offset_bits  = 16,
248 	  .size_bits    = 16 },
249 	{ PATH_REC_FIELD(qos_class),
250 	  .offset_words = 13,
251 	  .offset_bits  = 0,
252 	  .size_bits    = 12 },
253 	{ PATH_REC_FIELD(sl),
254 	  .offset_words = 13,
255 	  .offset_bits  = 12,
256 	  .size_bits    = 4 },
257 	{ PATH_REC_FIELD(mtu_selector),
258 	  .offset_words = 13,
259 	  .offset_bits  = 16,
260 	  .size_bits    = 2 },
261 	{ PATH_REC_FIELD(mtu),
262 	  .offset_words = 13,
263 	  .offset_bits  = 18,
264 	  .size_bits    = 6 },
265 	{ PATH_REC_FIELD(rate_selector),
266 	  .offset_words = 13,
267 	  .offset_bits  = 24,
268 	  .size_bits    = 2 },
269 	{ PATH_REC_FIELD(rate),
270 	  .offset_words = 13,
271 	  .offset_bits  = 26,
272 	  .size_bits    = 6 },
273 	{ PATH_REC_FIELD(packet_life_time_selector),
274 	  .offset_words = 14,
275 	  .offset_bits  = 0,
276 	  .size_bits    = 2 },
277 	{ PATH_REC_FIELD(packet_life_time),
278 	  .offset_words = 14,
279 	  .offset_bits  = 2,
280 	  .size_bits    = 6 },
281 	{ PATH_REC_FIELD(preference),
282 	  .offset_words = 14,
283 	  .offset_bits  = 8,
284 	  .size_bits    = 8 },
285 	{ RESERVED,
286 	  .offset_words = 14,
287 	  .offset_bits  = 16,
288 	  .size_bits    = 48 },
289 };
290 
291 #define OPA_PATH_REC_FIELD(field) \
292 	.struct_offset_bytes = \
293 		offsetof(struct sa_path_rec, field), \
294 	.struct_size_bytes   = \
295 		sizeof_field(struct sa_path_rec, field),	\
296 	.field_name          = "sa_path_rec:" #field
297 
298 static const struct ib_field opa_path_rec_table[] = {
299 	{ OPA_PATH_REC_FIELD(service_id),
300 	  .offset_words = 0,
301 	  .offset_bits  = 0,
302 	  .size_bits    = 64 },
303 	{ OPA_PATH_REC_FIELD(dgid),
304 	  .offset_words = 2,
305 	  .offset_bits  = 0,
306 	  .size_bits    = 128 },
307 	{ OPA_PATH_REC_FIELD(sgid),
308 	  .offset_words = 6,
309 	  .offset_bits  = 0,
310 	  .size_bits    = 128 },
311 	{ OPA_PATH_REC_FIELD(opa.dlid),
312 	  .offset_words = 10,
313 	  .offset_bits  = 0,
314 	  .size_bits    = 32 },
315 	{ OPA_PATH_REC_FIELD(opa.slid),
316 	  .offset_words = 11,
317 	  .offset_bits  = 0,
318 	  .size_bits    = 32 },
319 	{ OPA_PATH_REC_FIELD(opa.raw_traffic),
320 	  .offset_words = 12,
321 	  .offset_bits  = 0,
322 	  .size_bits    = 1 },
323 	{ RESERVED,
324 	  .offset_words = 12,
325 	  .offset_bits  = 1,
326 	  .size_bits    = 3 },
327 	{ OPA_PATH_REC_FIELD(flow_label),
328 	  .offset_words = 12,
329 	  .offset_bits  = 4,
330 	  .size_bits    = 20 },
331 	{ OPA_PATH_REC_FIELD(hop_limit),
332 	  .offset_words = 12,
333 	  .offset_bits  = 24,
334 	  .size_bits    = 8 },
335 	{ OPA_PATH_REC_FIELD(traffic_class),
336 	  .offset_words = 13,
337 	  .offset_bits  = 0,
338 	  .size_bits    = 8 },
339 	{ OPA_PATH_REC_FIELD(reversible),
340 	  .offset_words = 13,
341 	  .offset_bits  = 8,
342 	  .size_bits    = 1 },
343 	{ OPA_PATH_REC_FIELD(numb_path),
344 	  .offset_words = 13,
345 	  .offset_bits  = 9,
346 	  .size_bits    = 7 },
347 	{ OPA_PATH_REC_FIELD(pkey),
348 	  .offset_words = 13,
349 	  .offset_bits  = 16,
350 	  .size_bits    = 16 },
351 	{ OPA_PATH_REC_FIELD(opa.l2_8B),
352 	  .offset_words = 14,
353 	  .offset_bits  = 0,
354 	  .size_bits    = 1 },
355 	{ OPA_PATH_REC_FIELD(opa.l2_10B),
356 	  .offset_words = 14,
357 	  .offset_bits  = 1,
358 	  .size_bits    = 1 },
359 	{ OPA_PATH_REC_FIELD(opa.l2_9B),
360 	  .offset_words = 14,
361 	  .offset_bits  = 2,
362 	  .size_bits    = 1 },
363 	{ OPA_PATH_REC_FIELD(opa.l2_16B),
364 	  .offset_words = 14,
365 	  .offset_bits  = 3,
366 	  .size_bits    = 1 },
367 	{ RESERVED,
368 	  .offset_words = 14,
369 	  .offset_bits  = 4,
370 	  .size_bits    = 2 },
371 	{ OPA_PATH_REC_FIELD(opa.qos_type),
372 	  .offset_words = 14,
373 	  .offset_bits  = 6,
374 	  .size_bits    = 2 },
375 	{ OPA_PATH_REC_FIELD(opa.qos_priority),
376 	  .offset_words = 14,
377 	  .offset_bits  = 8,
378 	  .size_bits    = 8 },
379 	{ RESERVED,
380 	  .offset_words = 14,
381 	  .offset_bits  = 16,
382 	  .size_bits    = 3 },
383 	{ OPA_PATH_REC_FIELD(sl),
384 	  .offset_words = 14,
385 	  .offset_bits  = 19,
386 	  .size_bits    = 5 },
387 	{ RESERVED,
388 	  .offset_words = 14,
389 	  .offset_bits  = 24,
390 	  .size_bits    = 8 },
391 	{ OPA_PATH_REC_FIELD(mtu_selector),
392 	  .offset_words = 15,
393 	  .offset_bits  = 0,
394 	  .size_bits    = 2 },
395 	{ OPA_PATH_REC_FIELD(mtu),
396 	  .offset_words = 15,
397 	  .offset_bits  = 2,
398 	  .size_bits    = 6 },
399 	{ OPA_PATH_REC_FIELD(rate_selector),
400 	  .offset_words = 15,
401 	  .offset_bits  = 8,
402 	  .size_bits    = 2 },
403 	{ OPA_PATH_REC_FIELD(rate),
404 	  .offset_words = 15,
405 	  .offset_bits  = 10,
406 	  .size_bits    = 6 },
407 	{ OPA_PATH_REC_FIELD(packet_life_time_selector),
408 	  .offset_words = 15,
409 	  .offset_bits  = 16,
410 	  .size_bits    = 2 },
411 	{ OPA_PATH_REC_FIELD(packet_life_time),
412 	  .offset_words = 15,
413 	  .offset_bits  = 18,
414 	  .size_bits    = 6 },
415 	{ OPA_PATH_REC_FIELD(preference),
416 	  .offset_words = 15,
417 	  .offset_bits  = 24,
418 	  .size_bits    = 8 },
419 };
420 
421 #define MCMEMBER_REC_FIELD(field) \
422 	.struct_offset_bytes = offsetof(struct ib_sa_mcmember_rec, field),	\
423 	.struct_size_bytes   = sizeof_field(struct ib_sa_mcmember_rec, field),	\
424 	.field_name          = "sa_mcmember_rec:" #field
425 
426 static const struct ib_field mcmember_rec_table[] = {
427 	{ MCMEMBER_REC_FIELD(mgid),
428 	  .offset_words = 0,
429 	  .offset_bits  = 0,
430 	  .size_bits    = 128 },
431 	{ MCMEMBER_REC_FIELD(port_gid),
432 	  .offset_words = 4,
433 	  .offset_bits  = 0,
434 	  .size_bits    = 128 },
435 	{ MCMEMBER_REC_FIELD(qkey),
436 	  .offset_words = 8,
437 	  .offset_bits  = 0,
438 	  .size_bits    = 32 },
439 	{ MCMEMBER_REC_FIELD(mlid),
440 	  .offset_words = 9,
441 	  .offset_bits  = 0,
442 	  .size_bits    = 16 },
443 	{ MCMEMBER_REC_FIELD(mtu_selector),
444 	  .offset_words = 9,
445 	  .offset_bits  = 16,
446 	  .size_bits    = 2 },
447 	{ MCMEMBER_REC_FIELD(mtu),
448 	  .offset_words = 9,
449 	  .offset_bits  = 18,
450 	  .size_bits    = 6 },
451 	{ MCMEMBER_REC_FIELD(traffic_class),
452 	  .offset_words = 9,
453 	  .offset_bits  = 24,
454 	  .size_bits    = 8 },
455 	{ MCMEMBER_REC_FIELD(pkey),
456 	  .offset_words = 10,
457 	  .offset_bits  = 0,
458 	  .size_bits    = 16 },
459 	{ MCMEMBER_REC_FIELD(rate_selector),
460 	  .offset_words = 10,
461 	  .offset_bits  = 16,
462 	  .size_bits    = 2 },
463 	{ MCMEMBER_REC_FIELD(rate),
464 	  .offset_words = 10,
465 	  .offset_bits  = 18,
466 	  .size_bits    = 6 },
467 	{ MCMEMBER_REC_FIELD(packet_life_time_selector),
468 	  .offset_words = 10,
469 	  .offset_bits  = 24,
470 	  .size_bits    = 2 },
471 	{ MCMEMBER_REC_FIELD(packet_life_time),
472 	  .offset_words = 10,
473 	  .offset_bits  = 26,
474 	  .size_bits    = 6 },
475 	{ MCMEMBER_REC_FIELD(sl),
476 	  .offset_words = 11,
477 	  .offset_bits  = 0,
478 	  .size_bits    = 4 },
479 	{ MCMEMBER_REC_FIELD(flow_label),
480 	  .offset_words = 11,
481 	  .offset_bits  = 4,
482 	  .size_bits    = 20 },
483 	{ MCMEMBER_REC_FIELD(hop_limit),
484 	  .offset_words = 11,
485 	  .offset_bits  = 24,
486 	  .size_bits    = 8 },
487 	{ MCMEMBER_REC_FIELD(scope),
488 	  .offset_words = 12,
489 	  .offset_bits  = 0,
490 	  .size_bits    = 4 },
491 	{ MCMEMBER_REC_FIELD(join_state),
492 	  .offset_words = 12,
493 	  .offset_bits  = 4,
494 	  .size_bits    = 4 },
495 	{ MCMEMBER_REC_FIELD(proxy_join),
496 	  .offset_words = 12,
497 	  .offset_bits  = 8,
498 	  .size_bits    = 1 },
499 	{ RESERVED,
500 	  .offset_words = 12,
501 	  .offset_bits  = 9,
502 	  .size_bits    = 23 },
503 };
504 
505 #define SERVICE_REC_FIELD(field) \
506 	.struct_offset_bytes = offsetof(struct ib_sa_service_rec, field),	\
507 	.struct_size_bytes   = sizeof_field(struct ib_sa_service_rec, field),	\
508 	.field_name          = "sa_service_rec:" #field
509 
510 static const struct ib_field service_rec_table[] = {
511 	{ SERVICE_REC_FIELD(id),
512 	  .offset_words = 0,
513 	  .offset_bits  = 0,
514 	  .size_bits    = 64 },
515 	{ SERVICE_REC_FIELD(gid),
516 	  .offset_words = 2,
517 	  .offset_bits  = 0,
518 	  .size_bits    = 128 },
519 	{ SERVICE_REC_FIELD(pkey),
520 	  .offset_words = 6,
521 	  .offset_bits  = 0,
522 	  .size_bits    = 16 },
523 	{ SERVICE_REC_FIELD(lease),
524 	  .offset_words = 7,
525 	  .offset_bits  = 0,
526 	  .size_bits    = 32 },
527 	{ SERVICE_REC_FIELD(key),
528 	  .offset_words = 8,
529 	  .offset_bits  = 0,
530 	  .size_bits    = 128 },
531 	{ SERVICE_REC_FIELD(name),
532 	  .offset_words = 12,
533 	  .offset_bits  = 0,
534 	  .size_bits    = 64*8 },
535 	{ SERVICE_REC_FIELD(data8),
536 	  .offset_words = 28,
537 	  .offset_bits  = 0,
538 	  .size_bits    = 16*8 },
539 	{ SERVICE_REC_FIELD(data16),
540 	  .offset_words = 32,
541 	  .offset_bits  = 0,
542 	  .size_bits    = 8*16 },
543 	{ SERVICE_REC_FIELD(data32),
544 	  .offset_words = 36,
545 	  .offset_bits  = 0,
546 	  .size_bits    = 4*32 },
547 	{ SERVICE_REC_FIELD(data64),
548 	  .offset_words = 40,
549 	  .offset_bits  = 0,
550 	  .size_bits    = 2*64 },
551 };
552 
553 #define CLASSPORTINFO_REC_FIELD(field) \
554 	.struct_offset_bytes = offsetof(struct ib_class_port_info, field),	\
555 	.struct_size_bytes   = sizeof_field(struct ib_class_port_info, field),	\
556 	.field_name          = "ib_class_port_info:" #field
557 
558 static const struct ib_field ib_classport_info_rec_table[] = {
559 	{ CLASSPORTINFO_REC_FIELD(base_version),
560 	  .offset_words = 0,
561 	  .offset_bits  = 0,
562 	  .size_bits    = 8 },
563 	{ CLASSPORTINFO_REC_FIELD(class_version),
564 	  .offset_words = 0,
565 	  .offset_bits  = 8,
566 	  .size_bits    = 8 },
567 	{ CLASSPORTINFO_REC_FIELD(capability_mask),
568 	  .offset_words = 0,
569 	  .offset_bits  = 16,
570 	  .size_bits    = 16 },
571 	{ CLASSPORTINFO_REC_FIELD(cap_mask2_resp_time),
572 	  .offset_words = 1,
573 	  .offset_bits  = 0,
574 	  .size_bits    = 32 },
575 	{ CLASSPORTINFO_REC_FIELD(redirect_gid),
576 	  .offset_words = 2,
577 	  .offset_bits  = 0,
578 	  .size_bits    = 128 },
579 	{ CLASSPORTINFO_REC_FIELD(redirect_tcslfl),
580 	  .offset_words = 6,
581 	  .offset_bits  = 0,
582 	  .size_bits    = 32 },
583 	{ CLASSPORTINFO_REC_FIELD(redirect_lid),
584 	  .offset_words = 7,
585 	  .offset_bits  = 0,
586 	  .size_bits    = 16 },
587 	{ CLASSPORTINFO_REC_FIELD(redirect_pkey),
588 	  .offset_words = 7,
589 	  .offset_bits  = 16,
590 	  .size_bits    = 16 },
591 
592 	{ CLASSPORTINFO_REC_FIELD(redirect_qp),
593 	  .offset_words = 8,
594 	  .offset_bits  = 0,
595 	  .size_bits    = 32 },
596 	{ CLASSPORTINFO_REC_FIELD(redirect_qkey),
597 	  .offset_words = 9,
598 	  .offset_bits  = 0,
599 	  .size_bits    = 32 },
600 
601 	{ CLASSPORTINFO_REC_FIELD(trap_gid),
602 	  .offset_words = 10,
603 	  .offset_bits  = 0,
604 	  .size_bits    = 128 },
605 	{ CLASSPORTINFO_REC_FIELD(trap_tcslfl),
606 	  .offset_words = 14,
607 	  .offset_bits  = 0,
608 	  .size_bits    = 32 },
609 
610 	{ CLASSPORTINFO_REC_FIELD(trap_lid),
611 	  .offset_words = 15,
612 	  .offset_bits  = 0,
613 	  .size_bits    = 16 },
614 	{ CLASSPORTINFO_REC_FIELD(trap_pkey),
615 	  .offset_words = 15,
616 	  .offset_bits  = 16,
617 	  .size_bits    = 16 },
618 
619 	{ CLASSPORTINFO_REC_FIELD(trap_hlqp),
620 	  .offset_words = 16,
621 	  .offset_bits  = 0,
622 	  .size_bits    = 32 },
623 	{ CLASSPORTINFO_REC_FIELD(trap_qkey),
624 	  .offset_words = 17,
625 	  .offset_bits  = 0,
626 	  .size_bits    = 32 },
627 };
628 
629 #define OPA_CLASSPORTINFO_REC_FIELD(field) \
630 	.struct_offset_bytes =\
631 		offsetof(struct opa_class_port_info, field),	\
632 	.struct_size_bytes   = \
633 		sizeof_field(struct opa_class_port_info, field),	\
634 	.field_name          = "opa_class_port_info:" #field
635 
636 static const struct ib_field opa_classport_info_rec_table[] = {
637 	{ OPA_CLASSPORTINFO_REC_FIELD(base_version),
638 	  .offset_words = 0,
639 	  .offset_bits  = 0,
640 	  .size_bits    = 8 },
641 	{ OPA_CLASSPORTINFO_REC_FIELD(class_version),
642 	  .offset_words = 0,
643 	  .offset_bits  = 8,
644 	  .size_bits    = 8 },
645 	{ OPA_CLASSPORTINFO_REC_FIELD(cap_mask),
646 	  .offset_words = 0,
647 	  .offset_bits  = 16,
648 	  .size_bits    = 16 },
649 	{ OPA_CLASSPORTINFO_REC_FIELD(cap_mask2_resp_time),
650 	  .offset_words = 1,
651 	  .offset_bits  = 0,
652 	  .size_bits    = 32 },
653 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_gid),
654 	  .offset_words = 2,
655 	  .offset_bits  = 0,
656 	  .size_bits    = 128 },
657 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_tc_fl),
658 	  .offset_words = 6,
659 	  .offset_bits  = 0,
660 	  .size_bits    = 32 },
661 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_lid),
662 	  .offset_words = 7,
663 	  .offset_bits  = 0,
664 	  .size_bits    = 32 },
665 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_sl_qp),
666 	  .offset_words = 8,
667 	  .offset_bits  = 0,
668 	  .size_bits    = 32 },
669 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_qkey),
670 	  .offset_words = 9,
671 	  .offset_bits  = 0,
672 	  .size_bits    = 32 },
673 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_gid),
674 	  .offset_words = 10,
675 	  .offset_bits  = 0,
676 	  .size_bits    = 128 },
677 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_tc_fl),
678 	  .offset_words = 14,
679 	  .offset_bits  = 0,
680 	  .size_bits    = 32 },
681 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_lid),
682 	  .offset_words = 15,
683 	  .offset_bits  = 0,
684 	  .size_bits    = 32 },
685 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_hl_qp),
686 	  .offset_words = 16,
687 	  .offset_bits  = 0,
688 	  .size_bits    = 32 },
689 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_qkey),
690 	  .offset_words = 17,
691 	  .offset_bits  = 0,
692 	  .size_bits    = 32 },
693 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_pkey),
694 	  .offset_words = 18,
695 	  .offset_bits  = 0,
696 	  .size_bits    = 16 },
697 	{ OPA_CLASSPORTINFO_REC_FIELD(redirect_pkey),
698 	  .offset_words = 18,
699 	  .offset_bits  = 16,
700 	  .size_bits    = 16 },
701 	{ OPA_CLASSPORTINFO_REC_FIELD(trap_sl_rsvd),
702 	  .offset_words = 19,
703 	  .offset_bits  = 0,
704 	  .size_bits    = 8 },
705 	{ RESERVED,
706 	  .offset_words = 19,
707 	  .offset_bits  = 8,
708 	  .size_bits    = 24 },
709 };
710 
711 #define GUIDINFO_REC_FIELD(field) \
712 	.struct_offset_bytes = offsetof(struct ib_sa_guidinfo_rec, field),	\
713 	.struct_size_bytes   = sizeof_field(struct ib_sa_guidinfo_rec, field),	\
714 	.field_name          = "sa_guidinfo_rec:" #field
715 
716 static const struct ib_field guidinfo_rec_table[] = {
717 	{ GUIDINFO_REC_FIELD(lid),
718 	  .offset_words = 0,
719 	  .offset_bits  = 0,
720 	  .size_bits    = 16 },
721 	{ GUIDINFO_REC_FIELD(block_num),
722 	  .offset_words = 0,
723 	  .offset_bits  = 16,
724 	  .size_bits    = 8 },
725 	{ GUIDINFO_REC_FIELD(res1),
726 	  .offset_words = 0,
727 	  .offset_bits  = 24,
728 	  .size_bits    = 8 },
729 	{ GUIDINFO_REC_FIELD(res2),
730 	  .offset_words = 1,
731 	  .offset_bits  = 0,
732 	  .size_bits    = 32 },
733 	{ GUIDINFO_REC_FIELD(guid_info_list),
734 	  .offset_words = 2,
735 	  .offset_bits  = 0,
736 	  .size_bits    = 512 },
737 };
738 
739 static inline void ib_sa_disable_local_svc(struct ib_sa_query *query)
740 {
741 	query->flags &= ~IB_SA_ENABLE_LOCAL_SERVICE;
742 }
743 
744 static inline int ib_sa_query_cancelled(struct ib_sa_query *query)
745 {
746 	return (query->flags & IB_SA_CANCEL);
747 }
748 
749 static void ib_nl_set_path_rec_attrs(struct sk_buff *skb,
750 				     struct ib_sa_query *query)
751 {
752 	struct sa_path_rec *sa_rec = query->mad_buf->context[1];
753 	struct ib_sa_mad *mad = query->mad_buf->mad;
754 	ib_sa_comp_mask comp_mask = mad->sa_hdr.comp_mask;
755 	u16 val16;
756 	u64 val64;
757 	struct rdma_ls_resolve_header *header;
758 
759 	query->mad_buf->context[1] = NULL;
760 
761 	/* Construct the family header first */
762 	header = skb_put(skb, NLMSG_ALIGN(sizeof(*header)));
763 	memcpy(header->device_name, dev_name(&query->port->agent->device->dev),
764 	       LS_DEVICE_NAME_MAX);
765 	header->port_num = query->port->port_num;
766 
767 	if ((comp_mask & IB_SA_PATH_REC_REVERSIBLE) &&
768 	    sa_rec->reversible != 0)
769 		query->path_use = LS_RESOLVE_PATH_USE_GMP;
770 	else
771 		query->path_use = LS_RESOLVE_PATH_USE_UNIDIRECTIONAL;
772 	header->path_use = query->path_use;
773 
774 	/* Now build the attributes */
775 	if (comp_mask & IB_SA_PATH_REC_SERVICE_ID) {
776 		val64 = be64_to_cpu(sa_rec->service_id);
777 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_SERVICE_ID,
778 			sizeof(val64), &val64);
779 	}
780 	if (comp_mask & IB_SA_PATH_REC_DGID)
781 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_DGID,
782 			sizeof(sa_rec->dgid), &sa_rec->dgid);
783 	if (comp_mask & IB_SA_PATH_REC_SGID)
784 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_SGID,
785 			sizeof(sa_rec->sgid), &sa_rec->sgid);
786 	if (comp_mask & IB_SA_PATH_REC_TRAFFIC_CLASS)
787 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_TCLASS,
788 			sizeof(sa_rec->traffic_class), &sa_rec->traffic_class);
789 
790 	if (comp_mask & IB_SA_PATH_REC_PKEY) {
791 		val16 = be16_to_cpu(sa_rec->pkey);
792 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_PKEY,
793 			sizeof(val16), &val16);
794 	}
795 	if (comp_mask & IB_SA_PATH_REC_QOS_CLASS) {
796 		val16 = be16_to_cpu(sa_rec->qos_class);
797 		nla_put(skb, RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_QOS_CLASS,
798 			sizeof(val16), &val16);
799 	}
800 }
801 
802 static int ib_nl_get_path_rec_attrs_len(ib_sa_comp_mask comp_mask)
803 {
804 	int len = 0;
805 
806 	if (comp_mask & IB_SA_PATH_REC_SERVICE_ID)
807 		len += nla_total_size(sizeof(u64));
808 	if (comp_mask & IB_SA_PATH_REC_DGID)
809 		len += nla_total_size(sizeof(struct rdma_nla_ls_gid));
810 	if (comp_mask & IB_SA_PATH_REC_SGID)
811 		len += nla_total_size(sizeof(struct rdma_nla_ls_gid));
812 	if (comp_mask & IB_SA_PATH_REC_TRAFFIC_CLASS)
813 		len += nla_total_size(sizeof(u8));
814 	if (comp_mask & IB_SA_PATH_REC_PKEY)
815 		len += nla_total_size(sizeof(u16));
816 	if (comp_mask & IB_SA_PATH_REC_QOS_CLASS)
817 		len += nla_total_size(sizeof(u16));
818 
819 	/*
820 	 * Make sure that at least some of the required comp_mask bits are
821 	 * set.
822 	 */
823 	if (WARN_ON(len == 0))
824 		return len;
825 
826 	/* Add the family header */
827 	len += NLMSG_ALIGN(sizeof(struct rdma_ls_resolve_header));
828 
829 	return len;
830 }
831 
832 static int ib_nl_make_request(struct ib_sa_query *query, gfp_t gfp_mask)
833 {
834 	struct sk_buff *skb = NULL;
835 	struct nlmsghdr *nlh;
836 	void *data;
837 	struct ib_sa_mad *mad;
838 	int len;
839 	unsigned long flags;
840 	unsigned long delay;
841 	gfp_t gfp_flag;
842 	int ret;
843 
844 	INIT_LIST_HEAD(&query->list);
845 	query->seq = (u32)atomic_inc_return(&ib_nl_sa_request_seq);
846 
847 	mad = query->mad_buf->mad;
848 	len = ib_nl_get_path_rec_attrs_len(mad->sa_hdr.comp_mask);
849 	if (len <= 0)
850 		return -EMSGSIZE;
851 
852 	skb = nlmsg_new(len, gfp_mask);
853 	if (!skb)
854 		return -ENOMEM;
855 
856 	/* Put nlmsg header only for now */
857 	data = ibnl_put_msg(skb, &nlh, query->seq, 0, RDMA_NL_LS,
858 			    RDMA_NL_LS_OP_RESOLVE, NLM_F_REQUEST);
859 	if (!data) {
860 		nlmsg_free(skb);
861 		return -EMSGSIZE;
862 	}
863 
864 	/* Add attributes */
865 	ib_nl_set_path_rec_attrs(skb, query);
866 
867 	/* Repair the nlmsg header length */
868 	nlmsg_end(skb, nlh);
869 
870 	gfp_flag = ((gfp_mask & GFP_ATOMIC) == GFP_ATOMIC) ? GFP_ATOMIC :
871 		GFP_NOWAIT;
872 
873 	spin_lock_irqsave(&ib_nl_request_lock, flags);
874 	ret = rdma_nl_multicast(&init_net, skb, RDMA_NL_GROUP_LS, gfp_flag);
875 
876 	if (ret)
877 		goto out;
878 
879 	/* Put the request on the list.*/
880 	delay = msecs_to_jiffies(sa_local_svc_timeout_ms);
881 	query->timeout = delay + jiffies;
882 	list_add_tail(&query->list, &ib_nl_request_list);
883 	/* Start the timeout if this is the only request */
884 	if (ib_nl_request_list.next == &query->list)
885 		queue_delayed_work(ib_nl_wq, &ib_nl_timed_work, delay);
886 
887 out:
888 	spin_unlock_irqrestore(&ib_nl_request_lock, flags);
889 
890 	return ret;
891 }
892 
893 static int ib_nl_cancel_request(struct ib_sa_query *query)
894 {
895 	unsigned long flags;
896 	struct ib_sa_query *wait_query;
897 	int found = 0;
898 
899 	spin_lock_irqsave(&ib_nl_request_lock, flags);
900 	list_for_each_entry(wait_query, &ib_nl_request_list, list) {
901 		/* Let the timeout to take care of the callback */
902 		if (query == wait_query) {
903 			query->flags |= IB_SA_CANCEL;
904 			query->timeout = jiffies;
905 			list_move(&query->list, &ib_nl_request_list);
906 			found = 1;
907 			mod_delayed_work(ib_nl_wq, &ib_nl_timed_work, 1);
908 			break;
909 		}
910 	}
911 	spin_unlock_irqrestore(&ib_nl_request_lock, flags);
912 
913 	return found;
914 }
915 
916 static void send_handler(struct ib_mad_agent *agent,
917 			 struct ib_mad_send_wc *mad_send_wc);
918 
919 static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query,
920 					   const struct nlmsghdr *nlh)
921 {
922 	struct ib_mad_send_wc mad_send_wc;
923 	struct ib_sa_mad *mad = NULL;
924 	const struct nlattr *head, *curr;
925 	struct ib_path_rec_data  *rec;
926 	int len, rem;
927 	u32 mask = 0;
928 	int status = -EIO;
929 
930 	if (query->callback) {
931 		head = (const struct nlattr *) nlmsg_data(nlh);
932 		len = nlmsg_len(nlh);
933 		switch (query->path_use) {
934 		case LS_RESOLVE_PATH_USE_UNIDIRECTIONAL:
935 			mask = IB_PATH_PRIMARY | IB_PATH_OUTBOUND;
936 			break;
937 
938 		case LS_RESOLVE_PATH_USE_ALL:
939 		case LS_RESOLVE_PATH_USE_GMP:
940 		default:
941 			mask = IB_PATH_PRIMARY | IB_PATH_GMP |
942 				IB_PATH_BIDIRECTIONAL;
943 			break;
944 		}
945 		nla_for_each_attr(curr, head, len, rem) {
946 			if (curr->nla_type == LS_NLA_TYPE_PATH_RECORD) {
947 				rec = nla_data(curr);
948 				/*
949 				 * Get the first one. In the future, we may
950 				 * need to get up to 6 pathrecords.
951 				 */
952 				if ((rec->flags & mask) == mask) {
953 					mad = query->mad_buf->mad;
954 					mad->mad_hdr.method |=
955 						IB_MGMT_METHOD_RESP;
956 					memcpy(mad->data, rec->path_rec,
957 					       sizeof(rec->path_rec));
958 					status = 0;
959 					break;
960 				}
961 			}
962 		}
963 		query->callback(query, status, mad);
964 	}
965 
966 	mad_send_wc.send_buf = query->mad_buf;
967 	mad_send_wc.status = IB_WC_SUCCESS;
968 	send_handler(query->mad_buf->mad_agent, &mad_send_wc);
969 }
970 
971 static void ib_nl_request_timeout(struct work_struct *work)
972 {
973 	unsigned long flags;
974 	struct ib_sa_query *query;
975 	unsigned long delay;
976 	struct ib_mad_send_wc mad_send_wc;
977 	int ret;
978 
979 	spin_lock_irqsave(&ib_nl_request_lock, flags);
980 	while (!list_empty(&ib_nl_request_list)) {
981 		query = list_entry(ib_nl_request_list.next,
982 				   struct ib_sa_query, list);
983 
984 		if (time_after(query->timeout, jiffies)) {
985 			delay = query->timeout - jiffies;
986 			if ((long)delay <= 0)
987 				delay = 1;
988 			queue_delayed_work(ib_nl_wq, &ib_nl_timed_work, delay);
989 			break;
990 		}
991 
992 		list_del(&query->list);
993 		ib_sa_disable_local_svc(query);
994 		/* Hold the lock to protect against query cancellation */
995 		if (ib_sa_query_cancelled(query))
996 			ret = -1;
997 		else
998 			ret = ib_post_send_mad(query->mad_buf, NULL);
999 		if (ret) {
1000 			mad_send_wc.send_buf = query->mad_buf;
1001 			mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
1002 			spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1003 			send_handler(query->port->agent, &mad_send_wc);
1004 			spin_lock_irqsave(&ib_nl_request_lock, flags);
1005 		}
1006 	}
1007 	spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1008 }
1009 
1010 int ib_nl_handle_set_timeout(struct sk_buff *skb,
1011 			     struct nlmsghdr *nlh,
1012 			     struct netlink_ext_ack *extack)
1013 {
1014 	int timeout, delta, abs_delta;
1015 	const struct nlattr *attr;
1016 	unsigned long flags;
1017 	struct ib_sa_query *query;
1018 	long delay = 0;
1019 	struct nlattr *tb[LS_NLA_TYPE_MAX];
1020 	int ret;
1021 
1022 	if (!(nlh->nlmsg_flags & NLM_F_REQUEST) ||
1023 	    !(NETLINK_CB(skb).sk))
1024 		return -EPERM;
1025 
1026 	ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
1027 				   nlmsg_len(nlh), ib_nl_policy, NULL);
1028 	attr = (const struct nlattr *)tb[LS_NLA_TYPE_TIMEOUT];
1029 	if (ret || !attr)
1030 		goto settimeout_out;
1031 
1032 	timeout = *(int *) nla_data(attr);
1033 	if (timeout < IB_SA_LOCAL_SVC_TIMEOUT_MIN)
1034 		timeout = IB_SA_LOCAL_SVC_TIMEOUT_MIN;
1035 	if (timeout > IB_SA_LOCAL_SVC_TIMEOUT_MAX)
1036 		timeout = IB_SA_LOCAL_SVC_TIMEOUT_MAX;
1037 
1038 	delta = timeout - sa_local_svc_timeout_ms;
1039 	if (delta < 0)
1040 		abs_delta = -delta;
1041 	else
1042 		abs_delta = delta;
1043 
1044 	if (delta != 0) {
1045 		spin_lock_irqsave(&ib_nl_request_lock, flags);
1046 		sa_local_svc_timeout_ms = timeout;
1047 		list_for_each_entry(query, &ib_nl_request_list, list) {
1048 			if (delta < 0 && abs_delta > query->timeout)
1049 				query->timeout = 0;
1050 			else
1051 				query->timeout += delta;
1052 
1053 			/* Get the new delay from the first entry */
1054 			if (!delay) {
1055 				delay = query->timeout - jiffies;
1056 				if (delay <= 0)
1057 					delay = 1;
1058 			}
1059 		}
1060 		if (delay)
1061 			mod_delayed_work(ib_nl_wq, &ib_nl_timed_work,
1062 					 (unsigned long)delay);
1063 		spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1064 	}
1065 
1066 settimeout_out:
1067 	return 0;
1068 }
1069 
1070 static inline int ib_nl_is_good_resolve_resp(const struct nlmsghdr *nlh)
1071 {
1072 	struct nlattr *tb[LS_NLA_TYPE_MAX];
1073 	int ret;
1074 
1075 	if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
1076 		return 0;
1077 
1078 	ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
1079 				   nlmsg_len(nlh), ib_nl_policy, NULL);
1080 	if (ret)
1081 		return 0;
1082 
1083 	return 1;
1084 }
1085 
1086 int ib_nl_handle_resolve_resp(struct sk_buff *skb,
1087 			      struct nlmsghdr *nlh,
1088 			      struct netlink_ext_ack *extack)
1089 {
1090 	unsigned long flags;
1091 	struct ib_sa_query *query;
1092 	struct ib_mad_send_buf *send_buf;
1093 	struct ib_mad_send_wc mad_send_wc;
1094 	int found = 0;
1095 	int ret;
1096 
1097 	if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
1098 	    !(NETLINK_CB(skb).sk))
1099 		return -EPERM;
1100 
1101 	spin_lock_irqsave(&ib_nl_request_lock, flags);
1102 	list_for_each_entry(query, &ib_nl_request_list, list) {
1103 		/*
1104 		 * If the query is cancelled, let the timeout routine
1105 		 * take care of it.
1106 		 */
1107 		if (nlh->nlmsg_seq == query->seq) {
1108 			found = !ib_sa_query_cancelled(query);
1109 			if (found)
1110 				list_del(&query->list);
1111 			break;
1112 		}
1113 	}
1114 
1115 	if (!found) {
1116 		spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1117 		goto resp_out;
1118 	}
1119 
1120 	send_buf = query->mad_buf;
1121 
1122 	if (!ib_nl_is_good_resolve_resp(nlh)) {
1123 		/* if the result is a failure, send out the packet via IB */
1124 		ib_sa_disable_local_svc(query);
1125 		ret = ib_post_send_mad(query->mad_buf, NULL);
1126 		spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1127 		if (ret) {
1128 			mad_send_wc.send_buf = send_buf;
1129 			mad_send_wc.status = IB_WC_GENERAL_ERR;
1130 			send_handler(query->port->agent, &mad_send_wc);
1131 		}
1132 	} else {
1133 		spin_unlock_irqrestore(&ib_nl_request_lock, flags);
1134 		ib_nl_process_good_resolve_rsp(query, nlh);
1135 	}
1136 
1137 resp_out:
1138 	return 0;
1139 }
1140 
1141 static void free_sm_ah(struct kref *kref)
1142 {
1143 	struct ib_sa_sm_ah *sm_ah = container_of(kref, struct ib_sa_sm_ah, ref);
1144 
1145 	rdma_destroy_ah(sm_ah->ah, 0);
1146 	kfree(sm_ah);
1147 }
1148 
1149 void ib_sa_register_client(struct ib_sa_client *client)
1150 {
1151 	atomic_set(&client->users, 1);
1152 	init_completion(&client->comp);
1153 }
1154 EXPORT_SYMBOL(ib_sa_register_client);
1155 
1156 void ib_sa_unregister_client(struct ib_sa_client *client)
1157 {
1158 	ib_sa_client_put(client);
1159 	wait_for_completion(&client->comp);
1160 }
1161 EXPORT_SYMBOL(ib_sa_unregister_client);
1162 
1163 /**
1164  * ib_sa_cancel_query - try to cancel an SA query
1165  * @id:ID of query to cancel
1166  * @query:query pointer to cancel
1167  *
1168  * Try to cancel an SA query.  If the id and query don't match up or
1169  * the query has already completed, nothing is done.  Otherwise the
1170  * query is canceled and will complete with a status of -EINTR.
1171  */
1172 void ib_sa_cancel_query(int id, struct ib_sa_query *query)
1173 {
1174 	unsigned long flags;
1175 	struct ib_mad_agent *agent;
1176 	struct ib_mad_send_buf *mad_buf;
1177 
1178 	xa_lock_irqsave(&queries, flags);
1179 	if (xa_load(&queries, id) != query) {
1180 		xa_unlock_irqrestore(&queries, flags);
1181 		return;
1182 	}
1183 	agent = query->port->agent;
1184 	mad_buf = query->mad_buf;
1185 	xa_unlock_irqrestore(&queries, flags);
1186 
1187 	/*
1188 	 * If the query is still on the netlink request list, schedule
1189 	 * it to be cancelled by the timeout routine. Otherwise, it has been
1190 	 * sent to the MAD layer and has to be cancelled from there.
1191 	 */
1192 	if (!ib_nl_cancel_request(query))
1193 		ib_cancel_mad(agent, mad_buf);
1194 }
1195 EXPORT_SYMBOL(ib_sa_cancel_query);
1196 
1197 static u8 get_src_path_mask(struct ib_device *device, u32 port_num)
1198 {
1199 	struct ib_sa_device *sa_dev;
1200 	struct ib_sa_port   *port;
1201 	unsigned long flags;
1202 	u8 src_path_mask;
1203 
1204 	sa_dev = ib_get_client_data(device, &sa_client);
1205 	if (!sa_dev)
1206 		return 0x7f;
1207 
1208 	port  = &sa_dev->port[port_num - sa_dev->start_port];
1209 	spin_lock_irqsave(&port->ah_lock, flags);
1210 	src_path_mask = port->sm_ah ? port->sm_ah->src_path_mask : 0x7f;
1211 	spin_unlock_irqrestore(&port->ah_lock, flags);
1212 
1213 	return src_path_mask;
1214 }
1215 
1216 static int init_ah_attr_grh_fields(struct ib_device *device, u32 port_num,
1217 				   struct sa_path_rec *rec,
1218 				   struct rdma_ah_attr *ah_attr,
1219 				   const struct ib_gid_attr *gid_attr)
1220 {
1221 	enum ib_gid_type type = sa_conv_pathrec_to_gid_type(rec);
1222 
1223 	if (!gid_attr) {
1224 		gid_attr = rdma_find_gid_by_port(device, &rec->sgid, type,
1225 						 port_num, NULL);
1226 		if (IS_ERR(gid_attr))
1227 			return PTR_ERR(gid_attr);
1228 	} else
1229 		rdma_hold_gid_attr(gid_attr);
1230 
1231 	rdma_move_grh_sgid_attr(ah_attr, &rec->dgid,
1232 				be32_to_cpu(rec->flow_label),
1233 				rec->hop_limit,	rec->traffic_class,
1234 				gid_attr);
1235 	return 0;
1236 }
1237 
1238 /**
1239  * ib_init_ah_attr_from_path - Initialize address handle attributes based on
1240  *   an SA path record.
1241  * @device: Device associated ah attributes initialization.
1242  * @port_num: Port on the specified device.
1243  * @rec: path record entry to use for ah attributes initialization.
1244  * @ah_attr: address handle attributes to initialization from path record.
1245  * @gid_attr: SGID attribute to consider during initialization.
1246  *
1247  * When ib_init_ah_attr_from_path() returns success,
1248  * (a) for IB link layer it optionally contains a reference to SGID attribute
1249  * when GRH is present for IB link layer.
1250  * (b) for RoCE link layer it contains a reference to SGID attribute.
1251  * User must invoke rdma_destroy_ah_attr() to release reference to SGID
1252  * attributes which are initialized using ib_init_ah_attr_from_path().
1253  */
1254 int ib_init_ah_attr_from_path(struct ib_device *device, u32 port_num,
1255 			      struct sa_path_rec *rec,
1256 			      struct rdma_ah_attr *ah_attr,
1257 			      const struct ib_gid_attr *gid_attr)
1258 {
1259 	int ret = 0;
1260 
1261 	memset(ah_attr, 0, sizeof(*ah_attr));
1262 	ah_attr->type = rdma_ah_find_type(device, port_num);
1263 	rdma_ah_set_sl(ah_attr, rec->sl);
1264 	rdma_ah_set_port_num(ah_attr, port_num);
1265 	rdma_ah_set_static_rate(ah_attr, rec->rate);
1266 
1267 	if (sa_path_is_roce(rec)) {
1268 		ret = roce_resolve_route_from_path(rec, gid_attr);
1269 		if (ret)
1270 			return ret;
1271 
1272 		memcpy(ah_attr->roce.dmac, sa_path_get_dmac(rec), ETH_ALEN);
1273 	} else {
1274 		rdma_ah_set_dlid(ah_attr, be32_to_cpu(sa_path_get_dlid(rec)));
1275 		if (sa_path_is_opa(rec) &&
1276 		    rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE))
1277 			rdma_ah_set_make_grd(ah_attr, true);
1278 
1279 		rdma_ah_set_path_bits(ah_attr,
1280 				      be32_to_cpu(sa_path_get_slid(rec)) &
1281 				      get_src_path_mask(device, port_num));
1282 	}
1283 
1284 	if (rec->hop_limit > 0 || sa_path_is_roce(rec))
1285 		ret = init_ah_attr_grh_fields(device, port_num,
1286 					      rec, ah_attr, gid_attr);
1287 	return ret;
1288 }
1289 EXPORT_SYMBOL(ib_init_ah_attr_from_path);
1290 
1291 static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask)
1292 {
1293 	struct rdma_ah_attr ah_attr;
1294 	unsigned long flags;
1295 
1296 	spin_lock_irqsave(&query->port->ah_lock, flags);
1297 	if (!query->port->sm_ah) {
1298 		spin_unlock_irqrestore(&query->port->ah_lock, flags);
1299 		return -EAGAIN;
1300 	}
1301 	kref_get(&query->port->sm_ah->ref);
1302 	query->sm_ah = query->port->sm_ah;
1303 	spin_unlock_irqrestore(&query->port->ah_lock, flags);
1304 
1305 	/*
1306 	 * Always check if sm_ah has valid dlid assigned,
1307 	 * before querying for class port info
1308 	 */
1309 	if ((rdma_query_ah(query->sm_ah->ah, &ah_attr) < 0) ||
1310 	    !rdma_is_valid_unicast_lid(&ah_attr)) {
1311 		kref_put(&query->sm_ah->ref, free_sm_ah);
1312 		return -EAGAIN;
1313 	}
1314 	query->mad_buf = ib_create_send_mad(query->port->agent, 1,
1315 					    query->sm_ah->pkey_index,
1316 					    0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA,
1317 					    gfp_mask,
1318 					    ((query->flags & IB_SA_QUERY_OPA) ?
1319 					     OPA_MGMT_BASE_VERSION :
1320 					     IB_MGMT_BASE_VERSION));
1321 	if (IS_ERR(query->mad_buf)) {
1322 		kref_put(&query->sm_ah->ref, free_sm_ah);
1323 		return -ENOMEM;
1324 	}
1325 
1326 	query->mad_buf->ah = query->sm_ah->ah;
1327 
1328 	return 0;
1329 }
1330 
1331 static void free_mad(struct ib_sa_query *query)
1332 {
1333 	ib_free_send_mad(query->mad_buf);
1334 	kref_put(&query->sm_ah->ref, free_sm_ah);
1335 }
1336 
1337 static void init_mad(struct ib_sa_query *query, struct ib_mad_agent *agent)
1338 {
1339 	struct ib_sa_mad *mad = query->mad_buf->mad;
1340 	unsigned long flags;
1341 
1342 	memset(mad, 0, sizeof *mad);
1343 
1344 	if (query->flags & IB_SA_QUERY_OPA) {
1345 		mad->mad_hdr.base_version  = OPA_MGMT_BASE_VERSION;
1346 		mad->mad_hdr.class_version = OPA_SA_CLASS_VERSION;
1347 	} else {
1348 		mad->mad_hdr.base_version  = IB_MGMT_BASE_VERSION;
1349 		mad->mad_hdr.class_version = IB_SA_CLASS_VERSION;
1350 	}
1351 	mad->mad_hdr.mgmt_class    = IB_MGMT_CLASS_SUBN_ADM;
1352 	spin_lock_irqsave(&tid_lock, flags);
1353 	mad->mad_hdr.tid           =
1354 		cpu_to_be64(((u64) agent->hi_tid) << 32 | tid++);
1355 	spin_unlock_irqrestore(&tid_lock, flags);
1356 }
1357 
1358 static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms,
1359 		    gfp_t gfp_mask)
1360 {
1361 	unsigned long flags;
1362 	int ret, id;
1363 
1364 	xa_lock_irqsave(&queries, flags);
1365 	ret = __xa_alloc(&queries, &id, query, xa_limit_32b, gfp_mask);
1366 	xa_unlock_irqrestore(&queries, flags);
1367 	if (ret < 0)
1368 		return ret;
1369 
1370 	query->mad_buf->timeout_ms  = timeout_ms;
1371 	query->mad_buf->context[0] = query;
1372 	query->id = id;
1373 
1374 	if ((query->flags & IB_SA_ENABLE_LOCAL_SERVICE) &&
1375 	    (!(query->flags & IB_SA_QUERY_OPA))) {
1376 		if (rdma_nl_chk_listeners(RDMA_NL_GROUP_LS)) {
1377 			if (!ib_nl_make_request(query, gfp_mask))
1378 				return id;
1379 		}
1380 		ib_sa_disable_local_svc(query);
1381 	}
1382 
1383 	ret = ib_post_send_mad(query->mad_buf, NULL);
1384 	if (ret) {
1385 		xa_lock_irqsave(&queries, flags);
1386 		__xa_erase(&queries, id);
1387 		xa_unlock_irqrestore(&queries, flags);
1388 	}
1389 
1390 	/*
1391 	 * It's not safe to dereference query any more, because the
1392 	 * send may already have completed and freed the query in
1393 	 * another context.
1394 	 */
1395 	return ret ? ret : id;
1396 }
1397 
1398 void ib_sa_unpack_path(void *attribute, struct sa_path_rec *rec)
1399 {
1400 	ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), attribute, rec);
1401 }
1402 EXPORT_SYMBOL(ib_sa_unpack_path);
1403 
1404 void ib_sa_pack_path(struct sa_path_rec *rec, void *attribute)
1405 {
1406 	ib_pack(path_rec_table, ARRAY_SIZE(path_rec_table), rec, attribute);
1407 }
1408 EXPORT_SYMBOL(ib_sa_pack_path);
1409 
1410 static bool ib_sa_opa_pathrecord_support(struct ib_sa_client *client,
1411 					 struct ib_sa_device *sa_dev,
1412 					 u32 port_num)
1413 {
1414 	struct ib_sa_port *port;
1415 	unsigned long flags;
1416 	bool ret = false;
1417 
1418 	port = &sa_dev->port[port_num - sa_dev->start_port];
1419 	spin_lock_irqsave(&port->classport_lock, flags);
1420 	if (!port->classport_info.valid)
1421 		goto ret;
1422 
1423 	if (port->classport_info.data.type == RDMA_CLASS_PORT_INFO_OPA)
1424 		ret = opa_get_cpi_capmask2(&port->classport_info.data.opa) &
1425 			OPA_CLASS_PORT_INFO_PR_SUPPORT;
1426 ret:
1427 	spin_unlock_irqrestore(&port->classport_lock, flags);
1428 	return ret;
1429 }
1430 
1431 enum opa_pr_supported {
1432 	PR_NOT_SUPPORTED,
1433 	PR_OPA_SUPPORTED,
1434 	PR_IB_SUPPORTED
1435 };
1436 
1437 /*
1438  * opa_pr_query_possible - Check if current PR query can be an OPA query.
1439  *
1440  * Retuns PR_NOT_SUPPORTED if a path record query is not
1441  * possible, PR_OPA_SUPPORTED if an OPA path record query
1442  * is possible and PR_IB_SUPPORTED if an IB path record
1443  * query is possible.
1444  */
1445 static int opa_pr_query_possible(struct ib_sa_client *client,
1446 				 struct ib_sa_device *sa_dev,
1447 				 struct ib_device *device, u32 port_num,
1448 				 struct sa_path_rec *rec)
1449 {
1450 	struct ib_port_attr port_attr;
1451 
1452 	if (ib_query_port(device, port_num, &port_attr))
1453 		return PR_NOT_SUPPORTED;
1454 
1455 	if (ib_sa_opa_pathrecord_support(client, sa_dev, port_num))
1456 		return PR_OPA_SUPPORTED;
1457 
1458 	if (port_attr.lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))
1459 		return PR_NOT_SUPPORTED;
1460 	else
1461 		return PR_IB_SUPPORTED;
1462 }
1463 
1464 static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query,
1465 				    int status,
1466 				    struct ib_sa_mad *mad)
1467 {
1468 	struct ib_sa_path_query *query =
1469 		container_of(sa_query, struct ib_sa_path_query, sa_query);
1470 
1471 	if (mad) {
1472 		struct sa_path_rec rec;
1473 
1474 		if (sa_query->flags & IB_SA_QUERY_OPA) {
1475 			ib_unpack(opa_path_rec_table,
1476 				  ARRAY_SIZE(opa_path_rec_table),
1477 				  mad->data, &rec);
1478 			rec.rec_type = SA_PATH_REC_TYPE_OPA;
1479 			query->callback(status, &rec, query->context);
1480 		} else {
1481 			ib_unpack(path_rec_table,
1482 				  ARRAY_SIZE(path_rec_table),
1483 				  mad->data, &rec);
1484 			rec.rec_type = SA_PATH_REC_TYPE_IB;
1485 			sa_path_set_dmac_zero(&rec);
1486 
1487 			if (query->conv_pr) {
1488 				struct sa_path_rec opa;
1489 
1490 				memset(&opa, 0, sizeof(struct sa_path_rec));
1491 				sa_convert_path_ib_to_opa(&opa, &rec);
1492 				query->callback(status, &opa, query->context);
1493 			} else {
1494 				query->callback(status, &rec, query->context);
1495 			}
1496 		}
1497 	} else
1498 		query->callback(status, NULL, query->context);
1499 }
1500 
1501 static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
1502 {
1503 	struct ib_sa_path_query *query =
1504 		container_of(sa_query, struct ib_sa_path_query, sa_query);
1505 
1506 	kfree(query->conv_pr);
1507 	kfree(query);
1508 }
1509 
1510 /**
1511  * ib_sa_path_rec_get - Start a Path get query
1512  * @client:SA client
1513  * @device:device to send query on
1514  * @port_num: port number to send query on
1515  * @rec:Path Record to send in query
1516  * @comp_mask:component mask to send in query
1517  * @timeout_ms:time to wait for response
1518  * @gfp_mask:GFP mask to use for internal allocations
1519  * @callback:function called when query completes, times out or is
1520  * canceled
1521  * @context:opaque user context passed to callback
1522  * @sa_query:query context, used to cancel query
1523  *
1524  * Send a Path Record Get query to the SA to look up a path.  The
1525  * callback function will be called when the query completes (or
1526  * fails); status is 0 for a successful response, -EINTR if the query
1527  * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error
1528  * occurred sending the query.  The resp parameter of the callback is
1529  * only valid if status is 0.
1530  *
1531  * If the return value of ib_sa_path_rec_get() is negative, it is an
1532  * error code.  Otherwise it is a query ID that can be used to cancel
1533  * the query.
1534  */
1535 int ib_sa_path_rec_get(struct ib_sa_client *client,
1536 		       struct ib_device *device, u32 port_num,
1537 		       struct sa_path_rec *rec,
1538 		       ib_sa_comp_mask comp_mask,
1539 		       unsigned long timeout_ms, gfp_t gfp_mask,
1540 		       void (*callback)(int status,
1541 					struct sa_path_rec *resp,
1542 					void *context),
1543 		       void *context,
1544 		       struct ib_sa_query **sa_query)
1545 {
1546 	struct ib_sa_path_query *query;
1547 	struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client);
1548 	struct ib_sa_port   *port;
1549 	struct ib_mad_agent *agent;
1550 	struct ib_sa_mad *mad;
1551 	enum opa_pr_supported status;
1552 	int ret;
1553 
1554 	if (!sa_dev)
1555 		return -ENODEV;
1556 
1557 	if ((rec->rec_type != SA_PATH_REC_TYPE_IB) &&
1558 	    (rec->rec_type != SA_PATH_REC_TYPE_OPA))
1559 		return -EINVAL;
1560 
1561 	port  = &sa_dev->port[port_num - sa_dev->start_port];
1562 	agent = port->agent;
1563 
1564 	query = kzalloc(sizeof(*query), gfp_mask);
1565 	if (!query)
1566 		return -ENOMEM;
1567 
1568 	query->sa_query.port     = port;
1569 	if (rec->rec_type == SA_PATH_REC_TYPE_OPA) {
1570 		status = opa_pr_query_possible(client, sa_dev, device, port_num,
1571 					       rec);
1572 		if (status == PR_NOT_SUPPORTED) {
1573 			ret = -EINVAL;
1574 			goto err1;
1575 		} else if (status == PR_OPA_SUPPORTED) {
1576 			query->sa_query.flags |= IB_SA_QUERY_OPA;
1577 		} else {
1578 			query->conv_pr =
1579 				kmalloc(sizeof(*query->conv_pr), gfp_mask);
1580 			if (!query->conv_pr) {
1581 				ret = -ENOMEM;
1582 				goto err1;
1583 			}
1584 		}
1585 	}
1586 
1587 	ret = alloc_mad(&query->sa_query, gfp_mask);
1588 	if (ret)
1589 		goto err2;
1590 
1591 	ib_sa_client_get(client);
1592 	query->sa_query.client = client;
1593 	query->callback        = callback;
1594 	query->context         = context;
1595 
1596 	mad = query->sa_query.mad_buf->mad;
1597 	init_mad(&query->sa_query, agent);
1598 
1599 	query->sa_query.callback = callback ? ib_sa_path_rec_callback : NULL;
1600 	query->sa_query.release  = ib_sa_path_rec_release;
1601 	mad->mad_hdr.method	 = IB_MGMT_METHOD_GET;
1602 	mad->mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_PATH_REC);
1603 	mad->sa_hdr.comp_mask	 = comp_mask;
1604 
1605 	if (query->sa_query.flags & IB_SA_QUERY_OPA) {
1606 		ib_pack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table),
1607 			rec, mad->data);
1608 	} else if (query->conv_pr) {
1609 		sa_convert_path_opa_to_ib(query->conv_pr, rec);
1610 		ib_pack(path_rec_table, ARRAY_SIZE(path_rec_table),
1611 			query->conv_pr, mad->data);
1612 	} else {
1613 		ib_pack(path_rec_table, ARRAY_SIZE(path_rec_table),
1614 			rec, mad->data);
1615 	}
1616 
1617 	*sa_query = &query->sa_query;
1618 
1619 	query->sa_query.flags |= IB_SA_ENABLE_LOCAL_SERVICE;
1620 	query->sa_query.mad_buf->context[1] = (query->conv_pr) ?
1621 						query->conv_pr : rec;
1622 
1623 	ret = send_mad(&query->sa_query, timeout_ms, gfp_mask);
1624 	if (ret < 0)
1625 		goto err3;
1626 
1627 	return ret;
1628 
1629 err3:
1630 	*sa_query = NULL;
1631 	ib_sa_client_put(query->sa_query.client);
1632 	free_mad(&query->sa_query);
1633 err2:
1634 	kfree(query->conv_pr);
1635 err1:
1636 	kfree(query);
1637 	return ret;
1638 }
1639 EXPORT_SYMBOL(ib_sa_path_rec_get);
1640 
1641 static void ib_sa_service_rec_callback(struct ib_sa_query *sa_query,
1642 				    int status,
1643 				    struct ib_sa_mad *mad)
1644 {
1645 	struct ib_sa_service_query *query =
1646 		container_of(sa_query, struct ib_sa_service_query, sa_query);
1647 
1648 	if (mad) {
1649 		struct ib_sa_service_rec rec;
1650 
1651 		ib_unpack(service_rec_table, ARRAY_SIZE(service_rec_table),
1652 			  mad->data, &rec);
1653 		query->callback(status, &rec, query->context);
1654 	} else
1655 		query->callback(status, NULL, query->context);
1656 }
1657 
1658 static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
1659 {
1660 	kfree(container_of(sa_query, struct ib_sa_service_query, sa_query));
1661 }
1662 
1663 /**
1664  * ib_sa_service_rec_query - Start Service Record operation
1665  * @client:SA client
1666  * @device:device to send request on
1667  * @port_num: port number to send request on
1668  * @method:SA method - should be get, set, or delete
1669  * @rec:Service Record to send in request
1670  * @comp_mask:component mask to send in request
1671  * @timeout_ms:time to wait for response
1672  * @gfp_mask:GFP mask to use for internal allocations
1673  * @callback:function called when request completes, times out or is
1674  * canceled
1675  * @context:opaque user context passed to callback
1676  * @sa_query:request context, used to cancel request
1677  *
1678  * Send a Service Record set/get/delete to the SA to register,
1679  * unregister or query a service record.
1680  * The callback function will be called when the request completes (or
1681  * fails); status is 0 for a successful response, -EINTR if the query
1682  * is canceled, -ETIMEDOUT is the query timed out, or -EIO if an error
1683  * occurred sending the query.  The resp parameter of the callback is
1684  * only valid if status is 0.
1685  *
1686  * If the return value of ib_sa_service_rec_query() is negative, it is an
1687  * error code.  Otherwise it is a request ID that can be used to cancel
1688  * the query.
1689  */
1690 int ib_sa_service_rec_query(struct ib_sa_client *client,
1691 			    struct ib_device *device, u32 port_num, u8 method,
1692 			    struct ib_sa_service_rec *rec,
1693 			    ib_sa_comp_mask comp_mask,
1694 			    unsigned long timeout_ms, gfp_t gfp_mask,
1695 			    void (*callback)(int status,
1696 					     struct ib_sa_service_rec *resp,
1697 					     void *context),
1698 			    void *context,
1699 			    struct ib_sa_query **sa_query)
1700 {
1701 	struct ib_sa_service_query *query;
1702 	struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client);
1703 	struct ib_sa_port   *port;
1704 	struct ib_mad_agent *agent;
1705 	struct ib_sa_mad *mad;
1706 	int ret;
1707 
1708 	if (!sa_dev)
1709 		return -ENODEV;
1710 
1711 	port  = &sa_dev->port[port_num - sa_dev->start_port];
1712 	agent = port->agent;
1713 
1714 	if (method != IB_MGMT_METHOD_GET &&
1715 	    method != IB_MGMT_METHOD_SET &&
1716 	    method != IB_SA_METHOD_DELETE)
1717 		return -EINVAL;
1718 
1719 	query = kzalloc(sizeof(*query), gfp_mask);
1720 	if (!query)
1721 		return -ENOMEM;
1722 
1723 	query->sa_query.port     = port;
1724 	ret = alloc_mad(&query->sa_query, gfp_mask);
1725 	if (ret)
1726 		goto err1;
1727 
1728 	ib_sa_client_get(client);
1729 	query->sa_query.client = client;
1730 	query->callback        = callback;
1731 	query->context         = context;
1732 
1733 	mad = query->sa_query.mad_buf->mad;
1734 	init_mad(&query->sa_query, agent);
1735 
1736 	query->sa_query.callback = callback ? ib_sa_service_rec_callback : NULL;
1737 	query->sa_query.release  = ib_sa_service_rec_release;
1738 	mad->mad_hdr.method	 = method;
1739 	mad->mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_SERVICE_REC);
1740 	mad->sa_hdr.comp_mask	 = comp_mask;
1741 
1742 	ib_pack(service_rec_table, ARRAY_SIZE(service_rec_table),
1743 		rec, mad->data);
1744 
1745 	*sa_query = &query->sa_query;
1746 
1747 	ret = send_mad(&query->sa_query, timeout_ms, gfp_mask);
1748 	if (ret < 0)
1749 		goto err2;
1750 
1751 	return ret;
1752 
1753 err2:
1754 	*sa_query = NULL;
1755 	ib_sa_client_put(query->sa_query.client);
1756 	free_mad(&query->sa_query);
1757 
1758 err1:
1759 	kfree(query);
1760 	return ret;
1761 }
1762 EXPORT_SYMBOL(ib_sa_service_rec_query);
1763 
1764 static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query,
1765 					int status,
1766 					struct ib_sa_mad *mad)
1767 {
1768 	struct ib_sa_mcmember_query *query =
1769 		container_of(sa_query, struct ib_sa_mcmember_query, sa_query);
1770 
1771 	if (mad) {
1772 		struct ib_sa_mcmember_rec rec;
1773 
1774 		ib_unpack(mcmember_rec_table, ARRAY_SIZE(mcmember_rec_table),
1775 			  mad->data, &rec);
1776 		query->callback(status, &rec, query->context);
1777 	} else
1778 		query->callback(status, NULL, query->context);
1779 }
1780 
1781 static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query)
1782 {
1783 	kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query));
1784 }
1785 
1786 int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
1787 			     struct ib_device *device, u32 port_num,
1788 			     u8 method,
1789 			     struct ib_sa_mcmember_rec *rec,
1790 			     ib_sa_comp_mask comp_mask,
1791 			     unsigned long timeout_ms, gfp_t gfp_mask,
1792 			     void (*callback)(int status,
1793 					      struct ib_sa_mcmember_rec *resp,
1794 					      void *context),
1795 			     void *context,
1796 			     struct ib_sa_query **sa_query)
1797 {
1798 	struct ib_sa_mcmember_query *query;
1799 	struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client);
1800 	struct ib_sa_port   *port;
1801 	struct ib_mad_agent *agent;
1802 	struct ib_sa_mad *mad;
1803 	int ret;
1804 
1805 	if (!sa_dev)
1806 		return -ENODEV;
1807 
1808 	port  = &sa_dev->port[port_num - sa_dev->start_port];
1809 	agent = port->agent;
1810 
1811 	query = kzalloc(sizeof(*query), gfp_mask);
1812 	if (!query)
1813 		return -ENOMEM;
1814 
1815 	query->sa_query.port     = port;
1816 	ret = alloc_mad(&query->sa_query, gfp_mask);
1817 	if (ret)
1818 		goto err1;
1819 
1820 	ib_sa_client_get(client);
1821 	query->sa_query.client = client;
1822 	query->callback        = callback;
1823 	query->context         = context;
1824 
1825 	mad = query->sa_query.mad_buf->mad;
1826 	init_mad(&query->sa_query, agent);
1827 
1828 	query->sa_query.callback = callback ? ib_sa_mcmember_rec_callback : NULL;
1829 	query->sa_query.release  = ib_sa_mcmember_rec_release;
1830 	mad->mad_hdr.method	 = method;
1831 	mad->mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_MC_MEMBER_REC);
1832 	mad->sa_hdr.comp_mask	 = comp_mask;
1833 
1834 	ib_pack(mcmember_rec_table, ARRAY_SIZE(mcmember_rec_table),
1835 		rec, mad->data);
1836 
1837 	*sa_query = &query->sa_query;
1838 
1839 	ret = send_mad(&query->sa_query, timeout_ms, gfp_mask);
1840 	if (ret < 0)
1841 		goto err2;
1842 
1843 	return ret;
1844 
1845 err2:
1846 	*sa_query = NULL;
1847 	ib_sa_client_put(query->sa_query.client);
1848 	free_mad(&query->sa_query);
1849 
1850 err1:
1851 	kfree(query);
1852 	return ret;
1853 }
1854 
1855 /* Support GuidInfoRecord */
1856 static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query,
1857 					int status,
1858 					struct ib_sa_mad *mad)
1859 {
1860 	struct ib_sa_guidinfo_query *query =
1861 		container_of(sa_query, struct ib_sa_guidinfo_query, sa_query);
1862 
1863 	if (mad) {
1864 		struct ib_sa_guidinfo_rec rec;
1865 
1866 		ib_unpack(guidinfo_rec_table, ARRAY_SIZE(guidinfo_rec_table),
1867 			  mad->data, &rec);
1868 		query->callback(status, &rec, query->context);
1869 	} else
1870 		query->callback(status, NULL, query->context);
1871 }
1872 
1873 static void ib_sa_guidinfo_rec_release(struct ib_sa_query *sa_query)
1874 {
1875 	kfree(container_of(sa_query, struct ib_sa_guidinfo_query, sa_query));
1876 }
1877 
1878 int ib_sa_guid_info_rec_query(struct ib_sa_client *client,
1879 			      struct ib_device *device, u32 port_num,
1880 			      struct ib_sa_guidinfo_rec *rec,
1881 			      ib_sa_comp_mask comp_mask, u8 method,
1882 			      unsigned long timeout_ms, gfp_t gfp_mask,
1883 			      void (*callback)(int status,
1884 					       struct ib_sa_guidinfo_rec *resp,
1885 					       void *context),
1886 			      void *context,
1887 			      struct ib_sa_query **sa_query)
1888 {
1889 	struct ib_sa_guidinfo_query *query;
1890 	struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client);
1891 	struct ib_sa_port *port;
1892 	struct ib_mad_agent *agent;
1893 	struct ib_sa_mad *mad;
1894 	int ret;
1895 
1896 	if (!sa_dev)
1897 		return -ENODEV;
1898 
1899 	if (method != IB_MGMT_METHOD_GET &&
1900 	    method != IB_MGMT_METHOD_SET &&
1901 	    method != IB_SA_METHOD_DELETE) {
1902 		return -EINVAL;
1903 	}
1904 
1905 	port  = &sa_dev->port[port_num - sa_dev->start_port];
1906 	agent = port->agent;
1907 
1908 	query = kzalloc(sizeof(*query), gfp_mask);
1909 	if (!query)
1910 		return -ENOMEM;
1911 
1912 	query->sa_query.port = port;
1913 	ret = alloc_mad(&query->sa_query, gfp_mask);
1914 	if (ret)
1915 		goto err1;
1916 
1917 	ib_sa_client_get(client);
1918 	query->sa_query.client = client;
1919 	query->callback        = callback;
1920 	query->context         = context;
1921 
1922 	mad = query->sa_query.mad_buf->mad;
1923 	init_mad(&query->sa_query, agent);
1924 
1925 	query->sa_query.callback = callback ? ib_sa_guidinfo_rec_callback : NULL;
1926 	query->sa_query.release  = ib_sa_guidinfo_rec_release;
1927 
1928 	mad->mad_hdr.method	 = method;
1929 	mad->mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_GUID_INFO_REC);
1930 	mad->sa_hdr.comp_mask	 = comp_mask;
1931 
1932 	ib_pack(guidinfo_rec_table, ARRAY_SIZE(guidinfo_rec_table), rec,
1933 		mad->data);
1934 
1935 	*sa_query = &query->sa_query;
1936 
1937 	ret = send_mad(&query->sa_query, timeout_ms, gfp_mask);
1938 	if (ret < 0)
1939 		goto err2;
1940 
1941 	return ret;
1942 
1943 err2:
1944 	*sa_query = NULL;
1945 	ib_sa_client_put(query->sa_query.client);
1946 	free_mad(&query->sa_query);
1947 
1948 err1:
1949 	kfree(query);
1950 	return ret;
1951 }
1952 EXPORT_SYMBOL(ib_sa_guid_info_rec_query);
1953 
1954 struct ib_classport_info_context {
1955 	struct completion	done;
1956 	struct ib_sa_query	*sa_query;
1957 };
1958 
1959 static void ib_classportinfo_cb(void *context)
1960 {
1961 	struct ib_classport_info_context *cb_ctx = context;
1962 
1963 	complete(&cb_ctx->done);
1964 }
1965 
1966 static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query,
1967 					      int status,
1968 					      struct ib_sa_mad *mad)
1969 {
1970 	unsigned long flags;
1971 	struct ib_sa_classport_info_query *query =
1972 		container_of(sa_query, struct ib_sa_classport_info_query, sa_query);
1973 	struct ib_sa_classport_cache *info = &sa_query->port->classport_info;
1974 
1975 	if (mad) {
1976 		if (sa_query->flags & IB_SA_QUERY_OPA) {
1977 			struct opa_class_port_info rec;
1978 
1979 			ib_unpack(opa_classport_info_rec_table,
1980 				  ARRAY_SIZE(opa_classport_info_rec_table),
1981 				  mad->data, &rec);
1982 
1983 			spin_lock_irqsave(&sa_query->port->classport_lock,
1984 					  flags);
1985 			if (!status && !info->valid) {
1986 				memcpy(&info->data.opa, &rec,
1987 				       sizeof(info->data.opa));
1988 
1989 				info->valid = true;
1990 				info->data.type = RDMA_CLASS_PORT_INFO_OPA;
1991 			}
1992 			spin_unlock_irqrestore(&sa_query->port->classport_lock,
1993 					       flags);
1994 
1995 		} else {
1996 			struct ib_class_port_info rec;
1997 
1998 			ib_unpack(ib_classport_info_rec_table,
1999 				  ARRAY_SIZE(ib_classport_info_rec_table),
2000 				  mad->data, &rec);
2001 
2002 			spin_lock_irqsave(&sa_query->port->classport_lock,
2003 					  flags);
2004 			if (!status && !info->valid) {
2005 				memcpy(&info->data.ib, &rec,
2006 				       sizeof(info->data.ib));
2007 
2008 				info->valid = true;
2009 				info->data.type = RDMA_CLASS_PORT_INFO_IB;
2010 			}
2011 			spin_unlock_irqrestore(&sa_query->port->classport_lock,
2012 					       flags);
2013 		}
2014 	}
2015 	query->callback(query->context);
2016 }
2017 
2018 static void ib_sa_classport_info_rec_release(struct ib_sa_query *sa_query)
2019 {
2020 	kfree(container_of(sa_query, struct ib_sa_classport_info_query,
2021 			   sa_query));
2022 }
2023 
2024 static int ib_sa_classport_info_rec_query(struct ib_sa_port *port,
2025 					  unsigned long timeout_ms,
2026 					  void (*callback)(void *context),
2027 					  void *context,
2028 					  struct ib_sa_query **sa_query)
2029 {
2030 	struct ib_mad_agent *agent;
2031 	struct ib_sa_classport_info_query *query;
2032 	struct ib_sa_mad *mad;
2033 	gfp_t gfp_mask = GFP_KERNEL;
2034 	int ret;
2035 
2036 	agent = port->agent;
2037 
2038 	query = kzalloc(sizeof(*query), gfp_mask);
2039 	if (!query)
2040 		return -ENOMEM;
2041 
2042 	query->sa_query.port = port;
2043 	query->sa_query.flags |= rdma_cap_opa_ah(port->agent->device,
2044 						 port->port_num) ?
2045 				 IB_SA_QUERY_OPA : 0;
2046 	ret = alloc_mad(&query->sa_query, gfp_mask);
2047 	if (ret)
2048 		goto err_free;
2049 
2050 	query->callback = callback;
2051 	query->context = context;
2052 
2053 	mad = query->sa_query.mad_buf->mad;
2054 	init_mad(&query->sa_query, agent);
2055 
2056 	query->sa_query.callback = ib_sa_classport_info_rec_callback;
2057 	query->sa_query.release  = ib_sa_classport_info_rec_release;
2058 	mad->mad_hdr.method	 = IB_MGMT_METHOD_GET;
2059 	mad->mad_hdr.attr_id	 = cpu_to_be16(IB_SA_ATTR_CLASS_PORTINFO);
2060 	mad->sa_hdr.comp_mask	 = 0;
2061 	*sa_query = &query->sa_query;
2062 
2063 	ret = send_mad(&query->sa_query, timeout_ms, gfp_mask);
2064 	if (ret < 0)
2065 		goto err_free_mad;
2066 
2067 	return ret;
2068 
2069 err_free_mad:
2070 	*sa_query = NULL;
2071 	free_mad(&query->sa_query);
2072 
2073 err_free:
2074 	kfree(query);
2075 	return ret;
2076 }
2077 
2078 static void update_ib_cpi(struct work_struct *work)
2079 {
2080 	struct ib_sa_port *port =
2081 		container_of(work, struct ib_sa_port, ib_cpi_work.work);
2082 	struct ib_classport_info_context *cb_context;
2083 	unsigned long flags;
2084 	int ret;
2085 
2086 	/* If the classport info is valid, nothing
2087 	 * to do here.
2088 	 */
2089 	spin_lock_irqsave(&port->classport_lock, flags);
2090 	if (port->classport_info.valid) {
2091 		spin_unlock_irqrestore(&port->classport_lock, flags);
2092 		return;
2093 	}
2094 	spin_unlock_irqrestore(&port->classport_lock, flags);
2095 
2096 	cb_context = kmalloc(sizeof(*cb_context), GFP_KERNEL);
2097 	if (!cb_context)
2098 		goto err_nomem;
2099 
2100 	init_completion(&cb_context->done);
2101 
2102 	ret = ib_sa_classport_info_rec_query(port, 3000,
2103 					     ib_classportinfo_cb, cb_context,
2104 					     &cb_context->sa_query);
2105 	if (ret < 0)
2106 		goto free_cb_err;
2107 	wait_for_completion(&cb_context->done);
2108 free_cb_err:
2109 	kfree(cb_context);
2110 	spin_lock_irqsave(&port->classport_lock, flags);
2111 
2112 	/* If the classport info is still not valid, the query should have
2113 	 * failed for some reason. Retry issuing the query
2114 	 */
2115 	if (!port->classport_info.valid) {
2116 		port->classport_info.retry_cnt++;
2117 		if (port->classport_info.retry_cnt <=
2118 		    IB_SA_CPI_MAX_RETRY_CNT) {
2119 			unsigned long delay =
2120 				msecs_to_jiffies(IB_SA_CPI_RETRY_WAIT);
2121 
2122 			queue_delayed_work(ib_wq, &port->ib_cpi_work, delay);
2123 		}
2124 	}
2125 	spin_unlock_irqrestore(&port->classport_lock, flags);
2126 
2127 err_nomem:
2128 	return;
2129 }
2130 
2131 static void send_handler(struct ib_mad_agent *agent,
2132 			 struct ib_mad_send_wc *mad_send_wc)
2133 {
2134 	struct ib_sa_query *query = mad_send_wc->send_buf->context[0];
2135 	unsigned long flags;
2136 
2137 	if (query->callback)
2138 		switch (mad_send_wc->status) {
2139 		case IB_WC_SUCCESS:
2140 			/* No callback -- already got recv */
2141 			break;
2142 		case IB_WC_RESP_TIMEOUT_ERR:
2143 			query->callback(query, -ETIMEDOUT, NULL);
2144 			break;
2145 		case IB_WC_WR_FLUSH_ERR:
2146 			query->callback(query, -EINTR, NULL);
2147 			break;
2148 		default:
2149 			query->callback(query, -EIO, NULL);
2150 			break;
2151 		}
2152 
2153 	xa_lock_irqsave(&queries, flags);
2154 	__xa_erase(&queries, query->id);
2155 	xa_unlock_irqrestore(&queries, flags);
2156 
2157 	free_mad(query);
2158 	if (query->client)
2159 		ib_sa_client_put(query->client);
2160 	query->release(query);
2161 }
2162 
2163 static void recv_handler(struct ib_mad_agent *mad_agent,
2164 			 struct ib_mad_send_buf *send_buf,
2165 			 struct ib_mad_recv_wc *mad_recv_wc)
2166 {
2167 	struct ib_sa_query *query;
2168 
2169 	if (!send_buf)
2170 		return;
2171 
2172 	query = send_buf->context[0];
2173 	if (query->callback) {
2174 		if (mad_recv_wc->wc->status == IB_WC_SUCCESS)
2175 			query->callback(query,
2176 					mad_recv_wc->recv_buf.mad->mad_hdr.status ?
2177 					-EINVAL : 0,
2178 					(struct ib_sa_mad *) mad_recv_wc->recv_buf.mad);
2179 		else
2180 			query->callback(query, -EIO, NULL);
2181 	}
2182 
2183 	ib_free_recv_mad(mad_recv_wc);
2184 }
2185 
2186 static void update_sm_ah(struct work_struct *work)
2187 {
2188 	struct ib_sa_port *port =
2189 		container_of(work, struct ib_sa_port, update_task);
2190 	struct ib_sa_sm_ah *new_ah;
2191 	struct ib_port_attr port_attr;
2192 	struct rdma_ah_attr   ah_attr;
2193 	bool grh_required;
2194 
2195 	if (ib_query_port(port->agent->device, port->port_num, &port_attr)) {
2196 		pr_warn("Couldn't query port\n");
2197 		return;
2198 	}
2199 
2200 	new_ah = kmalloc(sizeof(*new_ah), GFP_KERNEL);
2201 	if (!new_ah)
2202 		return;
2203 
2204 	kref_init(&new_ah->ref);
2205 	new_ah->src_path_mask = (1 << port_attr.lmc) - 1;
2206 
2207 	new_ah->pkey_index = 0;
2208 	if (ib_find_pkey(port->agent->device, port->port_num,
2209 			 IB_DEFAULT_PKEY_FULL, &new_ah->pkey_index))
2210 		pr_err("Couldn't find index for default PKey\n");
2211 
2212 	memset(&ah_attr, 0, sizeof(ah_attr));
2213 	ah_attr.type = rdma_ah_find_type(port->agent->device,
2214 					 port->port_num);
2215 	rdma_ah_set_dlid(&ah_attr, port_attr.sm_lid);
2216 	rdma_ah_set_sl(&ah_attr, port_attr.sm_sl);
2217 	rdma_ah_set_port_num(&ah_attr, port->port_num);
2218 
2219 	grh_required = rdma_is_grh_required(port->agent->device,
2220 					    port->port_num);
2221 
2222 	/*
2223 	 * The OPA sm_lid of 0xFFFF needs special handling so that it can be
2224 	 * differentiated from a permissive LID of 0xFFFF.  We set the
2225 	 * grh_required flag here so the SA can program the DGID in the
2226 	 * address handle appropriately
2227 	 */
2228 	if (ah_attr.type == RDMA_AH_ATTR_TYPE_OPA &&
2229 	    (grh_required ||
2230 	     port_attr.sm_lid == be16_to_cpu(IB_LID_PERMISSIVE)))
2231 		rdma_ah_set_make_grd(&ah_attr, true);
2232 
2233 	if (ah_attr.type == RDMA_AH_ATTR_TYPE_IB && grh_required) {
2234 		rdma_ah_set_ah_flags(&ah_attr, IB_AH_GRH);
2235 		rdma_ah_set_subnet_prefix(&ah_attr,
2236 					  cpu_to_be64(port_attr.subnet_prefix));
2237 		rdma_ah_set_interface_id(&ah_attr,
2238 					 cpu_to_be64(IB_SA_WELL_KNOWN_GUID));
2239 	}
2240 
2241 	new_ah->ah = rdma_create_ah(port->agent->qp->pd, &ah_attr,
2242 				    RDMA_CREATE_AH_SLEEPABLE);
2243 	if (IS_ERR(new_ah->ah)) {
2244 		pr_warn("Couldn't create new SM AH\n");
2245 		kfree(new_ah);
2246 		return;
2247 	}
2248 
2249 	spin_lock_irq(&port->ah_lock);
2250 	if (port->sm_ah)
2251 		kref_put(&port->sm_ah->ref, free_sm_ah);
2252 	port->sm_ah = new_ah;
2253 	spin_unlock_irq(&port->ah_lock);
2254 }
2255 
2256 static void ib_sa_event(struct ib_event_handler *handler,
2257 			struct ib_event *event)
2258 {
2259 	if (event->event == IB_EVENT_PORT_ERR    ||
2260 	    event->event == IB_EVENT_PORT_ACTIVE ||
2261 	    event->event == IB_EVENT_LID_CHANGE  ||
2262 	    event->event == IB_EVENT_PKEY_CHANGE ||
2263 	    event->event == IB_EVENT_SM_CHANGE   ||
2264 	    event->event == IB_EVENT_CLIENT_REREGISTER) {
2265 		unsigned long flags;
2266 		struct ib_sa_device *sa_dev =
2267 			container_of(handler, typeof(*sa_dev), event_handler);
2268 		u32 port_num = event->element.port_num - sa_dev->start_port;
2269 		struct ib_sa_port *port = &sa_dev->port[port_num];
2270 
2271 		if (!rdma_cap_ib_sa(handler->device, port->port_num))
2272 			return;
2273 
2274 		spin_lock_irqsave(&port->ah_lock, flags);
2275 		if (port->sm_ah)
2276 			kref_put(&port->sm_ah->ref, free_sm_ah);
2277 		port->sm_ah = NULL;
2278 		spin_unlock_irqrestore(&port->ah_lock, flags);
2279 
2280 		if (event->event == IB_EVENT_SM_CHANGE ||
2281 		    event->event == IB_EVENT_CLIENT_REREGISTER ||
2282 		    event->event == IB_EVENT_LID_CHANGE ||
2283 		    event->event == IB_EVENT_PORT_ACTIVE) {
2284 			unsigned long delay =
2285 				msecs_to_jiffies(IB_SA_CPI_RETRY_WAIT);
2286 
2287 			spin_lock_irqsave(&port->classport_lock, flags);
2288 			port->classport_info.valid = false;
2289 			port->classport_info.retry_cnt = 0;
2290 			spin_unlock_irqrestore(&port->classport_lock, flags);
2291 			queue_delayed_work(ib_wq,
2292 					   &port->ib_cpi_work, delay);
2293 		}
2294 		queue_work(ib_wq, &sa_dev->port[port_num].update_task);
2295 	}
2296 }
2297 
2298 static int ib_sa_add_one(struct ib_device *device)
2299 {
2300 	struct ib_sa_device *sa_dev;
2301 	int s, e, i;
2302 	int count = 0;
2303 	int ret;
2304 
2305 	s = rdma_start_port(device);
2306 	e = rdma_end_port(device);
2307 
2308 	sa_dev = kzalloc(struct_size(sa_dev, port, e - s + 1), GFP_KERNEL);
2309 	if (!sa_dev)
2310 		return -ENOMEM;
2311 
2312 	sa_dev->start_port = s;
2313 	sa_dev->end_port   = e;
2314 
2315 	for (i = 0; i <= e - s; ++i) {
2316 		spin_lock_init(&sa_dev->port[i].ah_lock);
2317 		if (!rdma_cap_ib_sa(device, i + 1))
2318 			continue;
2319 
2320 		sa_dev->port[i].sm_ah    = NULL;
2321 		sa_dev->port[i].port_num = i + s;
2322 
2323 		spin_lock_init(&sa_dev->port[i].classport_lock);
2324 		sa_dev->port[i].classport_info.valid = false;
2325 
2326 		sa_dev->port[i].agent =
2327 			ib_register_mad_agent(device, i + s, IB_QPT_GSI,
2328 					      NULL, 0, send_handler,
2329 					      recv_handler, sa_dev, 0);
2330 		if (IS_ERR(sa_dev->port[i].agent)) {
2331 			ret = PTR_ERR(sa_dev->port[i].agent);
2332 			goto err;
2333 		}
2334 
2335 		INIT_WORK(&sa_dev->port[i].update_task, update_sm_ah);
2336 		INIT_DELAYED_WORK(&sa_dev->port[i].ib_cpi_work,
2337 				  update_ib_cpi);
2338 
2339 		count++;
2340 	}
2341 
2342 	if (!count) {
2343 		ret = -EOPNOTSUPP;
2344 		goto free;
2345 	}
2346 
2347 	ib_set_client_data(device, &sa_client, sa_dev);
2348 
2349 	/*
2350 	 * We register our event handler after everything is set up,
2351 	 * and then update our cached info after the event handler is
2352 	 * registered to avoid any problems if a port changes state
2353 	 * during our initialization.
2354 	 */
2355 
2356 	INIT_IB_EVENT_HANDLER(&sa_dev->event_handler, device, ib_sa_event);
2357 	ib_register_event_handler(&sa_dev->event_handler);
2358 
2359 	for (i = 0; i <= e - s; ++i) {
2360 		if (rdma_cap_ib_sa(device, i + 1))
2361 			update_sm_ah(&sa_dev->port[i].update_task);
2362 	}
2363 
2364 	return 0;
2365 
2366 err:
2367 	while (--i >= 0) {
2368 		if (rdma_cap_ib_sa(device, i + 1))
2369 			ib_unregister_mad_agent(sa_dev->port[i].agent);
2370 	}
2371 free:
2372 	kfree(sa_dev);
2373 	return ret;
2374 }
2375 
2376 static void ib_sa_remove_one(struct ib_device *device, void *client_data)
2377 {
2378 	struct ib_sa_device *sa_dev = client_data;
2379 	int i;
2380 
2381 	ib_unregister_event_handler(&sa_dev->event_handler);
2382 	flush_workqueue(ib_wq);
2383 
2384 	for (i = 0; i <= sa_dev->end_port - sa_dev->start_port; ++i) {
2385 		if (rdma_cap_ib_sa(device, i + 1)) {
2386 			cancel_delayed_work_sync(&sa_dev->port[i].ib_cpi_work);
2387 			ib_unregister_mad_agent(sa_dev->port[i].agent);
2388 			if (sa_dev->port[i].sm_ah)
2389 				kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah);
2390 		}
2391 
2392 	}
2393 
2394 	kfree(sa_dev);
2395 }
2396 
2397 int ib_sa_init(void)
2398 {
2399 	int ret;
2400 
2401 	get_random_bytes(&tid, sizeof tid);
2402 
2403 	atomic_set(&ib_nl_sa_request_seq, 0);
2404 
2405 	ret = ib_register_client(&sa_client);
2406 	if (ret) {
2407 		pr_err("Couldn't register ib_sa client\n");
2408 		goto err1;
2409 	}
2410 
2411 	ret = mcast_init();
2412 	if (ret) {
2413 		pr_err("Couldn't initialize multicast handling\n");
2414 		goto err2;
2415 	}
2416 
2417 	ib_nl_wq = alloc_ordered_workqueue("ib_nl_sa_wq", WQ_MEM_RECLAIM);
2418 	if (!ib_nl_wq) {
2419 		ret = -ENOMEM;
2420 		goto err3;
2421 	}
2422 
2423 	INIT_DELAYED_WORK(&ib_nl_timed_work, ib_nl_request_timeout);
2424 
2425 	return 0;
2426 
2427 err3:
2428 	mcast_cleanup();
2429 err2:
2430 	ib_unregister_client(&sa_client);
2431 err1:
2432 	return ret;
2433 }
2434 
2435 void ib_sa_cleanup(void)
2436 {
2437 	cancel_delayed_work(&ib_nl_timed_work);
2438 	flush_workqueue(ib_nl_wq);
2439 	destroy_workqueue(ib_nl_wq);
2440 	mcast_cleanup();
2441 	ib_unregister_client(&sa_client);
2442 	WARN_ON(!xa_empty(&queries));
2443 }
2444