1 /* 2 * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2005 Intel Corporation. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. 5 * Copyright (c) 2009 HNR Consulting. All rights reserved. 6 * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 * 36 */ 37 38 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 39 40 #include <linux/dma-mapping.h> 41 #include <linux/slab.h> 42 #include <linux/module.h> 43 #include <linux/security.h> 44 #include <linux/xarray.h> 45 #include <rdma/ib_cache.h> 46 47 #include "mad_priv.h" 48 #include "core_priv.h" 49 #include "mad_rmpp.h" 50 #include "smi.h" 51 #include "opa_smi.h" 52 #include "agent.h" 53 54 #define CREATE_TRACE_POINTS 55 #include <trace/events/ib_mad.h> 56 57 #ifdef CONFIG_TRACEPOINTS 58 static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, 59 struct ib_mad_qp_info *qp_info, 60 struct trace_event_raw_ib_mad_send_template *entry) 61 { 62 u16 pkey; 63 struct ib_device *dev = qp_info->port_priv->device; 64 u8 pnum = qp_info->port_priv->port_num; 65 struct ib_ud_wr *wr = &mad_send_wr->send_wr; 66 struct rdma_ah_attr attr = {}; 67 68 rdma_query_ah(wr->ah, &attr); 69 70 /* These are common */ 71 entry->sl = attr.sl; 72 ib_query_pkey(dev, pnum, wr->pkey_index, &pkey); 73 entry->pkey = pkey; 74 entry->rqpn = wr->remote_qpn; 75 entry->rqkey = wr->remote_qkey; 76 entry->dlid = rdma_ah_get_dlid(&attr); 77 } 78 #endif 79 80 static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; 81 static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; 82 83 module_param_named(send_queue_size, mad_sendq_size, int, 0444); 84 MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests"); 85 module_param_named(recv_queue_size, mad_recvq_size, int, 0444); 86 MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests"); 87 88 /* Client ID 0 is used for snoop-only clients */ 89 static DEFINE_XARRAY_ALLOC1(ib_mad_clients); 90 static u32 ib_mad_client_next; 91 static struct list_head ib_mad_port_list; 92 93 /* Port list lock */ 94 static DEFINE_SPINLOCK(ib_mad_port_list_lock); 95 96 /* Forward declarations */ 97 static int method_in_use(struct ib_mad_mgmt_method_table **method, 98 struct ib_mad_reg_req *mad_reg_req); 99 static void remove_mad_reg_req(struct ib_mad_agent_private *priv); 100 static struct ib_mad_agent_private *find_mad_agent( 101 struct ib_mad_port_private *port_priv, 102 const struct ib_mad_hdr *mad); 103 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info, 104 struct ib_mad_private *mad); 105 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv); 106 static void timeout_sends(struct work_struct *work); 107 static void local_completions(struct work_struct *work); 108 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req, 109 struct ib_mad_agent_private *agent_priv, 110 u8 mgmt_class); 111 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req, 112 struct ib_mad_agent_private *agent_priv); 113 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv, 114 struct ib_wc *wc); 115 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc); 116 117 /* 118 * Returns a ib_mad_port_private structure or NULL for a device/port 119 * Assumes ib_mad_port_list_lock is being held 120 */ 121 static inline struct ib_mad_port_private * 122 __ib_get_mad_port(struct ib_device *device, int port_num) 123 { 124 struct ib_mad_port_private *entry; 125 126 list_for_each_entry(entry, &ib_mad_port_list, port_list) { 127 if (entry->device == device && entry->port_num == port_num) 128 return entry; 129 } 130 return NULL; 131 } 132 133 /* 134 * Wrapper function to return a ib_mad_port_private structure or NULL 135 * for a device/port 136 */ 137 static inline struct ib_mad_port_private * 138 ib_get_mad_port(struct ib_device *device, int port_num) 139 { 140 struct ib_mad_port_private *entry; 141 unsigned long flags; 142 143 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 144 entry = __ib_get_mad_port(device, port_num); 145 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 146 147 return entry; 148 } 149 150 static inline u8 convert_mgmt_class(u8 mgmt_class) 151 { 152 /* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */ 153 return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ? 154 0 : mgmt_class; 155 } 156 157 static int get_spl_qp_index(enum ib_qp_type qp_type) 158 { 159 switch (qp_type) 160 { 161 case IB_QPT_SMI: 162 return 0; 163 case IB_QPT_GSI: 164 return 1; 165 default: 166 return -1; 167 } 168 } 169 170 static int vendor_class_index(u8 mgmt_class) 171 { 172 return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START; 173 } 174 175 static int is_vendor_class(u8 mgmt_class) 176 { 177 if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) || 178 (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END)) 179 return 0; 180 return 1; 181 } 182 183 static int is_vendor_oui(char *oui) 184 { 185 if (oui[0] || oui[1] || oui[2]) 186 return 1; 187 return 0; 188 } 189 190 static int is_vendor_method_in_use( 191 struct ib_mad_mgmt_vendor_class *vendor_class, 192 struct ib_mad_reg_req *mad_reg_req) 193 { 194 struct ib_mad_mgmt_method_table *method; 195 int i; 196 197 for (i = 0; i < MAX_MGMT_OUI; i++) { 198 if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) { 199 method = vendor_class->method_table[i]; 200 if (method) { 201 if (method_in_use(&method, mad_reg_req)) 202 return 1; 203 else 204 break; 205 } 206 } 207 } 208 return 0; 209 } 210 211 int ib_response_mad(const struct ib_mad_hdr *hdr) 212 { 213 return ((hdr->method & IB_MGMT_METHOD_RESP) || 214 (hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) || 215 ((hdr->mgmt_class == IB_MGMT_CLASS_BM) && 216 (hdr->attr_mod & IB_BM_ATTR_MOD_RESP))); 217 } 218 EXPORT_SYMBOL(ib_response_mad); 219 220 /* 221 * ib_register_mad_agent - Register to send/receive MADs 222 * 223 * Context: Process context. 224 */ 225 struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, 226 u8 port_num, 227 enum ib_qp_type qp_type, 228 struct ib_mad_reg_req *mad_reg_req, 229 u8 rmpp_version, 230 ib_mad_send_handler send_handler, 231 ib_mad_recv_handler recv_handler, 232 void *context, 233 u32 registration_flags) 234 { 235 struct ib_mad_port_private *port_priv; 236 struct ib_mad_agent *ret = ERR_PTR(-EINVAL); 237 struct ib_mad_agent_private *mad_agent_priv; 238 struct ib_mad_reg_req *reg_req = NULL; 239 struct ib_mad_mgmt_class_table *class; 240 struct ib_mad_mgmt_vendor_class_table *vendor; 241 struct ib_mad_mgmt_vendor_class *vendor_class; 242 struct ib_mad_mgmt_method_table *method; 243 int ret2, qpn; 244 u8 mgmt_class, vclass; 245 246 if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) || 247 (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num))) 248 return ERR_PTR(-EPROTONOSUPPORT); 249 250 /* Validate parameters */ 251 qpn = get_spl_qp_index(qp_type); 252 if (qpn == -1) { 253 dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n", 254 __func__, qp_type); 255 goto error1; 256 } 257 258 if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) { 259 dev_dbg_ratelimited(&device->dev, 260 "%s: invalid RMPP Version %u\n", 261 __func__, rmpp_version); 262 goto error1; 263 } 264 265 /* Validate MAD registration request if supplied */ 266 if (mad_reg_req) { 267 if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) { 268 dev_dbg_ratelimited(&device->dev, 269 "%s: invalid Class Version %u\n", 270 __func__, 271 mad_reg_req->mgmt_class_version); 272 goto error1; 273 } 274 if (!recv_handler) { 275 dev_dbg_ratelimited(&device->dev, 276 "%s: no recv_handler\n", __func__); 277 goto error1; 278 } 279 if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) { 280 /* 281 * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only 282 * one in this range currently allowed 283 */ 284 if (mad_reg_req->mgmt_class != 285 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 286 dev_dbg_ratelimited(&device->dev, 287 "%s: Invalid Mgmt Class 0x%x\n", 288 __func__, mad_reg_req->mgmt_class); 289 goto error1; 290 } 291 } else if (mad_reg_req->mgmt_class == 0) { 292 /* 293 * Class 0 is reserved in IBA and is used for 294 * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 295 */ 296 dev_dbg_ratelimited(&device->dev, 297 "%s: Invalid Mgmt Class 0\n", 298 __func__); 299 goto error1; 300 } else if (is_vendor_class(mad_reg_req->mgmt_class)) { 301 /* 302 * If class is in "new" vendor range, 303 * ensure supplied OUI is not zero 304 */ 305 if (!is_vendor_oui(mad_reg_req->oui)) { 306 dev_dbg_ratelimited(&device->dev, 307 "%s: No OUI specified for class 0x%x\n", 308 __func__, 309 mad_reg_req->mgmt_class); 310 goto error1; 311 } 312 } 313 /* Make sure class supplied is consistent with RMPP */ 314 if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) { 315 if (rmpp_version) { 316 dev_dbg_ratelimited(&device->dev, 317 "%s: RMPP version for non-RMPP class 0x%x\n", 318 __func__, mad_reg_req->mgmt_class); 319 goto error1; 320 } 321 } 322 323 /* Make sure class supplied is consistent with QP type */ 324 if (qp_type == IB_QPT_SMI) { 325 if ((mad_reg_req->mgmt_class != 326 IB_MGMT_CLASS_SUBN_LID_ROUTED) && 327 (mad_reg_req->mgmt_class != 328 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 329 dev_dbg_ratelimited(&device->dev, 330 "%s: Invalid SM QP type: class 0x%x\n", 331 __func__, mad_reg_req->mgmt_class); 332 goto error1; 333 } 334 } else { 335 if ((mad_reg_req->mgmt_class == 336 IB_MGMT_CLASS_SUBN_LID_ROUTED) || 337 (mad_reg_req->mgmt_class == 338 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 339 dev_dbg_ratelimited(&device->dev, 340 "%s: Invalid GS QP type: class 0x%x\n", 341 __func__, mad_reg_req->mgmt_class); 342 goto error1; 343 } 344 } 345 } else { 346 /* No registration request supplied */ 347 if (!send_handler) 348 goto error1; 349 if (registration_flags & IB_MAD_USER_RMPP) 350 goto error1; 351 } 352 353 /* Validate device and port */ 354 port_priv = ib_get_mad_port(device, port_num); 355 if (!port_priv) { 356 dev_dbg_ratelimited(&device->dev, "%s: Invalid port %d\n", 357 __func__, port_num); 358 ret = ERR_PTR(-ENODEV); 359 goto error1; 360 } 361 362 /* Verify the QP requested is supported. For example, Ethernet devices 363 * will not have QP0. 364 */ 365 if (!port_priv->qp_info[qpn].qp) { 366 dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n", 367 __func__, qpn); 368 ret = ERR_PTR(-EPROTONOSUPPORT); 369 goto error1; 370 } 371 372 /* Allocate structures */ 373 mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL); 374 if (!mad_agent_priv) { 375 ret = ERR_PTR(-ENOMEM); 376 goto error1; 377 } 378 379 if (mad_reg_req) { 380 reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL); 381 if (!reg_req) { 382 ret = ERR_PTR(-ENOMEM); 383 goto error3; 384 } 385 } 386 387 /* Now, fill in the various structures */ 388 mad_agent_priv->qp_info = &port_priv->qp_info[qpn]; 389 mad_agent_priv->reg_req = reg_req; 390 mad_agent_priv->agent.rmpp_version = rmpp_version; 391 mad_agent_priv->agent.device = device; 392 mad_agent_priv->agent.recv_handler = recv_handler; 393 mad_agent_priv->agent.send_handler = send_handler; 394 mad_agent_priv->agent.context = context; 395 mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp; 396 mad_agent_priv->agent.port_num = port_num; 397 mad_agent_priv->agent.flags = registration_flags; 398 spin_lock_init(&mad_agent_priv->lock); 399 INIT_LIST_HEAD(&mad_agent_priv->send_list); 400 INIT_LIST_HEAD(&mad_agent_priv->wait_list); 401 INIT_LIST_HEAD(&mad_agent_priv->done_list); 402 INIT_LIST_HEAD(&mad_agent_priv->rmpp_list); 403 INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends); 404 INIT_LIST_HEAD(&mad_agent_priv->local_list); 405 INIT_WORK(&mad_agent_priv->local_work, local_completions); 406 atomic_set(&mad_agent_priv->refcount, 1); 407 init_completion(&mad_agent_priv->comp); 408 409 ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type); 410 if (ret2) { 411 ret = ERR_PTR(ret2); 412 goto error4; 413 } 414 415 /* 416 * The mlx4 driver uses the top byte to distinguish which virtual 417 * function generated the MAD, so we must avoid using it. 418 */ 419 ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid, 420 mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1), 421 &ib_mad_client_next, GFP_KERNEL); 422 if (ret2 < 0) { 423 ret = ERR_PTR(ret2); 424 goto error5; 425 } 426 427 /* 428 * Make sure MAD registration (if supplied) 429 * is non overlapping with any existing ones 430 */ 431 spin_lock_irq(&port_priv->reg_lock); 432 if (mad_reg_req) { 433 mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class); 434 if (!is_vendor_class(mgmt_class)) { 435 class = port_priv->version[mad_reg_req-> 436 mgmt_class_version].class; 437 if (class) { 438 method = class->method_table[mgmt_class]; 439 if (method) { 440 if (method_in_use(&method, 441 mad_reg_req)) 442 goto error6; 443 } 444 } 445 ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv, 446 mgmt_class); 447 } else { 448 /* "New" vendor class range */ 449 vendor = port_priv->version[mad_reg_req-> 450 mgmt_class_version].vendor; 451 if (vendor) { 452 vclass = vendor_class_index(mgmt_class); 453 vendor_class = vendor->vendor_class[vclass]; 454 if (vendor_class) { 455 if (is_vendor_method_in_use( 456 vendor_class, 457 mad_reg_req)) 458 goto error6; 459 } 460 } 461 ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv); 462 } 463 if (ret2) { 464 ret = ERR_PTR(ret2); 465 goto error6; 466 } 467 } 468 spin_unlock_irq(&port_priv->reg_lock); 469 470 trace_ib_mad_create_agent(mad_agent_priv); 471 return &mad_agent_priv->agent; 472 error6: 473 spin_unlock_irq(&port_priv->reg_lock); 474 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid); 475 error5: 476 ib_mad_agent_security_cleanup(&mad_agent_priv->agent); 477 error4: 478 kfree(reg_req); 479 error3: 480 kfree(mad_agent_priv); 481 error1: 482 return ret; 483 } 484 EXPORT_SYMBOL(ib_register_mad_agent); 485 486 static inline int is_snooping_sends(int mad_snoop_flags) 487 { 488 return (mad_snoop_flags & 489 (/*IB_MAD_SNOOP_POSTED_SENDS | 490 IB_MAD_SNOOP_RMPP_SENDS |*/ 491 IB_MAD_SNOOP_SEND_COMPLETIONS /*| 492 IB_MAD_SNOOP_RMPP_SEND_COMPLETIONS*/)); 493 } 494 495 static inline int is_snooping_recvs(int mad_snoop_flags) 496 { 497 return (mad_snoop_flags & 498 (IB_MAD_SNOOP_RECVS /*| 499 IB_MAD_SNOOP_RMPP_RECVS*/)); 500 } 501 502 static int register_snoop_agent(struct ib_mad_qp_info *qp_info, 503 struct ib_mad_snoop_private *mad_snoop_priv) 504 { 505 struct ib_mad_snoop_private **new_snoop_table; 506 unsigned long flags; 507 int i; 508 509 spin_lock_irqsave(&qp_info->snoop_lock, flags); 510 /* Check for empty slot in array. */ 511 for (i = 0; i < qp_info->snoop_table_size; i++) 512 if (!qp_info->snoop_table[i]) 513 break; 514 515 if (i == qp_info->snoop_table_size) { 516 /* Grow table. */ 517 new_snoop_table = krealloc(qp_info->snoop_table, 518 sizeof mad_snoop_priv * 519 (qp_info->snoop_table_size + 1), 520 GFP_ATOMIC); 521 if (!new_snoop_table) { 522 i = -ENOMEM; 523 goto out; 524 } 525 526 qp_info->snoop_table = new_snoop_table; 527 qp_info->snoop_table_size++; 528 } 529 qp_info->snoop_table[i] = mad_snoop_priv; 530 atomic_inc(&qp_info->snoop_count); 531 out: 532 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 533 return i; 534 } 535 536 struct ib_mad_agent *ib_register_mad_snoop(struct ib_device *device, 537 u8 port_num, 538 enum ib_qp_type qp_type, 539 int mad_snoop_flags, 540 ib_mad_snoop_handler snoop_handler, 541 ib_mad_recv_handler recv_handler, 542 void *context) 543 { 544 struct ib_mad_port_private *port_priv; 545 struct ib_mad_agent *ret; 546 struct ib_mad_snoop_private *mad_snoop_priv; 547 int qpn; 548 int err; 549 550 /* Validate parameters */ 551 if ((is_snooping_sends(mad_snoop_flags) && !snoop_handler) || 552 (is_snooping_recvs(mad_snoop_flags) && !recv_handler)) { 553 ret = ERR_PTR(-EINVAL); 554 goto error1; 555 } 556 qpn = get_spl_qp_index(qp_type); 557 if (qpn == -1) { 558 ret = ERR_PTR(-EINVAL); 559 goto error1; 560 } 561 port_priv = ib_get_mad_port(device, port_num); 562 if (!port_priv) { 563 ret = ERR_PTR(-ENODEV); 564 goto error1; 565 } 566 /* Allocate structures */ 567 mad_snoop_priv = kzalloc(sizeof *mad_snoop_priv, GFP_KERNEL); 568 if (!mad_snoop_priv) { 569 ret = ERR_PTR(-ENOMEM); 570 goto error1; 571 } 572 573 /* Now, fill in the various structures */ 574 mad_snoop_priv->qp_info = &port_priv->qp_info[qpn]; 575 mad_snoop_priv->agent.device = device; 576 mad_snoop_priv->agent.recv_handler = recv_handler; 577 mad_snoop_priv->agent.snoop_handler = snoop_handler; 578 mad_snoop_priv->agent.context = context; 579 mad_snoop_priv->agent.qp = port_priv->qp_info[qpn].qp; 580 mad_snoop_priv->agent.port_num = port_num; 581 mad_snoop_priv->mad_snoop_flags = mad_snoop_flags; 582 init_completion(&mad_snoop_priv->comp); 583 584 err = ib_mad_agent_security_setup(&mad_snoop_priv->agent, qp_type); 585 if (err) { 586 ret = ERR_PTR(err); 587 goto error2; 588 } 589 590 mad_snoop_priv->snoop_index = register_snoop_agent( 591 &port_priv->qp_info[qpn], 592 mad_snoop_priv); 593 if (mad_snoop_priv->snoop_index < 0) { 594 ret = ERR_PTR(mad_snoop_priv->snoop_index); 595 goto error3; 596 } 597 598 atomic_set(&mad_snoop_priv->refcount, 1); 599 return &mad_snoop_priv->agent; 600 error3: 601 ib_mad_agent_security_cleanup(&mad_snoop_priv->agent); 602 error2: 603 kfree(mad_snoop_priv); 604 error1: 605 return ret; 606 } 607 EXPORT_SYMBOL(ib_register_mad_snoop); 608 609 static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv) 610 { 611 if (atomic_dec_and_test(&mad_agent_priv->refcount)) 612 complete(&mad_agent_priv->comp); 613 } 614 615 static inline void deref_snoop_agent(struct ib_mad_snoop_private *mad_snoop_priv) 616 { 617 if (atomic_dec_and_test(&mad_snoop_priv->refcount)) 618 complete(&mad_snoop_priv->comp); 619 } 620 621 static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv) 622 { 623 struct ib_mad_port_private *port_priv; 624 625 /* Note that we could still be handling received MADs */ 626 trace_ib_mad_unregister_agent(mad_agent_priv); 627 628 /* 629 * Canceling all sends results in dropping received response 630 * MADs, preventing us from queuing additional work 631 */ 632 cancel_mads(mad_agent_priv); 633 port_priv = mad_agent_priv->qp_info->port_priv; 634 cancel_delayed_work(&mad_agent_priv->timed_work); 635 636 spin_lock_irq(&port_priv->reg_lock); 637 remove_mad_reg_req(mad_agent_priv); 638 spin_unlock_irq(&port_priv->reg_lock); 639 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid); 640 641 flush_workqueue(port_priv->wq); 642 ib_cancel_rmpp_recvs(mad_agent_priv); 643 644 deref_mad_agent(mad_agent_priv); 645 wait_for_completion(&mad_agent_priv->comp); 646 647 ib_mad_agent_security_cleanup(&mad_agent_priv->agent); 648 649 kfree(mad_agent_priv->reg_req); 650 kfree_rcu(mad_agent_priv, rcu); 651 } 652 653 static void unregister_mad_snoop(struct ib_mad_snoop_private *mad_snoop_priv) 654 { 655 struct ib_mad_qp_info *qp_info; 656 unsigned long flags; 657 658 qp_info = mad_snoop_priv->qp_info; 659 spin_lock_irqsave(&qp_info->snoop_lock, flags); 660 qp_info->snoop_table[mad_snoop_priv->snoop_index] = NULL; 661 atomic_dec(&qp_info->snoop_count); 662 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 663 664 deref_snoop_agent(mad_snoop_priv); 665 wait_for_completion(&mad_snoop_priv->comp); 666 667 ib_mad_agent_security_cleanup(&mad_snoop_priv->agent); 668 669 kfree(mad_snoop_priv); 670 } 671 672 /* 673 * ib_unregister_mad_agent - Unregisters a client from using MAD services 674 * 675 * Context: Process context. 676 */ 677 void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent) 678 { 679 struct ib_mad_agent_private *mad_agent_priv; 680 struct ib_mad_snoop_private *mad_snoop_priv; 681 682 /* If the TID is zero, the agent can only snoop. */ 683 if (mad_agent->hi_tid) { 684 mad_agent_priv = container_of(mad_agent, 685 struct ib_mad_agent_private, 686 agent); 687 unregister_mad_agent(mad_agent_priv); 688 } else { 689 mad_snoop_priv = container_of(mad_agent, 690 struct ib_mad_snoop_private, 691 agent); 692 unregister_mad_snoop(mad_snoop_priv); 693 } 694 } 695 EXPORT_SYMBOL(ib_unregister_mad_agent); 696 697 static void dequeue_mad(struct ib_mad_list_head *mad_list) 698 { 699 struct ib_mad_queue *mad_queue; 700 unsigned long flags; 701 702 mad_queue = mad_list->mad_queue; 703 spin_lock_irqsave(&mad_queue->lock, flags); 704 list_del(&mad_list->list); 705 mad_queue->count--; 706 spin_unlock_irqrestore(&mad_queue->lock, flags); 707 } 708 709 static void snoop_send(struct ib_mad_qp_info *qp_info, 710 struct ib_mad_send_buf *send_buf, 711 struct ib_mad_send_wc *mad_send_wc, 712 int mad_snoop_flags) 713 { 714 struct ib_mad_snoop_private *mad_snoop_priv; 715 unsigned long flags; 716 int i; 717 718 spin_lock_irqsave(&qp_info->snoop_lock, flags); 719 for (i = 0; i < qp_info->snoop_table_size; i++) { 720 mad_snoop_priv = qp_info->snoop_table[i]; 721 if (!mad_snoop_priv || 722 !(mad_snoop_priv->mad_snoop_flags & mad_snoop_flags)) 723 continue; 724 725 atomic_inc(&mad_snoop_priv->refcount); 726 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 727 mad_snoop_priv->agent.snoop_handler(&mad_snoop_priv->agent, 728 send_buf, mad_send_wc); 729 deref_snoop_agent(mad_snoop_priv); 730 spin_lock_irqsave(&qp_info->snoop_lock, flags); 731 } 732 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 733 } 734 735 static void snoop_recv(struct ib_mad_qp_info *qp_info, 736 struct ib_mad_recv_wc *mad_recv_wc, 737 int mad_snoop_flags) 738 { 739 struct ib_mad_snoop_private *mad_snoop_priv; 740 unsigned long flags; 741 int i; 742 743 spin_lock_irqsave(&qp_info->snoop_lock, flags); 744 for (i = 0; i < qp_info->snoop_table_size; i++) { 745 mad_snoop_priv = qp_info->snoop_table[i]; 746 if (!mad_snoop_priv || 747 !(mad_snoop_priv->mad_snoop_flags & mad_snoop_flags)) 748 continue; 749 750 atomic_inc(&mad_snoop_priv->refcount); 751 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 752 mad_snoop_priv->agent.recv_handler(&mad_snoop_priv->agent, NULL, 753 mad_recv_wc); 754 deref_snoop_agent(mad_snoop_priv); 755 spin_lock_irqsave(&qp_info->snoop_lock, flags); 756 } 757 spin_unlock_irqrestore(&qp_info->snoop_lock, flags); 758 } 759 760 static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid, 761 u16 pkey_index, u8 port_num, struct ib_wc *wc) 762 { 763 memset(wc, 0, sizeof *wc); 764 wc->wr_cqe = cqe; 765 wc->status = IB_WC_SUCCESS; 766 wc->opcode = IB_WC_RECV; 767 wc->pkey_index = pkey_index; 768 wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh); 769 wc->src_qp = IB_QP0; 770 wc->qp = qp; 771 wc->slid = slid; 772 wc->sl = 0; 773 wc->dlid_path_bits = 0; 774 wc->port_num = port_num; 775 } 776 777 static size_t mad_priv_size(const struct ib_mad_private *mp) 778 { 779 return sizeof(struct ib_mad_private) + mp->mad_size; 780 } 781 782 static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags) 783 { 784 size_t size = sizeof(struct ib_mad_private) + mad_size; 785 struct ib_mad_private *ret = kzalloc(size, flags); 786 787 if (ret) 788 ret->mad_size = mad_size; 789 790 return ret; 791 } 792 793 static size_t port_mad_size(const struct ib_mad_port_private *port_priv) 794 { 795 return rdma_max_mad_size(port_priv->device, port_priv->port_num); 796 } 797 798 static size_t mad_priv_dma_size(const struct ib_mad_private *mp) 799 { 800 return sizeof(struct ib_grh) + mp->mad_size; 801 } 802 803 /* 804 * Return 0 if SMP is to be sent 805 * Return 1 if SMP was consumed locally (whether or not solicited) 806 * Return < 0 if error 807 */ 808 static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, 809 struct ib_mad_send_wr_private *mad_send_wr) 810 { 811 int ret = 0; 812 struct ib_smp *smp = mad_send_wr->send_buf.mad; 813 struct opa_smp *opa_smp = (struct opa_smp *)smp; 814 unsigned long flags; 815 struct ib_mad_local_private *local; 816 struct ib_mad_private *mad_priv; 817 struct ib_mad_port_private *port_priv; 818 struct ib_mad_agent_private *recv_mad_agent = NULL; 819 struct ib_device *device = mad_agent_priv->agent.device; 820 u8 port_num; 821 struct ib_wc mad_wc; 822 struct ib_ud_wr *send_wr = &mad_send_wr->send_wr; 823 size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv); 824 u16 out_mad_pkey_index = 0; 825 u16 drslid; 826 bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device, 827 mad_agent_priv->qp_info->port_priv->port_num); 828 829 if (rdma_cap_ib_switch(device) && 830 smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 831 port_num = send_wr->port_num; 832 else 833 port_num = mad_agent_priv->agent.port_num; 834 835 /* 836 * Directed route handling starts if the initial LID routed part of 837 * a request or the ending LID routed part of a response is empty. 838 * If we are at the start of the LID routed part, don't update the 839 * hop_ptr or hop_cnt. See section 14.2.2, Vol 1 IB spec. 840 */ 841 if (opa && smp->class_version == OPA_SM_CLASS_VERSION) { 842 u32 opa_drslid; 843 844 trace_ib_mad_handle_out_opa_smi(opa_smp); 845 846 if ((opa_get_smp_direction(opa_smp) 847 ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) == 848 OPA_LID_PERMISSIVE && 849 opa_smi_handle_dr_smp_send(opa_smp, 850 rdma_cap_ib_switch(device), 851 port_num) == IB_SMI_DISCARD) { 852 ret = -EINVAL; 853 dev_err(&device->dev, "OPA Invalid directed route\n"); 854 goto out; 855 } 856 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid); 857 if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) && 858 opa_drslid & 0xffff0000) { 859 ret = -EINVAL; 860 dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n", 861 opa_drslid); 862 goto out; 863 } 864 drslid = (u16)(opa_drslid & 0x0000ffff); 865 866 /* Check to post send on QP or process locally */ 867 if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD && 868 opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD) 869 goto out; 870 } else { 871 trace_ib_mad_handle_out_ib_smi(smp); 872 873 if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) == 874 IB_LID_PERMISSIVE && 875 smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) == 876 IB_SMI_DISCARD) { 877 ret = -EINVAL; 878 dev_err(&device->dev, "Invalid directed route\n"); 879 goto out; 880 } 881 drslid = be16_to_cpu(smp->dr_slid); 882 883 /* Check to post send on QP or process locally */ 884 if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD && 885 smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD) 886 goto out; 887 } 888 889 local = kmalloc(sizeof *local, GFP_ATOMIC); 890 if (!local) { 891 ret = -ENOMEM; 892 goto out; 893 } 894 local->mad_priv = NULL; 895 local->recv_mad_agent = NULL; 896 mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC); 897 if (!mad_priv) { 898 ret = -ENOMEM; 899 kfree(local); 900 goto out; 901 } 902 903 build_smp_wc(mad_agent_priv->agent.qp, 904 send_wr->wr.wr_cqe, drslid, 905 send_wr->pkey_index, 906 send_wr->port_num, &mad_wc); 907 908 if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) { 909 mad_wc.byte_len = mad_send_wr->send_buf.hdr_len 910 + mad_send_wr->send_buf.data_len 911 + sizeof(struct ib_grh); 912 } 913 914 /* No GRH for DR SMP */ 915 ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL, 916 (const struct ib_mad *)smp, 917 (struct ib_mad *)mad_priv->mad, &mad_size, 918 &out_mad_pkey_index); 919 switch (ret) 920 { 921 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY: 922 if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) && 923 mad_agent_priv->agent.recv_handler) { 924 local->mad_priv = mad_priv; 925 local->recv_mad_agent = mad_agent_priv; 926 /* 927 * Reference MAD agent until receive 928 * side of local completion handled 929 */ 930 atomic_inc(&mad_agent_priv->refcount); 931 } else 932 kfree(mad_priv); 933 break; 934 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED: 935 kfree(mad_priv); 936 break; 937 case IB_MAD_RESULT_SUCCESS: 938 /* Treat like an incoming receive MAD */ 939 port_priv = ib_get_mad_port(mad_agent_priv->agent.device, 940 mad_agent_priv->agent.port_num); 941 if (port_priv) { 942 memcpy(mad_priv->mad, smp, mad_priv->mad_size); 943 recv_mad_agent = find_mad_agent(port_priv, 944 (const struct ib_mad_hdr *)mad_priv->mad); 945 } 946 if (!port_priv || !recv_mad_agent) { 947 /* 948 * No receiving agent so drop packet and 949 * generate send completion. 950 */ 951 kfree(mad_priv); 952 break; 953 } 954 local->mad_priv = mad_priv; 955 local->recv_mad_agent = recv_mad_agent; 956 break; 957 default: 958 kfree(mad_priv); 959 kfree(local); 960 ret = -EINVAL; 961 goto out; 962 } 963 964 local->mad_send_wr = mad_send_wr; 965 if (opa) { 966 local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index; 967 local->return_wc_byte_len = mad_size; 968 } 969 /* Reference MAD agent until send side of local completion handled */ 970 atomic_inc(&mad_agent_priv->refcount); 971 /* Queue local completion to local list */ 972 spin_lock_irqsave(&mad_agent_priv->lock, flags); 973 list_add_tail(&local->completion_list, &mad_agent_priv->local_list); 974 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 975 queue_work(mad_agent_priv->qp_info->port_priv->wq, 976 &mad_agent_priv->local_work); 977 ret = 1; 978 out: 979 return ret; 980 } 981 982 static int get_pad_size(int hdr_len, int data_len, size_t mad_size) 983 { 984 int seg_size, pad; 985 986 seg_size = mad_size - hdr_len; 987 if (data_len && seg_size) { 988 pad = seg_size - data_len % seg_size; 989 return pad == seg_size ? 0 : pad; 990 } else 991 return seg_size; 992 } 993 994 static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr) 995 { 996 struct ib_rmpp_segment *s, *t; 997 998 list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) { 999 list_del(&s->list); 1000 kfree(s); 1001 } 1002 } 1003 1004 static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr, 1005 size_t mad_size, gfp_t gfp_mask) 1006 { 1007 struct ib_mad_send_buf *send_buf = &send_wr->send_buf; 1008 struct ib_rmpp_mad *rmpp_mad = send_buf->mad; 1009 struct ib_rmpp_segment *seg = NULL; 1010 int left, seg_size, pad; 1011 1012 send_buf->seg_size = mad_size - send_buf->hdr_len; 1013 send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR; 1014 seg_size = send_buf->seg_size; 1015 pad = send_wr->pad; 1016 1017 /* Allocate data segments. */ 1018 for (left = send_buf->data_len + pad; left > 0; left -= seg_size) { 1019 seg = kmalloc(sizeof (*seg) + seg_size, gfp_mask); 1020 if (!seg) { 1021 free_send_rmpp_list(send_wr); 1022 return -ENOMEM; 1023 } 1024 seg->num = ++send_buf->seg_count; 1025 list_add_tail(&seg->list, &send_wr->rmpp_list); 1026 } 1027 1028 /* Zero any padding */ 1029 if (pad) 1030 memset(seg->data + seg_size - pad, 0, pad); 1031 1032 rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv-> 1033 agent.rmpp_version; 1034 rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA; 1035 ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE); 1036 1037 send_wr->cur_seg = container_of(send_wr->rmpp_list.next, 1038 struct ib_rmpp_segment, list); 1039 send_wr->last_ack_seg = send_wr->cur_seg; 1040 return 0; 1041 } 1042 1043 int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent) 1044 { 1045 return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP); 1046 } 1047 EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent); 1048 1049 struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent, 1050 u32 remote_qpn, u16 pkey_index, 1051 int rmpp_active, 1052 int hdr_len, int data_len, 1053 gfp_t gfp_mask, 1054 u8 base_version) 1055 { 1056 struct ib_mad_agent_private *mad_agent_priv; 1057 struct ib_mad_send_wr_private *mad_send_wr; 1058 int pad, message_size, ret, size; 1059 void *buf; 1060 size_t mad_size; 1061 bool opa; 1062 1063 mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private, 1064 agent); 1065 1066 opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num); 1067 1068 if (opa && base_version == OPA_MGMT_BASE_VERSION) 1069 mad_size = sizeof(struct opa_mad); 1070 else 1071 mad_size = sizeof(struct ib_mad); 1072 1073 pad = get_pad_size(hdr_len, data_len, mad_size); 1074 message_size = hdr_len + data_len + pad; 1075 1076 if (ib_mad_kernel_rmpp_agent(mad_agent)) { 1077 if (!rmpp_active && message_size > mad_size) 1078 return ERR_PTR(-EINVAL); 1079 } else 1080 if (rmpp_active || message_size > mad_size) 1081 return ERR_PTR(-EINVAL); 1082 1083 size = rmpp_active ? hdr_len : mad_size; 1084 buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask); 1085 if (!buf) 1086 return ERR_PTR(-ENOMEM); 1087 1088 mad_send_wr = buf + size; 1089 INIT_LIST_HEAD(&mad_send_wr->rmpp_list); 1090 mad_send_wr->send_buf.mad = buf; 1091 mad_send_wr->send_buf.hdr_len = hdr_len; 1092 mad_send_wr->send_buf.data_len = data_len; 1093 mad_send_wr->pad = pad; 1094 1095 mad_send_wr->mad_agent_priv = mad_agent_priv; 1096 mad_send_wr->sg_list[0].length = hdr_len; 1097 mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey; 1098 1099 /* OPA MADs don't have to be the full 2048 bytes */ 1100 if (opa && base_version == OPA_MGMT_BASE_VERSION && 1101 data_len < mad_size - hdr_len) 1102 mad_send_wr->sg_list[1].length = data_len; 1103 else 1104 mad_send_wr->sg_list[1].length = mad_size - hdr_len; 1105 1106 mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey; 1107 1108 mad_send_wr->mad_list.cqe.done = ib_mad_send_done; 1109 1110 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe; 1111 mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list; 1112 mad_send_wr->send_wr.wr.num_sge = 2; 1113 mad_send_wr->send_wr.wr.opcode = IB_WR_SEND; 1114 mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED; 1115 mad_send_wr->send_wr.remote_qpn = remote_qpn; 1116 mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY; 1117 mad_send_wr->send_wr.pkey_index = pkey_index; 1118 1119 if (rmpp_active) { 1120 ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask); 1121 if (ret) { 1122 kfree(buf); 1123 return ERR_PTR(ret); 1124 } 1125 } 1126 1127 mad_send_wr->send_buf.mad_agent = mad_agent; 1128 atomic_inc(&mad_agent_priv->refcount); 1129 return &mad_send_wr->send_buf; 1130 } 1131 EXPORT_SYMBOL(ib_create_send_mad); 1132 1133 int ib_get_mad_data_offset(u8 mgmt_class) 1134 { 1135 if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM) 1136 return IB_MGMT_SA_HDR; 1137 else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) || 1138 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) || 1139 (mgmt_class == IB_MGMT_CLASS_BIS)) 1140 return IB_MGMT_DEVICE_HDR; 1141 else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) && 1142 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)) 1143 return IB_MGMT_VENDOR_HDR; 1144 else 1145 return IB_MGMT_MAD_HDR; 1146 } 1147 EXPORT_SYMBOL(ib_get_mad_data_offset); 1148 1149 int ib_is_mad_class_rmpp(u8 mgmt_class) 1150 { 1151 if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) || 1152 (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) || 1153 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) || 1154 (mgmt_class == IB_MGMT_CLASS_BIS) || 1155 ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) && 1156 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))) 1157 return 1; 1158 return 0; 1159 } 1160 EXPORT_SYMBOL(ib_is_mad_class_rmpp); 1161 1162 void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num) 1163 { 1164 struct ib_mad_send_wr_private *mad_send_wr; 1165 struct list_head *list; 1166 1167 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private, 1168 send_buf); 1169 list = &mad_send_wr->cur_seg->list; 1170 1171 if (mad_send_wr->cur_seg->num < seg_num) { 1172 list_for_each_entry(mad_send_wr->cur_seg, list, list) 1173 if (mad_send_wr->cur_seg->num == seg_num) 1174 break; 1175 } else if (mad_send_wr->cur_seg->num > seg_num) { 1176 list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list) 1177 if (mad_send_wr->cur_seg->num == seg_num) 1178 break; 1179 } 1180 return mad_send_wr->cur_seg->data; 1181 } 1182 EXPORT_SYMBOL(ib_get_rmpp_segment); 1183 1184 static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr) 1185 { 1186 if (mad_send_wr->send_buf.seg_count) 1187 return ib_get_rmpp_segment(&mad_send_wr->send_buf, 1188 mad_send_wr->seg_num); 1189 else 1190 return mad_send_wr->send_buf.mad + 1191 mad_send_wr->send_buf.hdr_len; 1192 } 1193 1194 void ib_free_send_mad(struct ib_mad_send_buf *send_buf) 1195 { 1196 struct ib_mad_agent_private *mad_agent_priv; 1197 struct ib_mad_send_wr_private *mad_send_wr; 1198 1199 mad_agent_priv = container_of(send_buf->mad_agent, 1200 struct ib_mad_agent_private, agent); 1201 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private, 1202 send_buf); 1203 1204 free_send_rmpp_list(mad_send_wr); 1205 kfree(send_buf->mad); 1206 deref_mad_agent(mad_agent_priv); 1207 } 1208 EXPORT_SYMBOL(ib_free_send_mad); 1209 1210 int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr) 1211 { 1212 struct ib_mad_qp_info *qp_info; 1213 struct list_head *list; 1214 struct ib_mad_agent *mad_agent; 1215 struct ib_sge *sge; 1216 unsigned long flags; 1217 int ret; 1218 1219 /* Set WR ID to find mad_send_wr upon completion */ 1220 qp_info = mad_send_wr->mad_agent_priv->qp_info; 1221 mad_send_wr->mad_list.mad_queue = &qp_info->send_queue; 1222 mad_send_wr->mad_list.cqe.done = ib_mad_send_done; 1223 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe; 1224 1225 mad_agent = mad_send_wr->send_buf.mad_agent; 1226 sge = mad_send_wr->sg_list; 1227 sge[0].addr = ib_dma_map_single(mad_agent->device, 1228 mad_send_wr->send_buf.mad, 1229 sge[0].length, 1230 DMA_TO_DEVICE); 1231 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr))) 1232 return -ENOMEM; 1233 1234 mad_send_wr->header_mapping = sge[0].addr; 1235 1236 sge[1].addr = ib_dma_map_single(mad_agent->device, 1237 ib_get_payload(mad_send_wr), 1238 sge[1].length, 1239 DMA_TO_DEVICE); 1240 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) { 1241 ib_dma_unmap_single(mad_agent->device, 1242 mad_send_wr->header_mapping, 1243 sge[0].length, DMA_TO_DEVICE); 1244 return -ENOMEM; 1245 } 1246 mad_send_wr->payload_mapping = sge[1].addr; 1247 1248 spin_lock_irqsave(&qp_info->send_queue.lock, flags); 1249 if (qp_info->send_queue.count < qp_info->send_queue.max_active) { 1250 trace_ib_mad_ib_send_mad(mad_send_wr, qp_info); 1251 ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr, 1252 NULL); 1253 list = &qp_info->send_queue.list; 1254 } else { 1255 ret = 0; 1256 list = &qp_info->overflow_list; 1257 } 1258 1259 if (!ret) { 1260 qp_info->send_queue.count++; 1261 list_add_tail(&mad_send_wr->mad_list.list, list); 1262 } 1263 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags); 1264 if (ret) { 1265 ib_dma_unmap_single(mad_agent->device, 1266 mad_send_wr->header_mapping, 1267 sge[0].length, DMA_TO_DEVICE); 1268 ib_dma_unmap_single(mad_agent->device, 1269 mad_send_wr->payload_mapping, 1270 sge[1].length, DMA_TO_DEVICE); 1271 } 1272 return ret; 1273 } 1274 1275 /* 1276 * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated 1277 * with the registered client 1278 */ 1279 int ib_post_send_mad(struct ib_mad_send_buf *send_buf, 1280 struct ib_mad_send_buf **bad_send_buf) 1281 { 1282 struct ib_mad_agent_private *mad_agent_priv; 1283 struct ib_mad_send_buf *next_send_buf; 1284 struct ib_mad_send_wr_private *mad_send_wr; 1285 unsigned long flags; 1286 int ret = -EINVAL; 1287 1288 /* Walk list of send WRs and post each on send list */ 1289 for (; send_buf; send_buf = next_send_buf) { 1290 mad_send_wr = container_of(send_buf, 1291 struct ib_mad_send_wr_private, 1292 send_buf); 1293 mad_agent_priv = mad_send_wr->mad_agent_priv; 1294 1295 ret = ib_mad_enforce_security(mad_agent_priv, 1296 mad_send_wr->send_wr.pkey_index); 1297 if (ret) 1298 goto error; 1299 1300 if (!send_buf->mad_agent->send_handler || 1301 (send_buf->timeout_ms && 1302 !send_buf->mad_agent->recv_handler)) { 1303 ret = -EINVAL; 1304 goto error; 1305 } 1306 1307 if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) { 1308 if (mad_agent_priv->agent.rmpp_version) { 1309 ret = -EINVAL; 1310 goto error; 1311 } 1312 } 1313 1314 /* 1315 * Save pointer to next work request to post in case the 1316 * current one completes, and the user modifies the work 1317 * request associated with the completion 1318 */ 1319 next_send_buf = send_buf->next; 1320 mad_send_wr->send_wr.ah = send_buf->ah; 1321 1322 if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class == 1323 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 1324 ret = handle_outgoing_dr_smp(mad_agent_priv, 1325 mad_send_wr); 1326 if (ret < 0) /* error */ 1327 goto error; 1328 else if (ret == 1) /* locally consumed */ 1329 continue; 1330 } 1331 1332 mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid; 1333 /* Timeout will be updated after send completes */ 1334 mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms); 1335 mad_send_wr->max_retries = send_buf->retries; 1336 mad_send_wr->retries_left = send_buf->retries; 1337 send_buf->retries = 0; 1338 /* Reference for work request to QP + response */ 1339 mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0); 1340 mad_send_wr->status = IB_WC_SUCCESS; 1341 1342 /* Reference MAD agent until send completes */ 1343 atomic_inc(&mad_agent_priv->refcount); 1344 spin_lock_irqsave(&mad_agent_priv->lock, flags); 1345 list_add_tail(&mad_send_wr->agent_list, 1346 &mad_agent_priv->send_list); 1347 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 1348 1349 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 1350 ret = ib_send_rmpp_mad(mad_send_wr); 1351 if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED) 1352 ret = ib_send_mad(mad_send_wr); 1353 } else 1354 ret = ib_send_mad(mad_send_wr); 1355 if (ret < 0) { 1356 /* Fail send request */ 1357 spin_lock_irqsave(&mad_agent_priv->lock, flags); 1358 list_del(&mad_send_wr->agent_list); 1359 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 1360 atomic_dec(&mad_agent_priv->refcount); 1361 goto error; 1362 } 1363 } 1364 return 0; 1365 error: 1366 if (bad_send_buf) 1367 *bad_send_buf = send_buf; 1368 return ret; 1369 } 1370 EXPORT_SYMBOL(ib_post_send_mad); 1371 1372 /* 1373 * ib_free_recv_mad - Returns data buffers used to receive 1374 * a MAD to the access layer 1375 */ 1376 void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc) 1377 { 1378 struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf; 1379 struct ib_mad_private_header *mad_priv_hdr; 1380 struct ib_mad_private *priv; 1381 struct list_head free_list; 1382 1383 INIT_LIST_HEAD(&free_list); 1384 list_splice_init(&mad_recv_wc->rmpp_list, &free_list); 1385 1386 list_for_each_entry_safe(mad_recv_buf, temp_recv_buf, 1387 &free_list, list) { 1388 mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc, 1389 recv_buf); 1390 mad_priv_hdr = container_of(mad_recv_wc, 1391 struct ib_mad_private_header, 1392 recv_wc); 1393 priv = container_of(mad_priv_hdr, struct ib_mad_private, 1394 header); 1395 kfree(priv); 1396 } 1397 } 1398 EXPORT_SYMBOL(ib_free_recv_mad); 1399 1400 static int method_in_use(struct ib_mad_mgmt_method_table **method, 1401 struct ib_mad_reg_req *mad_reg_req) 1402 { 1403 int i; 1404 1405 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) { 1406 if ((*method)->agent[i]) { 1407 pr_err("Method %d already in use\n", i); 1408 return -EINVAL; 1409 } 1410 } 1411 return 0; 1412 } 1413 1414 static int allocate_method_table(struct ib_mad_mgmt_method_table **method) 1415 { 1416 /* Allocate management method table */ 1417 *method = kzalloc(sizeof **method, GFP_ATOMIC); 1418 return (*method) ? 0 : (-ENOMEM); 1419 } 1420 1421 /* 1422 * Check to see if there are any methods still in use 1423 */ 1424 static int check_method_table(struct ib_mad_mgmt_method_table *method) 1425 { 1426 int i; 1427 1428 for (i = 0; i < IB_MGMT_MAX_METHODS; i++) 1429 if (method->agent[i]) 1430 return 1; 1431 return 0; 1432 } 1433 1434 /* 1435 * Check to see if there are any method tables for this class still in use 1436 */ 1437 static int check_class_table(struct ib_mad_mgmt_class_table *class) 1438 { 1439 int i; 1440 1441 for (i = 0; i < MAX_MGMT_CLASS; i++) 1442 if (class->method_table[i]) 1443 return 1; 1444 return 0; 1445 } 1446 1447 static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class) 1448 { 1449 int i; 1450 1451 for (i = 0; i < MAX_MGMT_OUI; i++) 1452 if (vendor_class->method_table[i]) 1453 return 1; 1454 return 0; 1455 } 1456 1457 static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class, 1458 const char *oui) 1459 { 1460 int i; 1461 1462 for (i = 0; i < MAX_MGMT_OUI; i++) 1463 /* Is there matching OUI for this vendor class ? */ 1464 if (!memcmp(vendor_class->oui[i], oui, 3)) 1465 return i; 1466 1467 return -1; 1468 } 1469 1470 static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor) 1471 { 1472 int i; 1473 1474 for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++) 1475 if (vendor->vendor_class[i]) 1476 return 1; 1477 1478 return 0; 1479 } 1480 1481 static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method, 1482 struct ib_mad_agent_private *agent) 1483 { 1484 int i; 1485 1486 /* Remove any methods for this mad agent */ 1487 for (i = 0; i < IB_MGMT_MAX_METHODS; i++) { 1488 if (method->agent[i] == agent) { 1489 method->agent[i] = NULL; 1490 } 1491 } 1492 } 1493 1494 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req, 1495 struct ib_mad_agent_private *agent_priv, 1496 u8 mgmt_class) 1497 { 1498 struct ib_mad_port_private *port_priv; 1499 struct ib_mad_mgmt_class_table **class; 1500 struct ib_mad_mgmt_method_table **method; 1501 int i, ret; 1502 1503 port_priv = agent_priv->qp_info->port_priv; 1504 class = &port_priv->version[mad_reg_req->mgmt_class_version].class; 1505 if (!*class) { 1506 /* Allocate management class table for "new" class version */ 1507 *class = kzalloc(sizeof **class, GFP_ATOMIC); 1508 if (!*class) { 1509 ret = -ENOMEM; 1510 goto error1; 1511 } 1512 1513 /* Allocate method table for this management class */ 1514 method = &(*class)->method_table[mgmt_class]; 1515 if ((ret = allocate_method_table(method))) 1516 goto error2; 1517 } else { 1518 method = &(*class)->method_table[mgmt_class]; 1519 if (!*method) { 1520 /* Allocate method table for this management class */ 1521 if ((ret = allocate_method_table(method))) 1522 goto error1; 1523 } 1524 } 1525 1526 /* Now, make sure methods are not already in use */ 1527 if (method_in_use(method, mad_reg_req)) 1528 goto error3; 1529 1530 /* Finally, add in methods being registered */ 1531 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) 1532 (*method)->agent[i] = agent_priv; 1533 1534 return 0; 1535 1536 error3: 1537 /* Remove any methods for this mad agent */ 1538 remove_methods_mad_agent(*method, agent_priv); 1539 /* Now, check to see if there are any methods in use */ 1540 if (!check_method_table(*method)) { 1541 /* If not, release management method table */ 1542 kfree(*method); 1543 *method = NULL; 1544 } 1545 ret = -EINVAL; 1546 goto error1; 1547 error2: 1548 kfree(*class); 1549 *class = NULL; 1550 error1: 1551 return ret; 1552 } 1553 1554 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req, 1555 struct ib_mad_agent_private *agent_priv) 1556 { 1557 struct ib_mad_port_private *port_priv; 1558 struct ib_mad_mgmt_vendor_class_table **vendor_table; 1559 struct ib_mad_mgmt_vendor_class_table *vendor = NULL; 1560 struct ib_mad_mgmt_vendor_class *vendor_class = NULL; 1561 struct ib_mad_mgmt_method_table **method; 1562 int i, ret = -ENOMEM; 1563 u8 vclass; 1564 1565 /* "New" vendor (with OUI) class */ 1566 vclass = vendor_class_index(mad_reg_req->mgmt_class); 1567 port_priv = agent_priv->qp_info->port_priv; 1568 vendor_table = &port_priv->version[ 1569 mad_reg_req->mgmt_class_version].vendor; 1570 if (!*vendor_table) { 1571 /* Allocate mgmt vendor class table for "new" class version */ 1572 vendor = kzalloc(sizeof *vendor, GFP_ATOMIC); 1573 if (!vendor) 1574 goto error1; 1575 1576 *vendor_table = vendor; 1577 } 1578 if (!(*vendor_table)->vendor_class[vclass]) { 1579 /* Allocate table for this management vendor class */ 1580 vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC); 1581 if (!vendor_class) 1582 goto error2; 1583 1584 (*vendor_table)->vendor_class[vclass] = vendor_class; 1585 } 1586 for (i = 0; i < MAX_MGMT_OUI; i++) { 1587 /* Is there matching OUI for this vendor class ? */ 1588 if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i], 1589 mad_reg_req->oui, 3)) { 1590 method = &(*vendor_table)->vendor_class[ 1591 vclass]->method_table[i]; 1592 if (!*method) 1593 goto error3; 1594 goto check_in_use; 1595 } 1596 } 1597 for (i = 0; i < MAX_MGMT_OUI; i++) { 1598 /* OUI slot available ? */ 1599 if (!is_vendor_oui((*vendor_table)->vendor_class[ 1600 vclass]->oui[i])) { 1601 method = &(*vendor_table)->vendor_class[ 1602 vclass]->method_table[i]; 1603 /* Allocate method table for this OUI */ 1604 if (!*method) { 1605 ret = allocate_method_table(method); 1606 if (ret) 1607 goto error3; 1608 } 1609 memcpy((*vendor_table)->vendor_class[vclass]->oui[i], 1610 mad_reg_req->oui, 3); 1611 goto check_in_use; 1612 } 1613 } 1614 dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n"); 1615 goto error3; 1616 1617 check_in_use: 1618 /* Now, make sure methods are not already in use */ 1619 if (method_in_use(method, mad_reg_req)) 1620 goto error4; 1621 1622 /* Finally, add in methods being registered */ 1623 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) 1624 (*method)->agent[i] = agent_priv; 1625 1626 return 0; 1627 1628 error4: 1629 /* Remove any methods for this mad agent */ 1630 remove_methods_mad_agent(*method, agent_priv); 1631 /* Now, check to see if there are any methods in use */ 1632 if (!check_method_table(*method)) { 1633 /* If not, release management method table */ 1634 kfree(*method); 1635 *method = NULL; 1636 } 1637 ret = -EINVAL; 1638 error3: 1639 if (vendor_class) { 1640 (*vendor_table)->vendor_class[vclass] = NULL; 1641 kfree(vendor_class); 1642 } 1643 error2: 1644 if (vendor) { 1645 *vendor_table = NULL; 1646 kfree(vendor); 1647 } 1648 error1: 1649 return ret; 1650 } 1651 1652 static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv) 1653 { 1654 struct ib_mad_port_private *port_priv; 1655 struct ib_mad_mgmt_class_table *class; 1656 struct ib_mad_mgmt_method_table *method; 1657 struct ib_mad_mgmt_vendor_class_table *vendor; 1658 struct ib_mad_mgmt_vendor_class *vendor_class; 1659 int index; 1660 u8 mgmt_class; 1661 1662 /* 1663 * Was MAD registration request supplied 1664 * with original registration ? 1665 */ 1666 if (!agent_priv->reg_req) { 1667 goto out; 1668 } 1669 1670 port_priv = agent_priv->qp_info->port_priv; 1671 mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class); 1672 class = port_priv->version[ 1673 agent_priv->reg_req->mgmt_class_version].class; 1674 if (!class) 1675 goto vendor_check; 1676 1677 method = class->method_table[mgmt_class]; 1678 if (method) { 1679 /* Remove any methods for this mad agent */ 1680 remove_methods_mad_agent(method, agent_priv); 1681 /* Now, check to see if there are any methods still in use */ 1682 if (!check_method_table(method)) { 1683 /* If not, release management method table */ 1684 kfree(method); 1685 class->method_table[mgmt_class] = NULL; 1686 /* Any management classes left ? */ 1687 if (!check_class_table(class)) { 1688 /* If not, release management class table */ 1689 kfree(class); 1690 port_priv->version[ 1691 agent_priv->reg_req-> 1692 mgmt_class_version].class = NULL; 1693 } 1694 } 1695 } 1696 1697 vendor_check: 1698 if (!is_vendor_class(mgmt_class)) 1699 goto out; 1700 1701 /* normalize mgmt_class to vendor range 2 */ 1702 mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class); 1703 vendor = port_priv->version[ 1704 agent_priv->reg_req->mgmt_class_version].vendor; 1705 1706 if (!vendor) 1707 goto out; 1708 1709 vendor_class = vendor->vendor_class[mgmt_class]; 1710 if (vendor_class) { 1711 index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui); 1712 if (index < 0) 1713 goto out; 1714 method = vendor_class->method_table[index]; 1715 if (method) { 1716 /* Remove any methods for this mad agent */ 1717 remove_methods_mad_agent(method, agent_priv); 1718 /* 1719 * Now, check to see if there are 1720 * any methods still in use 1721 */ 1722 if (!check_method_table(method)) { 1723 /* If not, release management method table */ 1724 kfree(method); 1725 vendor_class->method_table[index] = NULL; 1726 memset(vendor_class->oui[index], 0, 3); 1727 /* Any OUIs left ? */ 1728 if (!check_vendor_class(vendor_class)) { 1729 /* If not, release vendor class table */ 1730 kfree(vendor_class); 1731 vendor->vendor_class[mgmt_class] = NULL; 1732 /* Any other vendor classes left ? */ 1733 if (!check_vendor_table(vendor)) { 1734 kfree(vendor); 1735 port_priv->version[ 1736 agent_priv->reg_req-> 1737 mgmt_class_version]. 1738 vendor = NULL; 1739 } 1740 } 1741 } 1742 } 1743 } 1744 1745 out: 1746 return; 1747 } 1748 1749 static struct ib_mad_agent_private * 1750 find_mad_agent(struct ib_mad_port_private *port_priv, 1751 const struct ib_mad_hdr *mad_hdr) 1752 { 1753 struct ib_mad_agent_private *mad_agent = NULL; 1754 unsigned long flags; 1755 1756 if (ib_response_mad(mad_hdr)) { 1757 u32 hi_tid; 1758 1759 /* 1760 * Routing is based on high 32 bits of transaction ID 1761 * of MAD. 1762 */ 1763 hi_tid = be64_to_cpu(mad_hdr->tid) >> 32; 1764 rcu_read_lock(); 1765 mad_agent = xa_load(&ib_mad_clients, hi_tid); 1766 if (mad_agent && !atomic_inc_not_zero(&mad_agent->refcount)) 1767 mad_agent = NULL; 1768 rcu_read_unlock(); 1769 } else { 1770 struct ib_mad_mgmt_class_table *class; 1771 struct ib_mad_mgmt_method_table *method; 1772 struct ib_mad_mgmt_vendor_class_table *vendor; 1773 struct ib_mad_mgmt_vendor_class *vendor_class; 1774 const struct ib_vendor_mad *vendor_mad; 1775 int index; 1776 1777 spin_lock_irqsave(&port_priv->reg_lock, flags); 1778 /* 1779 * Routing is based on version, class, and method 1780 * For "newer" vendor MADs, also based on OUI 1781 */ 1782 if (mad_hdr->class_version >= MAX_MGMT_VERSION) 1783 goto out; 1784 if (!is_vendor_class(mad_hdr->mgmt_class)) { 1785 class = port_priv->version[ 1786 mad_hdr->class_version].class; 1787 if (!class) 1788 goto out; 1789 if (convert_mgmt_class(mad_hdr->mgmt_class) >= 1790 ARRAY_SIZE(class->method_table)) 1791 goto out; 1792 method = class->method_table[convert_mgmt_class( 1793 mad_hdr->mgmt_class)]; 1794 if (method) 1795 mad_agent = method->agent[mad_hdr->method & 1796 ~IB_MGMT_METHOD_RESP]; 1797 } else { 1798 vendor = port_priv->version[ 1799 mad_hdr->class_version].vendor; 1800 if (!vendor) 1801 goto out; 1802 vendor_class = vendor->vendor_class[vendor_class_index( 1803 mad_hdr->mgmt_class)]; 1804 if (!vendor_class) 1805 goto out; 1806 /* Find matching OUI */ 1807 vendor_mad = (const struct ib_vendor_mad *)mad_hdr; 1808 index = find_vendor_oui(vendor_class, vendor_mad->oui); 1809 if (index == -1) 1810 goto out; 1811 method = vendor_class->method_table[index]; 1812 if (method) { 1813 mad_agent = method->agent[mad_hdr->method & 1814 ~IB_MGMT_METHOD_RESP]; 1815 } 1816 } 1817 if (mad_agent) 1818 atomic_inc(&mad_agent->refcount); 1819 out: 1820 spin_unlock_irqrestore(&port_priv->reg_lock, flags); 1821 } 1822 1823 if (mad_agent && !mad_agent->agent.recv_handler) { 1824 dev_notice(&port_priv->device->dev, 1825 "No receive handler for client %p on port %d\n", 1826 &mad_agent->agent, port_priv->port_num); 1827 deref_mad_agent(mad_agent); 1828 mad_agent = NULL; 1829 } 1830 1831 return mad_agent; 1832 } 1833 1834 static int validate_mad(const struct ib_mad_hdr *mad_hdr, 1835 const struct ib_mad_qp_info *qp_info, 1836 bool opa) 1837 { 1838 int valid = 0; 1839 u32 qp_num = qp_info->qp->qp_num; 1840 1841 /* Make sure MAD base version is understood */ 1842 if (mad_hdr->base_version != IB_MGMT_BASE_VERSION && 1843 (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) { 1844 pr_err("MAD received with unsupported base version %d %s\n", 1845 mad_hdr->base_version, opa ? "(opa)" : ""); 1846 goto out; 1847 } 1848 1849 /* Filter SMI packets sent to other than QP0 */ 1850 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) || 1851 (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 1852 if (qp_num == 0) 1853 valid = 1; 1854 } else { 1855 /* CM attributes other than ClassPortInfo only use Send method */ 1856 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) && 1857 (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) && 1858 (mad_hdr->method != IB_MGMT_METHOD_SEND)) 1859 goto out; 1860 /* Filter GSI packets sent to QP0 */ 1861 if (qp_num != 0) 1862 valid = 1; 1863 } 1864 1865 out: 1866 return valid; 1867 } 1868 1869 static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv, 1870 const struct ib_mad_hdr *mad_hdr) 1871 { 1872 struct ib_rmpp_mad *rmpp_mad; 1873 1874 rmpp_mad = (struct ib_rmpp_mad *)mad_hdr; 1875 return !mad_agent_priv->agent.rmpp_version || 1876 !ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) || 1877 !(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & 1878 IB_MGMT_RMPP_FLAG_ACTIVE) || 1879 (rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA); 1880 } 1881 1882 static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr, 1883 const struct ib_mad_recv_wc *rwc) 1884 { 1885 return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class == 1886 rwc->recv_buf.mad->mad_hdr.mgmt_class; 1887 } 1888 1889 static inline int rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv, 1890 const struct ib_mad_send_wr_private *wr, 1891 const struct ib_mad_recv_wc *rwc ) 1892 { 1893 struct rdma_ah_attr attr; 1894 u8 send_resp, rcv_resp; 1895 union ib_gid sgid; 1896 struct ib_device *device = mad_agent_priv->agent.device; 1897 u8 port_num = mad_agent_priv->agent.port_num; 1898 u8 lmc; 1899 bool has_grh; 1900 1901 send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad); 1902 rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr); 1903 1904 if (send_resp == rcv_resp) 1905 /* both requests, or both responses. GIDs different */ 1906 return 0; 1907 1908 if (rdma_query_ah(wr->send_buf.ah, &attr)) 1909 /* Assume not equal, to avoid false positives. */ 1910 return 0; 1911 1912 has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH); 1913 if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH)) 1914 /* one has GID, other does not. Assume different */ 1915 return 0; 1916 1917 if (!send_resp && rcv_resp) { 1918 /* is request/response. */ 1919 if (!has_grh) { 1920 if (ib_get_cached_lmc(device, port_num, &lmc)) 1921 return 0; 1922 return (!lmc || !((rdma_ah_get_path_bits(&attr) ^ 1923 rwc->wc->dlid_path_bits) & 1924 ((1 << lmc) - 1))); 1925 } else { 1926 const struct ib_global_route *grh = 1927 rdma_ah_read_grh(&attr); 1928 1929 if (rdma_query_gid(device, port_num, 1930 grh->sgid_index, &sgid)) 1931 return 0; 1932 return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw, 1933 16); 1934 } 1935 } 1936 1937 if (!has_grh) 1938 return rdma_ah_get_dlid(&attr) == rwc->wc->slid; 1939 else 1940 return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw, 1941 rwc->recv_buf.grh->sgid.raw, 1942 16); 1943 } 1944 1945 static inline int is_direct(u8 class) 1946 { 1947 return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE); 1948 } 1949 1950 struct ib_mad_send_wr_private* 1951 ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv, 1952 const struct ib_mad_recv_wc *wc) 1953 { 1954 struct ib_mad_send_wr_private *wr; 1955 const struct ib_mad_hdr *mad_hdr; 1956 1957 mad_hdr = &wc->recv_buf.mad->mad_hdr; 1958 1959 list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) { 1960 if ((wr->tid == mad_hdr->tid) && 1961 rcv_has_same_class(wr, wc) && 1962 /* 1963 * Don't check GID for direct routed MADs. 1964 * These might have permissive LIDs. 1965 */ 1966 (is_direct(mad_hdr->mgmt_class) || 1967 rcv_has_same_gid(mad_agent_priv, wr, wc))) 1968 return (wr->status == IB_WC_SUCCESS) ? wr : NULL; 1969 } 1970 1971 /* 1972 * It's possible to receive the response before we've 1973 * been notified that the send has completed 1974 */ 1975 list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) { 1976 if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) && 1977 wr->tid == mad_hdr->tid && 1978 wr->timeout && 1979 rcv_has_same_class(wr, wc) && 1980 /* 1981 * Don't check GID for direct routed MADs. 1982 * These might have permissive LIDs. 1983 */ 1984 (is_direct(mad_hdr->mgmt_class) || 1985 rcv_has_same_gid(mad_agent_priv, wr, wc))) 1986 /* Verify request has not been canceled */ 1987 return (wr->status == IB_WC_SUCCESS) ? wr : NULL; 1988 } 1989 return NULL; 1990 } 1991 1992 void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr) 1993 { 1994 mad_send_wr->timeout = 0; 1995 if (mad_send_wr->refcount == 1) 1996 list_move_tail(&mad_send_wr->agent_list, 1997 &mad_send_wr->mad_agent_priv->done_list); 1998 } 1999 2000 static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, 2001 struct ib_mad_recv_wc *mad_recv_wc) 2002 { 2003 struct ib_mad_send_wr_private *mad_send_wr; 2004 struct ib_mad_send_wc mad_send_wc; 2005 unsigned long flags; 2006 int ret; 2007 2008 INIT_LIST_HEAD(&mad_recv_wc->rmpp_list); 2009 ret = ib_mad_enforce_security(mad_agent_priv, 2010 mad_recv_wc->wc->pkey_index); 2011 if (ret) { 2012 ib_free_recv_mad(mad_recv_wc); 2013 deref_mad_agent(mad_agent_priv); 2014 return; 2015 } 2016 2017 list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list); 2018 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 2019 mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv, 2020 mad_recv_wc); 2021 if (!mad_recv_wc) { 2022 deref_mad_agent(mad_agent_priv); 2023 return; 2024 } 2025 } 2026 2027 /* Complete corresponding request */ 2028 if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) { 2029 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2030 mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc); 2031 if (!mad_send_wr) { 2032 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2033 if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) 2034 && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class) 2035 && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr) 2036 & IB_MGMT_RMPP_FLAG_ACTIVE)) { 2037 /* user rmpp is in effect 2038 * and this is an active RMPP MAD 2039 */ 2040 mad_agent_priv->agent.recv_handler( 2041 &mad_agent_priv->agent, NULL, 2042 mad_recv_wc); 2043 atomic_dec(&mad_agent_priv->refcount); 2044 } else { 2045 /* not user rmpp, revert to normal behavior and 2046 * drop the mad */ 2047 ib_free_recv_mad(mad_recv_wc); 2048 deref_mad_agent(mad_agent_priv); 2049 return; 2050 } 2051 } else { 2052 ib_mark_mad_done(mad_send_wr); 2053 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2054 2055 /* Defined behavior is to complete response before request */ 2056 mad_agent_priv->agent.recv_handler( 2057 &mad_agent_priv->agent, 2058 &mad_send_wr->send_buf, 2059 mad_recv_wc); 2060 atomic_dec(&mad_agent_priv->refcount); 2061 2062 mad_send_wc.status = IB_WC_SUCCESS; 2063 mad_send_wc.vendor_err = 0; 2064 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2065 ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); 2066 } 2067 } else { 2068 mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL, 2069 mad_recv_wc); 2070 deref_mad_agent(mad_agent_priv); 2071 } 2072 2073 return; 2074 } 2075 2076 static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv, 2077 const struct ib_mad_qp_info *qp_info, 2078 const struct ib_wc *wc, 2079 int port_num, 2080 struct ib_mad_private *recv, 2081 struct ib_mad_private *response) 2082 { 2083 enum smi_forward_action retsmi; 2084 struct ib_smp *smp = (struct ib_smp *)recv->mad; 2085 2086 trace_ib_mad_handle_ib_smi(smp); 2087 2088 if (smi_handle_dr_smp_recv(smp, 2089 rdma_cap_ib_switch(port_priv->device), 2090 port_num, 2091 port_priv->device->phys_port_cnt) == 2092 IB_SMI_DISCARD) 2093 return IB_SMI_DISCARD; 2094 2095 retsmi = smi_check_forward_dr_smp(smp); 2096 if (retsmi == IB_SMI_LOCAL) 2097 return IB_SMI_HANDLE; 2098 2099 if (retsmi == IB_SMI_SEND) { /* don't forward */ 2100 if (smi_handle_dr_smp_send(smp, 2101 rdma_cap_ib_switch(port_priv->device), 2102 port_num) == IB_SMI_DISCARD) 2103 return IB_SMI_DISCARD; 2104 2105 if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD) 2106 return IB_SMI_DISCARD; 2107 } else if (rdma_cap_ib_switch(port_priv->device)) { 2108 /* forward case for switches */ 2109 memcpy(response, recv, mad_priv_size(response)); 2110 response->header.recv_wc.wc = &response->header.wc; 2111 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad; 2112 response->header.recv_wc.recv_buf.grh = &response->grh; 2113 2114 agent_send_response((const struct ib_mad_hdr *)response->mad, 2115 &response->grh, wc, 2116 port_priv->device, 2117 smi_get_fwd_port(smp), 2118 qp_info->qp->qp_num, 2119 response->mad_size, 2120 false); 2121 2122 return IB_SMI_DISCARD; 2123 } 2124 return IB_SMI_HANDLE; 2125 } 2126 2127 static bool generate_unmatched_resp(const struct ib_mad_private *recv, 2128 struct ib_mad_private *response, 2129 size_t *resp_len, bool opa) 2130 { 2131 const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad; 2132 struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad; 2133 2134 if (recv_hdr->method == IB_MGMT_METHOD_GET || 2135 recv_hdr->method == IB_MGMT_METHOD_SET) { 2136 memcpy(response, recv, mad_priv_size(response)); 2137 response->header.recv_wc.wc = &response->header.wc; 2138 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad; 2139 response->header.recv_wc.recv_buf.grh = &response->grh; 2140 resp_hdr->method = IB_MGMT_METHOD_GET_RESP; 2141 resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB); 2142 if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2143 resp_hdr->status |= IB_SMP_DIRECTION; 2144 2145 if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) { 2146 if (recv_hdr->mgmt_class == 2147 IB_MGMT_CLASS_SUBN_LID_ROUTED || 2148 recv_hdr->mgmt_class == 2149 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2150 *resp_len = opa_get_smp_header_size( 2151 (struct opa_smp *)recv->mad); 2152 else 2153 *resp_len = sizeof(struct ib_mad_hdr); 2154 } 2155 2156 return true; 2157 } else { 2158 return false; 2159 } 2160 } 2161 2162 static enum smi_action 2163 handle_opa_smi(struct ib_mad_port_private *port_priv, 2164 struct ib_mad_qp_info *qp_info, 2165 struct ib_wc *wc, 2166 int port_num, 2167 struct ib_mad_private *recv, 2168 struct ib_mad_private *response) 2169 { 2170 enum smi_forward_action retsmi; 2171 struct opa_smp *smp = (struct opa_smp *)recv->mad; 2172 2173 trace_ib_mad_handle_opa_smi(smp); 2174 2175 if (opa_smi_handle_dr_smp_recv(smp, 2176 rdma_cap_ib_switch(port_priv->device), 2177 port_num, 2178 port_priv->device->phys_port_cnt) == 2179 IB_SMI_DISCARD) 2180 return IB_SMI_DISCARD; 2181 2182 retsmi = opa_smi_check_forward_dr_smp(smp); 2183 if (retsmi == IB_SMI_LOCAL) 2184 return IB_SMI_HANDLE; 2185 2186 if (retsmi == IB_SMI_SEND) { /* don't forward */ 2187 if (opa_smi_handle_dr_smp_send(smp, 2188 rdma_cap_ib_switch(port_priv->device), 2189 port_num) == IB_SMI_DISCARD) 2190 return IB_SMI_DISCARD; 2191 2192 if (opa_smi_check_local_smp(smp, port_priv->device) == 2193 IB_SMI_DISCARD) 2194 return IB_SMI_DISCARD; 2195 2196 } else if (rdma_cap_ib_switch(port_priv->device)) { 2197 /* forward case for switches */ 2198 memcpy(response, recv, mad_priv_size(response)); 2199 response->header.recv_wc.wc = &response->header.wc; 2200 response->header.recv_wc.recv_buf.opa_mad = 2201 (struct opa_mad *)response->mad; 2202 response->header.recv_wc.recv_buf.grh = &response->grh; 2203 2204 agent_send_response((const struct ib_mad_hdr *)response->mad, 2205 &response->grh, wc, 2206 port_priv->device, 2207 opa_smi_get_fwd_port(smp), 2208 qp_info->qp->qp_num, 2209 recv->header.wc.byte_len, 2210 true); 2211 2212 return IB_SMI_DISCARD; 2213 } 2214 2215 return IB_SMI_HANDLE; 2216 } 2217 2218 static enum smi_action 2219 handle_smi(struct ib_mad_port_private *port_priv, 2220 struct ib_mad_qp_info *qp_info, 2221 struct ib_wc *wc, 2222 int port_num, 2223 struct ib_mad_private *recv, 2224 struct ib_mad_private *response, 2225 bool opa) 2226 { 2227 struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad; 2228 2229 if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION && 2230 mad_hdr->class_version == OPA_SM_CLASS_VERSION) 2231 return handle_opa_smi(port_priv, qp_info, wc, port_num, recv, 2232 response); 2233 2234 return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response); 2235 } 2236 2237 static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2238 { 2239 struct ib_mad_port_private *port_priv = cq->cq_context; 2240 struct ib_mad_list_head *mad_list = 2241 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2242 struct ib_mad_qp_info *qp_info; 2243 struct ib_mad_private_header *mad_priv_hdr; 2244 struct ib_mad_private *recv, *response = NULL; 2245 struct ib_mad_agent_private *mad_agent; 2246 int port_num; 2247 int ret = IB_MAD_RESULT_SUCCESS; 2248 size_t mad_size; 2249 u16 resp_mad_pkey_index = 0; 2250 bool opa; 2251 2252 if (list_empty_careful(&port_priv->port_list)) 2253 return; 2254 2255 if (wc->status != IB_WC_SUCCESS) { 2256 /* 2257 * Receive errors indicate that the QP has entered the error 2258 * state - error handling/shutdown code will cleanup 2259 */ 2260 return; 2261 } 2262 2263 qp_info = mad_list->mad_queue->qp_info; 2264 dequeue_mad(mad_list); 2265 2266 opa = rdma_cap_opa_mad(qp_info->port_priv->device, 2267 qp_info->port_priv->port_num); 2268 2269 mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header, 2270 mad_list); 2271 recv = container_of(mad_priv_hdr, struct ib_mad_private, header); 2272 ib_dma_unmap_single(port_priv->device, 2273 recv->header.mapping, 2274 mad_priv_dma_size(recv), 2275 DMA_FROM_DEVICE); 2276 2277 /* Setup MAD receive work completion from "normal" work completion */ 2278 recv->header.wc = *wc; 2279 recv->header.recv_wc.wc = &recv->header.wc; 2280 2281 if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) { 2282 recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh); 2283 recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad); 2284 } else { 2285 recv->header.recv_wc.mad_len = sizeof(struct ib_mad); 2286 recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad); 2287 } 2288 2289 recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad; 2290 recv->header.recv_wc.recv_buf.grh = &recv->grh; 2291 2292 if (atomic_read(&qp_info->snoop_count)) 2293 snoop_recv(qp_info, &recv->header.recv_wc, IB_MAD_SNOOP_RECVS); 2294 2295 /* Validate MAD */ 2296 if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa)) 2297 goto out; 2298 2299 trace_ib_mad_recv_done_handler(qp_info, wc, 2300 (struct ib_mad_hdr *)recv->mad); 2301 2302 mad_size = recv->mad_size; 2303 response = alloc_mad_private(mad_size, GFP_KERNEL); 2304 if (!response) 2305 goto out; 2306 2307 if (rdma_cap_ib_switch(port_priv->device)) 2308 port_num = wc->port_num; 2309 else 2310 port_num = port_priv->port_num; 2311 2312 if (((struct ib_mad_hdr *)recv->mad)->mgmt_class == 2313 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 2314 if (handle_smi(port_priv, qp_info, wc, port_num, recv, 2315 response, opa) 2316 == IB_SMI_DISCARD) 2317 goto out; 2318 } 2319 2320 /* Give driver "right of first refusal" on incoming MAD */ 2321 if (port_priv->device->ops.process_mad) { 2322 ret = port_priv->device->ops.process_mad( 2323 port_priv->device, 0, port_priv->port_num, wc, 2324 &recv->grh, (const struct ib_mad *)recv->mad, 2325 (struct ib_mad *)response->mad, &mad_size, 2326 &resp_mad_pkey_index); 2327 2328 if (opa) 2329 wc->pkey_index = resp_mad_pkey_index; 2330 2331 if (ret & IB_MAD_RESULT_SUCCESS) { 2332 if (ret & IB_MAD_RESULT_CONSUMED) 2333 goto out; 2334 if (ret & IB_MAD_RESULT_REPLY) { 2335 agent_send_response((const struct ib_mad_hdr *)response->mad, 2336 &recv->grh, wc, 2337 port_priv->device, 2338 port_num, 2339 qp_info->qp->qp_num, 2340 mad_size, opa); 2341 goto out; 2342 } 2343 } 2344 } 2345 2346 mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad); 2347 if (mad_agent) { 2348 trace_ib_mad_recv_done_agent(mad_agent); 2349 ib_mad_complete_recv(mad_agent, &recv->header.recv_wc); 2350 /* 2351 * recv is freed up in error cases in ib_mad_complete_recv 2352 * or via recv_handler in ib_mad_complete_recv() 2353 */ 2354 recv = NULL; 2355 } else if ((ret & IB_MAD_RESULT_SUCCESS) && 2356 generate_unmatched_resp(recv, response, &mad_size, opa)) { 2357 agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc, 2358 port_priv->device, port_num, 2359 qp_info->qp->qp_num, mad_size, opa); 2360 } 2361 2362 out: 2363 /* Post another receive request for this QP */ 2364 if (response) { 2365 ib_mad_post_receive_mads(qp_info, response); 2366 kfree(recv); 2367 } else 2368 ib_mad_post_receive_mads(qp_info, recv); 2369 } 2370 2371 static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv) 2372 { 2373 struct ib_mad_send_wr_private *mad_send_wr; 2374 unsigned long delay; 2375 2376 if (list_empty(&mad_agent_priv->wait_list)) { 2377 cancel_delayed_work(&mad_agent_priv->timed_work); 2378 } else { 2379 mad_send_wr = list_entry(mad_agent_priv->wait_list.next, 2380 struct ib_mad_send_wr_private, 2381 agent_list); 2382 2383 if (time_after(mad_agent_priv->timeout, 2384 mad_send_wr->timeout)) { 2385 mad_agent_priv->timeout = mad_send_wr->timeout; 2386 delay = mad_send_wr->timeout - jiffies; 2387 if ((long)delay <= 0) 2388 delay = 1; 2389 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq, 2390 &mad_agent_priv->timed_work, delay); 2391 } 2392 } 2393 } 2394 2395 static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr) 2396 { 2397 struct ib_mad_agent_private *mad_agent_priv; 2398 struct ib_mad_send_wr_private *temp_mad_send_wr; 2399 struct list_head *list_item; 2400 unsigned long delay; 2401 2402 mad_agent_priv = mad_send_wr->mad_agent_priv; 2403 list_del(&mad_send_wr->agent_list); 2404 2405 delay = mad_send_wr->timeout; 2406 mad_send_wr->timeout += jiffies; 2407 2408 if (delay) { 2409 list_for_each_prev(list_item, &mad_agent_priv->wait_list) { 2410 temp_mad_send_wr = list_entry(list_item, 2411 struct ib_mad_send_wr_private, 2412 agent_list); 2413 if (time_after(mad_send_wr->timeout, 2414 temp_mad_send_wr->timeout)) 2415 break; 2416 } 2417 } 2418 else 2419 list_item = &mad_agent_priv->wait_list; 2420 list_add(&mad_send_wr->agent_list, list_item); 2421 2422 /* Reschedule a work item if we have a shorter timeout */ 2423 if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list) 2424 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq, 2425 &mad_agent_priv->timed_work, delay); 2426 } 2427 2428 void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr, 2429 unsigned long timeout_ms) 2430 { 2431 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms); 2432 wait_for_response(mad_send_wr); 2433 } 2434 2435 /* 2436 * Process a send work completion 2437 */ 2438 void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr, 2439 struct ib_mad_send_wc *mad_send_wc) 2440 { 2441 struct ib_mad_agent_private *mad_agent_priv; 2442 unsigned long flags; 2443 int ret; 2444 2445 mad_agent_priv = mad_send_wr->mad_agent_priv; 2446 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2447 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 2448 ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc); 2449 if (ret == IB_RMPP_RESULT_CONSUMED) 2450 goto done; 2451 } else 2452 ret = IB_RMPP_RESULT_UNHANDLED; 2453 2454 if (mad_send_wc->status != IB_WC_SUCCESS && 2455 mad_send_wr->status == IB_WC_SUCCESS) { 2456 mad_send_wr->status = mad_send_wc->status; 2457 mad_send_wr->refcount -= (mad_send_wr->timeout > 0); 2458 } 2459 2460 if (--mad_send_wr->refcount > 0) { 2461 if (mad_send_wr->refcount == 1 && mad_send_wr->timeout && 2462 mad_send_wr->status == IB_WC_SUCCESS) { 2463 wait_for_response(mad_send_wr); 2464 } 2465 goto done; 2466 } 2467 2468 /* Remove send from MAD agent and notify client of completion */ 2469 list_del(&mad_send_wr->agent_list); 2470 adjust_timeout(mad_agent_priv); 2471 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2472 2473 if (mad_send_wr->status != IB_WC_SUCCESS ) 2474 mad_send_wc->status = mad_send_wr->status; 2475 if (ret == IB_RMPP_RESULT_INTERNAL) 2476 ib_rmpp_send_handler(mad_send_wc); 2477 else 2478 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2479 mad_send_wc); 2480 2481 /* Release reference on agent taken when sending */ 2482 deref_mad_agent(mad_agent_priv); 2483 return; 2484 done: 2485 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2486 } 2487 2488 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc) 2489 { 2490 struct ib_mad_port_private *port_priv = cq->cq_context; 2491 struct ib_mad_list_head *mad_list = 2492 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2493 struct ib_mad_send_wr_private *mad_send_wr, *queued_send_wr; 2494 struct ib_mad_qp_info *qp_info; 2495 struct ib_mad_queue *send_queue; 2496 struct ib_mad_send_wc mad_send_wc; 2497 unsigned long flags; 2498 int ret; 2499 2500 if (list_empty_careful(&port_priv->port_list)) 2501 return; 2502 2503 if (wc->status != IB_WC_SUCCESS) { 2504 if (!ib_mad_send_error(port_priv, wc)) 2505 return; 2506 } 2507 2508 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private, 2509 mad_list); 2510 send_queue = mad_list->mad_queue; 2511 qp_info = send_queue->qp_info; 2512 2513 trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv); 2514 trace_ib_mad_send_done_handler(mad_send_wr, wc); 2515 2516 retry: 2517 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, 2518 mad_send_wr->header_mapping, 2519 mad_send_wr->sg_list[0].length, DMA_TO_DEVICE); 2520 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, 2521 mad_send_wr->payload_mapping, 2522 mad_send_wr->sg_list[1].length, DMA_TO_DEVICE); 2523 queued_send_wr = NULL; 2524 spin_lock_irqsave(&send_queue->lock, flags); 2525 list_del(&mad_list->list); 2526 2527 /* Move queued send to the send queue */ 2528 if (send_queue->count-- > send_queue->max_active) { 2529 mad_list = container_of(qp_info->overflow_list.next, 2530 struct ib_mad_list_head, list); 2531 queued_send_wr = container_of(mad_list, 2532 struct ib_mad_send_wr_private, 2533 mad_list); 2534 list_move_tail(&mad_list->list, &send_queue->list); 2535 } 2536 spin_unlock_irqrestore(&send_queue->lock, flags); 2537 2538 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2539 mad_send_wc.status = wc->status; 2540 mad_send_wc.vendor_err = wc->vendor_err; 2541 if (atomic_read(&qp_info->snoop_count)) 2542 snoop_send(qp_info, &mad_send_wr->send_buf, &mad_send_wc, 2543 IB_MAD_SNOOP_SEND_COMPLETIONS); 2544 ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); 2545 2546 if (queued_send_wr) { 2547 trace_ib_mad_send_done_resend(queued_send_wr, qp_info); 2548 ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr, 2549 NULL); 2550 if (ret) { 2551 dev_err(&port_priv->device->dev, 2552 "ib_post_send failed: %d\n", ret); 2553 mad_send_wr = queued_send_wr; 2554 wc->status = IB_WC_LOC_QP_OP_ERR; 2555 goto retry; 2556 } 2557 } 2558 } 2559 2560 static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info) 2561 { 2562 struct ib_mad_send_wr_private *mad_send_wr; 2563 struct ib_mad_list_head *mad_list; 2564 unsigned long flags; 2565 2566 spin_lock_irqsave(&qp_info->send_queue.lock, flags); 2567 list_for_each_entry(mad_list, &qp_info->send_queue.list, list) { 2568 mad_send_wr = container_of(mad_list, 2569 struct ib_mad_send_wr_private, 2570 mad_list); 2571 mad_send_wr->retry = 1; 2572 } 2573 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags); 2574 } 2575 2576 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv, 2577 struct ib_wc *wc) 2578 { 2579 struct ib_mad_list_head *mad_list = 2580 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2581 struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info; 2582 struct ib_mad_send_wr_private *mad_send_wr; 2583 int ret; 2584 2585 /* 2586 * Send errors will transition the QP to SQE - move 2587 * QP to RTS and repost flushed work requests 2588 */ 2589 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private, 2590 mad_list); 2591 if (wc->status == IB_WC_WR_FLUSH_ERR) { 2592 if (mad_send_wr->retry) { 2593 /* Repost send */ 2594 mad_send_wr->retry = 0; 2595 trace_ib_mad_error_handler(mad_send_wr, qp_info); 2596 ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr, 2597 NULL); 2598 if (!ret) 2599 return false; 2600 } 2601 } else { 2602 struct ib_qp_attr *attr; 2603 2604 /* Transition QP to RTS and fail offending send */ 2605 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2606 if (attr) { 2607 attr->qp_state = IB_QPS_RTS; 2608 attr->cur_qp_state = IB_QPS_SQE; 2609 ret = ib_modify_qp(qp_info->qp, attr, 2610 IB_QP_STATE | IB_QP_CUR_STATE); 2611 kfree(attr); 2612 if (ret) 2613 dev_err(&port_priv->device->dev, 2614 "%s - ib_modify_qp to RTS: %d\n", 2615 __func__, ret); 2616 else 2617 mark_sends_for_retry(qp_info); 2618 } 2619 } 2620 2621 return true; 2622 } 2623 2624 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv) 2625 { 2626 unsigned long flags; 2627 struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr; 2628 struct ib_mad_send_wc mad_send_wc; 2629 struct list_head cancel_list; 2630 2631 INIT_LIST_HEAD(&cancel_list); 2632 2633 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2634 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr, 2635 &mad_agent_priv->send_list, agent_list) { 2636 if (mad_send_wr->status == IB_WC_SUCCESS) { 2637 mad_send_wr->status = IB_WC_WR_FLUSH_ERR; 2638 mad_send_wr->refcount -= (mad_send_wr->timeout > 0); 2639 } 2640 } 2641 2642 /* Empty wait list to prevent receives from finding a request */ 2643 list_splice_init(&mad_agent_priv->wait_list, &cancel_list); 2644 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2645 2646 /* Report all cancelled requests */ 2647 mad_send_wc.status = IB_WC_WR_FLUSH_ERR; 2648 mad_send_wc.vendor_err = 0; 2649 2650 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr, 2651 &cancel_list, agent_list) { 2652 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2653 list_del(&mad_send_wr->agent_list); 2654 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2655 &mad_send_wc); 2656 atomic_dec(&mad_agent_priv->refcount); 2657 } 2658 } 2659 2660 static struct ib_mad_send_wr_private* 2661 find_send_wr(struct ib_mad_agent_private *mad_agent_priv, 2662 struct ib_mad_send_buf *send_buf) 2663 { 2664 struct ib_mad_send_wr_private *mad_send_wr; 2665 2666 list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list, 2667 agent_list) { 2668 if (&mad_send_wr->send_buf == send_buf) 2669 return mad_send_wr; 2670 } 2671 2672 list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list, 2673 agent_list) { 2674 if (is_rmpp_data_mad(mad_agent_priv, 2675 mad_send_wr->send_buf.mad) && 2676 &mad_send_wr->send_buf == send_buf) 2677 return mad_send_wr; 2678 } 2679 return NULL; 2680 } 2681 2682 int ib_modify_mad(struct ib_mad_agent *mad_agent, 2683 struct ib_mad_send_buf *send_buf, u32 timeout_ms) 2684 { 2685 struct ib_mad_agent_private *mad_agent_priv; 2686 struct ib_mad_send_wr_private *mad_send_wr; 2687 unsigned long flags; 2688 int active; 2689 2690 mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private, 2691 agent); 2692 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2693 mad_send_wr = find_send_wr(mad_agent_priv, send_buf); 2694 if (!mad_send_wr || mad_send_wr->status != IB_WC_SUCCESS) { 2695 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2696 return -EINVAL; 2697 } 2698 2699 active = (!mad_send_wr->timeout || mad_send_wr->refcount > 1); 2700 if (!timeout_ms) { 2701 mad_send_wr->status = IB_WC_WR_FLUSH_ERR; 2702 mad_send_wr->refcount -= (mad_send_wr->timeout > 0); 2703 } 2704 2705 mad_send_wr->send_buf.timeout_ms = timeout_ms; 2706 if (active) 2707 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms); 2708 else 2709 ib_reset_mad_timeout(mad_send_wr, timeout_ms); 2710 2711 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2712 return 0; 2713 } 2714 EXPORT_SYMBOL(ib_modify_mad); 2715 2716 void ib_cancel_mad(struct ib_mad_agent *mad_agent, 2717 struct ib_mad_send_buf *send_buf) 2718 { 2719 ib_modify_mad(mad_agent, send_buf, 0); 2720 } 2721 EXPORT_SYMBOL(ib_cancel_mad); 2722 2723 static void local_completions(struct work_struct *work) 2724 { 2725 struct ib_mad_agent_private *mad_agent_priv; 2726 struct ib_mad_local_private *local; 2727 struct ib_mad_agent_private *recv_mad_agent; 2728 unsigned long flags; 2729 int free_mad; 2730 struct ib_wc wc; 2731 struct ib_mad_send_wc mad_send_wc; 2732 bool opa; 2733 2734 mad_agent_priv = 2735 container_of(work, struct ib_mad_agent_private, local_work); 2736 2737 opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device, 2738 mad_agent_priv->qp_info->port_priv->port_num); 2739 2740 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2741 while (!list_empty(&mad_agent_priv->local_list)) { 2742 local = list_entry(mad_agent_priv->local_list.next, 2743 struct ib_mad_local_private, 2744 completion_list); 2745 list_del(&local->completion_list); 2746 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2747 free_mad = 0; 2748 if (local->mad_priv) { 2749 u8 base_version; 2750 recv_mad_agent = local->recv_mad_agent; 2751 if (!recv_mad_agent) { 2752 dev_err(&mad_agent_priv->agent.device->dev, 2753 "No receive MAD agent for local completion\n"); 2754 free_mad = 1; 2755 goto local_send_completion; 2756 } 2757 2758 /* 2759 * Defined behavior is to complete response 2760 * before request 2761 */ 2762 build_smp_wc(recv_mad_agent->agent.qp, 2763 local->mad_send_wr->send_wr.wr.wr_cqe, 2764 be16_to_cpu(IB_LID_PERMISSIVE), 2765 local->mad_send_wr->send_wr.pkey_index, 2766 recv_mad_agent->agent.port_num, &wc); 2767 2768 local->mad_priv->header.recv_wc.wc = &wc; 2769 2770 base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version; 2771 if (opa && base_version == OPA_MGMT_BASE_VERSION) { 2772 local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len; 2773 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad); 2774 } else { 2775 local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad); 2776 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad); 2777 } 2778 2779 INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list); 2780 list_add(&local->mad_priv->header.recv_wc.recv_buf.list, 2781 &local->mad_priv->header.recv_wc.rmpp_list); 2782 local->mad_priv->header.recv_wc.recv_buf.grh = NULL; 2783 local->mad_priv->header.recv_wc.recv_buf.mad = 2784 (struct ib_mad *)local->mad_priv->mad; 2785 if (atomic_read(&recv_mad_agent->qp_info->snoop_count)) 2786 snoop_recv(recv_mad_agent->qp_info, 2787 &local->mad_priv->header.recv_wc, 2788 IB_MAD_SNOOP_RECVS); 2789 recv_mad_agent->agent.recv_handler( 2790 &recv_mad_agent->agent, 2791 &local->mad_send_wr->send_buf, 2792 &local->mad_priv->header.recv_wc); 2793 spin_lock_irqsave(&recv_mad_agent->lock, flags); 2794 atomic_dec(&recv_mad_agent->refcount); 2795 spin_unlock_irqrestore(&recv_mad_agent->lock, flags); 2796 } 2797 2798 local_send_completion: 2799 /* Complete send */ 2800 mad_send_wc.status = IB_WC_SUCCESS; 2801 mad_send_wc.vendor_err = 0; 2802 mad_send_wc.send_buf = &local->mad_send_wr->send_buf; 2803 if (atomic_read(&mad_agent_priv->qp_info->snoop_count)) 2804 snoop_send(mad_agent_priv->qp_info, 2805 &local->mad_send_wr->send_buf, 2806 &mad_send_wc, IB_MAD_SNOOP_SEND_COMPLETIONS); 2807 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2808 &mad_send_wc); 2809 2810 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2811 atomic_dec(&mad_agent_priv->refcount); 2812 if (free_mad) 2813 kfree(local->mad_priv); 2814 kfree(local); 2815 } 2816 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2817 } 2818 2819 static int retry_send(struct ib_mad_send_wr_private *mad_send_wr) 2820 { 2821 int ret; 2822 2823 if (!mad_send_wr->retries_left) 2824 return -ETIMEDOUT; 2825 2826 mad_send_wr->retries_left--; 2827 mad_send_wr->send_buf.retries++; 2828 2829 mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms); 2830 2831 if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) { 2832 ret = ib_retry_rmpp(mad_send_wr); 2833 switch (ret) { 2834 case IB_RMPP_RESULT_UNHANDLED: 2835 ret = ib_send_mad(mad_send_wr); 2836 break; 2837 case IB_RMPP_RESULT_CONSUMED: 2838 ret = 0; 2839 break; 2840 default: 2841 ret = -ECOMM; 2842 break; 2843 } 2844 } else 2845 ret = ib_send_mad(mad_send_wr); 2846 2847 if (!ret) { 2848 mad_send_wr->refcount++; 2849 list_add_tail(&mad_send_wr->agent_list, 2850 &mad_send_wr->mad_agent_priv->send_list); 2851 } 2852 return ret; 2853 } 2854 2855 static void timeout_sends(struct work_struct *work) 2856 { 2857 struct ib_mad_agent_private *mad_agent_priv; 2858 struct ib_mad_send_wr_private *mad_send_wr; 2859 struct ib_mad_send_wc mad_send_wc; 2860 unsigned long flags, delay; 2861 2862 mad_agent_priv = container_of(work, struct ib_mad_agent_private, 2863 timed_work.work); 2864 mad_send_wc.vendor_err = 0; 2865 2866 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2867 while (!list_empty(&mad_agent_priv->wait_list)) { 2868 mad_send_wr = list_entry(mad_agent_priv->wait_list.next, 2869 struct ib_mad_send_wr_private, 2870 agent_list); 2871 2872 if (time_after(mad_send_wr->timeout, jiffies)) { 2873 delay = mad_send_wr->timeout - jiffies; 2874 if ((long)delay <= 0) 2875 delay = 1; 2876 queue_delayed_work(mad_agent_priv->qp_info-> 2877 port_priv->wq, 2878 &mad_agent_priv->timed_work, delay); 2879 break; 2880 } 2881 2882 list_del(&mad_send_wr->agent_list); 2883 if (mad_send_wr->status == IB_WC_SUCCESS && 2884 !retry_send(mad_send_wr)) 2885 continue; 2886 2887 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2888 2889 if (mad_send_wr->status == IB_WC_SUCCESS) 2890 mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR; 2891 else 2892 mad_send_wc.status = mad_send_wr->status; 2893 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2894 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2895 &mad_send_wc); 2896 2897 atomic_dec(&mad_agent_priv->refcount); 2898 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2899 } 2900 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2901 } 2902 2903 /* 2904 * Allocate receive MADs and post receive WRs for them 2905 */ 2906 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info, 2907 struct ib_mad_private *mad) 2908 { 2909 unsigned long flags; 2910 int post, ret; 2911 struct ib_mad_private *mad_priv; 2912 struct ib_sge sg_list; 2913 struct ib_recv_wr recv_wr; 2914 struct ib_mad_queue *recv_queue = &qp_info->recv_queue; 2915 2916 /* Initialize common scatter list fields */ 2917 sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey; 2918 2919 /* Initialize common receive WR fields */ 2920 recv_wr.next = NULL; 2921 recv_wr.sg_list = &sg_list; 2922 recv_wr.num_sge = 1; 2923 2924 do { 2925 /* Allocate and map receive buffer */ 2926 if (mad) { 2927 mad_priv = mad; 2928 mad = NULL; 2929 } else { 2930 mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv), 2931 GFP_ATOMIC); 2932 if (!mad_priv) { 2933 ret = -ENOMEM; 2934 break; 2935 } 2936 } 2937 sg_list.length = mad_priv_dma_size(mad_priv); 2938 sg_list.addr = ib_dma_map_single(qp_info->port_priv->device, 2939 &mad_priv->grh, 2940 mad_priv_dma_size(mad_priv), 2941 DMA_FROM_DEVICE); 2942 if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device, 2943 sg_list.addr))) { 2944 ret = -ENOMEM; 2945 break; 2946 } 2947 mad_priv->header.mapping = sg_list.addr; 2948 mad_priv->header.mad_list.mad_queue = recv_queue; 2949 mad_priv->header.mad_list.cqe.done = ib_mad_recv_done; 2950 recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe; 2951 2952 /* Post receive WR */ 2953 spin_lock_irqsave(&recv_queue->lock, flags); 2954 post = (++recv_queue->count < recv_queue->max_active); 2955 list_add_tail(&mad_priv->header.mad_list.list, &recv_queue->list); 2956 spin_unlock_irqrestore(&recv_queue->lock, flags); 2957 ret = ib_post_recv(qp_info->qp, &recv_wr, NULL); 2958 if (ret) { 2959 spin_lock_irqsave(&recv_queue->lock, flags); 2960 list_del(&mad_priv->header.mad_list.list); 2961 recv_queue->count--; 2962 spin_unlock_irqrestore(&recv_queue->lock, flags); 2963 ib_dma_unmap_single(qp_info->port_priv->device, 2964 mad_priv->header.mapping, 2965 mad_priv_dma_size(mad_priv), 2966 DMA_FROM_DEVICE); 2967 kfree(mad_priv); 2968 dev_err(&qp_info->port_priv->device->dev, 2969 "ib_post_recv failed: %d\n", ret); 2970 break; 2971 } 2972 } while (post); 2973 2974 return ret; 2975 } 2976 2977 /* 2978 * Return all the posted receive MADs 2979 */ 2980 static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info) 2981 { 2982 struct ib_mad_private_header *mad_priv_hdr; 2983 struct ib_mad_private *recv; 2984 struct ib_mad_list_head *mad_list; 2985 2986 if (!qp_info->qp) 2987 return; 2988 2989 while (!list_empty(&qp_info->recv_queue.list)) { 2990 2991 mad_list = list_entry(qp_info->recv_queue.list.next, 2992 struct ib_mad_list_head, list); 2993 mad_priv_hdr = container_of(mad_list, 2994 struct ib_mad_private_header, 2995 mad_list); 2996 recv = container_of(mad_priv_hdr, struct ib_mad_private, 2997 header); 2998 2999 /* Remove from posted receive MAD list */ 3000 list_del(&mad_list->list); 3001 3002 ib_dma_unmap_single(qp_info->port_priv->device, 3003 recv->header.mapping, 3004 mad_priv_dma_size(recv), 3005 DMA_FROM_DEVICE); 3006 kfree(recv); 3007 } 3008 3009 qp_info->recv_queue.count = 0; 3010 } 3011 3012 /* 3013 * Start the port 3014 */ 3015 static int ib_mad_port_start(struct ib_mad_port_private *port_priv) 3016 { 3017 int ret, i; 3018 struct ib_qp_attr *attr; 3019 struct ib_qp *qp; 3020 u16 pkey_index; 3021 3022 attr = kmalloc(sizeof *attr, GFP_KERNEL); 3023 if (!attr) 3024 return -ENOMEM; 3025 3026 ret = ib_find_pkey(port_priv->device, port_priv->port_num, 3027 IB_DEFAULT_PKEY_FULL, &pkey_index); 3028 if (ret) 3029 pkey_index = 0; 3030 3031 for (i = 0; i < IB_MAD_QPS_CORE; i++) { 3032 qp = port_priv->qp_info[i].qp; 3033 if (!qp) 3034 continue; 3035 3036 /* 3037 * PKey index for QP1 is irrelevant but 3038 * one is needed for the Reset to Init transition 3039 */ 3040 attr->qp_state = IB_QPS_INIT; 3041 attr->pkey_index = pkey_index; 3042 attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY; 3043 ret = ib_modify_qp(qp, attr, IB_QP_STATE | 3044 IB_QP_PKEY_INDEX | IB_QP_QKEY); 3045 if (ret) { 3046 dev_err(&port_priv->device->dev, 3047 "Couldn't change QP%d state to INIT: %d\n", 3048 i, ret); 3049 goto out; 3050 } 3051 3052 attr->qp_state = IB_QPS_RTR; 3053 ret = ib_modify_qp(qp, attr, IB_QP_STATE); 3054 if (ret) { 3055 dev_err(&port_priv->device->dev, 3056 "Couldn't change QP%d state to RTR: %d\n", 3057 i, ret); 3058 goto out; 3059 } 3060 3061 attr->qp_state = IB_QPS_RTS; 3062 attr->sq_psn = IB_MAD_SEND_Q_PSN; 3063 ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN); 3064 if (ret) { 3065 dev_err(&port_priv->device->dev, 3066 "Couldn't change QP%d state to RTS: %d\n", 3067 i, ret); 3068 goto out; 3069 } 3070 } 3071 3072 ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP); 3073 if (ret) { 3074 dev_err(&port_priv->device->dev, 3075 "Failed to request completion notification: %d\n", 3076 ret); 3077 goto out; 3078 } 3079 3080 for (i = 0; i < IB_MAD_QPS_CORE; i++) { 3081 if (!port_priv->qp_info[i].qp) 3082 continue; 3083 3084 ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL); 3085 if (ret) { 3086 dev_err(&port_priv->device->dev, 3087 "Couldn't post receive WRs\n"); 3088 goto out; 3089 } 3090 } 3091 out: 3092 kfree(attr); 3093 return ret; 3094 } 3095 3096 static void qp_event_handler(struct ib_event *event, void *qp_context) 3097 { 3098 struct ib_mad_qp_info *qp_info = qp_context; 3099 3100 /* It's worse than that! He's dead, Jim! */ 3101 dev_err(&qp_info->port_priv->device->dev, 3102 "Fatal error (%d) on MAD QP (%d)\n", 3103 event->event, qp_info->qp->qp_num); 3104 } 3105 3106 static void init_mad_queue(struct ib_mad_qp_info *qp_info, 3107 struct ib_mad_queue *mad_queue) 3108 { 3109 mad_queue->qp_info = qp_info; 3110 mad_queue->count = 0; 3111 spin_lock_init(&mad_queue->lock); 3112 INIT_LIST_HEAD(&mad_queue->list); 3113 } 3114 3115 static void init_mad_qp(struct ib_mad_port_private *port_priv, 3116 struct ib_mad_qp_info *qp_info) 3117 { 3118 qp_info->port_priv = port_priv; 3119 init_mad_queue(qp_info, &qp_info->send_queue); 3120 init_mad_queue(qp_info, &qp_info->recv_queue); 3121 INIT_LIST_HEAD(&qp_info->overflow_list); 3122 spin_lock_init(&qp_info->snoop_lock); 3123 qp_info->snoop_table = NULL; 3124 qp_info->snoop_table_size = 0; 3125 atomic_set(&qp_info->snoop_count, 0); 3126 } 3127 3128 static int create_mad_qp(struct ib_mad_qp_info *qp_info, 3129 enum ib_qp_type qp_type) 3130 { 3131 struct ib_qp_init_attr qp_init_attr; 3132 int ret; 3133 3134 memset(&qp_init_attr, 0, sizeof qp_init_attr); 3135 qp_init_attr.send_cq = qp_info->port_priv->cq; 3136 qp_init_attr.recv_cq = qp_info->port_priv->cq; 3137 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; 3138 qp_init_attr.cap.max_send_wr = mad_sendq_size; 3139 qp_init_attr.cap.max_recv_wr = mad_recvq_size; 3140 qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG; 3141 qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG; 3142 qp_init_attr.qp_type = qp_type; 3143 qp_init_attr.port_num = qp_info->port_priv->port_num; 3144 qp_init_attr.qp_context = qp_info; 3145 qp_init_attr.event_handler = qp_event_handler; 3146 qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr); 3147 if (IS_ERR(qp_info->qp)) { 3148 dev_err(&qp_info->port_priv->device->dev, 3149 "Couldn't create ib_mad QP%d\n", 3150 get_spl_qp_index(qp_type)); 3151 ret = PTR_ERR(qp_info->qp); 3152 goto error; 3153 } 3154 /* Use minimum queue sizes unless the CQ is resized */ 3155 qp_info->send_queue.max_active = mad_sendq_size; 3156 qp_info->recv_queue.max_active = mad_recvq_size; 3157 return 0; 3158 3159 error: 3160 return ret; 3161 } 3162 3163 static void destroy_mad_qp(struct ib_mad_qp_info *qp_info) 3164 { 3165 if (!qp_info->qp) 3166 return; 3167 3168 ib_destroy_qp(qp_info->qp); 3169 kfree(qp_info->snoop_table); 3170 } 3171 3172 /* 3173 * Open the port 3174 * Create the QP, PD, MR, and CQ if needed 3175 */ 3176 static int ib_mad_port_open(struct ib_device *device, 3177 int port_num) 3178 { 3179 int ret, cq_size; 3180 struct ib_mad_port_private *port_priv; 3181 unsigned long flags; 3182 char name[sizeof "ib_mad123"]; 3183 int has_smi; 3184 3185 if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE)) 3186 return -EFAULT; 3187 3188 if (WARN_ON(rdma_cap_opa_mad(device, port_num) && 3189 rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE)) 3190 return -EFAULT; 3191 3192 /* Create new device info */ 3193 port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL); 3194 if (!port_priv) 3195 return -ENOMEM; 3196 3197 port_priv->device = device; 3198 port_priv->port_num = port_num; 3199 spin_lock_init(&port_priv->reg_lock); 3200 init_mad_qp(port_priv, &port_priv->qp_info[0]); 3201 init_mad_qp(port_priv, &port_priv->qp_info[1]); 3202 3203 cq_size = mad_sendq_size + mad_recvq_size; 3204 has_smi = rdma_cap_ib_smi(device, port_num); 3205 if (has_smi) 3206 cq_size *= 2; 3207 3208 port_priv->pd = ib_alloc_pd(device, 0); 3209 if (IS_ERR(port_priv->pd)) { 3210 dev_err(&device->dev, "Couldn't create ib_mad PD\n"); 3211 ret = PTR_ERR(port_priv->pd); 3212 goto error3; 3213 } 3214 3215 port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0, 3216 IB_POLL_UNBOUND_WORKQUEUE); 3217 if (IS_ERR(port_priv->cq)) { 3218 dev_err(&device->dev, "Couldn't create ib_mad CQ\n"); 3219 ret = PTR_ERR(port_priv->cq); 3220 goto error4; 3221 } 3222 3223 if (has_smi) { 3224 ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); 3225 if (ret) 3226 goto error6; 3227 } 3228 ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI); 3229 if (ret) 3230 goto error7; 3231 3232 snprintf(name, sizeof name, "ib_mad%d", port_num); 3233 port_priv->wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM); 3234 if (!port_priv->wq) { 3235 ret = -ENOMEM; 3236 goto error8; 3237 } 3238 3239 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3240 list_add_tail(&port_priv->port_list, &ib_mad_port_list); 3241 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3242 3243 ret = ib_mad_port_start(port_priv); 3244 if (ret) { 3245 dev_err(&device->dev, "Couldn't start port\n"); 3246 goto error9; 3247 } 3248 3249 return 0; 3250 3251 error9: 3252 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3253 list_del_init(&port_priv->port_list); 3254 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3255 3256 destroy_workqueue(port_priv->wq); 3257 error8: 3258 destroy_mad_qp(&port_priv->qp_info[1]); 3259 error7: 3260 destroy_mad_qp(&port_priv->qp_info[0]); 3261 error6: 3262 ib_free_cq(port_priv->cq); 3263 cleanup_recv_queue(&port_priv->qp_info[1]); 3264 cleanup_recv_queue(&port_priv->qp_info[0]); 3265 error4: 3266 ib_dealloc_pd(port_priv->pd); 3267 error3: 3268 kfree(port_priv); 3269 3270 return ret; 3271 } 3272 3273 /* 3274 * Close the port 3275 * If there are no classes using the port, free the port 3276 * resources (CQ, MR, PD, QP) and remove the port's info structure 3277 */ 3278 static int ib_mad_port_close(struct ib_device *device, int port_num) 3279 { 3280 struct ib_mad_port_private *port_priv; 3281 unsigned long flags; 3282 3283 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3284 port_priv = __ib_get_mad_port(device, port_num); 3285 if (port_priv == NULL) { 3286 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3287 dev_err(&device->dev, "Port %d not found\n", port_num); 3288 return -ENODEV; 3289 } 3290 list_del_init(&port_priv->port_list); 3291 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3292 3293 destroy_workqueue(port_priv->wq); 3294 destroy_mad_qp(&port_priv->qp_info[1]); 3295 destroy_mad_qp(&port_priv->qp_info[0]); 3296 ib_free_cq(port_priv->cq); 3297 ib_dealloc_pd(port_priv->pd); 3298 cleanup_recv_queue(&port_priv->qp_info[1]); 3299 cleanup_recv_queue(&port_priv->qp_info[0]); 3300 /* XXX: Handle deallocation of MAD registration tables */ 3301 3302 kfree(port_priv); 3303 3304 return 0; 3305 } 3306 3307 static void ib_mad_init_device(struct ib_device *device) 3308 { 3309 int start, i; 3310 3311 start = rdma_start_port(device); 3312 3313 for (i = start; i <= rdma_end_port(device); i++) { 3314 if (!rdma_cap_ib_mad(device, i)) 3315 continue; 3316 3317 if (ib_mad_port_open(device, i)) { 3318 dev_err(&device->dev, "Couldn't open port %d\n", i); 3319 goto error; 3320 } 3321 if (ib_agent_port_open(device, i)) { 3322 dev_err(&device->dev, 3323 "Couldn't open port %d for agents\n", i); 3324 goto error_agent; 3325 } 3326 } 3327 return; 3328 3329 error_agent: 3330 if (ib_mad_port_close(device, i)) 3331 dev_err(&device->dev, "Couldn't close port %d\n", i); 3332 3333 error: 3334 while (--i >= start) { 3335 if (!rdma_cap_ib_mad(device, i)) 3336 continue; 3337 3338 if (ib_agent_port_close(device, i)) 3339 dev_err(&device->dev, 3340 "Couldn't close port %d for agents\n", i); 3341 if (ib_mad_port_close(device, i)) 3342 dev_err(&device->dev, "Couldn't close port %d\n", i); 3343 } 3344 } 3345 3346 static void ib_mad_remove_device(struct ib_device *device, void *client_data) 3347 { 3348 unsigned int i; 3349 3350 rdma_for_each_port (device, i) { 3351 if (!rdma_cap_ib_mad(device, i)) 3352 continue; 3353 3354 if (ib_agent_port_close(device, i)) 3355 dev_err(&device->dev, 3356 "Couldn't close port %d for agents\n", i); 3357 if (ib_mad_port_close(device, i)) 3358 dev_err(&device->dev, "Couldn't close port %d\n", i); 3359 } 3360 } 3361 3362 static struct ib_client mad_client = { 3363 .name = "mad", 3364 .add = ib_mad_init_device, 3365 .remove = ib_mad_remove_device 3366 }; 3367 3368 int ib_mad_init(void) 3369 { 3370 mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE); 3371 mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE); 3372 3373 mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE); 3374 mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE); 3375 3376 INIT_LIST_HEAD(&ib_mad_port_list); 3377 3378 if (ib_register_client(&mad_client)) { 3379 pr_err("Couldn't register ib_mad client\n"); 3380 return -EINVAL; 3381 } 3382 3383 return 0; 3384 } 3385 3386 void ib_mad_cleanup(void) 3387 { 3388 ib_unregister_client(&mad_client); 3389 } 3390