1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_arp_table -manage arp table 7 * @rf: RDMA PCI function 8 * @ip_addr: ip address for device 9 * @ipv4: IPv4 flag 10 * @mac_addr: mac address ptr 11 * @action: modify, delete or add 12 */ 13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, 14 u8 *mac_addr, u32 action) 15 { 16 unsigned long flags; 17 int arp_index; 18 u32 ip[4] = {}; 19 20 if (ipv4) 21 ip[0] = *ip_addr; 22 else 23 memcpy(ip, ip_addr, sizeof(ip)); 24 25 spin_lock_irqsave(&rf->arp_lock, flags); 26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 28 break; 29 } 30 31 switch (action) { 32 case IRDMA_ARP_ADD: 33 if (arp_index != rf->arp_table_size) { 34 arp_index = -1; 35 break; 36 } 37 38 arp_index = 0; 39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 40 (u32 *)&arp_index, &rf->next_arp_index)) { 41 arp_index = -1; 42 break; 43 } 44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip, 46 sizeof(rf->arp_table[arp_index].ip_addr)); 47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 48 break; 49 case IRDMA_ARP_RESOLVE: 50 if (arp_index == rf->arp_table_size) 51 arp_index = -1; 52 break; 53 case IRDMA_ARP_DELETE: 54 if (arp_index == rf->arp_table_size) { 55 arp_index = -1; 56 break; 57 } 58 59 memset(rf->arp_table[arp_index].ip_addr, 0, 60 sizeof(rf->arp_table[arp_index].ip_addr)); 61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 63 break; 64 default: 65 arp_index = -1; 66 break; 67 } 68 69 spin_unlock_irqrestore(&rf->arp_lock, flags); 70 return arp_index; 71 } 72 73 /** 74 * irdma_add_arp - add a new arp entry if needed 75 * @rf: RDMA function 76 * @ip: IP address 77 * @ipv4: IPv4 flag 78 * @mac: MAC address 79 */ 80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, u8 *mac) 81 { 82 int arpidx; 83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); 85 if (arpidx >= 0) { 86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 87 return arpidx; 88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 90 ipv4, IRDMA_ARP_DELETE); 91 } 92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD); 94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE); 96 } 97 98 /** 99 * wr32 - write 32 bits to hw register 100 * @hw: hardware information including registers 101 * @reg: register offset 102 * @val: value to write to register 103 */ 104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) 105 { 106 writel(val, hw->hw_addr + reg); 107 } 108 109 /** 110 * rd32 - read a 32 bit hw register 111 * @hw: hardware information including registers 112 * @reg: register offset 113 * 114 * Return value of register content 115 */ 116 inline u32 rd32(struct irdma_hw *hw, u32 reg) 117 { 118 return readl(hw->hw_addr + reg); 119 } 120 121 /** 122 * rd64 - read a 64 bit hw register 123 * @hw: hardware information including registers 124 * @reg: register offset 125 * 126 * Return value of register content 127 */ 128 inline u64 rd64(struct irdma_hw *hw, u32 reg) 129 { 130 return readq(hw->hw_addr + reg); 131 } 132 133 static void irdma_gid_change_event(struct ib_device *ibdev) 134 { 135 struct ib_event ib_event; 136 137 ib_event.event = IB_EVENT_GID_CHANGE; 138 ib_event.device = ibdev; 139 ib_event.element.port_num = 1; 140 ib_dispatch_event(&ib_event); 141 } 142 143 /** 144 * irdma_inetaddr_event - system notifier for ipv4 addr events 145 * @notifier: not used 146 * @event: event for notifier 147 * @ptr: if address 148 */ 149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, 150 void *ptr) 151 { 152 struct in_ifaddr *ifa = ptr; 153 struct net_device *netdev = ifa->ifa_dev->dev; 154 struct irdma_device *iwdev; 155 struct ib_device *ibdev; 156 u32 local_ipaddr; 157 158 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 159 if (!ibdev) 160 return NOTIFY_DONE; 161 162 iwdev = to_iwdev(ibdev); 163 local_ipaddr = ntohl(ifa->ifa_address); 164 ibdev_dbg(&iwdev->ibdev, 165 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", netdev, 166 event, &local_ipaddr, netdev->dev_addr); 167 switch (event) { 168 case NETDEV_DOWN: 169 irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr, 170 &local_ipaddr, true, IRDMA_ARP_DELETE); 171 irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false); 172 irdma_gid_change_event(&iwdev->ibdev); 173 break; 174 case NETDEV_UP: 175 case NETDEV_CHANGEADDR: 176 irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr); 177 irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true); 178 irdma_gid_change_event(&iwdev->ibdev); 179 break; 180 default: 181 break; 182 } 183 184 ib_device_put(ibdev); 185 186 return NOTIFY_DONE; 187 } 188 189 /** 190 * irdma_inet6addr_event - system notifier for ipv6 addr events 191 * @notifier: not used 192 * @event: event for notifier 193 * @ptr: if address 194 */ 195 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, 196 void *ptr) 197 { 198 struct inet6_ifaddr *ifa = ptr; 199 struct net_device *netdev = ifa->idev->dev; 200 struct irdma_device *iwdev; 201 struct ib_device *ibdev; 202 u32 local_ipaddr6[4]; 203 204 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 205 if (!ibdev) 206 return NOTIFY_DONE; 207 208 iwdev = to_iwdev(ibdev); 209 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); 210 ibdev_dbg(&iwdev->ibdev, 211 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", netdev, 212 event, local_ipaddr6, netdev->dev_addr); 213 switch (event) { 214 case NETDEV_DOWN: 215 irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr, 216 local_ipaddr6, false, IRDMA_ARP_DELETE); 217 irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false); 218 irdma_gid_change_event(&iwdev->ibdev); 219 break; 220 case NETDEV_UP: 221 case NETDEV_CHANGEADDR: 222 irdma_add_arp(iwdev->rf, local_ipaddr6, false, 223 netdev->dev_addr); 224 irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true); 225 irdma_gid_change_event(&iwdev->ibdev); 226 break; 227 default: 228 break; 229 } 230 231 ib_device_put(ibdev); 232 233 return NOTIFY_DONE; 234 } 235 236 /** 237 * irdma_net_event - system notifier for net events 238 * @notifier: not used 239 * @event: event for notifier 240 * @ptr: neighbor 241 */ 242 int irdma_net_event(struct notifier_block *notifier, unsigned long event, 243 void *ptr) 244 { 245 struct neighbour *neigh = ptr; 246 struct irdma_device *iwdev; 247 struct ib_device *ibdev; 248 __be32 *p; 249 u32 local_ipaddr[4] = {}; 250 bool ipv4 = true; 251 252 ibdev = ib_device_get_by_netdev((struct net_device *)neigh->dev, 253 RDMA_DRIVER_IRDMA); 254 if (!ibdev) 255 return NOTIFY_DONE; 256 257 iwdev = to_iwdev(ibdev); 258 259 switch (event) { 260 case NETEVENT_NEIGH_UPDATE: 261 p = (__be32 *)neigh->primary_key; 262 if (neigh->tbl->family == AF_INET6) { 263 ipv4 = false; 264 irdma_copy_ip_ntohl(local_ipaddr, p); 265 } else { 266 local_ipaddr[0] = ntohl(*p); 267 } 268 269 ibdev_dbg(&iwdev->ibdev, 270 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n", 271 iwdev->netdev, neigh->nud_state, local_ipaddr, 272 neigh->ha); 273 274 if (neigh->nud_state & NUD_VALID) 275 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); 276 277 else 278 irdma_manage_arp_cache(iwdev->rf, neigh->ha, 279 local_ipaddr, ipv4, 280 IRDMA_ARP_DELETE); 281 break; 282 default: 283 break; 284 } 285 286 ib_device_put(ibdev); 287 288 return NOTIFY_DONE; 289 } 290 291 /** 292 * irdma_netdevice_event - system notifier for netdev events 293 * @notifier: not used 294 * @event: event for notifier 295 * @ptr: netdev 296 */ 297 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 298 void *ptr) 299 { 300 struct irdma_device *iwdev; 301 struct ib_device *ibdev; 302 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 303 304 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 305 if (!ibdev) 306 return NOTIFY_DONE; 307 308 iwdev = to_iwdev(ibdev); 309 iwdev->iw_status = 1; 310 switch (event) { 311 case NETDEV_DOWN: 312 iwdev->iw_status = 0; 313 fallthrough; 314 case NETDEV_UP: 315 irdma_port_ibevent(iwdev); 316 break; 317 default: 318 break; 319 } 320 ib_device_put(ibdev); 321 322 return NOTIFY_DONE; 323 } 324 325 /** 326 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table 327 * @iwdev: irdma device 328 */ 329 static void irdma_add_ipv6_addr(struct irdma_device *iwdev) 330 { 331 struct net_device *ip_dev; 332 struct inet6_dev *idev; 333 struct inet6_ifaddr *ifp, *tmp; 334 u32 local_ipaddr6[4]; 335 336 rcu_read_lock(); 337 for_each_netdev_rcu (&init_net, ip_dev) { 338 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF && 339 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || 340 ip_dev == iwdev->netdev) && 341 (READ_ONCE(ip_dev->flags) & IFF_UP)) { 342 idev = __in6_dev_get(ip_dev); 343 if (!idev) { 344 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); 345 break; 346 } 347 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, 348 if_list) { 349 ibdev_dbg(&iwdev->ibdev, 350 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n", 351 &ifp->addr, 352 rdma_vlan_dev_vlan_id(ip_dev), 353 ip_dev->dev_addr); 354 355 irdma_copy_ip_ntohl(local_ipaddr6, 356 ifp->addr.in6_u.u6_addr32); 357 irdma_manage_arp_cache(iwdev->rf, 358 ip_dev->dev_addr, 359 local_ipaddr6, false, 360 IRDMA_ARP_ADD); 361 } 362 } 363 } 364 rcu_read_unlock(); 365 } 366 367 /** 368 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table 369 * @iwdev: irdma device 370 */ 371 static void irdma_add_ipv4_addr(struct irdma_device *iwdev) 372 { 373 struct net_device *dev; 374 struct in_device *idev; 375 u32 ip_addr; 376 377 rcu_read_lock(); 378 for_each_netdev_rcu (&init_net, dev) { 379 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && 380 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || 381 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { 382 const struct in_ifaddr *ifa; 383 384 idev = __in_dev_get_rcu(dev); 385 if (!idev) 386 continue; 387 388 in_dev_for_each_ifa_rcu(ifa, idev) { 389 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", 390 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), 391 dev->dev_addr); 392 393 ip_addr = ntohl(ifa->ifa_address); 394 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, 395 &ip_addr, true, 396 IRDMA_ARP_ADD); 397 } 398 } 399 } 400 rcu_read_unlock(); 401 } 402 403 /** 404 * irdma_add_ip - add ip addresses 405 * @iwdev: irdma device 406 * 407 * Add ipv4/ipv6 addresses to the arp cache 408 */ 409 void irdma_add_ip(struct irdma_device *iwdev) 410 { 411 irdma_add_ipv4_addr(iwdev); 412 irdma_add_ipv6_addr(iwdev); 413 } 414 415 /** 416 * irdma_alloc_and_get_cqp_request - get cqp struct 417 * @cqp: device cqp ptr 418 * @wait: cqp to be used in wait mode 419 */ 420 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 421 bool wait) 422 { 423 struct irdma_cqp_request *cqp_request = NULL; 424 unsigned long flags; 425 426 spin_lock_irqsave(&cqp->req_lock, flags); 427 if (!list_empty(&cqp->cqp_avail_reqs)) { 428 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, 429 struct irdma_cqp_request, list); 430 list_del_init(&cqp_request->list); 431 } 432 spin_unlock_irqrestore(&cqp->req_lock, flags); 433 if (!cqp_request) { 434 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 435 if (cqp_request) { 436 cqp_request->dynamic = true; 437 if (wait) 438 init_waitqueue_head(&cqp_request->waitq); 439 } 440 } 441 if (!cqp_request) { 442 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); 443 return NULL; 444 } 445 446 cqp_request->waiting = wait; 447 refcount_set(&cqp_request->refcnt, 1); 448 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 449 450 return cqp_request; 451 } 452 453 /** 454 * irdma_get_cqp_request - increase refcount for cqp_request 455 * @cqp_request: pointer to cqp_request instance 456 */ 457 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 458 { 459 refcount_inc(&cqp_request->refcnt); 460 } 461 462 /** 463 * irdma_free_cqp_request - free cqp request 464 * @cqp: cqp ptr 465 * @cqp_request: to be put back in cqp list 466 */ 467 void irdma_free_cqp_request(struct irdma_cqp *cqp, 468 struct irdma_cqp_request *cqp_request) 469 { 470 unsigned long flags; 471 472 if (cqp_request->dynamic) { 473 kfree(cqp_request); 474 } else { 475 cqp_request->request_done = false; 476 cqp_request->callback_fcn = NULL; 477 cqp_request->waiting = false; 478 479 spin_lock_irqsave(&cqp->req_lock, flags); 480 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 481 spin_unlock_irqrestore(&cqp->req_lock, flags); 482 } 483 wake_up(&cqp->remove_wq); 484 } 485 486 /** 487 * irdma_put_cqp_request - dec ref count and free if 0 488 * @cqp: cqp ptr 489 * @cqp_request: to be put back in cqp list 490 */ 491 void irdma_put_cqp_request(struct irdma_cqp *cqp, 492 struct irdma_cqp_request *cqp_request) 493 { 494 if (refcount_dec_and_test(&cqp_request->refcnt)) 495 irdma_free_cqp_request(cqp, cqp_request); 496 } 497 498 /** 499 * irdma_free_pending_cqp_request -free pending cqp request objs 500 * @cqp: cqp ptr 501 * @cqp_request: to be put back in cqp list 502 */ 503 static void 504 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 505 struct irdma_cqp_request *cqp_request) 506 { 507 if (cqp_request->waiting) { 508 cqp_request->compl_info.error = true; 509 cqp_request->request_done = true; 510 wake_up(&cqp_request->waitq); 511 } 512 wait_event_timeout(cqp->remove_wq, 513 refcount_read(&cqp_request->refcnt) == 1, 1000); 514 irdma_put_cqp_request(cqp, cqp_request); 515 } 516 517 /** 518 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 519 * completions 520 * @rf: RDMA PCI function 521 */ 522 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 523 { 524 struct irdma_sc_dev *dev = &rf->sc_dev; 525 struct irdma_cqp *cqp = &rf->cqp; 526 struct irdma_cqp_request *cqp_request = NULL; 527 struct cqp_cmds_info *pcmdinfo = NULL; 528 u32 i, pending_work, wqe_idx; 529 530 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 531 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 532 for (i = 0; i < pending_work; i++) { 533 cqp_request = (struct irdma_cqp_request *)(unsigned long) 534 cqp->scratch_array[wqe_idx]; 535 if (cqp_request) 536 irdma_free_pending_cqp_request(cqp, cqp_request); 537 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 538 } 539 540 while (!list_empty(&dev->cqp_cmd_head)) { 541 pcmdinfo = irdma_remove_cqp_head(dev); 542 cqp_request = 543 container_of(pcmdinfo, struct irdma_cqp_request, info); 544 if (cqp_request) 545 irdma_free_pending_cqp_request(cqp, cqp_request); 546 } 547 } 548 549 /** 550 * irdma_wait_event - wait for completion 551 * @rf: RDMA PCI function 552 * @cqp_request: cqp request to wait 553 */ 554 static enum irdma_status_code irdma_wait_event(struct irdma_pci_f *rf, 555 struct irdma_cqp_request *cqp_request) 556 { 557 struct irdma_cqp_timeout cqp_timeout = {}; 558 bool cqp_error = false; 559 enum irdma_status_code err_code = 0; 560 561 cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]; 562 do { 563 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 564 if (wait_event_timeout(cqp_request->waitq, 565 cqp_request->request_done, 566 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) 567 break; 568 569 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 570 571 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 572 continue; 573 574 if (!rf->reset) { 575 rf->reset = true; 576 rf->gen_ops.request_reset(rf); 577 } 578 return IRDMA_ERR_TIMEOUT; 579 } while (1); 580 581 cqp_error = cqp_request->compl_info.error; 582 if (cqp_error) { 583 err_code = IRDMA_ERR_CQP_COMPL_ERROR; 584 if (cqp_request->compl_info.maj_err_code == 0xFFFF && 585 cqp_request->compl_info.min_err_code == 0x8029) { 586 if (!rf->reset) { 587 rf->reset = true; 588 rf->gen_ops.request_reset(rf); 589 } 590 } 591 } 592 593 return err_code; 594 } 595 596 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 597 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 598 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 599 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 600 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 601 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 602 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 603 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 604 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 605 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 606 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 607 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 608 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 609 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 610 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 611 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 612 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 613 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 614 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 615 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 616 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 617 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 618 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 619 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 620 [IRDMA_OP_RESUME] = "Resume QP Cmd", 621 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", 622 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 623 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 624 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 625 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 626 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 627 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 628 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 629 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 630 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 631 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 632 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 633 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 634 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 635 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 636 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 637 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 638 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 639 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 640 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 641 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 642 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 643 }; 644 645 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 646 {0xffff, 0x8006, "Flush No Wqe Pending"}, 647 {0xffff, 0x8007, "Modify QP Bad Close"}, 648 {0xffff, 0x8009, "LLP Closed"}, 649 {0xffff, 0x800a, "Reset Not Sent"} 650 }; 651 652 /** 653 * irdma_cqp_crit_err - check if CQP error is critical 654 * @dev: pointer to dev structure 655 * @cqp_cmd: code for last CQP operation 656 * @maj_err_code: major error code 657 * @min_err_code: minot error code 658 */ 659 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 660 u16 maj_err_code, u16 min_err_code) 661 { 662 int i; 663 664 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 665 if (maj_err_code == irdma_noncrit_err_list[i].maj && 666 min_err_code == irdma_noncrit_err_list[i].min) { 667 ibdev_dbg(to_ibdev(dev), 668 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n", 669 irdma_noncrit_err_list[i].desc, 670 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 671 min_err_code); 672 return false; 673 } 674 } 675 return true; 676 } 677 678 /** 679 * irdma_handle_cqp_op - process cqp command 680 * @rf: RDMA PCI function 681 * @cqp_request: cqp request to process 682 */ 683 enum irdma_status_code irdma_handle_cqp_op(struct irdma_pci_f *rf, 684 struct irdma_cqp_request *cqp_request) 685 { 686 struct irdma_sc_dev *dev = &rf->sc_dev; 687 struct cqp_cmds_info *info = &cqp_request->info; 688 enum irdma_status_code status; 689 bool put_cqp_request = true; 690 691 if (rf->reset) 692 return IRDMA_ERR_NOT_READY; 693 694 irdma_get_cqp_request(cqp_request); 695 status = irdma_process_cqp_cmd(dev, info); 696 if (status) 697 goto err; 698 699 if (cqp_request->waiting) { 700 put_cqp_request = false; 701 status = irdma_wait_event(rf, cqp_request); 702 if (status) 703 goto err; 704 } 705 706 return 0; 707 708 err: 709 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 710 cqp_request->compl_info.maj_err_code, 711 cqp_request->compl_info.min_err_code)) 712 ibdev_err(&rf->iwdev->ibdev, 713 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 714 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, 715 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, 716 cqp_request->compl_info.min_err_code); 717 718 if (put_cqp_request) 719 irdma_put_cqp_request(&rf->cqp, cqp_request); 720 721 return status; 722 } 723 724 void irdma_qp_add_ref(struct ib_qp *ibqp) 725 { 726 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; 727 728 refcount_inc(&iwqp->refcnt); 729 } 730 731 void irdma_qp_rem_ref(struct ib_qp *ibqp) 732 { 733 struct irdma_qp *iwqp = to_iwqp(ibqp); 734 struct irdma_device *iwdev = iwqp->iwdev; 735 u32 qp_num; 736 unsigned long flags; 737 738 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 739 if (!refcount_dec_and_test(&iwqp->refcnt)) { 740 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 741 return; 742 } 743 744 qp_num = iwqp->ibqp.qp_num; 745 iwdev->rf->qp_table[qp_num] = NULL; 746 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 747 complete(&iwqp->free_qp); 748 } 749 750 struct ib_device *to_ibdev(struct irdma_sc_dev *dev) 751 { 752 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 753 } 754 755 /** 756 * irdma_get_qp - get qp address 757 * @device: iwarp device 758 * @qpn: qp number 759 */ 760 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) 761 { 762 struct irdma_device *iwdev = to_iwdev(device); 763 764 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 765 return NULL; 766 767 return &iwdev->rf->qp_table[qpn]->ibqp; 768 } 769 770 /** 771 * irdma_get_hw_addr - return hw addr 772 * @par: points to shared dev 773 */ 774 u8 __iomem *irdma_get_hw_addr(void *par) 775 { 776 struct irdma_sc_dev *dev = par; 777 778 return dev->hw->hw_addr; 779 } 780 781 /** 782 * irdma_remove_cqp_head - return head entry and remove 783 * @dev: device 784 */ 785 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) 786 { 787 struct list_head *entry; 788 struct list_head *list = &dev->cqp_cmd_head; 789 790 if (list_empty(list)) 791 return NULL; 792 793 entry = list->next; 794 list_del(entry); 795 796 return entry; 797 } 798 799 /** 800 * irdma_cqp_sds_cmd - create cqp command for sd 801 * @dev: hardware control device structure 802 * @sdinfo: information for sd cqp 803 * 804 */ 805 enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 806 struct irdma_update_sds_info *sdinfo) 807 { 808 struct irdma_cqp_request *cqp_request; 809 struct cqp_cmds_info *cqp_info; 810 struct irdma_pci_f *rf = dev_to_rf(dev); 811 enum irdma_status_code status; 812 813 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 814 if (!cqp_request) 815 return IRDMA_ERR_NO_MEMORY; 816 817 cqp_info = &cqp_request->info; 818 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 819 sizeof(cqp_info->in.u.update_pe_sds.info)); 820 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 821 cqp_info->post_sq = 1; 822 cqp_info->in.u.update_pe_sds.dev = dev; 823 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 824 825 status = irdma_handle_cqp_op(rf, cqp_request); 826 irdma_put_cqp_request(&rf->cqp, cqp_request); 827 828 return status; 829 } 830 831 /** 832 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 833 * @qp: hardware control qp 834 * @op: suspend or resume 835 */ 836 enum irdma_status_code irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, 837 u8 op) 838 { 839 struct irdma_sc_dev *dev = qp->dev; 840 struct irdma_cqp_request *cqp_request; 841 struct irdma_sc_cqp *cqp = dev->cqp; 842 struct cqp_cmds_info *cqp_info; 843 struct irdma_pci_f *rf = dev_to_rf(dev); 844 enum irdma_status_code status; 845 846 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 847 if (!cqp_request) 848 return IRDMA_ERR_NO_MEMORY; 849 850 cqp_info = &cqp_request->info; 851 cqp_info->cqp_cmd = op; 852 cqp_info->in.u.suspend_resume.cqp = cqp; 853 cqp_info->in.u.suspend_resume.qp = qp; 854 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 855 856 status = irdma_handle_cqp_op(rf, cqp_request); 857 irdma_put_cqp_request(&rf->cqp, cqp_request); 858 859 return status; 860 } 861 862 /** 863 * irdma_term_modify_qp - modify qp for term message 864 * @qp: hardware control qp 865 * @next_state: qp's next state 866 * @term: terminate code 867 * @term_len: length 868 */ 869 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 870 u8 term_len) 871 { 872 struct irdma_qp *iwqp; 873 874 iwqp = qp->qp_uk.back_qp; 875 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 876 }; 877 878 /** 879 * irdma_terminate_done - after terminate is completed 880 * @qp: hardware control qp 881 * @timeout_occurred: indicates if terminate timer expired 882 */ 883 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 884 { 885 struct irdma_qp *iwqp; 886 u8 hte = 0; 887 bool first_time; 888 unsigned long flags; 889 890 iwqp = qp->qp_uk.back_qp; 891 spin_lock_irqsave(&iwqp->lock, flags); 892 if (iwqp->hte_added) { 893 iwqp->hte_added = 0; 894 hte = 1; 895 } 896 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 897 qp->term_flags |= IRDMA_TERM_DONE; 898 spin_unlock_irqrestore(&iwqp->lock, flags); 899 if (first_time) { 900 if (!timeout_occurred) 901 irdma_terminate_del_timer(qp); 902 903 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 904 irdma_cm_disconn(iwqp); 905 } 906 } 907 908 static void irdma_terminate_timeout(struct timer_list *t) 909 { 910 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 911 struct irdma_sc_qp *qp = &iwqp->sc_qp; 912 913 irdma_terminate_done(qp, 1); 914 irdma_qp_rem_ref(&iwqp->ibqp); 915 } 916 917 /** 918 * irdma_terminate_start_timer - start terminate timeout 919 * @qp: hardware control qp 920 */ 921 void irdma_terminate_start_timer(struct irdma_sc_qp *qp) 922 { 923 struct irdma_qp *iwqp; 924 925 iwqp = qp->qp_uk.back_qp; 926 irdma_qp_add_ref(&iwqp->ibqp); 927 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 928 iwqp->terminate_timer.expires = jiffies + HZ; 929 930 add_timer(&iwqp->terminate_timer); 931 } 932 933 /** 934 * irdma_terminate_del_timer - delete terminate timeout 935 * @qp: hardware control qp 936 */ 937 void irdma_terminate_del_timer(struct irdma_sc_qp *qp) 938 { 939 struct irdma_qp *iwqp; 940 int ret; 941 942 iwqp = qp->qp_uk.back_qp; 943 ret = del_timer(&iwqp->terminate_timer); 944 if (ret) 945 irdma_qp_rem_ref(&iwqp->ibqp); 946 } 947 948 /** 949 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 950 * @dev: function device struct 951 * @val_mem: buffer for fpm 952 * @hmc_fn_id: function id for fpm 953 */ 954 enum irdma_status_code 955 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 956 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 957 { 958 struct irdma_cqp_request *cqp_request; 959 struct cqp_cmds_info *cqp_info; 960 struct irdma_pci_f *rf = dev_to_rf(dev); 961 enum irdma_status_code status; 962 963 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 964 if (!cqp_request) 965 return IRDMA_ERR_NO_MEMORY; 966 967 cqp_info = &cqp_request->info; 968 cqp_request->param = NULL; 969 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 970 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 971 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 972 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 973 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 974 cqp_info->post_sq = 1; 975 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 976 977 status = irdma_handle_cqp_op(rf, cqp_request); 978 irdma_put_cqp_request(&rf->cqp, cqp_request); 979 980 return status; 981 } 982 983 /** 984 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 985 * @dev: hardware control device structure 986 * @val_mem: buffer with fpm values 987 * @hmc_fn_id: function id for fpm 988 */ 989 enum irdma_status_code 990 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 991 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 992 { 993 struct irdma_cqp_request *cqp_request; 994 struct cqp_cmds_info *cqp_info; 995 struct irdma_pci_f *rf = dev_to_rf(dev); 996 enum irdma_status_code status; 997 998 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 999 if (!cqp_request) 1000 return IRDMA_ERR_NO_MEMORY; 1001 1002 cqp_info = &cqp_request->info; 1003 cqp_request->param = NULL; 1004 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 1005 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 1006 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 1007 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 1008 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 1009 cqp_info->post_sq = 1; 1010 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 1011 1012 status = irdma_handle_cqp_op(rf, cqp_request); 1013 irdma_put_cqp_request(&rf->cqp, cqp_request); 1014 1015 return status; 1016 } 1017 1018 /** 1019 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1020 * @dev: device pointer 1021 * @cq: pointer to created cq 1022 */ 1023 enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, 1024 struct irdma_sc_cq *cq) 1025 { 1026 struct irdma_pci_f *rf = dev_to_rf(dev); 1027 struct irdma_cqp *iwcqp = &rf->cqp; 1028 struct irdma_cqp_request *cqp_request; 1029 struct cqp_cmds_info *cqp_info; 1030 enum irdma_status_code status; 1031 1032 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1033 if (!cqp_request) 1034 return IRDMA_ERR_NO_MEMORY; 1035 1036 cqp_info = &cqp_request->info; 1037 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1038 cqp_info->post_sq = 1; 1039 cqp_info->in.u.cq_create.cq = cq; 1040 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1041 1042 status = irdma_handle_cqp_op(rf, cqp_request); 1043 irdma_put_cqp_request(iwcqp, cqp_request); 1044 1045 return status; 1046 } 1047 1048 /** 1049 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1050 * @dev: device pointer 1051 * @qp: pointer to created qp 1052 */ 1053 enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, 1054 struct irdma_sc_qp *qp) 1055 { 1056 struct irdma_pci_f *rf = dev_to_rf(dev); 1057 struct irdma_cqp *iwcqp = &rf->cqp; 1058 struct irdma_cqp_request *cqp_request; 1059 struct cqp_cmds_info *cqp_info; 1060 struct irdma_create_qp_info *qp_info; 1061 enum irdma_status_code status; 1062 1063 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1064 if (!cqp_request) 1065 return IRDMA_ERR_NO_MEMORY; 1066 1067 cqp_info = &cqp_request->info; 1068 qp_info = &cqp_request->info.in.u.qp_create.info; 1069 memset(qp_info, 0, sizeof(*qp_info)); 1070 qp_info->cq_num_valid = true; 1071 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1072 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1073 cqp_info->post_sq = 1; 1074 cqp_info->in.u.qp_create.qp = qp; 1075 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1076 1077 status = irdma_handle_cqp_op(rf, cqp_request); 1078 irdma_put_cqp_request(iwcqp, cqp_request); 1079 1080 return status; 1081 } 1082 1083 /** 1084 * irdma_dealloc_push_page - free a push page for qp 1085 * @rf: RDMA PCI function 1086 * @qp: hardware control qp 1087 */ 1088 static void irdma_dealloc_push_page(struct irdma_pci_f *rf, 1089 struct irdma_sc_qp *qp) 1090 { 1091 struct irdma_cqp_request *cqp_request; 1092 struct cqp_cmds_info *cqp_info; 1093 enum irdma_status_code status; 1094 1095 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1096 return; 1097 1098 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1099 if (!cqp_request) 1100 return; 1101 1102 cqp_info = &cqp_request->info; 1103 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1104 cqp_info->post_sq = 1; 1105 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1106 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1107 cqp_info->in.u.manage_push_page.info.free_page = 1; 1108 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1109 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1110 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1111 status = irdma_handle_cqp_op(rf, cqp_request); 1112 if (!status) 1113 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1114 irdma_put_cqp_request(&rf->cqp, cqp_request); 1115 } 1116 1117 /** 1118 * irdma_free_qp_rsrc - free up memory resources for qp 1119 * @iwqp: qp ptr (user or kernel) 1120 */ 1121 void irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1122 { 1123 struct irdma_device *iwdev = iwqp->iwdev; 1124 struct irdma_pci_f *rf = iwdev->rf; 1125 u32 qp_num = iwqp->ibqp.qp_num; 1126 1127 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1128 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1129 if (iwqp->sc_qp.vsi) { 1130 irdma_qp_rem_qos(&iwqp->sc_qp); 1131 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1132 iwqp->sc_qp.user_pri); 1133 } 1134 1135 if (qp_num > 2) 1136 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1137 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, 1138 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); 1139 iwqp->q2_ctx_mem.va = NULL; 1140 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, 1141 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); 1142 iwqp->kqp.dma_mem.va = NULL; 1143 kfree(iwqp->kqp.sq_wrid_mem); 1144 iwqp->kqp.sq_wrid_mem = NULL; 1145 kfree(iwqp->kqp.rq_wrid_mem); 1146 iwqp->kqp.rq_wrid_mem = NULL; 1147 kfree(iwqp); 1148 } 1149 1150 /** 1151 * irdma_cq_wq_destroy - send cq destroy cqp 1152 * @rf: RDMA PCI function 1153 * @cq: hardware control cq 1154 */ 1155 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1156 { 1157 struct irdma_cqp_request *cqp_request; 1158 struct cqp_cmds_info *cqp_info; 1159 1160 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1161 if (!cqp_request) 1162 return; 1163 1164 cqp_info = &cqp_request->info; 1165 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1166 cqp_info->post_sq = 1; 1167 cqp_info->in.u.cq_destroy.cq = cq; 1168 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1169 1170 irdma_handle_cqp_op(rf, cqp_request); 1171 irdma_put_cqp_request(&rf->cqp, cqp_request); 1172 } 1173 1174 /** 1175 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1176 * @cqp_request: modify QP completion 1177 */ 1178 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1179 { 1180 struct cqp_cmds_info *cqp_info; 1181 struct irdma_qp *iwqp; 1182 1183 cqp_info = &cqp_request->info; 1184 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1185 atomic_dec(&iwqp->hw_mod_qp_pend); 1186 wake_up(&iwqp->mod_qp_waitq); 1187 } 1188 1189 /** 1190 * irdma_hw_modify_qp - setup cqp for modify qp 1191 * @iwdev: RDMA device 1192 * @iwqp: qp ptr (user or kernel) 1193 * @info: info for modify qp 1194 * @wait: flag to wait or not for modify qp completion 1195 */ 1196 enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev, 1197 struct irdma_qp *iwqp, 1198 struct irdma_modify_qp_info *info, 1199 bool wait) 1200 { 1201 enum irdma_status_code status; 1202 struct irdma_pci_f *rf = iwdev->rf; 1203 struct irdma_cqp_request *cqp_request; 1204 struct cqp_cmds_info *cqp_info; 1205 struct irdma_modify_qp_info *m_info; 1206 1207 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1208 if (!cqp_request) 1209 return IRDMA_ERR_NO_MEMORY; 1210 1211 if (!wait) { 1212 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1213 atomic_inc(&iwqp->hw_mod_qp_pend); 1214 } 1215 cqp_info = &cqp_request->info; 1216 m_info = &cqp_info->in.u.qp_modify.info; 1217 memcpy(m_info, info, sizeof(*m_info)); 1218 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1219 cqp_info->post_sq = 1; 1220 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1221 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1222 status = irdma_handle_cqp_op(rf, cqp_request); 1223 irdma_put_cqp_request(&rf->cqp, cqp_request); 1224 if (status) { 1225 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1226 return status; 1227 1228 switch (m_info->next_iwarp_state) { 1229 struct irdma_gen_ae_info ae_info; 1230 1231 case IRDMA_QP_STATE_RTS: 1232 case IRDMA_QP_STATE_IDLE: 1233 case IRDMA_QP_STATE_TERMINATE: 1234 case IRDMA_QP_STATE_CLOSING: 1235 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1236 irdma_send_reset(iwqp->cm_node); 1237 else 1238 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1239 if (!wait) { 1240 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1241 ae_info.ae_src = 0; 1242 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1243 } else { 1244 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1245 wait); 1246 if (!cqp_request) 1247 return IRDMA_ERR_NO_MEMORY; 1248 1249 cqp_info = &cqp_request->info; 1250 m_info = &cqp_info->in.u.qp_modify.info; 1251 memcpy(m_info, info, sizeof(*m_info)); 1252 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1253 cqp_info->post_sq = 1; 1254 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1255 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1256 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1257 m_info->reset_tcp_conn = true; 1258 irdma_handle_cqp_op(rf, cqp_request); 1259 irdma_put_cqp_request(&rf->cqp, cqp_request); 1260 } 1261 break; 1262 case IRDMA_QP_STATE_ERROR: 1263 default: 1264 break; 1265 } 1266 } 1267 1268 return status; 1269 } 1270 1271 /** 1272 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1273 * @dev: device pointer 1274 * @cq: pointer to cq 1275 */ 1276 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1277 { 1278 struct irdma_pci_f *rf = dev_to_rf(dev); 1279 1280 irdma_cq_wq_destroy(rf, cq); 1281 } 1282 1283 /** 1284 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1285 * @dev: device pointer 1286 * @qp: pointer to qp 1287 */ 1288 enum irdma_status_code irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1289 { 1290 struct irdma_pci_f *rf = dev_to_rf(dev); 1291 struct irdma_cqp *iwcqp = &rf->cqp; 1292 struct irdma_cqp_request *cqp_request; 1293 struct cqp_cmds_info *cqp_info; 1294 enum irdma_status_code status; 1295 1296 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1297 if (!cqp_request) 1298 return IRDMA_ERR_NO_MEMORY; 1299 1300 cqp_info = &cqp_request->info; 1301 memset(cqp_info, 0, sizeof(*cqp_info)); 1302 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1303 cqp_info->post_sq = 1; 1304 cqp_info->in.u.qp_destroy.qp = qp; 1305 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1306 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1307 1308 status = irdma_handle_cqp_op(rf, cqp_request); 1309 irdma_put_cqp_request(&rf->cqp, cqp_request); 1310 1311 return status; 1312 } 1313 1314 /** 1315 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1316 * @dev: hardware control device structure 1317 * @qp: hardware control qp 1318 */ 1319 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1320 { 1321 struct irdma_gen_ae_info info = {}; 1322 struct irdma_pci_f *rf = dev_to_rf(dev); 1323 1324 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); 1325 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1326 info.ae_src = IRDMA_AE_SOURCE_RQ; 1327 irdma_gen_ae(rf, qp, &info, false); 1328 } 1329 1330 /** 1331 * irdma_init_hash_desc - initialize hash for crc calculation 1332 * @desc: cryption type 1333 */ 1334 enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc) 1335 { 1336 struct crypto_shash *tfm; 1337 struct shash_desc *tdesc; 1338 1339 tfm = crypto_alloc_shash("crc32c", 0, 0); 1340 if (IS_ERR(tfm)) 1341 return IRDMA_ERR_MPA_CRC; 1342 1343 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), 1344 GFP_KERNEL); 1345 if (!tdesc) { 1346 crypto_free_shash(tfm); 1347 return IRDMA_ERR_MPA_CRC; 1348 } 1349 1350 tdesc->tfm = tfm; 1351 *desc = tdesc; 1352 1353 return 0; 1354 } 1355 1356 /** 1357 * irdma_free_hash_desc - free hash desc 1358 * @desc: to be freed 1359 */ 1360 void irdma_free_hash_desc(struct shash_desc *desc) 1361 { 1362 if (desc) { 1363 crypto_free_shash(desc->tfm); 1364 kfree(desc); 1365 } 1366 } 1367 1368 /** 1369 * irdma_ieq_check_mpacrc - check if mpa crc is OK 1370 * @desc: desc for hash 1371 * @addr: address of buffer for crc 1372 * @len: length of buffer 1373 * @val: value to be compared 1374 */ 1375 enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc, 1376 void *addr, u32 len, u32 val) 1377 { 1378 u32 crc = 0; 1379 int ret; 1380 enum irdma_status_code ret_code = 0; 1381 1382 crypto_shash_init(desc); 1383 ret = crypto_shash_update(desc, addr, len); 1384 if (!ret) 1385 crypto_shash_final(desc, (u8 *)&crc); 1386 if (crc != val) 1387 ret_code = IRDMA_ERR_MPA_CRC; 1388 1389 return ret_code; 1390 } 1391 1392 /** 1393 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1394 * @dev: hardware control device structure 1395 * @buf: receive puda buffer on exception q 1396 */ 1397 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1398 struct irdma_puda_buf *buf) 1399 { 1400 struct irdma_qp *iwqp; 1401 struct irdma_cm_node *cm_node; 1402 struct irdma_device *iwdev = buf->vsi->back_vsi; 1403 u32 loc_addr[4] = {}; 1404 u32 rem_addr[4] = {}; 1405 u16 loc_port, rem_port; 1406 struct ipv6hdr *ip6h; 1407 struct iphdr *iph = (struct iphdr *)buf->iph; 1408 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1409 1410 if (iph->version == 4) { 1411 loc_addr[0] = ntohl(iph->daddr); 1412 rem_addr[0] = ntohl(iph->saddr); 1413 } else { 1414 ip6h = (struct ipv6hdr *)buf->iph; 1415 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); 1416 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); 1417 } 1418 loc_port = ntohs(tcph->dest); 1419 rem_port = ntohs(tcph->source); 1420 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1421 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1422 if (!cm_node) 1423 return NULL; 1424 1425 iwqp = cm_node->iwqp; 1426 irdma_rem_ref_cm_node(cm_node); 1427 1428 return &iwqp->sc_qp; 1429 } 1430 1431 /** 1432 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1433 * @qp: qp ptr 1434 */ 1435 void irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1436 { 1437 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1438 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1439 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1440 1441 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1442 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); 1443 1444 irdma_send_ack(cm_node); 1445 } 1446 1447 /** 1448 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1449 * @qp: qp pointer 1450 * @ah_info: AH info pointer 1451 */ 1452 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1453 struct irdma_ah_info *ah_info) 1454 { 1455 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1456 struct iphdr *iph; 1457 struct ipv6hdr *ip6h; 1458 1459 memset(ah_info, 0, sizeof(*ah_info)); 1460 ah_info->do_lpbk = true; 1461 ah_info->vlan_tag = buf->vlan_id; 1462 ah_info->insert_vlan_tag = buf->vlan_valid; 1463 ah_info->ipv4_valid = buf->ipv4; 1464 ah_info->vsi = qp->vsi; 1465 1466 if (buf->smac_valid) 1467 ether_addr_copy(ah_info->mac_addr, buf->smac); 1468 1469 if (buf->ipv4) { 1470 ah_info->ipv4_valid = true; 1471 iph = (struct iphdr *)buf->iph; 1472 ah_info->hop_ttl = iph->ttl; 1473 ah_info->tc_tos = iph->tos; 1474 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); 1475 ah_info->src_ip_addr[0] = ntohl(iph->saddr); 1476 } else { 1477 ip6h = (struct ipv6hdr *)buf->iph; 1478 ah_info->hop_ttl = ip6h->hop_limit; 1479 ah_info->tc_tos = ip6h->priority; 1480 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1481 ip6h->daddr.in6_u.u6_addr32); 1482 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1483 ip6h->saddr.in6_u.u6_addr32); 1484 } 1485 1486 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1487 ah_info->dest_ip_addr, 1488 ah_info->ipv4_valid, 1489 NULL, IRDMA_ARP_RESOLVE); 1490 } 1491 1492 /** 1493 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1494 * @buf: puda to update 1495 * @len: length of buffer 1496 * @seqnum: seq number for tcp 1497 */ 1498 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1499 u16 len, u32 seqnum) 1500 { 1501 struct tcphdr *tcph; 1502 struct iphdr *iph; 1503 u16 iphlen; 1504 u16 pktsize; 1505 u8 *addr = buf->mem.va; 1506 1507 iphlen = (buf->ipv4) ? 20 : 40; 1508 iph = (struct iphdr *)(addr + buf->maclen); 1509 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1510 pktsize = len + buf->tcphlen + iphlen; 1511 iph->tot_len = htons(pktsize); 1512 tcph->seq = htonl(seqnum); 1513 } 1514 1515 /** 1516 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1517 * @buf: puda to update 1518 * @len: length of buffer 1519 * @seqnum: seq number for tcp 1520 */ 1521 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1522 u32 seqnum) 1523 { 1524 struct tcphdr *tcph; 1525 u8 *addr; 1526 1527 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1528 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1529 1530 addr = buf->mem.va; 1531 tcph = (struct tcphdr *)addr; 1532 tcph->seq = htonl(seqnum); 1533 } 1534 1535 /** 1536 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1537 * buffer 1538 * @info: to get information 1539 * @buf: puda buffer 1540 */ 1541 static enum irdma_status_code 1542 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1543 struct irdma_puda_buf *buf) 1544 { 1545 struct iphdr *iph; 1546 struct ipv6hdr *ip6h; 1547 struct tcphdr *tcph; 1548 u16 iphlen; 1549 u16 pkt_len; 1550 u8 *mem = buf->mem.va; 1551 struct ethhdr *ethh = buf->mem.va; 1552 1553 if (ethh->h_proto == htons(0x8100)) { 1554 info->vlan_valid = true; 1555 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & 1556 VLAN_VID_MASK; 1557 } 1558 1559 buf->maclen = (info->vlan_valid) ? 18 : 14; 1560 iphlen = (info->l3proto) ? 40 : 20; 1561 buf->ipv4 = (info->l3proto) ? false : true; 1562 buf->iph = mem + buf->maclen; 1563 iph = (struct iphdr *)buf->iph; 1564 buf->tcph = buf->iph + iphlen; 1565 tcph = (struct tcphdr *)buf->tcph; 1566 1567 if (buf->ipv4) { 1568 pkt_len = ntohs(iph->tot_len); 1569 } else { 1570 ip6h = (struct ipv6hdr *)buf->iph; 1571 pkt_len = ntohs(ip6h->payload_len) + iphlen; 1572 } 1573 1574 buf->totallen = pkt_len + buf->maclen; 1575 1576 if (info->payload_len < buf->totallen) { 1577 ibdev_dbg(to_ibdev(buf->vsi->dev), 1578 "ERR: payload_len = 0x%x totallen expected0x%x\n", 1579 info->payload_len, buf->totallen); 1580 return IRDMA_ERR_INVALID_SIZE; 1581 } 1582 1583 buf->tcphlen = tcph->doff << 2; 1584 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1585 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1586 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1587 buf->seqnum = ntohl(tcph->seq); 1588 1589 return 0; 1590 } 1591 1592 /** 1593 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1594 * @info: to get information 1595 * @buf: puda buffer 1596 */ 1597 enum irdma_status_code 1598 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1599 struct irdma_puda_buf *buf) 1600 { 1601 struct tcphdr *tcph; 1602 u32 pkt_len; 1603 u8 *mem; 1604 1605 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1606 return irdma_gen1_puda_get_tcpip_info(info, buf); 1607 1608 mem = buf->mem.va; 1609 buf->vlan_valid = info->vlan_valid; 1610 if (info->vlan_valid) 1611 buf->vlan_id = info->vlan; 1612 1613 buf->ipv4 = info->ipv4; 1614 if (buf->ipv4) 1615 buf->iph = mem + IRDMA_IPV4_PAD; 1616 else 1617 buf->iph = mem; 1618 1619 buf->tcph = mem + IRDMA_TCP_OFFSET; 1620 tcph = (struct tcphdr *)buf->tcph; 1621 pkt_len = info->payload_len; 1622 buf->totallen = pkt_len; 1623 buf->tcphlen = tcph->doff << 2; 1624 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1625 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1626 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1627 buf->seqnum = ntohl(tcph->seq); 1628 1629 if (info->smac_valid) { 1630 ether_addr_copy(buf->smac, info->smac); 1631 buf->smac_valid = true; 1632 } 1633 1634 return 0; 1635 } 1636 1637 /** 1638 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1639 * @t: timer_list pointer 1640 */ 1641 static void irdma_hw_stats_timeout(struct timer_list *t) 1642 { 1643 struct irdma_vsi_pestat *pf_devstat = 1644 from_timer(pf_devstat, t, stats_timer); 1645 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1646 1647 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1648 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); 1649 else 1650 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1651 1652 mod_timer(&pf_devstat->stats_timer, 1653 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1654 } 1655 1656 /** 1657 * irdma_hw_stats_start_timer - Start periodic stats timer 1658 * @vsi: vsi structure pointer 1659 */ 1660 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1661 { 1662 struct irdma_vsi_pestat *devstat = vsi->pestat; 1663 1664 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1665 mod_timer(&devstat->stats_timer, 1666 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1667 } 1668 1669 /** 1670 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1671 * @vsi: pointer to vsi structure 1672 */ 1673 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1674 { 1675 struct irdma_vsi_pestat *devstat = vsi->pestat; 1676 1677 del_timer_sync(&devstat->stats_timer); 1678 } 1679 1680 /** 1681 * irdma_process_stats - Checking for wrap and update stats 1682 * @pestat: stats structure pointer 1683 */ 1684 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) 1685 { 1686 sc_vsi_update_stats(pestat->vsi); 1687 } 1688 1689 /** 1690 * irdma_cqp_gather_stats_gen1 - Gather stats 1691 * @dev: pointer to device structure 1692 * @pestat: statistics structure 1693 */ 1694 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, 1695 struct irdma_vsi_pestat *pestat) 1696 { 1697 struct irdma_gather_stats *gather_stats = 1698 pestat->gather_info.gather_stats_va; 1699 u32 stats_inst_offset_32; 1700 u32 stats_inst_offset_64; 1701 1702 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? 1703 pestat->gather_info.stats_inst_index : 1704 pestat->hw->hmc.hmc_fn_id; 1705 stats_inst_offset_32 *= 4; 1706 stats_inst_offset_64 = stats_inst_offset_32 * 2; 1707 1708 gather_stats->rxvlanerr = 1709 rd32(dev->hw, 1710 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR] 1711 + stats_inst_offset_32); 1712 gather_stats->ip4rxdiscard = 1713 rd32(dev->hw, 1714 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] 1715 + stats_inst_offset_32); 1716 gather_stats->ip4rxtrunc = 1717 rd32(dev->hw, 1718 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] 1719 + stats_inst_offset_32); 1720 gather_stats->ip4txnoroute = 1721 rd32(dev->hw, 1722 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] 1723 + stats_inst_offset_32); 1724 gather_stats->ip6rxdiscard = 1725 rd32(dev->hw, 1726 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] 1727 + stats_inst_offset_32); 1728 gather_stats->ip6rxtrunc = 1729 rd32(dev->hw, 1730 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] 1731 + stats_inst_offset_32); 1732 gather_stats->ip6txnoroute = 1733 rd32(dev->hw, 1734 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] 1735 + stats_inst_offset_32); 1736 gather_stats->tcprtxseg = 1737 rd32(dev->hw, 1738 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG] 1739 + stats_inst_offset_32); 1740 gather_stats->tcprxopterr = 1741 rd32(dev->hw, 1742 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] 1743 + stats_inst_offset_32); 1744 1745 gather_stats->ip4rxocts = 1746 rd64(dev->hw, 1747 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS] 1748 + stats_inst_offset_64); 1749 gather_stats->ip4rxpkts = 1750 rd64(dev->hw, 1751 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS] 1752 + stats_inst_offset_64); 1753 gather_stats->ip4txfrag = 1754 rd64(dev->hw, 1755 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] 1756 + stats_inst_offset_64); 1757 gather_stats->ip4rxmcpkts = 1758 rd64(dev->hw, 1759 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] 1760 + stats_inst_offset_64); 1761 gather_stats->ip4txocts = 1762 rd64(dev->hw, 1763 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS] 1764 + stats_inst_offset_64); 1765 gather_stats->ip4txpkts = 1766 rd64(dev->hw, 1767 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS] 1768 + stats_inst_offset_64); 1769 gather_stats->ip4txfrag = 1770 rd64(dev->hw, 1771 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] 1772 + stats_inst_offset_64); 1773 gather_stats->ip4txmcpkts = 1774 rd64(dev->hw, 1775 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] 1776 + stats_inst_offset_64); 1777 gather_stats->ip6rxocts = 1778 rd64(dev->hw, 1779 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS] 1780 + stats_inst_offset_64); 1781 gather_stats->ip6rxpkts = 1782 rd64(dev->hw, 1783 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS] 1784 + stats_inst_offset_64); 1785 gather_stats->ip6txfrags = 1786 rd64(dev->hw, 1787 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] 1788 + stats_inst_offset_64); 1789 gather_stats->ip6rxmcpkts = 1790 rd64(dev->hw, 1791 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] 1792 + stats_inst_offset_64); 1793 gather_stats->ip6txocts = 1794 rd64(dev->hw, 1795 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS] 1796 + stats_inst_offset_64); 1797 gather_stats->ip6txpkts = 1798 rd64(dev->hw, 1799 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS] 1800 + stats_inst_offset_64); 1801 gather_stats->ip6txfrags = 1802 rd64(dev->hw, 1803 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] 1804 + stats_inst_offset_64); 1805 gather_stats->ip6txmcpkts = 1806 rd64(dev->hw, 1807 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] 1808 + stats_inst_offset_64); 1809 gather_stats->tcprxsegs = 1810 rd64(dev->hw, 1811 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS] 1812 + stats_inst_offset_64); 1813 gather_stats->tcptxsegs = 1814 rd64(dev->hw, 1815 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG] 1816 + stats_inst_offset_64); 1817 gather_stats->rdmarxrds = 1818 rd64(dev->hw, 1819 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS] 1820 + stats_inst_offset_64); 1821 gather_stats->rdmarxsnds = 1822 rd64(dev->hw, 1823 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS] 1824 + stats_inst_offset_64); 1825 gather_stats->rdmarxwrs = 1826 rd64(dev->hw, 1827 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS] 1828 + stats_inst_offset_64); 1829 gather_stats->rdmatxrds = 1830 rd64(dev->hw, 1831 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS] 1832 + stats_inst_offset_64); 1833 gather_stats->rdmatxsnds = 1834 rd64(dev->hw, 1835 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS] 1836 + stats_inst_offset_64); 1837 gather_stats->rdmatxwrs = 1838 rd64(dev->hw, 1839 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS] 1840 + stats_inst_offset_64); 1841 gather_stats->rdmavbn = 1842 rd64(dev->hw, 1843 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND] 1844 + stats_inst_offset_64); 1845 gather_stats->rdmavinv = 1846 rd64(dev->hw, 1847 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV] 1848 + stats_inst_offset_64); 1849 gather_stats->udprxpkts = 1850 rd64(dev->hw, 1851 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS] 1852 + stats_inst_offset_64); 1853 gather_stats->udptxpkts = 1854 rd64(dev->hw, 1855 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS] 1856 + stats_inst_offset_64); 1857 1858 irdma_process_stats(pestat); 1859 } 1860 1861 /** 1862 * irdma_process_cqp_stats - Checking for wrap and update stats 1863 * @cqp_request: cqp_request structure pointer 1864 */ 1865 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1866 { 1867 struct irdma_vsi_pestat *pestat = cqp_request->param; 1868 1869 irdma_process_stats(pestat); 1870 } 1871 1872 /** 1873 * irdma_cqp_gather_stats_cmd - Gather stats 1874 * @dev: pointer to device structure 1875 * @pestat: pointer to stats info 1876 * @wait: flag to wait or not wait for stats 1877 */ 1878 enum irdma_status_code 1879 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1880 struct irdma_vsi_pestat *pestat, bool wait) 1881 1882 { 1883 struct irdma_pci_f *rf = dev_to_rf(dev); 1884 struct irdma_cqp *iwcqp = &rf->cqp; 1885 struct irdma_cqp_request *cqp_request; 1886 struct cqp_cmds_info *cqp_info; 1887 enum irdma_status_code status; 1888 1889 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1890 if (!cqp_request) 1891 return IRDMA_ERR_NO_MEMORY; 1892 1893 cqp_info = &cqp_request->info; 1894 memset(cqp_info, 0, sizeof(*cqp_info)); 1895 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1896 cqp_info->post_sq = 1; 1897 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1898 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1899 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1900 cqp_request->param = pestat; 1901 if (!wait) 1902 cqp_request->callback_fcn = irdma_process_cqp_stats; 1903 status = irdma_handle_cqp_op(rf, cqp_request); 1904 if (wait) 1905 irdma_process_stats(pestat); 1906 irdma_put_cqp_request(&rf->cqp, cqp_request); 1907 1908 return status; 1909 } 1910 1911 /** 1912 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1913 * @vsi: pointer to vsi structure 1914 * @cmd: command to allocate or free 1915 * @stats_info: pointer to allocate stats info 1916 */ 1917 enum irdma_status_code 1918 irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1919 struct irdma_stats_inst_info *stats_info) 1920 { 1921 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1922 struct irdma_cqp *iwcqp = &rf->cqp; 1923 struct irdma_cqp_request *cqp_request; 1924 struct cqp_cmds_info *cqp_info; 1925 enum irdma_status_code status; 1926 bool wait = false; 1927 1928 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1929 wait = true; 1930 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1931 if (!cqp_request) 1932 return IRDMA_ERR_NO_MEMORY; 1933 1934 cqp_info = &cqp_request->info; 1935 memset(cqp_info, 0, sizeof(*cqp_info)); 1936 cqp_info->cqp_cmd = cmd; 1937 cqp_info->post_sq = 1; 1938 cqp_info->in.u.stats_manage.info = *stats_info; 1939 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1940 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1941 status = irdma_handle_cqp_op(rf, cqp_request); 1942 if (wait) 1943 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1944 irdma_put_cqp_request(iwcqp, cqp_request); 1945 1946 return status; 1947 } 1948 1949 /** 1950 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1951 * @dev: pointer to device info 1952 * @sc_ceq: pointer to ceq structure 1953 * @op: Create or Destroy 1954 */ 1955 enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, 1956 struct irdma_sc_ceq *sc_ceq, u8 op) 1957 { 1958 struct irdma_cqp_request *cqp_request; 1959 struct cqp_cmds_info *cqp_info; 1960 struct irdma_pci_f *rf = dev_to_rf(dev); 1961 enum irdma_status_code status; 1962 1963 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1964 if (!cqp_request) 1965 return IRDMA_ERR_NO_MEMORY; 1966 1967 cqp_info = &cqp_request->info; 1968 cqp_info->post_sq = 1; 1969 cqp_info->cqp_cmd = op; 1970 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1971 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1972 1973 status = irdma_handle_cqp_op(rf, cqp_request); 1974 irdma_put_cqp_request(&rf->cqp, cqp_request); 1975 1976 return status; 1977 } 1978 1979 /** 1980 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1981 * @dev: pointer to device info 1982 * @sc_aeq: pointer to aeq structure 1983 * @op: Create or Destroy 1984 */ 1985 enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, 1986 struct irdma_sc_aeq *sc_aeq, u8 op) 1987 { 1988 struct irdma_cqp_request *cqp_request; 1989 struct cqp_cmds_info *cqp_info; 1990 struct irdma_pci_f *rf = dev_to_rf(dev); 1991 enum irdma_status_code status; 1992 1993 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1994 if (!cqp_request) 1995 return IRDMA_ERR_NO_MEMORY; 1996 1997 cqp_info = &cqp_request->info; 1998 cqp_info->post_sq = 1; 1999 cqp_info->cqp_cmd = op; 2000 cqp_info->in.u.aeq_create.aeq = sc_aeq; 2001 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 2002 2003 status = irdma_handle_cqp_op(rf, cqp_request); 2004 irdma_put_cqp_request(&rf->cqp, cqp_request); 2005 2006 return status; 2007 } 2008 2009 /** 2010 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 2011 * @dev: pointer to device structure 2012 * @cmd: Add, modify or delete 2013 * @node_info: pointer to ws node info 2014 */ 2015 enum irdma_status_code 2016 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 2017 struct irdma_ws_node_info *node_info) 2018 { 2019 struct irdma_pci_f *rf = dev_to_rf(dev); 2020 struct irdma_cqp *iwcqp = &rf->cqp; 2021 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 2022 struct irdma_cqp_request *cqp_request; 2023 struct cqp_cmds_info *cqp_info; 2024 enum irdma_status_code status; 2025 bool poll; 2026 2027 if (!rf->sc_dev.ceq_valid) 2028 poll = true; 2029 else 2030 poll = false; 2031 2032 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 2033 if (!cqp_request) 2034 return IRDMA_ERR_NO_MEMORY; 2035 2036 cqp_info = &cqp_request->info; 2037 memset(cqp_info, 0, sizeof(*cqp_info)); 2038 cqp_info->cqp_cmd = cmd; 2039 cqp_info->post_sq = 1; 2040 cqp_info->in.u.ws_node.info = *node_info; 2041 cqp_info->in.u.ws_node.cqp = cqp; 2042 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 2043 status = irdma_handle_cqp_op(rf, cqp_request); 2044 if (status) 2045 goto exit; 2046 2047 if (poll) { 2048 struct irdma_ccq_cqe_info compl_info; 2049 2050 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 2051 &compl_info); 2052 node_info->qs_handle = compl_info.op_ret_val; 2053 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", 2054 compl_info.op_code, compl_info.op_ret_val); 2055 } else { 2056 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 2057 } 2058 2059 exit: 2060 irdma_put_cqp_request(&rf->cqp, cqp_request); 2061 2062 return status; 2063 } 2064 2065 /** 2066 * irdma_cqp_up_map_cmd - Set the up-up mapping 2067 * @dev: pointer to device structure 2068 * @cmd: map command 2069 * @map_info: pointer to up map info 2070 */ 2071 enum irdma_status_code irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev, u8 cmd, 2072 struct irdma_up_info *map_info) 2073 { 2074 struct irdma_pci_f *rf = dev_to_rf(dev); 2075 struct irdma_cqp *iwcqp = &rf->cqp; 2076 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 2077 struct irdma_cqp_request *cqp_request; 2078 struct cqp_cmds_info *cqp_info; 2079 enum irdma_status_code status; 2080 2081 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, false); 2082 if (!cqp_request) 2083 return IRDMA_ERR_NO_MEMORY; 2084 2085 cqp_info = &cqp_request->info; 2086 memset(cqp_info, 0, sizeof(*cqp_info)); 2087 cqp_info->cqp_cmd = cmd; 2088 cqp_info->post_sq = 1; 2089 cqp_info->in.u.up_map.info = *map_info; 2090 cqp_info->in.u.up_map.cqp = cqp; 2091 cqp_info->in.u.up_map.scratch = (uintptr_t)cqp_request; 2092 2093 status = irdma_handle_cqp_op(rf, cqp_request); 2094 irdma_put_cqp_request(&rf->cqp, cqp_request); 2095 2096 return status; 2097 } 2098 2099 /** 2100 * irdma_ah_cqp_op - perform an AH cqp operation 2101 * @rf: RDMA PCI function 2102 * @sc_ah: address handle 2103 * @cmd: AH operation 2104 * @wait: wait if true 2105 * @callback_fcn: Callback function on CQP op completion 2106 * @cb_param: parameter for callback function 2107 * 2108 * returns errno 2109 */ 2110 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 2111 bool wait, 2112 void (*callback_fcn)(struct irdma_cqp_request *), 2113 void *cb_param) 2114 { 2115 struct irdma_cqp_request *cqp_request; 2116 struct cqp_cmds_info *cqp_info; 2117 enum irdma_status_code status; 2118 2119 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 2120 return -EINVAL; 2121 2122 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 2123 if (!cqp_request) 2124 return -ENOMEM; 2125 2126 cqp_info = &cqp_request->info; 2127 cqp_info->cqp_cmd = cmd; 2128 cqp_info->post_sq = 1; 2129 if (cmd == IRDMA_OP_AH_CREATE) { 2130 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 2131 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 2132 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 2133 } else if (cmd == IRDMA_OP_AH_DESTROY) { 2134 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 2135 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 2136 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 2137 } 2138 2139 if (!wait) { 2140 cqp_request->callback_fcn = callback_fcn; 2141 cqp_request->param = cb_param; 2142 } 2143 status = irdma_handle_cqp_op(rf, cqp_request); 2144 irdma_put_cqp_request(&rf->cqp, cqp_request); 2145 2146 if (status) 2147 return -ENOMEM; 2148 2149 if (wait) 2150 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); 2151 2152 return 0; 2153 } 2154 2155 /** 2156 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 2157 * @cqp_request: pointer to cqp_request of create AH 2158 */ 2159 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 2160 { 2161 struct irdma_sc_qp *qp = cqp_request->param; 2162 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 2163 unsigned long flags; 2164 2165 spin_lock_irqsave(&qp->pfpdu.lock, flags); 2166 if (!cqp_request->compl_info.op_ret_val) { 2167 sc_ah->ah_info.ah_valid = true; 2168 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 2169 } else { 2170 sc_ah->ah_info.ah_valid = false; 2171 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 2172 } 2173 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 2174 } 2175 2176 /** 2177 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 2178 * @cqp_request: pointer to cqp_request of create AH 2179 */ 2180 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 2181 { 2182 struct irdma_cm_node *cm_node = cqp_request->param; 2183 struct irdma_sc_ah *sc_ah = cm_node->ah; 2184 2185 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 2186 irdma_add_conn_est_qh(cm_node); 2187 } 2188 2189 /** 2190 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 2191 * @dev: device pointer 2192 * @ah_info: Address handle info 2193 * @wait: When true will wait for operation to complete 2194 * @type: ILQ/IEQ 2195 * @cb_param: Callback param when not waiting 2196 * @ah_ret: Returned pointer to address handle if created 2197 * 2198 */ 2199 enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev, 2200 struct irdma_ah_info *ah_info, 2201 bool wait, enum puda_rsrc_type type, 2202 void *cb_param, 2203 struct irdma_sc_ah **ah_ret) 2204 { 2205 struct irdma_sc_ah *ah; 2206 struct irdma_pci_f *rf = dev_to_rf(dev); 2207 int err; 2208 2209 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2210 *ah_ret = ah; 2211 if (!ah) 2212 return IRDMA_ERR_NO_MEMORY; 2213 2214 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 2215 &ah_info->ah_idx, &rf->next_ah); 2216 if (err) 2217 goto err_free; 2218 2219 ah->dev = dev; 2220 ah->ah_info = *ah_info; 2221 2222 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 2223 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2224 irdma_ilq_ah_cb, cb_param); 2225 else 2226 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2227 irdma_ieq_ah_cb, cb_param); 2228 2229 if (err) 2230 goto error; 2231 return 0; 2232 2233 error: 2234 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2235 err_free: 2236 kfree(ah); 2237 *ah_ret = NULL; 2238 return IRDMA_ERR_NO_MEMORY; 2239 } 2240 2241 /** 2242 * irdma_puda_free_ah - free a puda address handle 2243 * @dev: device pointer 2244 * @ah: The address handle to free 2245 */ 2246 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2247 { 2248 struct irdma_pci_f *rf = dev_to_rf(dev); 2249 2250 if (!ah) 2251 return; 2252 2253 if (ah->ah_info.ah_valid) { 2254 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2255 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2256 } 2257 2258 kfree(ah); 2259 } 2260 2261 /** 2262 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 2263 * @cqp_request: pointer to cqp_request of create AH 2264 */ 2265 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 2266 { 2267 struct irdma_sc_ah *sc_ah = cqp_request->param; 2268 2269 if (!cqp_request->compl_info.op_ret_val) 2270 sc_ah->ah_info.ah_valid = true; 2271 else 2272 sc_ah->ah_info.ah_valid = false; 2273 } 2274 2275 /** 2276 * irdma_prm_add_pble_mem - add moemory to pble resources 2277 * @pprm: pble resource manager 2278 * @pchunk: chunk of memory to add 2279 */ 2280 enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2281 struct irdma_chunk *pchunk) 2282 { 2283 u64 sizeofbitmap; 2284 2285 if (pchunk->size & 0xfff) 2286 return IRDMA_ERR_PARAM; 2287 2288 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2289 2290 pchunk->bitmapmem.size = sizeofbitmap >> 3; 2291 pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL); 2292 2293 if (!pchunk->bitmapmem.va) 2294 return IRDMA_ERR_NO_MEMORY; 2295 2296 pchunk->bitmapbuf = pchunk->bitmapmem.va; 2297 bitmap_zero(pchunk->bitmapbuf, sizeofbitmap); 2298 2299 pchunk->sizeofbitmap = sizeofbitmap; 2300 /* each pble is 8 bytes hence shift by 3 */ 2301 pprm->total_pble_alloc += pchunk->size >> 3; 2302 pprm->free_pble_cnt += pchunk->size >> 3; 2303 2304 return 0; 2305 } 2306 2307 /** 2308 * irdma_prm_get_pbles - get pble's from prm 2309 * @pprm: pble resource manager 2310 * @chunkinfo: nformation about chunk where pble's were acquired 2311 * @mem_size: size of pble memory needed 2312 * @vaddr: returns virtual address of pble memory 2313 * @fpm_addr: returns fpm address of pble memory 2314 */ 2315 enum irdma_status_code 2316 irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2317 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2318 u64 **vaddr, u64 *fpm_addr) 2319 { 2320 u64 bits_needed; 2321 u64 bit_idx = PBLE_INVALID_IDX; 2322 struct irdma_chunk *pchunk = NULL; 2323 struct list_head *chunk_entry = pprm->clist.next; 2324 u32 offset; 2325 unsigned long flags; 2326 *vaddr = NULL; 2327 *fpm_addr = 0; 2328 2329 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2330 2331 spin_lock_irqsave(&pprm->prm_lock, flags); 2332 while (chunk_entry != &pprm->clist) { 2333 pchunk = (struct irdma_chunk *)chunk_entry; 2334 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2335 pchunk->sizeofbitmap, 0, 2336 bits_needed, 0); 2337 if (bit_idx < pchunk->sizeofbitmap) 2338 break; 2339 2340 /* list.next used macro */ 2341 chunk_entry = pchunk->list.next; 2342 } 2343 2344 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2345 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2346 return IRDMA_ERR_NO_MEMORY; 2347 } 2348 2349 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2350 offset = bit_idx << pprm->pble_shift; 2351 *vaddr = pchunk->vaddr + offset; 2352 *fpm_addr = pchunk->fpm_addr + offset; 2353 2354 chunkinfo->pchunk = pchunk; 2355 chunkinfo->bit_idx = bit_idx; 2356 chunkinfo->bits_used = bits_needed; 2357 /* 3 is sizeof pble divide */ 2358 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2359 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2360 2361 return 0; 2362 } 2363 2364 /** 2365 * irdma_prm_return_pbles - return pbles back to prm 2366 * @pprm: pble resource manager 2367 * @chunkinfo: chunk where pble's were acquired and to be freed 2368 */ 2369 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2370 struct irdma_pble_chunkinfo *chunkinfo) 2371 { 2372 unsigned long flags; 2373 2374 spin_lock_irqsave(&pprm->prm_lock, flags); 2375 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2376 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2377 chunkinfo->bits_used); 2378 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2379 } 2380 2381 enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va, 2382 dma_addr_t *pg_dma, u32 pg_cnt) 2383 { 2384 struct page *vm_page; 2385 int i; 2386 u8 *addr; 2387 2388 addr = (u8 *)(uintptr_t)va; 2389 for (i = 0; i < pg_cnt; i++) { 2390 vm_page = vmalloc_to_page(addr); 2391 if (!vm_page) 2392 goto err; 2393 2394 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, 2395 DMA_BIDIRECTIONAL); 2396 if (dma_mapping_error(hw->device, pg_dma[i])) 2397 goto err; 2398 2399 addr += PAGE_SIZE; 2400 } 2401 2402 return 0; 2403 2404 err: 2405 irdma_unmap_vm_page_list(hw, pg_dma, i); 2406 return IRDMA_ERR_NO_MEMORY; 2407 } 2408 2409 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) 2410 { 2411 int i; 2412 2413 for (i = 0; i < pg_cnt; i++) 2414 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2415 } 2416 2417 /** 2418 * irdma_pble_free_paged_mem - free virtual paged memory 2419 * @chunk: chunk to free with paged memory 2420 */ 2421 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2422 { 2423 if (!chunk->pg_cnt) 2424 goto done; 2425 2426 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2427 chunk->pg_cnt); 2428 2429 done: 2430 kfree(chunk->dmainfo.dmaaddrs); 2431 chunk->dmainfo.dmaaddrs = NULL; 2432 vfree(chunk->vaddr); 2433 chunk->vaddr = NULL; 2434 chunk->type = 0; 2435 } 2436 2437 /** 2438 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2439 * @chunk: chunk to add for paged memory 2440 * @pg_cnt: number of pages needed 2441 */ 2442 enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk, 2443 u32 pg_cnt) 2444 { 2445 u32 size; 2446 void *va; 2447 2448 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2449 if (!chunk->dmainfo.dmaaddrs) 2450 return IRDMA_ERR_NO_MEMORY; 2451 2452 size = PAGE_SIZE * pg_cnt; 2453 va = vmalloc(size); 2454 if (!va) 2455 goto err; 2456 2457 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2458 pg_cnt)) { 2459 vfree(va); 2460 goto err; 2461 } 2462 chunk->vaddr = va; 2463 chunk->size = size; 2464 chunk->pg_cnt = pg_cnt; 2465 chunk->type = PBLE_SD_PAGED; 2466 2467 return 0; 2468 err: 2469 kfree(chunk->dmainfo.dmaaddrs); 2470 chunk->dmainfo.dmaaddrs = NULL; 2471 2472 return IRDMA_ERR_NO_MEMORY; 2473 } 2474 2475 /** 2476 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2477 * @dev: device pointer 2478 */ 2479 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2480 { 2481 struct irdma_pci_f *rf = dev_to_rf(dev); 2482 u32 next = 1; 2483 u32 node_id; 2484 2485 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2486 &node_id, &next)) 2487 return IRDMA_WS_NODE_INVALID; 2488 2489 return (u16)node_id; 2490 } 2491 2492 /** 2493 * irdma_free_ws_node_id - Free a tx scheduler node ID 2494 * @dev: device pointer 2495 * @node_id: Work scheduler node ID 2496 */ 2497 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2498 { 2499 struct irdma_pci_f *rf = dev_to_rf(dev); 2500 2501 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2502 } 2503 2504 /** 2505 * irdma_modify_qp_to_err - Modify a QP to error 2506 * @sc_qp: qp structure 2507 */ 2508 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2509 { 2510 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2511 struct ib_qp_attr attr; 2512 2513 if (qp->iwdev->reset) 2514 return; 2515 attr.qp_state = IB_QPS_ERR; 2516 2517 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2518 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2519 else 2520 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2521 } 2522 2523 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2524 { 2525 struct ib_event ibevent; 2526 2527 if (!iwqp->ibqp.event_handler) 2528 return; 2529 2530 switch (event) { 2531 case IRDMA_QP_EVENT_CATASTROPHIC: 2532 ibevent.event = IB_EVENT_QP_FATAL; 2533 break; 2534 case IRDMA_QP_EVENT_ACCESS_ERR: 2535 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2536 break; 2537 } 2538 ibevent.device = iwqp->ibqp.device; 2539 ibevent.element.qp = &iwqp->ibqp; 2540 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2541 } 2542