1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_arp_table -manage arp table 7 * @rf: RDMA PCI function 8 * @ip_addr: ip address for device 9 * @ipv4: IPv4 flag 10 * @mac_addr: mac address ptr 11 * @action: modify, delete or add 12 */ 13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, 14 const u8 *mac_addr, u32 action) 15 { 16 unsigned long flags; 17 int arp_index; 18 u32 ip[4] = {}; 19 20 if (ipv4) 21 ip[0] = *ip_addr; 22 else 23 memcpy(ip, ip_addr, sizeof(ip)); 24 25 spin_lock_irqsave(&rf->arp_lock, flags); 26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 28 break; 29 } 30 31 switch (action) { 32 case IRDMA_ARP_ADD: 33 if (arp_index != rf->arp_table_size) { 34 arp_index = -1; 35 break; 36 } 37 38 arp_index = 0; 39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 40 (u32 *)&arp_index, &rf->next_arp_index)) { 41 arp_index = -1; 42 break; 43 } 44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip, 46 sizeof(rf->arp_table[arp_index].ip_addr)); 47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 48 break; 49 case IRDMA_ARP_RESOLVE: 50 if (arp_index == rf->arp_table_size) 51 arp_index = -1; 52 break; 53 case IRDMA_ARP_DELETE: 54 if (arp_index == rf->arp_table_size) { 55 arp_index = -1; 56 break; 57 } 58 59 memset(rf->arp_table[arp_index].ip_addr, 0, 60 sizeof(rf->arp_table[arp_index].ip_addr)); 61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 63 break; 64 default: 65 arp_index = -1; 66 break; 67 } 68 69 spin_unlock_irqrestore(&rf->arp_lock, flags); 70 return arp_index; 71 } 72 73 /** 74 * irdma_add_arp - add a new arp entry if needed 75 * @rf: RDMA function 76 * @ip: IP address 77 * @ipv4: IPv4 flag 78 * @mac: MAC address 79 */ 80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac) 81 { 82 int arpidx; 83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); 85 if (arpidx >= 0) { 86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 87 return arpidx; 88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 90 ipv4, IRDMA_ARP_DELETE); 91 } 92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD); 94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE); 96 } 97 98 /** 99 * wr32 - write 32 bits to hw register 100 * @hw: hardware information including registers 101 * @reg: register offset 102 * @val: value to write to register 103 */ 104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) 105 { 106 writel(val, hw->hw_addr + reg); 107 } 108 109 /** 110 * rd32 - read a 32 bit hw register 111 * @hw: hardware information including registers 112 * @reg: register offset 113 * 114 * Return value of register content 115 */ 116 inline u32 rd32(struct irdma_hw *hw, u32 reg) 117 { 118 return readl(hw->hw_addr + reg); 119 } 120 121 /** 122 * rd64 - read a 64 bit hw register 123 * @hw: hardware information including registers 124 * @reg: register offset 125 * 126 * Return value of register content 127 */ 128 inline u64 rd64(struct irdma_hw *hw, u32 reg) 129 { 130 return readq(hw->hw_addr + reg); 131 } 132 133 static void irdma_gid_change_event(struct ib_device *ibdev) 134 { 135 struct ib_event ib_event; 136 137 ib_event.event = IB_EVENT_GID_CHANGE; 138 ib_event.device = ibdev; 139 ib_event.element.port_num = 1; 140 ib_dispatch_event(&ib_event); 141 } 142 143 /** 144 * irdma_inetaddr_event - system notifier for ipv4 addr events 145 * @notifier: not used 146 * @event: event for notifier 147 * @ptr: if address 148 */ 149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, 150 void *ptr) 151 { 152 struct in_ifaddr *ifa = ptr; 153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; 154 struct irdma_device *iwdev; 155 struct ib_device *ibdev; 156 u32 local_ipaddr; 157 158 real_dev = rdma_vlan_dev_real_dev(netdev); 159 if (!real_dev) 160 real_dev = netdev; 161 162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 163 if (!ibdev) 164 return NOTIFY_DONE; 165 166 iwdev = to_iwdev(ibdev); 167 local_ipaddr = ntohl(ifa->ifa_address); 168 ibdev_dbg(&iwdev->ibdev, 169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev, 170 event, &local_ipaddr, real_dev->dev_addr); 171 switch (event) { 172 case NETDEV_DOWN: 173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 174 &local_ipaddr, true, IRDMA_ARP_DELETE); 175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false); 176 irdma_gid_change_event(&iwdev->ibdev); 177 break; 178 case NETDEV_UP: 179 case NETDEV_CHANGEADDR: 180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); 181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true); 182 irdma_gid_change_event(&iwdev->ibdev); 183 break; 184 default: 185 break; 186 } 187 188 ib_device_put(ibdev); 189 190 return NOTIFY_DONE; 191 } 192 193 /** 194 * irdma_inet6addr_event - system notifier for ipv6 addr events 195 * @notifier: not used 196 * @event: event for notifier 197 * @ptr: if address 198 */ 199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, 200 void *ptr) 201 { 202 struct inet6_ifaddr *ifa = ptr; 203 struct net_device *real_dev, *netdev = ifa->idev->dev; 204 struct irdma_device *iwdev; 205 struct ib_device *ibdev; 206 u32 local_ipaddr6[4]; 207 208 real_dev = rdma_vlan_dev_real_dev(netdev); 209 if (!real_dev) 210 real_dev = netdev; 211 212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 213 if (!ibdev) 214 return NOTIFY_DONE; 215 216 iwdev = to_iwdev(ibdev); 217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); 218 ibdev_dbg(&iwdev->ibdev, 219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev, 220 event, local_ipaddr6, real_dev->dev_addr); 221 switch (event) { 222 case NETDEV_DOWN: 223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 224 local_ipaddr6, false, IRDMA_ARP_DELETE); 225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false); 226 irdma_gid_change_event(&iwdev->ibdev); 227 break; 228 case NETDEV_UP: 229 case NETDEV_CHANGEADDR: 230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, 231 real_dev->dev_addr); 232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true); 233 irdma_gid_change_event(&iwdev->ibdev); 234 break; 235 default: 236 break; 237 } 238 239 ib_device_put(ibdev); 240 241 return NOTIFY_DONE; 242 } 243 244 /** 245 * irdma_net_event - system notifier for net events 246 * @notifier: not used 247 * @event: event for notifier 248 * @ptr: neighbor 249 */ 250 int irdma_net_event(struct notifier_block *notifier, unsigned long event, 251 void *ptr) 252 { 253 struct neighbour *neigh = ptr; 254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; 255 struct irdma_device *iwdev; 256 struct ib_device *ibdev; 257 __be32 *p; 258 u32 local_ipaddr[4] = {}; 259 bool ipv4 = true; 260 261 real_dev = rdma_vlan_dev_real_dev(netdev); 262 if (!real_dev) 263 real_dev = netdev; 264 265 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 266 if (!ibdev) 267 return NOTIFY_DONE; 268 269 iwdev = to_iwdev(ibdev); 270 271 switch (event) { 272 case NETEVENT_NEIGH_UPDATE: 273 p = (__be32 *)neigh->primary_key; 274 if (neigh->tbl->family == AF_INET6) { 275 ipv4 = false; 276 irdma_copy_ip_ntohl(local_ipaddr, p); 277 } else { 278 local_ipaddr[0] = ntohl(*p); 279 } 280 281 ibdev_dbg(&iwdev->ibdev, 282 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n", 283 iwdev->netdev, neigh->nud_state, local_ipaddr, 284 neigh->ha); 285 286 if (neigh->nud_state & NUD_VALID) 287 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); 288 289 else 290 irdma_manage_arp_cache(iwdev->rf, neigh->ha, 291 local_ipaddr, ipv4, 292 IRDMA_ARP_DELETE); 293 break; 294 default: 295 break; 296 } 297 298 ib_device_put(ibdev); 299 300 return NOTIFY_DONE; 301 } 302 303 /** 304 * irdma_netdevice_event - system notifier for netdev events 305 * @notifier: not used 306 * @event: event for notifier 307 * @ptr: netdev 308 */ 309 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 310 void *ptr) 311 { 312 struct irdma_device *iwdev; 313 struct ib_device *ibdev; 314 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 315 316 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 317 if (!ibdev) 318 return NOTIFY_DONE; 319 320 iwdev = to_iwdev(ibdev); 321 iwdev->iw_status = 1; 322 switch (event) { 323 case NETDEV_DOWN: 324 iwdev->iw_status = 0; 325 fallthrough; 326 case NETDEV_UP: 327 irdma_port_ibevent(iwdev); 328 break; 329 default: 330 break; 331 } 332 ib_device_put(ibdev); 333 334 return NOTIFY_DONE; 335 } 336 337 /** 338 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table 339 * @iwdev: irdma device 340 */ 341 static void irdma_add_ipv6_addr(struct irdma_device *iwdev) 342 { 343 struct net_device *ip_dev; 344 struct inet6_dev *idev; 345 struct inet6_ifaddr *ifp, *tmp; 346 u32 local_ipaddr6[4]; 347 348 rcu_read_lock(); 349 for_each_netdev_rcu (&init_net, ip_dev) { 350 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF && 351 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || 352 ip_dev == iwdev->netdev) && 353 (READ_ONCE(ip_dev->flags) & IFF_UP)) { 354 idev = __in6_dev_get(ip_dev); 355 if (!idev) { 356 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); 357 break; 358 } 359 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, 360 if_list) { 361 ibdev_dbg(&iwdev->ibdev, 362 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n", 363 &ifp->addr, 364 rdma_vlan_dev_vlan_id(ip_dev), 365 ip_dev->dev_addr); 366 367 irdma_copy_ip_ntohl(local_ipaddr6, 368 ifp->addr.in6_u.u6_addr32); 369 irdma_manage_arp_cache(iwdev->rf, 370 ip_dev->dev_addr, 371 local_ipaddr6, false, 372 IRDMA_ARP_ADD); 373 } 374 } 375 } 376 rcu_read_unlock(); 377 } 378 379 /** 380 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table 381 * @iwdev: irdma device 382 */ 383 static void irdma_add_ipv4_addr(struct irdma_device *iwdev) 384 { 385 struct net_device *dev; 386 struct in_device *idev; 387 u32 ip_addr; 388 389 rcu_read_lock(); 390 for_each_netdev_rcu (&init_net, dev) { 391 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && 392 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || 393 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { 394 const struct in_ifaddr *ifa; 395 396 idev = __in_dev_get_rcu(dev); 397 if (!idev) 398 continue; 399 400 in_dev_for_each_ifa_rcu(ifa, idev) { 401 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", 402 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), 403 dev->dev_addr); 404 405 ip_addr = ntohl(ifa->ifa_address); 406 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, 407 &ip_addr, true, 408 IRDMA_ARP_ADD); 409 } 410 } 411 } 412 rcu_read_unlock(); 413 } 414 415 /** 416 * irdma_add_ip - add ip addresses 417 * @iwdev: irdma device 418 * 419 * Add ipv4/ipv6 addresses to the arp cache 420 */ 421 void irdma_add_ip(struct irdma_device *iwdev) 422 { 423 irdma_add_ipv4_addr(iwdev); 424 irdma_add_ipv6_addr(iwdev); 425 } 426 427 /** 428 * irdma_alloc_and_get_cqp_request - get cqp struct 429 * @cqp: device cqp ptr 430 * @wait: cqp to be used in wait mode 431 */ 432 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 433 bool wait) 434 { 435 struct irdma_cqp_request *cqp_request = NULL; 436 unsigned long flags; 437 438 spin_lock_irqsave(&cqp->req_lock, flags); 439 if (!list_empty(&cqp->cqp_avail_reqs)) { 440 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, 441 struct irdma_cqp_request, list); 442 list_del_init(&cqp_request->list); 443 } 444 spin_unlock_irqrestore(&cqp->req_lock, flags); 445 if (!cqp_request) { 446 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 447 if (cqp_request) { 448 cqp_request->dynamic = true; 449 if (wait) 450 init_waitqueue_head(&cqp_request->waitq); 451 } 452 } 453 if (!cqp_request) { 454 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); 455 return NULL; 456 } 457 458 cqp_request->waiting = wait; 459 refcount_set(&cqp_request->refcnt, 1); 460 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 461 462 return cqp_request; 463 } 464 465 /** 466 * irdma_get_cqp_request - increase refcount for cqp_request 467 * @cqp_request: pointer to cqp_request instance 468 */ 469 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 470 { 471 refcount_inc(&cqp_request->refcnt); 472 } 473 474 /** 475 * irdma_free_cqp_request - free cqp request 476 * @cqp: cqp ptr 477 * @cqp_request: to be put back in cqp list 478 */ 479 void irdma_free_cqp_request(struct irdma_cqp *cqp, 480 struct irdma_cqp_request *cqp_request) 481 { 482 unsigned long flags; 483 484 if (cqp_request->dynamic) { 485 kfree(cqp_request); 486 } else { 487 cqp_request->request_done = false; 488 cqp_request->callback_fcn = NULL; 489 cqp_request->waiting = false; 490 491 spin_lock_irqsave(&cqp->req_lock, flags); 492 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 493 spin_unlock_irqrestore(&cqp->req_lock, flags); 494 } 495 wake_up(&cqp->remove_wq); 496 } 497 498 /** 499 * irdma_put_cqp_request - dec ref count and free if 0 500 * @cqp: cqp ptr 501 * @cqp_request: to be put back in cqp list 502 */ 503 void irdma_put_cqp_request(struct irdma_cqp *cqp, 504 struct irdma_cqp_request *cqp_request) 505 { 506 if (refcount_dec_and_test(&cqp_request->refcnt)) 507 irdma_free_cqp_request(cqp, cqp_request); 508 } 509 510 /** 511 * irdma_free_pending_cqp_request -free pending cqp request objs 512 * @cqp: cqp ptr 513 * @cqp_request: to be put back in cqp list 514 */ 515 static void 516 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 517 struct irdma_cqp_request *cqp_request) 518 { 519 if (cqp_request->waiting) { 520 cqp_request->compl_info.error = true; 521 cqp_request->request_done = true; 522 wake_up(&cqp_request->waitq); 523 } 524 wait_event_timeout(cqp->remove_wq, 525 refcount_read(&cqp_request->refcnt) == 1, 1000); 526 irdma_put_cqp_request(cqp, cqp_request); 527 } 528 529 /** 530 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 531 * completions 532 * @rf: RDMA PCI function 533 */ 534 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 535 { 536 struct irdma_sc_dev *dev = &rf->sc_dev; 537 struct irdma_cqp *cqp = &rf->cqp; 538 struct irdma_cqp_request *cqp_request = NULL; 539 struct cqp_cmds_info *pcmdinfo = NULL; 540 u32 i, pending_work, wqe_idx; 541 542 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 543 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 544 for (i = 0; i < pending_work; i++) { 545 cqp_request = (struct irdma_cqp_request *)(unsigned long) 546 cqp->scratch_array[wqe_idx]; 547 if (cqp_request) 548 irdma_free_pending_cqp_request(cqp, cqp_request); 549 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 550 } 551 552 while (!list_empty(&dev->cqp_cmd_head)) { 553 pcmdinfo = irdma_remove_cqp_head(dev); 554 cqp_request = 555 container_of(pcmdinfo, struct irdma_cqp_request, info); 556 if (cqp_request) 557 irdma_free_pending_cqp_request(cqp, cqp_request); 558 } 559 } 560 561 /** 562 * irdma_wait_event - wait for completion 563 * @rf: RDMA PCI function 564 * @cqp_request: cqp request to wait 565 */ 566 static int irdma_wait_event(struct irdma_pci_f *rf, 567 struct irdma_cqp_request *cqp_request) 568 { 569 struct irdma_cqp_timeout cqp_timeout = {}; 570 bool cqp_error = false; 571 int err_code = 0; 572 573 cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]; 574 do { 575 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 576 if (wait_event_timeout(cqp_request->waitq, 577 cqp_request->request_done, 578 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) 579 break; 580 581 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 582 583 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 584 continue; 585 586 if (!rf->reset) { 587 rf->reset = true; 588 rf->gen_ops.request_reset(rf); 589 } 590 return -ETIMEDOUT; 591 } while (1); 592 593 cqp_error = cqp_request->compl_info.error; 594 if (cqp_error) { 595 err_code = -EIO; 596 if (cqp_request->compl_info.maj_err_code == 0xFFFF && 597 cqp_request->compl_info.min_err_code == 0x8029) { 598 if (!rf->reset) { 599 rf->reset = true; 600 rf->gen_ops.request_reset(rf); 601 } 602 } 603 } 604 605 return err_code; 606 } 607 608 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 609 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 610 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 611 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 612 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 613 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 614 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 615 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 616 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 617 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 618 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 619 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 620 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 621 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 622 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 623 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 624 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 625 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 626 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 627 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 628 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 629 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 630 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 631 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 632 [IRDMA_OP_RESUME] = "Resume QP Cmd", 633 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", 634 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 635 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 636 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 637 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 638 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 639 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 640 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 641 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 642 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 643 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 644 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 645 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 646 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 647 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 648 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 649 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 650 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 651 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 652 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 653 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 654 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 655 }; 656 657 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 658 {0xffff, 0x8006, "Flush No Wqe Pending"}, 659 {0xffff, 0x8007, "Modify QP Bad Close"}, 660 {0xffff, 0x8009, "LLP Closed"}, 661 {0xffff, 0x800a, "Reset Not Sent"} 662 }; 663 664 /** 665 * irdma_cqp_crit_err - check if CQP error is critical 666 * @dev: pointer to dev structure 667 * @cqp_cmd: code for last CQP operation 668 * @maj_err_code: major error code 669 * @min_err_code: minot error code 670 */ 671 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 672 u16 maj_err_code, u16 min_err_code) 673 { 674 int i; 675 676 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 677 if (maj_err_code == irdma_noncrit_err_list[i].maj && 678 min_err_code == irdma_noncrit_err_list[i].min) { 679 ibdev_dbg(to_ibdev(dev), 680 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n", 681 irdma_noncrit_err_list[i].desc, 682 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 683 min_err_code); 684 return false; 685 } 686 } 687 return true; 688 } 689 690 /** 691 * irdma_handle_cqp_op - process cqp command 692 * @rf: RDMA PCI function 693 * @cqp_request: cqp request to process 694 */ 695 int irdma_handle_cqp_op(struct irdma_pci_f *rf, 696 struct irdma_cqp_request *cqp_request) 697 { 698 struct irdma_sc_dev *dev = &rf->sc_dev; 699 struct cqp_cmds_info *info = &cqp_request->info; 700 int status; 701 bool put_cqp_request = true; 702 703 if (rf->reset) 704 return -EBUSY; 705 706 irdma_get_cqp_request(cqp_request); 707 status = irdma_process_cqp_cmd(dev, info); 708 if (status) 709 goto err; 710 711 if (cqp_request->waiting) { 712 put_cqp_request = false; 713 status = irdma_wait_event(rf, cqp_request); 714 if (status) 715 goto err; 716 } 717 718 return 0; 719 720 err: 721 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 722 cqp_request->compl_info.maj_err_code, 723 cqp_request->compl_info.min_err_code)) 724 ibdev_err(&rf->iwdev->ibdev, 725 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 726 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, 727 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, 728 cqp_request->compl_info.min_err_code); 729 730 if (put_cqp_request) 731 irdma_put_cqp_request(&rf->cqp, cqp_request); 732 733 return status; 734 } 735 736 void irdma_qp_add_ref(struct ib_qp *ibqp) 737 { 738 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; 739 740 refcount_inc(&iwqp->refcnt); 741 } 742 743 void irdma_qp_rem_ref(struct ib_qp *ibqp) 744 { 745 struct irdma_qp *iwqp = to_iwqp(ibqp); 746 struct irdma_device *iwdev = iwqp->iwdev; 747 u32 qp_num; 748 unsigned long flags; 749 750 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 751 if (!refcount_dec_and_test(&iwqp->refcnt)) { 752 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 753 return; 754 } 755 756 qp_num = iwqp->ibqp.qp_num; 757 iwdev->rf->qp_table[qp_num] = NULL; 758 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 759 complete(&iwqp->free_qp); 760 } 761 762 struct ib_device *to_ibdev(struct irdma_sc_dev *dev) 763 { 764 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 765 } 766 767 /** 768 * irdma_get_qp - get qp address 769 * @device: iwarp device 770 * @qpn: qp number 771 */ 772 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) 773 { 774 struct irdma_device *iwdev = to_iwdev(device); 775 776 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 777 return NULL; 778 779 return &iwdev->rf->qp_table[qpn]->ibqp; 780 } 781 782 /** 783 * irdma_remove_cqp_head - return head entry and remove 784 * @dev: device 785 */ 786 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) 787 { 788 struct list_head *entry; 789 struct list_head *list = &dev->cqp_cmd_head; 790 791 if (list_empty(list)) 792 return NULL; 793 794 entry = list->next; 795 list_del(entry); 796 797 return entry; 798 } 799 800 /** 801 * irdma_cqp_sds_cmd - create cqp command for sd 802 * @dev: hardware control device structure 803 * @sdinfo: information for sd cqp 804 * 805 */ 806 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 807 struct irdma_update_sds_info *sdinfo) 808 { 809 struct irdma_cqp_request *cqp_request; 810 struct cqp_cmds_info *cqp_info; 811 struct irdma_pci_f *rf = dev_to_rf(dev); 812 int status; 813 814 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 815 if (!cqp_request) 816 return -ENOMEM; 817 818 cqp_info = &cqp_request->info; 819 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 820 sizeof(cqp_info->in.u.update_pe_sds.info)); 821 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 822 cqp_info->post_sq = 1; 823 cqp_info->in.u.update_pe_sds.dev = dev; 824 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 825 826 status = irdma_handle_cqp_op(rf, cqp_request); 827 irdma_put_cqp_request(&rf->cqp, cqp_request); 828 829 return status; 830 } 831 832 /** 833 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 834 * @qp: hardware control qp 835 * @op: suspend or resume 836 */ 837 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 838 { 839 struct irdma_sc_dev *dev = qp->dev; 840 struct irdma_cqp_request *cqp_request; 841 struct irdma_sc_cqp *cqp = dev->cqp; 842 struct cqp_cmds_info *cqp_info; 843 struct irdma_pci_f *rf = dev_to_rf(dev); 844 int status; 845 846 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 847 if (!cqp_request) 848 return -ENOMEM; 849 850 cqp_info = &cqp_request->info; 851 cqp_info->cqp_cmd = op; 852 cqp_info->in.u.suspend_resume.cqp = cqp; 853 cqp_info->in.u.suspend_resume.qp = qp; 854 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 855 856 status = irdma_handle_cqp_op(rf, cqp_request); 857 irdma_put_cqp_request(&rf->cqp, cqp_request); 858 859 return status; 860 } 861 862 /** 863 * irdma_term_modify_qp - modify qp for term message 864 * @qp: hardware control qp 865 * @next_state: qp's next state 866 * @term: terminate code 867 * @term_len: length 868 */ 869 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 870 u8 term_len) 871 { 872 struct irdma_qp *iwqp; 873 874 iwqp = qp->qp_uk.back_qp; 875 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 876 }; 877 878 /** 879 * irdma_terminate_done - after terminate is completed 880 * @qp: hardware control qp 881 * @timeout_occurred: indicates if terminate timer expired 882 */ 883 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 884 { 885 struct irdma_qp *iwqp; 886 u8 hte = 0; 887 bool first_time; 888 unsigned long flags; 889 890 iwqp = qp->qp_uk.back_qp; 891 spin_lock_irqsave(&iwqp->lock, flags); 892 if (iwqp->hte_added) { 893 iwqp->hte_added = 0; 894 hte = 1; 895 } 896 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 897 qp->term_flags |= IRDMA_TERM_DONE; 898 spin_unlock_irqrestore(&iwqp->lock, flags); 899 if (first_time) { 900 if (!timeout_occurred) 901 irdma_terminate_del_timer(qp); 902 903 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 904 irdma_cm_disconn(iwqp); 905 } 906 } 907 908 static void irdma_terminate_timeout(struct timer_list *t) 909 { 910 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 911 struct irdma_sc_qp *qp = &iwqp->sc_qp; 912 913 irdma_terminate_done(qp, 1); 914 irdma_qp_rem_ref(&iwqp->ibqp); 915 } 916 917 /** 918 * irdma_terminate_start_timer - start terminate timeout 919 * @qp: hardware control qp 920 */ 921 void irdma_terminate_start_timer(struct irdma_sc_qp *qp) 922 { 923 struct irdma_qp *iwqp; 924 925 iwqp = qp->qp_uk.back_qp; 926 irdma_qp_add_ref(&iwqp->ibqp); 927 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 928 iwqp->terminate_timer.expires = jiffies + HZ; 929 930 add_timer(&iwqp->terminate_timer); 931 } 932 933 /** 934 * irdma_terminate_del_timer - delete terminate timeout 935 * @qp: hardware control qp 936 */ 937 void irdma_terminate_del_timer(struct irdma_sc_qp *qp) 938 { 939 struct irdma_qp *iwqp; 940 int ret; 941 942 iwqp = qp->qp_uk.back_qp; 943 ret = del_timer(&iwqp->terminate_timer); 944 if (ret) 945 irdma_qp_rem_ref(&iwqp->ibqp); 946 } 947 948 /** 949 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 950 * @dev: function device struct 951 * @val_mem: buffer for fpm 952 * @hmc_fn_id: function id for fpm 953 */ 954 int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 955 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 956 { 957 struct irdma_cqp_request *cqp_request; 958 struct cqp_cmds_info *cqp_info; 959 struct irdma_pci_f *rf = dev_to_rf(dev); 960 int status; 961 962 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 963 if (!cqp_request) 964 return -ENOMEM; 965 966 cqp_info = &cqp_request->info; 967 cqp_request->param = NULL; 968 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 969 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 970 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 971 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 972 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 973 cqp_info->post_sq = 1; 974 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 975 976 status = irdma_handle_cqp_op(rf, cqp_request); 977 irdma_put_cqp_request(&rf->cqp, cqp_request); 978 979 return status; 980 } 981 982 /** 983 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 984 * @dev: hardware control device structure 985 * @val_mem: buffer with fpm values 986 * @hmc_fn_id: function id for fpm 987 */ 988 int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 989 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 990 { 991 struct irdma_cqp_request *cqp_request; 992 struct cqp_cmds_info *cqp_info; 993 struct irdma_pci_f *rf = dev_to_rf(dev); 994 int status; 995 996 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 997 if (!cqp_request) 998 return -ENOMEM; 999 1000 cqp_info = &cqp_request->info; 1001 cqp_request->param = NULL; 1002 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 1003 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 1004 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 1005 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 1006 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 1007 cqp_info->post_sq = 1; 1008 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 1009 1010 status = irdma_handle_cqp_op(rf, cqp_request); 1011 irdma_put_cqp_request(&rf->cqp, cqp_request); 1012 1013 return status; 1014 } 1015 1016 /** 1017 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1018 * @dev: device pointer 1019 * @cq: pointer to created cq 1020 */ 1021 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1022 { 1023 struct irdma_pci_f *rf = dev_to_rf(dev); 1024 struct irdma_cqp *iwcqp = &rf->cqp; 1025 struct irdma_cqp_request *cqp_request; 1026 struct cqp_cmds_info *cqp_info; 1027 int status; 1028 1029 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1030 if (!cqp_request) 1031 return -ENOMEM; 1032 1033 cqp_info = &cqp_request->info; 1034 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1035 cqp_info->post_sq = 1; 1036 cqp_info->in.u.cq_create.cq = cq; 1037 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1038 1039 status = irdma_handle_cqp_op(rf, cqp_request); 1040 irdma_put_cqp_request(iwcqp, cqp_request); 1041 1042 return status; 1043 } 1044 1045 /** 1046 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1047 * @dev: device pointer 1048 * @qp: pointer to created qp 1049 */ 1050 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1051 { 1052 struct irdma_pci_f *rf = dev_to_rf(dev); 1053 struct irdma_cqp *iwcqp = &rf->cqp; 1054 struct irdma_cqp_request *cqp_request; 1055 struct cqp_cmds_info *cqp_info; 1056 struct irdma_create_qp_info *qp_info; 1057 int status; 1058 1059 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1060 if (!cqp_request) 1061 return -ENOMEM; 1062 1063 cqp_info = &cqp_request->info; 1064 qp_info = &cqp_request->info.in.u.qp_create.info; 1065 memset(qp_info, 0, sizeof(*qp_info)); 1066 qp_info->cq_num_valid = true; 1067 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1068 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1069 cqp_info->post_sq = 1; 1070 cqp_info->in.u.qp_create.qp = qp; 1071 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1072 1073 status = irdma_handle_cqp_op(rf, cqp_request); 1074 irdma_put_cqp_request(iwcqp, cqp_request); 1075 1076 return status; 1077 } 1078 1079 /** 1080 * irdma_dealloc_push_page - free a push page for qp 1081 * @rf: RDMA PCI function 1082 * @qp: hardware control qp 1083 */ 1084 static void irdma_dealloc_push_page(struct irdma_pci_f *rf, 1085 struct irdma_sc_qp *qp) 1086 { 1087 struct irdma_cqp_request *cqp_request; 1088 struct cqp_cmds_info *cqp_info; 1089 int status; 1090 1091 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1092 return; 1093 1094 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1095 if (!cqp_request) 1096 return; 1097 1098 cqp_info = &cqp_request->info; 1099 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1100 cqp_info->post_sq = 1; 1101 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1102 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1103 cqp_info->in.u.manage_push_page.info.free_page = 1; 1104 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1105 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1106 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1107 status = irdma_handle_cqp_op(rf, cqp_request); 1108 if (!status) 1109 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1110 irdma_put_cqp_request(&rf->cqp, cqp_request); 1111 } 1112 1113 /** 1114 * irdma_free_qp_rsrc - free up memory resources for qp 1115 * @iwqp: qp ptr (user or kernel) 1116 */ 1117 void irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1118 { 1119 struct irdma_device *iwdev = iwqp->iwdev; 1120 struct irdma_pci_f *rf = iwdev->rf; 1121 u32 qp_num = iwqp->ibqp.qp_num; 1122 1123 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1124 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1125 if (iwqp->sc_qp.vsi) { 1126 irdma_qp_rem_qos(&iwqp->sc_qp); 1127 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1128 iwqp->sc_qp.user_pri); 1129 } 1130 1131 if (qp_num > 2) 1132 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1133 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, 1134 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); 1135 iwqp->q2_ctx_mem.va = NULL; 1136 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, 1137 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); 1138 iwqp->kqp.dma_mem.va = NULL; 1139 kfree(iwqp->kqp.sq_wrid_mem); 1140 kfree(iwqp->kqp.rq_wrid_mem); 1141 } 1142 1143 /** 1144 * irdma_cq_wq_destroy - send cq destroy cqp 1145 * @rf: RDMA PCI function 1146 * @cq: hardware control cq 1147 */ 1148 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1149 { 1150 struct irdma_cqp_request *cqp_request; 1151 struct cqp_cmds_info *cqp_info; 1152 1153 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1154 if (!cqp_request) 1155 return; 1156 1157 cqp_info = &cqp_request->info; 1158 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1159 cqp_info->post_sq = 1; 1160 cqp_info->in.u.cq_destroy.cq = cq; 1161 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1162 1163 irdma_handle_cqp_op(rf, cqp_request); 1164 irdma_put_cqp_request(&rf->cqp, cqp_request); 1165 } 1166 1167 /** 1168 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1169 * @cqp_request: modify QP completion 1170 */ 1171 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1172 { 1173 struct cqp_cmds_info *cqp_info; 1174 struct irdma_qp *iwqp; 1175 1176 cqp_info = &cqp_request->info; 1177 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1178 atomic_dec(&iwqp->hw_mod_qp_pend); 1179 wake_up(&iwqp->mod_qp_waitq); 1180 } 1181 1182 /** 1183 * irdma_hw_modify_qp - setup cqp for modify qp 1184 * @iwdev: RDMA device 1185 * @iwqp: qp ptr (user or kernel) 1186 * @info: info for modify qp 1187 * @wait: flag to wait or not for modify qp completion 1188 */ 1189 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1190 struct irdma_modify_qp_info *info, bool wait) 1191 { 1192 int status; 1193 struct irdma_pci_f *rf = iwdev->rf; 1194 struct irdma_cqp_request *cqp_request; 1195 struct cqp_cmds_info *cqp_info; 1196 struct irdma_modify_qp_info *m_info; 1197 1198 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1199 if (!cqp_request) 1200 return -ENOMEM; 1201 1202 if (!wait) { 1203 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1204 atomic_inc(&iwqp->hw_mod_qp_pend); 1205 } 1206 cqp_info = &cqp_request->info; 1207 m_info = &cqp_info->in.u.qp_modify.info; 1208 memcpy(m_info, info, sizeof(*m_info)); 1209 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1210 cqp_info->post_sq = 1; 1211 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1212 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1213 status = irdma_handle_cqp_op(rf, cqp_request); 1214 irdma_put_cqp_request(&rf->cqp, cqp_request); 1215 if (status) { 1216 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1217 return status; 1218 1219 switch (m_info->next_iwarp_state) { 1220 struct irdma_gen_ae_info ae_info; 1221 1222 case IRDMA_QP_STATE_RTS: 1223 case IRDMA_QP_STATE_IDLE: 1224 case IRDMA_QP_STATE_TERMINATE: 1225 case IRDMA_QP_STATE_CLOSING: 1226 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1227 irdma_send_reset(iwqp->cm_node); 1228 else 1229 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1230 if (!wait) { 1231 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1232 ae_info.ae_src = 0; 1233 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1234 } else { 1235 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1236 wait); 1237 if (!cqp_request) 1238 return -ENOMEM; 1239 1240 cqp_info = &cqp_request->info; 1241 m_info = &cqp_info->in.u.qp_modify.info; 1242 memcpy(m_info, info, sizeof(*m_info)); 1243 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1244 cqp_info->post_sq = 1; 1245 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1246 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1247 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1248 m_info->reset_tcp_conn = true; 1249 irdma_handle_cqp_op(rf, cqp_request); 1250 irdma_put_cqp_request(&rf->cqp, cqp_request); 1251 } 1252 break; 1253 case IRDMA_QP_STATE_ERROR: 1254 default: 1255 break; 1256 } 1257 } 1258 1259 return status; 1260 } 1261 1262 /** 1263 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1264 * @dev: device pointer 1265 * @cq: pointer to cq 1266 */ 1267 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1268 { 1269 struct irdma_pci_f *rf = dev_to_rf(dev); 1270 1271 irdma_cq_wq_destroy(rf, cq); 1272 } 1273 1274 /** 1275 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1276 * @dev: device pointer 1277 * @qp: pointer to qp 1278 */ 1279 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1280 { 1281 struct irdma_pci_f *rf = dev_to_rf(dev); 1282 struct irdma_cqp *iwcqp = &rf->cqp; 1283 struct irdma_cqp_request *cqp_request; 1284 struct cqp_cmds_info *cqp_info; 1285 int status; 1286 1287 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1288 if (!cqp_request) 1289 return -ENOMEM; 1290 1291 cqp_info = &cqp_request->info; 1292 memset(cqp_info, 0, sizeof(*cqp_info)); 1293 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1294 cqp_info->post_sq = 1; 1295 cqp_info->in.u.qp_destroy.qp = qp; 1296 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1297 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1298 1299 status = irdma_handle_cqp_op(rf, cqp_request); 1300 irdma_put_cqp_request(&rf->cqp, cqp_request); 1301 1302 return status; 1303 } 1304 1305 /** 1306 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1307 * @dev: hardware control device structure 1308 * @qp: hardware control qp 1309 */ 1310 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1311 { 1312 struct irdma_gen_ae_info info = {}; 1313 struct irdma_pci_f *rf = dev_to_rf(dev); 1314 1315 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); 1316 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1317 info.ae_src = IRDMA_AE_SOURCE_RQ; 1318 irdma_gen_ae(rf, qp, &info, false); 1319 } 1320 1321 /** 1322 * irdma_init_hash_desc - initialize hash for crc calculation 1323 * @desc: cryption type 1324 */ 1325 int irdma_init_hash_desc(struct shash_desc **desc) 1326 { 1327 struct crypto_shash *tfm; 1328 struct shash_desc *tdesc; 1329 1330 tfm = crypto_alloc_shash("crc32c", 0, 0); 1331 if (IS_ERR(tfm)) 1332 return -EINVAL; 1333 1334 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), 1335 GFP_KERNEL); 1336 if (!tdesc) { 1337 crypto_free_shash(tfm); 1338 return -EINVAL; 1339 } 1340 1341 tdesc->tfm = tfm; 1342 *desc = tdesc; 1343 1344 return 0; 1345 } 1346 1347 /** 1348 * irdma_free_hash_desc - free hash desc 1349 * @desc: to be freed 1350 */ 1351 void irdma_free_hash_desc(struct shash_desc *desc) 1352 { 1353 if (desc) { 1354 crypto_free_shash(desc->tfm); 1355 kfree(desc); 1356 } 1357 } 1358 1359 /** 1360 * irdma_ieq_check_mpacrc - check if mpa crc is OK 1361 * @desc: desc for hash 1362 * @addr: address of buffer for crc 1363 * @len: length of buffer 1364 * @val: value to be compared 1365 */ 1366 int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len, 1367 u32 val) 1368 { 1369 u32 crc = 0; 1370 int ret; 1371 int ret_code = 0; 1372 1373 crypto_shash_init(desc); 1374 ret = crypto_shash_update(desc, addr, len); 1375 if (!ret) 1376 crypto_shash_final(desc, (u8 *)&crc); 1377 if (crc != val) 1378 ret_code = -EINVAL; 1379 1380 return ret_code; 1381 } 1382 1383 /** 1384 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1385 * @dev: hardware control device structure 1386 * @buf: receive puda buffer on exception q 1387 */ 1388 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1389 struct irdma_puda_buf *buf) 1390 { 1391 struct irdma_qp *iwqp; 1392 struct irdma_cm_node *cm_node; 1393 struct irdma_device *iwdev = buf->vsi->back_vsi; 1394 u32 loc_addr[4] = {}; 1395 u32 rem_addr[4] = {}; 1396 u16 loc_port, rem_port; 1397 struct ipv6hdr *ip6h; 1398 struct iphdr *iph = (struct iphdr *)buf->iph; 1399 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1400 1401 if (iph->version == 4) { 1402 loc_addr[0] = ntohl(iph->daddr); 1403 rem_addr[0] = ntohl(iph->saddr); 1404 } else { 1405 ip6h = (struct ipv6hdr *)buf->iph; 1406 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); 1407 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); 1408 } 1409 loc_port = ntohs(tcph->dest); 1410 rem_port = ntohs(tcph->source); 1411 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1412 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1413 if (!cm_node) 1414 return NULL; 1415 1416 iwqp = cm_node->iwqp; 1417 irdma_rem_ref_cm_node(cm_node); 1418 1419 return &iwqp->sc_qp; 1420 } 1421 1422 /** 1423 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1424 * @qp: qp ptr 1425 */ 1426 void irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1427 { 1428 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1429 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1430 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1431 1432 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1433 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); 1434 1435 irdma_send_ack(cm_node); 1436 } 1437 1438 /** 1439 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1440 * @qp: qp pointer 1441 * @ah_info: AH info pointer 1442 */ 1443 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1444 struct irdma_ah_info *ah_info) 1445 { 1446 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1447 struct iphdr *iph; 1448 struct ipv6hdr *ip6h; 1449 1450 memset(ah_info, 0, sizeof(*ah_info)); 1451 ah_info->do_lpbk = true; 1452 ah_info->vlan_tag = buf->vlan_id; 1453 ah_info->insert_vlan_tag = buf->vlan_valid; 1454 ah_info->ipv4_valid = buf->ipv4; 1455 ah_info->vsi = qp->vsi; 1456 1457 if (buf->smac_valid) 1458 ether_addr_copy(ah_info->mac_addr, buf->smac); 1459 1460 if (buf->ipv4) { 1461 ah_info->ipv4_valid = true; 1462 iph = (struct iphdr *)buf->iph; 1463 ah_info->hop_ttl = iph->ttl; 1464 ah_info->tc_tos = iph->tos; 1465 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); 1466 ah_info->src_ip_addr[0] = ntohl(iph->saddr); 1467 } else { 1468 ip6h = (struct ipv6hdr *)buf->iph; 1469 ah_info->hop_ttl = ip6h->hop_limit; 1470 ah_info->tc_tos = ip6h->priority; 1471 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1472 ip6h->daddr.in6_u.u6_addr32); 1473 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1474 ip6h->saddr.in6_u.u6_addr32); 1475 } 1476 1477 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1478 ah_info->dest_ip_addr, 1479 ah_info->ipv4_valid, 1480 NULL, IRDMA_ARP_RESOLVE); 1481 } 1482 1483 /** 1484 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1485 * @buf: puda to update 1486 * @len: length of buffer 1487 * @seqnum: seq number for tcp 1488 */ 1489 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1490 u16 len, u32 seqnum) 1491 { 1492 struct tcphdr *tcph; 1493 struct iphdr *iph; 1494 u16 iphlen; 1495 u16 pktsize; 1496 u8 *addr = buf->mem.va; 1497 1498 iphlen = (buf->ipv4) ? 20 : 40; 1499 iph = (struct iphdr *)(addr + buf->maclen); 1500 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1501 pktsize = len + buf->tcphlen + iphlen; 1502 iph->tot_len = htons(pktsize); 1503 tcph->seq = htonl(seqnum); 1504 } 1505 1506 /** 1507 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1508 * @buf: puda to update 1509 * @len: length of buffer 1510 * @seqnum: seq number for tcp 1511 */ 1512 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1513 u32 seqnum) 1514 { 1515 struct tcphdr *tcph; 1516 u8 *addr; 1517 1518 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1519 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1520 1521 addr = buf->mem.va; 1522 tcph = (struct tcphdr *)addr; 1523 tcph->seq = htonl(seqnum); 1524 } 1525 1526 /** 1527 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1528 * buffer 1529 * @info: to get information 1530 * @buf: puda buffer 1531 */ 1532 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1533 struct irdma_puda_buf *buf) 1534 { 1535 struct iphdr *iph; 1536 struct ipv6hdr *ip6h; 1537 struct tcphdr *tcph; 1538 u16 iphlen; 1539 u16 pkt_len; 1540 u8 *mem = buf->mem.va; 1541 struct ethhdr *ethh = buf->mem.va; 1542 1543 if (ethh->h_proto == htons(0x8100)) { 1544 info->vlan_valid = true; 1545 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & 1546 VLAN_VID_MASK; 1547 } 1548 1549 buf->maclen = (info->vlan_valid) ? 18 : 14; 1550 iphlen = (info->l3proto) ? 40 : 20; 1551 buf->ipv4 = (info->l3proto) ? false : true; 1552 buf->iph = mem + buf->maclen; 1553 iph = (struct iphdr *)buf->iph; 1554 buf->tcph = buf->iph + iphlen; 1555 tcph = (struct tcphdr *)buf->tcph; 1556 1557 if (buf->ipv4) { 1558 pkt_len = ntohs(iph->tot_len); 1559 } else { 1560 ip6h = (struct ipv6hdr *)buf->iph; 1561 pkt_len = ntohs(ip6h->payload_len) + iphlen; 1562 } 1563 1564 buf->totallen = pkt_len + buf->maclen; 1565 1566 if (info->payload_len < buf->totallen) { 1567 ibdev_dbg(to_ibdev(buf->vsi->dev), 1568 "ERR: payload_len = 0x%x totallen expected0x%x\n", 1569 info->payload_len, buf->totallen); 1570 return -EINVAL; 1571 } 1572 1573 buf->tcphlen = tcph->doff << 2; 1574 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1575 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1576 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1577 buf->seqnum = ntohl(tcph->seq); 1578 1579 return 0; 1580 } 1581 1582 /** 1583 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1584 * @info: to get information 1585 * @buf: puda buffer 1586 */ 1587 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1588 struct irdma_puda_buf *buf) 1589 { 1590 struct tcphdr *tcph; 1591 u32 pkt_len; 1592 u8 *mem; 1593 1594 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1595 return irdma_gen1_puda_get_tcpip_info(info, buf); 1596 1597 mem = buf->mem.va; 1598 buf->vlan_valid = info->vlan_valid; 1599 if (info->vlan_valid) 1600 buf->vlan_id = info->vlan; 1601 1602 buf->ipv4 = info->ipv4; 1603 if (buf->ipv4) 1604 buf->iph = mem + IRDMA_IPV4_PAD; 1605 else 1606 buf->iph = mem; 1607 1608 buf->tcph = mem + IRDMA_TCP_OFFSET; 1609 tcph = (struct tcphdr *)buf->tcph; 1610 pkt_len = info->payload_len; 1611 buf->totallen = pkt_len; 1612 buf->tcphlen = tcph->doff << 2; 1613 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1614 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1615 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1616 buf->seqnum = ntohl(tcph->seq); 1617 1618 if (info->smac_valid) { 1619 ether_addr_copy(buf->smac, info->smac); 1620 buf->smac_valid = true; 1621 } 1622 1623 return 0; 1624 } 1625 1626 /** 1627 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1628 * @t: timer_list pointer 1629 */ 1630 static void irdma_hw_stats_timeout(struct timer_list *t) 1631 { 1632 struct irdma_vsi_pestat *pf_devstat = 1633 from_timer(pf_devstat, t, stats_timer); 1634 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1635 1636 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1637 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); 1638 else 1639 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1640 1641 mod_timer(&pf_devstat->stats_timer, 1642 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1643 } 1644 1645 /** 1646 * irdma_hw_stats_start_timer - Start periodic stats timer 1647 * @vsi: vsi structure pointer 1648 */ 1649 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1650 { 1651 struct irdma_vsi_pestat *devstat = vsi->pestat; 1652 1653 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1654 mod_timer(&devstat->stats_timer, 1655 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1656 } 1657 1658 /** 1659 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1660 * @vsi: pointer to vsi structure 1661 */ 1662 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1663 { 1664 struct irdma_vsi_pestat *devstat = vsi->pestat; 1665 1666 del_timer_sync(&devstat->stats_timer); 1667 } 1668 1669 /** 1670 * irdma_process_stats - Checking for wrap and update stats 1671 * @pestat: stats structure pointer 1672 */ 1673 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) 1674 { 1675 sc_vsi_update_stats(pestat->vsi); 1676 } 1677 1678 /** 1679 * irdma_cqp_gather_stats_gen1 - Gather stats 1680 * @dev: pointer to device structure 1681 * @pestat: statistics structure 1682 */ 1683 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, 1684 struct irdma_vsi_pestat *pestat) 1685 { 1686 struct irdma_gather_stats *gather_stats = 1687 pestat->gather_info.gather_stats_va; 1688 u32 stats_inst_offset_32; 1689 u32 stats_inst_offset_64; 1690 1691 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? 1692 pestat->gather_info.stats_inst_index : 1693 pestat->hw->hmc.hmc_fn_id; 1694 stats_inst_offset_32 *= 4; 1695 stats_inst_offset_64 = stats_inst_offset_32 * 2; 1696 1697 gather_stats->rxvlanerr = 1698 rd32(dev->hw, 1699 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR] 1700 + stats_inst_offset_32); 1701 gather_stats->ip4rxdiscard = 1702 rd32(dev->hw, 1703 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] 1704 + stats_inst_offset_32); 1705 gather_stats->ip4rxtrunc = 1706 rd32(dev->hw, 1707 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] 1708 + stats_inst_offset_32); 1709 gather_stats->ip4txnoroute = 1710 rd32(dev->hw, 1711 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] 1712 + stats_inst_offset_32); 1713 gather_stats->ip6rxdiscard = 1714 rd32(dev->hw, 1715 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] 1716 + stats_inst_offset_32); 1717 gather_stats->ip6rxtrunc = 1718 rd32(dev->hw, 1719 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] 1720 + stats_inst_offset_32); 1721 gather_stats->ip6txnoroute = 1722 rd32(dev->hw, 1723 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] 1724 + stats_inst_offset_32); 1725 gather_stats->tcprtxseg = 1726 rd32(dev->hw, 1727 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG] 1728 + stats_inst_offset_32); 1729 gather_stats->tcprxopterr = 1730 rd32(dev->hw, 1731 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] 1732 + stats_inst_offset_32); 1733 1734 gather_stats->ip4rxocts = 1735 rd64(dev->hw, 1736 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS] 1737 + stats_inst_offset_64); 1738 gather_stats->ip4rxpkts = 1739 rd64(dev->hw, 1740 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS] 1741 + stats_inst_offset_64); 1742 gather_stats->ip4txfrag = 1743 rd64(dev->hw, 1744 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] 1745 + stats_inst_offset_64); 1746 gather_stats->ip4rxmcpkts = 1747 rd64(dev->hw, 1748 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] 1749 + stats_inst_offset_64); 1750 gather_stats->ip4txocts = 1751 rd64(dev->hw, 1752 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS] 1753 + stats_inst_offset_64); 1754 gather_stats->ip4txpkts = 1755 rd64(dev->hw, 1756 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS] 1757 + stats_inst_offset_64); 1758 gather_stats->ip4txfrag = 1759 rd64(dev->hw, 1760 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] 1761 + stats_inst_offset_64); 1762 gather_stats->ip4txmcpkts = 1763 rd64(dev->hw, 1764 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] 1765 + stats_inst_offset_64); 1766 gather_stats->ip6rxocts = 1767 rd64(dev->hw, 1768 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS] 1769 + stats_inst_offset_64); 1770 gather_stats->ip6rxpkts = 1771 rd64(dev->hw, 1772 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS] 1773 + stats_inst_offset_64); 1774 gather_stats->ip6txfrags = 1775 rd64(dev->hw, 1776 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] 1777 + stats_inst_offset_64); 1778 gather_stats->ip6rxmcpkts = 1779 rd64(dev->hw, 1780 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] 1781 + stats_inst_offset_64); 1782 gather_stats->ip6txocts = 1783 rd64(dev->hw, 1784 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS] 1785 + stats_inst_offset_64); 1786 gather_stats->ip6txpkts = 1787 rd64(dev->hw, 1788 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS] 1789 + stats_inst_offset_64); 1790 gather_stats->ip6txfrags = 1791 rd64(dev->hw, 1792 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] 1793 + stats_inst_offset_64); 1794 gather_stats->ip6txmcpkts = 1795 rd64(dev->hw, 1796 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] 1797 + stats_inst_offset_64); 1798 gather_stats->tcprxsegs = 1799 rd64(dev->hw, 1800 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS] 1801 + stats_inst_offset_64); 1802 gather_stats->tcptxsegs = 1803 rd64(dev->hw, 1804 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG] 1805 + stats_inst_offset_64); 1806 gather_stats->rdmarxrds = 1807 rd64(dev->hw, 1808 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS] 1809 + stats_inst_offset_64); 1810 gather_stats->rdmarxsnds = 1811 rd64(dev->hw, 1812 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS] 1813 + stats_inst_offset_64); 1814 gather_stats->rdmarxwrs = 1815 rd64(dev->hw, 1816 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS] 1817 + stats_inst_offset_64); 1818 gather_stats->rdmatxrds = 1819 rd64(dev->hw, 1820 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS] 1821 + stats_inst_offset_64); 1822 gather_stats->rdmatxsnds = 1823 rd64(dev->hw, 1824 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS] 1825 + stats_inst_offset_64); 1826 gather_stats->rdmatxwrs = 1827 rd64(dev->hw, 1828 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS] 1829 + stats_inst_offset_64); 1830 gather_stats->rdmavbn = 1831 rd64(dev->hw, 1832 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND] 1833 + stats_inst_offset_64); 1834 gather_stats->rdmavinv = 1835 rd64(dev->hw, 1836 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV] 1837 + stats_inst_offset_64); 1838 gather_stats->udprxpkts = 1839 rd64(dev->hw, 1840 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS] 1841 + stats_inst_offset_64); 1842 gather_stats->udptxpkts = 1843 rd64(dev->hw, 1844 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS] 1845 + stats_inst_offset_64); 1846 1847 irdma_process_stats(pestat); 1848 } 1849 1850 /** 1851 * irdma_process_cqp_stats - Checking for wrap and update stats 1852 * @cqp_request: cqp_request structure pointer 1853 */ 1854 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1855 { 1856 struct irdma_vsi_pestat *pestat = cqp_request->param; 1857 1858 irdma_process_stats(pestat); 1859 } 1860 1861 /** 1862 * irdma_cqp_gather_stats_cmd - Gather stats 1863 * @dev: pointer to device structure 1864 * @pestat: pointer to stats info 1865 * @wait: flag to wait or not wait for stats 1866 */ 1867 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1868 struct irdma_vsi_pestat *pestat, bool wait) 1869 1870 { 1871 struct irdma_pci_f *rf = dev_to_rf(dev); 1872 struct irdma_cqp *iwcqp = &rf->cqp; 1873 struct irdma_cqp_request *cqp_request; 1874 struct cqp_cmds_info *cqp_info; 1875 int status; 1876 1877 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1878 if (!cqp_request) 1879 return -ENOMEM; 1880 1881 cqp_info = &cqp_request->info; 1882 memset(cqp_info, 0, sizeof(*cqp_info)); 1883 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1884 cqp_info->post_sq = 1; 1885 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1886 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1887 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1888 cqp_request->param = pestat; 1889 if (!wait) 1890 cqp_request->callback_fcn = irdma_process_cqp_stats; 1891 status = irdma_handle_cqp_op(rf, cqp_request); 1892 if (wait) 1893 irdma_process_stats(pestat); 1894 irdma_put_cqp_request(&rf->cqp, cqp_request); 1895 1896 return status; 1897 } 1898 1899 /** 1900 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1901 * @vsi: pointer to vsi structure 1902 * @cmd: command to allocate or free 1903 * @stats_info: pointer to allocate stats info 1904 */ 1905 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1906 struct irdma_stats_inst_info *stats_info) 1907 { 1908 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1909 struct irdma_cqp *iwcqp = &rf->cqp; 1910 struct irdma_cqp_request *cqp_request; 1911 struct cqp_cmds_info *cqp_info; 1912 int status; 1913 bool wait = false; 1914 1915 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1916 wait = true; 1917 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1918 if (!cqp_request) 1919 return -ENOMEM; 1920 1921 cqp_info = &cqp_request->info; 1922 memset(cqp_info, 0, sizeof(*cqp_info)); 1923 cqp_info->cqp_cmd = cmd; 1924 cqp_info->post_sq = 1; 1925 cqp_info->in.u.stats_manage.info = *stats_info; 1926 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1927 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1928 status = irdma_handle_cqp_op(rf, cqp_request); 1929 if (wait) 1930 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1931 irdma_put_cqp_request(iwcqp, cqp_request); 1932 1933 return status; 1934 } 1935 1936 /** 1937 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1938 * @dev: pointer to device info 1939 * @sc_ceq: pointer to ceq structure 1940 * @op: Create or Destroy 1941 */ 1942 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1943 u8 op) 1944 { 1945 struct irdma_cqp_request *cqp_request; 1946 struct cqp_cmds_info *cqp_info; 1947 struct irdma_pci_f *rf = dev_to_rf(dev); 1948 int status; 1949 1950 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1951 if (!cqp_request) 1952 return -ENOMEM; 1953 1954 cqp_info = &cqp_request->info; 1955 cqp_info->post_sq = 1; 1956 cqp_info->cqp_cmd = op; 1957 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1958 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1959 1960 status = irdma_handle_cqp_op(rf, cqp_request); 1961 irdma_put_cqp_request(&rf->cqp, cqp_request); 1962 1963 return status; 1964 } 1965 1966 /** 1967 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1968 * @dev: pointer to device info 1969 * @sc_aeq: pointer to aeq structure 1970 * @op: Create or Destroy 1971 */ 1972 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1973 u8 op) 1974 { 1975 struct irdma_cqp_request *cqp_request; 1976 struct cqp_cmds_info *cqp_info; 1977 struct irdma_pci_f *rf = dev_to_rf(dev); 1978 int status; 1979 1980 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1981 if (!cqp_request) 1982 return -ENOMEM; 1983 1984 cqp_info = &cqp_request->info; 1985 cqp_info->post_sq = 1; 1986 cqp_info->cqp_cmd = op; 1987 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1988 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1989 1990 status = irdma_handle_cqp_op(rf, cqp_request); 1991 irdma_put_cqp_request(&rf->cqp, cqp_request); 1992 1993 return status; 1994 } 1995 1996 /** 1997 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1998 * @dev: pointer to device structure 1999 * @cmd: Add, modify or delete 2000 * @node_info: pointer to ws node info 2001 */ 2002 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 2003 struct irdma_ws_node_info *node_info) 2004 { 2005 struct irdma_pci_f *rf = dev_to_rf(dev); 2006 struct irdma_cqp *iwcqp = &rf->cqp; 2007 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 2008 struct irdma_cqp_request *cqp_request; 2009 struct cqp_cmds_info *cqp_info; 2010 int status; 2011 bool poll; 2012 2013 if (!rf->sc_dev.ceq_valid) 2014 poll = true; 2015 else 2016 poll = false; 2017 2018 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 2019 if (!cqp_request) 2020 return -ENOMEM; 2021 2022 cqp_info = &cqp_request->info; 2023 memset(cqp_info, 0, sizeof(*cqp_info)); 2024 cqp_info->cqp_cmd = cmd; 2025 cqp_info->post_sq = 1; 2026 cqp_info->in.u.ws_node.info = *node_info; 2027 cqp_info->in.u.ws_node.cqp = cqp; 2028 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 2029 status = irdma_handle_cqp_op(rf, cqp_request); 2030 if (status) 2031 goto exit; 2032 2033 if (poll) { 2034 struct irdma_ccq_cqe_info compl_info; 2035 2036 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 2037 &compl_info); 2038 node_info->qs_handle = compl_info.op_ret_val; 2039 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", 2040 compl_info.op_code, compl_info.op_ret_val); 2041 } else { 2042 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 2043 } 2044 2045 exit: 2046 irdma_put_cqp_request(&rf->cqp, cqp_request); 2047 2048 return status; 2049 } 2050 2051 /** 2052 * irdma_ah_cqp_op - perform an AH cqp operation 2053 * @rf: RDMA PCI function 2054 * @sc_ah: address handle 2055 * @cmd: AH operation 2056 * @wait: wait if true 2057 * @callback_fcn: Callback function on CQP op completion 2058 * @cb_param: parameter for callback function 2059 * 2060 * returns errno 2061 */ 2062 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 2063 bool wait, 2064 void (*callback_fcn)(struct irdma_cqp_request *), 2065 void *cb_param) 2066 { 2067 struct irdma_cqp_request *cqp_request; 2068 struct cqp_cmds_info *cqp_info; 2069 int status; 2070 2071 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 2072 return -EINVAL; 2073 2074 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 2075 if (!cqp_request) 2076 return -ENOMEM; 2077 2078 cqp_info = &cqp_request->info; 2079 cqp_info->cqp_cmd = cmd; 2080 cqp_info->post_sq = 1; 2081 if (cmd == IRDMA_OP_AH_CREATE) { 2082 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 2083 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 2084 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 2085 } else if (cmd == IRDMA_OP_AH_DESTROY) { 2086 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 2087 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 2088 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 2089 } 2090 2091 if (!wait) { 2092 cqp_request->callback_fcn = callback_fcn; 2093 cqp_request->param = cb_param; 2094 } 2095 status = irdma_handle_cqp_op(rf, cqp_request); 2096 irdma_put_cqp_request(&rf->cqp, cqp_request); 2097 2098 if (status) 2099 return -ENOMEM; 2100 2101 if (wait) 2102 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); 2103 2104 return 0; 2105 } 2106 2107 /** 2108 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 2109 * @cqp_request: pointer to cqp_request of create AH 2110 */ 2111 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 2112 { 2113 struct irdma_sc_qp *qp = cqp_request->param; 2114 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 2115 unsigned long flags; 2116 2117 spin_lock_irqsave(&qp->pfpdu.lock, flags); 2118 if (!cqp_request->compl_info.op_ret_val) { 2119 sc_ah->ah_info.ah_valid = true; 2120 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 2121 } else { 2122 sc_ah->ah_info.ah_valid = false; 2123 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 2124 } 2125 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 2126 } 2127 2128 /** 2129 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 2130 * @cqp_request: pointer to cqp_request of create AH 2131 */ 2132 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 2133 { 2134 struct irdma_cm_node *cm_node = cqp_request->param; 2135 struct irdma_sc_ah *sc_ah = cm_node->ah; 2136 2137 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 2138 irdma_add_conn_est_qh(cm_node); 2139 } 2140 2141 /** 2142 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 2143 * @dev: device pointer 2144 * @ah_info: Address handle info 2145 * @wait: When true will wait for operation to complete 2146 * @type: ILQ/IEQ 2147 * @cb_param: Callback param when not waiting 2148 * @ah_ret: Returned pointer to address handle if created 2149 * 2150 */ 2151 int irdma_puda_create_ah(struct irdma_sc_dev *dev, 2152 struct irdma_ah_info *ah_info, bool wait, 2153 enum puda_rsrc_type type, void *cb_param, 2154 struct irdma_sc_ah **ah_ret) 2155 { 2156 struct irdma_sc_ah *ah; 2157 struct irdma_pci_f *rf = dev_to_rf(dev); 2158 int err; 2159 2160 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2161 *ah_ret = ah; 2162 if (!ah) 2163 return -ENOMEM; 2164 2165 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 2166 &ah_info->ah_idx, &rf->next_ah); 2167 if (err) 2168 goto err_free; 2169 2170 ah->dev = dev; 2171 ah->ah_info = *ah_info; 2172 2173 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 2174 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2175 irdma_ilq_ah_cb, cb_param); 2176 else 2177 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2178 irdma_ieq_ah_cb, cb_param); 2179 2180 if (err) 2181 goto error; 2182 return 0; 2183 2184 error: 2185 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2186 err_free: 2187 kfree(ah); 2188 *ah_ret = NULL; 2189 return -ENOMEM; 2190 } 2191 2192 /** 2193 * irdma_puda_free_ah - free a puda address handle 2194 * @dev: device pointer 2195 * @ah: The address handle to free 2196 */ 2197 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2198 { 2199 struct irdma_pci_f *rf = dev_to_rf(dev); 2200 2201 if (!ah) 2202 return; 2203 2204 if (ah->ah_info.ah_valid) { 2205 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2206 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2207 } 2208 2209 kfree(ah); 2210 } 2211 2212 /** 2213 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 2214 * @cqp_request: pointer to cqp_request of create AH 2215 */ 2216 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 2217 { 2218 struct irdma_sc_ah *sc_ah = cqp_request->param; 2219 2220 if (!cqp_request->compl_info.op_ret_val) 2221 sc_ah->ah_info.ah_valid = true; 2222 else 2223 sc_ah->ah_info.ah_valid = false; 2224 } 2225 2226 /** 2227 * irdma_prm_add_pble_mem - add moemory to pble resources 2228 * @pprm: pble resource manager 2229 * @pchunk: chunk of memory to add 2230 */ 2231 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2232 struct irdma_chunk *pchunk) 2233 { 2234 u64 sizeofbitmap; 2235 2236 if (pchunk->size & 0xfff) 2237 return -EINVAL; 2238 2239 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2240 2241 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 2242 if (!pchunk->bitmapbuf) 2243 return -ENOMEM; 2244 2245 pchunk->sizeofbitmap = sizeofbitmap; 2246 /* each pble is 8 bytes hence shift by 3 */ 2247 pprm->total_pble_alloc += pchunk->size >> 3; 2248 pprm->free_pble_cnt += pchunk->size >> 3; 2249 2250 return 0; 2251 } 2252 2253 /** 2254 * irdma_prm_get_pbles - get pble's from prm 2255 * @pprm: pble resource manager 2256 * @chunkinfo: nformation about chunk where pble's were acquired 2257 * @mem_size: size of pble memory needed 2258 * @vaddr: returns virtual address of pble memory 2259 * @fpm_addr: returns fpm address of pble memory 2260 */ 2261 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2262 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2263 u64 **vaddr, u64 *fpm_addr) 2264 { 2265 u64 bits_needed; 2266 u64 bit_idx = PBLE_INVALID_IDX; 2267 struct irdma_chunk *pchunk = NULL; 2268 struct list_head *chunk_entry = pprm->clist.next; 2269 u32 offset; 2270 unsigned long flags; 2271 *vaddr = NULL; 2272 *fpm_addr = 0; 2273 2274 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2275 2276 spin_lock_irqsave(&pprm->prm_lock, flags); 2277 while (chunk_entry != &pprm->clist) { 2278 pchunk = (struct irdma_chunk *)chunk_entry; 2279 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2280 pchunk->sizeofbitmap, 0, 2281 bits_needed, 0); 2282 if (bit_idx < pchunk->sizeofbitmap) 2283 break; 2284 2285 /* list.next used macro */ 2286 chunk_entry = pchunk->list.next; 2287 } 2288 2289 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2290 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2291 return -ENOMEM; 2292 } 2293 2294 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2295 offset = bit_idx << pprm->pble_shift; 2296 *vaddr = pchunk->vaddr + offset; 2297 *fpm_addr = pchunk->fpm_addr + offset; 2298 2299 chunkinfo->pchunk = pchunk; 2300 chunkinfo->bit_idx = bit_idx; 2301 chunkinfo->bits_used = bits_needed; 2302 /* 3 is sizeof pble divide */ 2303 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2304 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2305 2306 return 0; 2307 } 2308 2309 /** 2310 * irdma_prm_return_pbles - return pbles back to prm 2311 * @pprm: pble resource manager 2312 * @chunkinfo: chunk where pble's were acquired and to be freed 2313 */ 2314 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2315 struct irdma_pble_chunkinfo *chunkinfo) 2316 { 2317 unsigned long flags; 2318 2319 spin_lock_irqsave(&pprm->prm_lock, flags); 2320 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2321 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2322 chunkinfo->bits_used); 2323 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2324 } 2325 2326 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma, 2327 u32 pg_cnt) 2328 { 2329 struct page *vm_page; 2330 int i; 2331 u8 *addr; 2332 2333 addr = (u8 *)(uintptr_t)va; 2334 for (i = 0; i < pg_cnt; i++) { 2335 vm_page = vmalloc_to_page(addr); 2336 if (!vm_page) 2337 goto err; 2338 2339 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, 2340 DMA_BIDIRECTIONAL); 2341 if (dma_mapping_error(hw->device, pg_dma[i])) 2342 goto err; 2343 2344 addr += PAGE_SIZE; 2345 } 2346 2347 return 0; 2348 2349 err: 2350 irdma_unmap_vm_page_list(hw, pg_dma, i); 2351 return -ENOMEM; 2352 } 2353 2354 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) 2355 { 2356 int i; 2357 2358 for (i = 0; i < pg_cnt; i++) 2359 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2360 } 2361 2362 /** 2363 * irdma_pble_free_paged_mem - free virtual paged memory 2364 * @chunk: chunk to free with paged memory 2365 */ 2366 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2367 { 2368 if (!chunk->pg_cnt) 2369 goto done; 2370 2371 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2372 chunk->pg_cnt); 2373 2374 done: 2375 kfree(chunk->dmainfo.dmaaddrs); 2376 chunk->dmainfo.dmaaddrs = NULL; 2377 vfree(chunk->vaddr); 2378 chunk->vaddr = NULL; 2379 chunk->type = 0; 2380 } 2381 2382 /** 2383 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2384 * @chunk: chunk to add for paged memory 2385 * @pg_cnt: number of pages needed 2386 */ 2387 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2388 { 2389 u32 size; 2390 void *va; 2391 2392 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2393 if (!chunk->dmainfo.dmaaddrs) 2394 return -ENOMEM; 2395 2396 size = PAGE_SIZE * pg_cnt; 2397 va = vmalloc(size); 2398 if (!va) 2399 goto err; 2400 2401 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2402 pg_cnt)) { 2403 vfree(va); 2404 goto err; 2405 } 2406 chunk->vaddr = va; 2407 chunk->size = size; 2408 chunk->pg_cnt = pg_cnt; 2409 chunk->type = PBLE_SD_PAGED; 2410 2411 return 0; 2412 err: 2413 kfree(chunk->dmainfo.dmaaddrs); 2414 chunk->dmainfo.dmaaddrs = NULL; 2415 2416 return -ENOMEM; 2417 } 2418 2419 /** 2420 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2421 * @dev: device pointer 2422 */ 2423 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2424 { 2425 struct irdma_pci_f *rf = dev_to_rf(dev); 2426 u32 next = 1; 2427 u32 node_id; 2428 2429 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2430 &node_id, &next)) 2431 return IRDMA_WS_NODE_INVALID; 2432 2433 return (u16)node_id; 2434 } 2435 2436 /** 2437 * irdma_free_ws_node_id - Free a tx scheduler node ID 2438 * @dev: device pointer 2439 * @node_id: Work scheduler node ID 2440 */ 2441 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2442 { 2443 struct irdma_pci_f *rf = dev_to_rf(dev); 2444 2445 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2446 } 2447 2448 /** 2449 * irdma_modify_qp_to_err - Modify a QP to error 2450 * @sc_qp: qp structure 2451 */ 2452 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2453 { 2454 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2455 struct ib_qp_attr attr; 2456 2457 if (qp->iwdev->rf->reset) 2458 return; 2459 attr.qp_state = IB_QPS_ERR; 2460 2461 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2462 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2463 else 2464 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2465 } 2466 2467 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2468 { 2469 struct ib_event ibevent; 2470 2471 if (!iwqp->ibqp.event_handler) 2472 return; 2473 2474 switch (event) { 2475 case IRDMA_QP_EVENT_CATASTROPHIC: 2476 ibevent.event = IB_EVENT_QP_FATAL; 2477 break; 2478 case IRDMA_QP_EVENT_ACCESS_ERR: 2479 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2480 break; 2481 } 2482 ibevent.device = iwqp->ibqp.device; 2483 ibevent.element.qp = &iwqp->ibqp; 2484 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2485 } 2486 2487 bool irdma_cq_empty(struct irdma_cq *iwcq) 2488 { 2489 struct irdma_cq_uk *ukcq; 2490 u64 qword3; 2491 __le64 *cqe; 2492 u8 polarity; 2493 2494 ukcq = &iwcq->sc_cq.cq_uk; 2495 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2496 get_64bit_val(cqe, 24, &qword3); 2497 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2498 2499 return polarity != ukcq->polarity; 2500 } 2501