1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_arp_table -manage arp table 7 * @rf: RDMA PCI function 8 * @ip_addr: ip address for device 9 * @ipv4: IPv4 flag 10 * @mac_addr: mac address ptr 11 * @action: modify, delete or add 12 */ 13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, 14 const u8 *mac_addr, u32 action) 15 { 16 unsigned long flags; 17 int arp_index; 18 u32 ip[4] = {}; 19 20 if (ipv4) 21 ip[0] = *ip_addr; 22 else 23 memcpy(ip, ip_addr, sizeof(ip)); 24 25 spin_lock_irqsave(&rf->arp_lock, flags); 26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 28 break; 29 } 30 31 switch (action) { 32 case IRDMA_ARP_ADD: 33 if (arp_index != rf->arp_table_size) { 34 arp_index = -1; 35 break; 36 } 37 38 arp_index = 0; 39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 40 (u32 *)&arp_index, &rf->next_arp_index)) { 41 arp_index = -1; 42 break; 43 } 44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip, 46 sizeof(rf->arp_table[arp_index].ip_addr)); 47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 48 break; 49 case IRDMA_ARP_RESOLVE: 50 if (arp_index == rf->arp_table_size) 51 arp_index = -1; 52 break; 53 case IRDMA_ARP_DELETE: 54 if (arp_index == rf->arp_table_size) { 55 arp_index = -1; 56 break; 57 } 58 59 memset(rf->arp_table[arp_index].ip_addr, 0, 60 sizeof(rf->arp_table[arp_index].ip_addr)); 61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 63 break; 64 default: 65 arp_index = -1; 66 break; 67 } 68 69 spin_unlock_irqrestore(&rf->arp_lock, flags); 70 return arp_index; 71 } 72 73 /** 74 * irdma_add_arp - add a new arp entry if needed 75 * @rf: RDMA function 76 * @ip: IP address 77 * @ipv4: IPv4 flag 78 * @mac: MAC address 79 */ 80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac) 81 { 82 int arpidx; 83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); 85 if (arpidx >= 0) { 86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 87 return arpidx; 88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 90 ipv4, IRDMA_ARP_DELETE); 91 } 92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD); 94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE); 96 } 97 98 /** 99 * wr32 - write 32 bits to hw register 100 * @hw: hardware information including registers 101 * @reg: register offset 102 * @val: value to write to register 103 */ 104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) 105 { 106 writel(val, hw->hw_addr + reg); 107 } 108 109 /** 110 * rd32 - read a 32 bit hw register 111 * @hw: hardware information including registers 112 * @reg: register offset 113 * 114 * Return value of register content 115 */ 116 inline u32 rd32(struct irdma_hw *hw, u32 reg) 117 { 118 return readl(hw->hw_addr + reg); 119 } 120 121 /** 122 * rd64 - read a 64 bit hw register 123 * @hw: hardware information including registers 124 * @reg: register offset 125 * 126 * Return value of register content 127 */ 128 inline u64 rd64(struct irdma_hw *hw, u32 reg) 129 { 130 return readq(hw->hw_addr + reg); 131 } 132 133 static void irdma_gid_change_event(struct ib_device *ibdev) 134 { 135 struct ib_event ib_event; 136 137 ib_event.event = IB_EVENT_GID_CHANGE; 138 ib_event.device = ibdev; 139 ib_event.element.port_num = 1; 140 ib_dispatch_event(&ib_event); 141 } 142 143 /** 144 * irdma_inetaddr_event - system notifier for ipv4 addr events 145 * @notifier: not used 146 * @event: event for notifier 147 * @ptr: if address 148 */ 149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, 150 void *ptr) 151 { 152 struct in_ifaddr *ifa = ptr; 153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; 154 struct irdma_device *iwdev; 155 struct ib_device *ibdev; 156 u32 local_ipaddr; 157 158 real_dev = rdma_vlan_dev_real_dev(netdev); 159 if (!real_dev) 160 real_dev = netdev; 161 162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 163 if (!ibdev) 164 return NOTIFY_DONE; 165 166 iwdev = to_iwdev(ibdev); 167 local_ipaddr = ntohl(ifa->ifa_address); 168 ibdev_dbg(&iwdev->ibdev, 169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev, 170 event, &local_ipaddr, real_dev->dev_addr); 171 switch (event) { 172 case NETDEV_DOWN: 173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 174 &local_ipaddr, true, IRDMA_ARP_DELETE); 175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false); 176 irdma_gid_change_event(&iwdev->ibdev); 177 break; 178 case NETDEV_UP: 179 case NETDEV_CHANGEADDR: 180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); 181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true); 182 irdma_gid_change_event(&iwdev->ibdev); 183 break; 184 default: 185 break; 186 } 187 188 ib_device_put(ibdev); 189 190 return NOTIFY_DONE; 191 } 192 193 /** 194 * irdma_inet6addr_event - system notifier for ipv6 addr events 195 * @notifier: not used 196 * @event: event for notifier 197 * @ptr: if address 198 */ 199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, 200 void *ptr) 201 { 202 struct inet6_ifaddr *ifa = ptr; 203 struct net_device *real_dev, *netdev = ifa->idev->dev; 204 struct irdma_device *iwdev; 205 struct ib_device *ibdev; 206 u32 local_ipaddr6[4]; 207 208 real_dev = rdma_vlan_dev_real_dev(netdev); 209 if (!real_dev) 210 real_dev = netdev; 211 212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 213 if (!ibdev) 214 return NOTIFY_DONE; 215 216 iwdev = to_iwdev(ibdev); 217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); 218 ibdev_dbg(&iwdev->ibdev, 219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev, 220 event, local_ipaddr6, real_dev->dev_addr); 221 switch (event) { 222 case NETDEV_DOWN: 223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 224 local_ipaddr6, false, IRDMA_ARP_DELETE); 225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false); 226 irdma_gid_change_event(&iwdev->ibdev); 227 break; 228 case NETDEV_UP: 229 case NETDEV_CHANGEADDR: 230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, 231 real_dev->dev_addr); 232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true); 233 irdma_gid_change_event(&iwdev->ibdev); 234 break; 235 default: 236 break; 237 } 238 239 ib_device_put(ibdev); 240 241 return NOTIFY_DONE; 242 } 243 244 /** 245 * irdma_net_event - system notifier for net events 246 * @notifier: not used 247 * @event: event for notifier 248 * @ptr: neighbor 249 */ 250 int irdma_net_event(struct notifier_block *notifier, unsigned long event, 251 void *ptr) 252 { 253 struct neighbour *neigh = ptr; 254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; 255 struct irdma_device *iwdev; 256 struct ib_device *ibdev; 257 __be32 *p; 258 u32 local_ipaddr[4] = {}; 259 bool ipv4 = true; 260 261 switch (event) { 262 case NETEVENT_NEIGH_UPDATE: 263 real_dev = rdma_vlan_dev_real_dev(netdev); 264 if (!real_dev) 265 real_dev = netdev; 266 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 267 if (!ibdev) 268 return NOTIFY_DONE; 269 270 iwdev = to_iwdev(ibdev); 271 p = (__be32 *)neigh->primary_key; 272 if (neigh->tbl->family == AF_INET6) { 273 ipv4 = false; 274 irdma_copy_ip_ntohl(local_ipaddr, p); 275 } else { 276 local_ipaddr[0] = ntohl(*p); 277 } 278 279 ibdev_dbg(&iwdev->ibdev, 280 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n", 281 iwdev->netdev, neigh->nud_state, local_ipaddr, 282 neigh->ha); 283 284 if (neigh->nud_state & NUD_VALID) 285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); 286 287 else 288 irdma_manage_arp_cache(iwdev->rf, neigh->ha, 289 local_ipaddr, ipv4, 290 IRDMA_ARP_DELETE); 291 ib_device_put(ibdev); 292 break; 293 default: 294 break; 295 } 296 297 return NOTIFY_DONE; 298 } 299 300 /** 301 * irdma_netdevice_event - system notifier for netdev events 302 * @notifier: not used 303 * @event: event for notifier 304 * @ptr: netdev 305 */ 306 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 307 void *ptr) 308 { 309 struct irdma_device *iwdev; 310 struct ib_device *ibdev; 311 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 312 313 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 314 if (!ibdev) 315 return NOTIFY_DONE; 316 317 iwdev = to_iwdev(ibdev); 318 iwdev->iw_status = 1; 319 switch (event) { 320 case NETDEV_DOWN: 321 iwdev->iw_status = 0; 322 fallthrough; 323 case NETDEV_UP: 324 irdma_port_ibevent(iwdev); 325 break; 326 default: 327 break; 328 } 329 ib_device_put(ibdev); 330 331 return NOTIFY_DONE; 332 } 333 334 /** 335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table 336 * @iwdev: irdma device 337 */ 338 static void irdma_add_ipv6_addr(struct irdma_device *iwdev) 339 { 340 struct net_device *ip_dev; 341 struct inet6_dev *idev; 342 struct inet6_ifaddr *ifp, *tmp; 343 u32 local_ipaddr6[4]; 344 345 rcu_read_lock(); 346 for_each_netdev_rcu (&init_net, ip_dev) { 347 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF && 348 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || 349 ip_dev == iwdev->netdev) && 350 (READ_ONCE(ip_dev->flags) & IFF_UP)) { 351 idev = __in6_dev_get(ip_dev); 352 if (!idev) { 353 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); 354 break; 355 } 356 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, 357 if_list) { 358 ibdev_dbg(&iwdev->ibdev, 359 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n", 360 &ifp->addr, 361 rdma_vlan_dev_vlan_id(ip_dev), 362 ip_dev->dev_addr); 363 364 irdma_copy_ip_ntohl(local_ipaddr6, 365 ifp->addr.in6_u.u6_addr32); 366 irdma_manage_arp_cache(iwdev->rf, 367 ip_dev->dev_addr, 368 local_ipaddr6, false, 369 IRDMA_ARP_ADD); 370 } 371 } 372 } 373 rcu_read_unlock(); 374 } 375 376 /** 377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table 378 * @iwdev: irdma device 379 */ 380 static void irdma_add_ipv4_addr(struct irdma_device *iwdev) 381 { 382 struct net_device *dev; 383 struct in_device *idev; 384 u32 ip_addr; 385 386 rcu_read_lock(); 387 for_each_netdev_rcu (&init_net, dev) { 388 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && 389 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || 390 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { 391 const struct in_ifaddr *ifa; 392 393 idev = __in_dev_get_rcu(dev); 394 if (!idev) 395 continue; 396 397 in_dev_for_each_ifa_rcu(ifa, idev) { 398 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", 399 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), 400 dev->dev_addr); 401 402 ip_addr = ntohl(ifa->ifa_address); 403 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, 404 &ip_addr, true, 405 IRDMA_ARP_ADD); 406 } 407 } 408 } 409 rcu_read_unlock(); 410 } 411 412 /** 413 * irdma_add_ip - add ip addresses 414 * @iwdev: irdma device 415 * 416 * Add ipv4/ipv6 addresses to the arp cache 417 */ 418 void irdma_add_ip(struct irdma_device *iwdev) 419 { 420 irdma_add_ipv4_addr(iwdev); 421 irdma_add_ipv6_addr(iwdev); 422 } 423 424 /** 425 * irdma_alloc_and_get_cqp_request - get cqp struct 426 * @cqp: device cqp ptr 427 * @wait: cqp to be used in wait mode 428 */ 429 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 430 bool wait) 431 { 432 struct irdma_cqp_request *cqp_request = NULL; 433 unsigned long flags; 434 435 spin_lock_irqsave(&cqp->req_lock, flags); 436 if (!list_empty(&cqp->cqp_avail_reqs)) { 437 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, 438 struct irdma_cqp_request, list); 439 list_del_init(&cqp_request->list); 440 } 441 spin_unlock_irqrestore(&cqp->req_lock, flags); 442 if (!cqp_request) { 443 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 444 if (cqp_request) { 445 cqp_request->dynamic = true; 446 if (wait) 447 init_waitqueue_head(&cqp_request->waitq); 448 } 449 } 450 if (!cqp_request) { 451 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); 452 return NULL; 453 } 454 455 cqp_request->waiting = wait; 456 refcount_set(&cqp_request->refcnt, 1); 457 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 458 459 return cqp_request; 460 } 461 462 /** 463 * irdma_get_cqp_request - increase refcount for cqp_request 464 * @cqp_request: pointer to cqp_request instance 465 */ 466 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 467 { 468 refcount_inc(&cqp_request->refcnt); 469 } 470 471 /** 472 * irdma_free_cqp_request - free cqp request 473 * @cqp: cqp ptr 474 * @cqp_request: to be put back in cqp list 475 */ 476 void irdma_free_cqp_request(struct irdma_cqp *cqp, 477 struct irdma_cqp_request *cqp_request) 478 { 479 unsigned long flags; 480 481 if (cqp_request->dynamic) { 482 kfree(cqp_request); 483 } else { 484 cqp_request->request_done = false; 485 cqp_request->callback_fcn = NULL; 486 cqp_request->waiting = false; 487 488 spin_lock_irqsave(&cqp->req_lock, flags); 489 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 490 spin_unlock_irqrestore(&cqp->req_lock, flags); 491 } 492 wake_up(&cqp->remove_wq); 493 } 494 495 /** 496 * irdma_put_cqp_request - dec ref count and free if 0 497 * @cqp: cqp ptr 498 * @cqp_request: to be put back in cqp list 499 */ 500 void irdma_put_cqp_request(struct irdma_cqp *cqp, 501 struct irdma_cqp_request *cqp_request) 502 { 503 if (refcount_dec_and_test(&cqp_request->refcnt)) 504 irdma_free_cqp_request(cqp, cqp_request); 505 } 506 507 /** 508 * irdma_free_pending_cqp_request -free pending cqp request objs 509 * @cqp: cqp ptr 510 * @cqp_request: to be put back in cqp list 511 */ 512 static void 513 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 514 struct irdma_cqp_request *cqp_request) 515 { 516 if (cqp_request->waiting) { 517 cqp_request->compl_info.error = true; 518 cqp_request->request_done = true; 519 wake_up(&cqp_request->waitq); 520 } 521 wait_event_timeout(cqp->remove_wq, 522 refcount_read(&cqp_request->refcnt) == 1, 1000); 523 irdma_put_cqp_request(cqp, cqp_request); 524 } 525 526 /** 527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 528 * completions 529 * @rf: RDMA PCI function 530 */ 531 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 532 { 533 struct irdma_sc_dev *dev = &rf->sc_dev; 534 struct irdma_cqp *cqp = &rf->cqp; 535 struct irdma_cqp_request *cqp_request = NULL; 536 struct cqp_cmds_info *pcmdinfo = NULL; 537 u32 i, pending_work, wqe_idx; 538 539 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 540 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 541 for (i = 0; i < pending_work; i++) { 542 cqp_request = (struct irdma_cqp_request *)(unsigned long) 543 cqp->scratch_array[wqe_idx]; 544 if (cqp_request) 545 irdma_free_pending_cqp_request(cqp, cqp_request); 546 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 547 } 548 549 while (!list_empty(&dev->cqp_cmd_head)) { 550 pcmdinfo = irdma_remove_cqp_head(dev); 551 cqp_request = 552 container_of(pcmdinfo, struct irdma_cqp_request, info); 553 if (cqp_request) 554 irdma_free_pending_cqp_request(cqp, cqp_request); 555 } 556 } 557 558 /** 559 * irdma_wait_event - wait for completion 560 * @rf: RDMA PCI function 561 * @cqp_request: cqp request to wait 562 */ 563 static int irdma_wait_event(struct irdma_pci_f *rf, 564 struct irdma_cqp_request *cqp_request) 565 { 566 struct irdma_cqp_timeout cqp_timeout = {}; 567 bool cqp_error = false; 568 int err_code = 0; 569 570 cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]; 571 do { 572 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 573 if (wait_event_timeout(cqp_request->waitq, 574 cqp_request->request_done, 575 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) 576 break; 577 578 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 579 580 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 581 continue; 582 583 if (!rf->reset) { 584 rf->reset = true; 585 rf->gen_ops.request_reset(rf); 586 } 587 return -ETIMEDOUT; 588 } while (1); 589 590 cqp_error = cqp_request->compl_info.error; 591 if (cqp_error) { 592 err_code = -EIO; 593 if (cqp_request->compl_info.maj_err_code == 0xFFFF && 594 cqp_request->compl_info.min_err_code == 0x8029) { 595 if (!rf->reset) { 596 rf->reset = true; 597 rf->gen_ops.request_reset(rf); 598 } 599 } 600 } 601 602 return err_code; 603 } 604 605 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 606 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 607 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 608 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 609 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 610 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 611 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 612 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 613 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 614 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 615 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 616 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 617 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 618 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 619 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 620 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 621 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 622 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 623 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 624 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 625 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 626 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 627 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 628 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 629 [IRDMA_OP_RESUME] = "Resume QP Cmd", 630 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", 631 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 632 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 633 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 634 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 635 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 636 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 637 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 638 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 639 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 640 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 641 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 642 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 643 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 644 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 645 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 646 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 647 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 648 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 649 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 650 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 651 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 652 }; 653 654 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 655 {0xffff, 0x8002, "Invalid State"}, 656 {0xffff, 0x8006, "Flush No Wqe Pending"}, 657 {0xffff, 0x8007, "Modify QP Bad Close"}, 658 {0xffff, 0x8009, "LLP Closed"}, 659 {0xffff, 0x800a, "Reset Not Sent"} 660 }; 661 662 /** 663 * irdma_cqp_crit_err - check if CQP error is critical 664 * @dev: pointer to dev structure 665 * @cqp_cmd: code for last CQP operation 666 * @maj_err_code: major error code 667 * @min_err_code: minot error code 668 */ 669 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 670 u16 maj_err_code, u16 min_err_code) 671 { 672 int i; 673 674 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 675 if (maj_err_code == irdma_noncrit_err_list[i].maj && 676 min_err_code == irdma_noncrit_err_list[i].min) { 677 ibdev_dbg(to_ibdev(dev), 678 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n", 679 irdma_noncrit_err_list[i].desc, 680 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 681 min_err_code); 682 return false; 683 } 684 } 685 return true; 686 } 687 688 /** 689 * irdma_handle_cqp_op - process cqp command 690 * @rf: RDMA PCI function 691 * @cqp_request: cqp request to process 692 */ 693 int irdma_handle_cqp_op(struct irdma_pci_f *rf, 694 struct irdma_cqp_request *cqp_request) 695 { 696 struct irdma_sc_dev *dev = &rf->sc_dev; 697 struct cqp_cmds_info *info = &cqp_request->info; 698 int status; 699 bool put_cqp_request = true; 700 701 if (rf->reset) 702 return -EBUSY; 703 704 irdma_get_cqp_request(cqp_request); 705 status = irdma_process_cqp_cmd(dev, info); 706 if (status) 707 goto err; 708 709 if (cqp_request->waiting) { 710 put_cqp_request = false; 711 status = irdma_wait_event(rf, cqp_request); 712 if (status) 713 goto err; 714 } 715 716 return 0; 717 718 err: 719 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 720 cqp_request->compl_info.maj_err_code, 721 cqp_request->compl_info.min_err_code)) 722 ibdev_err(&rf->iwdev->ibdev, 723 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 724 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, 725 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, 726 cqp_request->compl_info.min_err_code); 727 728 if (put_cqp_request) 729 irdma_put_cqp_request(&rf->cqp, cqp_request); 730 731 return status; 732 } 733 734 void irdma_qp_add_ref(struct ib_qp *ibqp) 735 { 736 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; 737 738 refcount_inc(&iwqp->refcnt); 739 } 740 741 void irdma_qp_rem_ref(struct ib_qp *ibqp) 742 { 743 struct irdma_qp *iwqp = to_iwqp(ibqp); 744 struct irdma_device *iwdev = iwqp->iwdev; 745 u32 qp_num; 746 unsigned long flags; 747 748 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 749 if (!refcount_dec_and_test(&iwqp->refcnt)) { 750 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 751 return; 752 } 753 754 qp_num = iwqp->ibqp.qp_num; 755 iwdev->rf->qp_table[qp_num] = NULL; 756 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 757 complete(&iwqp->free_qp); 758 } 759 760 struct ib_device *to_ibdev(struct irdma_sc_dev *dev) 761 { 762 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 763 } 764 765 /** 766 * irdma_get_qp - get qp address 767 * @device: iwarp device 768 * @qpn: qp number 769 */ 770 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) 771 { 772 struct irdma_device *iwdev = to_iwdev(device); 773 774 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 775 return NULL; 776 777 return &iwdev->rf->qp_table[qpn]->ibqp; 778 } 779 780 /** 781 * irdma_remove_cqp_head - return head entry and remove 782 * @dev: device 783 */ 784 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) 785 { 786 struct list_head *entry; 787 struct list_head *list = &dev->cqp_cmd_head; 788 789 if (list_empty(list)) 790 return NULL; 791 792 entry = list->next; 793 list_del(entry); 794 795 return entry; 796 } 797 798 /** 799 * irdma_cqp_sds_cmd - create cqp command for sd 800 * @dev: hardware control device structure 801 * @sdinfo: information for sd cqp 802 * 803 */ 804 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 805 struct irdma_update_sds_info *sdinfo) 806 { 807 struct irdma_cqp_request *cqp_request; 808 struct cqp_cmds_info *cqp_info; 809 struct irdma_pci_f *rf = dev_to_rf(dev); 810 int status; 811 812 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 813 if (!cqp_request) 814 return -ENOMEM; 815 816 cqp_info = &cqp_request->info; 817 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 818 sizeof(cqp_info->in.u.update_pe_sds.info)); 819 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 820 cqp_info->post_sq = 1; 821 cqp_info->in.u.update_pe_sds.dev = dev; 822 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 823 824 status = irdma_handle_cqp_op(rf, cqp_request); 825 irdma_put_cqp_request(&rf->cqp, cqp_request); 826 827 return status; 828 } 829 830 /** 831 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 832 * @qp: hardware control qp 833 * @op: suspend or resume 834 */ 835 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 836 { 837 struct irdma_sc_dev *dev = qp->dev; 838 struct irdma_cqp_request *cqp_request; 839 struct irdma_sc_cqp *cqp = dev->cqp; 840 struct cqp_cmds_info *cqp_info; 841 struct irdma_pci_f *rf = dev_to_rf(dev); 842 int status; 843 844 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 845 if (!cqp_request) 846 return -ENOMEM; 847 848 cqp_info = &cqp_request->info; 849 cqp_info->cqp_cmd = op; 850 cqp_info->in.u.suspend_resume.cqp = cqp; 851 cqp_info->in.u.suspend_resume.qp = qp; 852 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 853 854 status = irdma_handle_cqp_op(rf, cqp_request); 855 irdma_put_cqp_request(&rf->cqp, cqp_request); 856 857 return status; 858 } 859 860 /** 861 * irdma_term_modify_qp - modify qp for term message 862 * @qp: hardware control qp 863 * @next_state: qp's next state 864 * @term: terminate code 865 * @term_len: length 866 */ 867 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 868 u8 term_len) 869 { 870 struct irdma_qp *iwqp; 871 872 iwqp = qp->qp_uk.back_qp; 873 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 874 }; 875 876 /** 877 * irdma_terminate_done - after terminate is completed 878 * @qp: hardware control qp 879 * @timeout_occurred: indicates if terminate timer expired 880 */ 881 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 882 { 883 struct irdma_qp *iwqp; 884 u8 hte = 0; 885 bool first_time; 886 unsigned long flags; 887 888 iwqp = qp->qp_uk.back_qp; 889 spin_lock_irqsave(&iwqp->lock, flags); 890 if (iwqp->hte_added) { 891 iwqp->hte_added = 0; 892 hte = 1; 893 } 894 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 895 qp->term_flags |= IRDMA_TERM_DONE; 896 spin_unlock_irqrestore(&iwqp->lock, flags); 897 if (first_time) { 898 if (!timeout_occurred) 899 irdma_terminate_del_timer(qp); 900 901 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 902 irdma_cm_disconn(iwqp); 903 } 904 } 905 906 static void irdma_terminate_timeout(struct timer_list *t) 907 { 908 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 909 struct irdma_sc_qp *qp = &iwqp->sc_qp; 910 911 irdma_terminate_done(qp, 1); 912 irdma_qp_rem_ref(&iwqp->ibqp); 913 } 914 915 /** 916 * irdma_terminate_start_timer - start terminate timeout 917 * @qp: hardware control qp 918 */ 919 void irdma_terminate_start_timer(struct irdma_sc_qp *qp) 920 { 921 struct irdma_qp *iwqp; 922 923 iwqp = qp->qp_uk.back_qp; 924 irdma_qp_add_ref(&iwqp->ibqp); 925 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 926 iwqp->terminate_timer.expires = jiffies + HZ; 927 928 add_timer(&iwqp->terminate_timer); 929 } 930 931 /** 932 * irdma_terminate_del_timer - delete terminate timeout 933 * @qp: hardware control qp 934 */ 935 void irdma_terminate_del_timer(struct irdma_sc_qp *qp) 936 { 937 struct irdma_qp *iwqp; 938 int ret; 939 940 iwqp = qp->qp_uk.back_qp; 941 ret = del_timer(&iwqp->terminate_timer); 942 if (ret) 943 irdma_qp_rem_ref(&iwqp->ibqp); 944 } 945 946 /** 947 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 948 * @dev: function device struct 949 * @val_mem: buffer for fpm 950 * @hmc_fn_id: function id for fpm 951 */ 952 int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 953 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 954 { 955 struct irdma_cqp_request *cqp_request; 956 struct cqp_cmds_info *cqp_info; 957 struct irdma_pci_f *rf = dev_to_rf(dev); 958 int status; 959 960 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 961 if (!cqp_request) 962 return -ENOMEM; 963 964 cqp_info = &cqp_request->info; 965 cqp_request->param = NULL; 966 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 967 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 968 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 969 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 970 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 971 cqp_info->post_sq = 1; 972 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 973 974 status = irdma_handle_cqp_op(rf, cqp_request); 975 irdma_put_cqp_request(&rf->cqp, cqp_request); 976 977 return status; 978 } 979 980 /** 981 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 982 * @dev: hardware control device structure 983 * @val_mem: buffer with fpm values 984 * @hmc_fn_id: function id for fpm 985 */ 986 int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 987 struct irdma_dma_mem *val_mem, u8 hmc_fn_id) 988 { 989 struct irdma_cqp_request *cqp_request; 990 struct cqp_cmds_info *cqp_info; 991 struct irdma_pci_f *rf = dev_to_rf(dev); 992 int status; 993 994 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 995 if (!cqp_request) 996 return -ENOMEM; 997 998 cqp_info = &cqp_request->info; 999 cqp_request->param = NULL; 1000 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 1001 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 1002 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 1003 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 1004 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 1005 cqp_info->post_sq = 1; 1006 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 1007 1008 status = irdma_handle_cqp_op(rf, cqp_request); 1009 irdma_put_cqp_request(&rf->cqp, cqp_request); 1010 1011 return status; 1012 } 1013 1014 /** 1015 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1016 * @dev: device pointer 1017 * @cq: pointer to created cq 1018 */ 1019 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1020 { 1021 struct irdma_pci_f *rf = dev_to_rf(dev); 1022 struct irdma_cqp *iwcqp = &rf->cqp; 1023 struct irdma_cqp_request *cqp_request; 1024 struct cqp_cmds_info *cqp_info; 1025 int status; 1026 1027 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1028 if (!cqp_request) 1029 return -ENOMEM; 1030 1031 cqp_info = &cqp_request->info; 1032 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1033 cqp_info->post_sq = 1; 1034 cqp_info->in.u.cq_create.cq = cq; 1035 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1036 1037 status = irdma_handle_cqp_op(rf, cqp_request); 1038 irdma_put_cqp_request(iwcqp, cqp_request); 1039 1040 return status; 1041 } 1042 1043 /** 1044 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1045 * @dev: device pointer 1046 * @qp: pointer to created qp 1047 */ 1048 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1049 { 1050 struct irdma_pci_f *rf = dev_to_rf(dev); 1051 struct irdma_cqp *iwcqp = &rf->cqp; 1052 struct irdma_cqp_request *cqp_request; 1053 struct cqp_cmds_info *cqp_info; 1054 struct irdma_create_qp_info *qp_info; 1055 int status; 1056 1057 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1058 if (!cqp_request) 1059 return -ENOMEM; 1060 1061 cqp_info = &cqp_request->info; 1062 qp_info = &cqp_request->info.in.u.qp_create.info; 1063 memset(qp_info, 0, sizeof(*qp_info)); 1064 qp_info->cq_num_valid = true; 1065 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1066 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1067 cqp_info->post_sq = 1; 1068 cqp_info->in.u.qp_create.qp = qp; 1069 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1070 1071 status = irdma_handle_cqp_op(rf, cqp_request); 1072 irdma_put_cqp_request(iwcqp, cqp_request); 1073 1074 return status; 1075 } 1076 1077 /** 1078 * irdma_dealloc_push_page - free a push page for qp 1079 * @rf: RDMA PCI function 1080 * @qp: hardware control qp 1081 */ 1082 static void irdma_dealloc_push_page(struct irdma_pci_f *rf, 1083 struct irdma_sc_qp *qp) 1084 { 1085 struct irdma_cqp_request *cqp_request; 1086 struct cqp_cmds_info *cqp_info; 1087 int status; 1088 1089 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1090 return; 1091 1092 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1093 if (!cqp_request) 1094 return; 1095 1096 cqp_info = &cqp_request->info; 1097 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1098 cqp_info->post_sq = 1; 1099 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1100 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1101 cqp_info->in.u.manage_push_page.info.free_page = 1; 1102 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1103 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1104 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1105 status = irdma_handle_cqp_op(rf, cqp_request); 1106 if (!status) 1107 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1108 irdma_put_cqp_request(&rf->cqp, cqp_request); 1109 } 1110 1111 /** 1112 * irdma_free_qp_rsrc - free up memory resources for qp 1113 * @iwqp: qp ptr (user or kernel) 1114 */ 1115 void irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1116 { 1117 struct irdma_device *iwdev = iwqp->iwdev; 1118 struct irdma_pci_f *rf = iwdev->rf; 1119 u32 qp_num = iwqp->ibqp.qp_num; 1120 1121 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1122 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1123 if (iwqp->sc_qp.vsi) { 1124 irdma_qp_rem_qos(&iwqp->sc_qp); 1125 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1126 iwqp->sc_qp.user_pri); 1127 } 1128 1129 if (qp_num > 2) 1130 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1131 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, 1132 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); 1133 iwqp->q2_ctx_mem.va = NULL; 1134 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, 1135 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); 1136 iwqp->kqp.dma_mem.va = NULL; 1137 kfree(iwqp->kqp.sq_wrid_mem); 1138 kfree(iwqp->kqp.rq_wrid_mem); 1139 } 1140 1141 /** 1142 * irdma_cq_wq_destroy - send cq destroy cqp 1143 * @rf: RDMA PCI function 1144 * @cq: hardware control cq 1145 */ 1146 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1147 { 1148 struct irdma_cqp_request *cqp_request; 1149 struct cqp_cmds_info *cqp_info; 1150 1151 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1152 if (!cqp_request) 1153 return; 1154 1155 cqp_info = &cqp_request->info; 1156 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1157 cqp_info->post_sq = 1; 1158 cqp_info->in.u.cq_destroy.cq = cq; 1159 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1160 1161 irdma_handle_cqp_op(rf, cqp_request); 1162 irdma_put_cqp_request(&rf->cqp, cqp_request); 1163 } 1164 1165 /** 1166 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1167 * @cqp_request: modify QP completion 1168 */ 1169 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1170 { 1171 struct cqp_cmds_info *cqp_info; 1172 struct irdma_qp *iwqp; 1173 1174 cqp_info = &cqp_request->info; 1175 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1176 atomic_dec(&iwqp->hw_mod_qp_pend); 1177 wake_up(&iwqp->mod_qp_waitq); 1178 } 1179 1180 /** 1181 * irdma_hw_modify_qp - setup cqp for modify qp 1182 * @iwdev: RDMA device 1183 * @iwqp: qp ptr (user or kernel) 1184 * @info: info for modify qp 1185 * @wait: flag to wait or not for modify qp completion 1186 */ 1187 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1188 struct irdma_modify_qp_info *info, bool wait) 1189 { 1190 int status; 1191 struct irdma_pci_f *rf = iwdev->rf; 1192 struct irdma_cqp_request *cqp_request; 1193 struct cqp_cmds_info *cqp_info; 1194 struct irdma_modify_qp_info *m_info; 1195 1196 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1197 if (!cqp_request) 1198 return -ENOMEM; 1199 1200 if (!wait) { 1201 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1202 atomic_inc(&iwqp->hw_mod_qp_pend); 1203 } 1204 cqp_info = &cqp_request->info; 1205 m_info = &cqp_info->in.u.qp_modify.info; 1206 memcpy(m_info, info, sizeof(*m_info)); 1207 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1208 cqp_info->post_sq = 1; 1209 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1210 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1211 status = irdma_handle_cqp_op(rf, cqp_request); 1212 irdma_put_cqp_request(&rf->cqp, cqp_request); 1213 if (status) { 1214 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1215 return status; 1216 1217 switch (m_info->next_iwarp_state) { 1218 struct irdma_gen_ae_info ae_info; 1219 1220 case IRDMA_QP_STATE_RTS: 1221 case IRDMA_QP_STATE_IDLE: 1222 case IRDMA_QP_STATE_TERMINATE: 1223 case IRDMA_QP_STATE_CLOSING: 1224 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1225 irdma_send_reset(iwqp->cm_node); 1226 else 1227 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1228 if (!wait) { 1229 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1230 ae_info.ae_src = 0; 1231 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1232 } else { 1233 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1234 wait); 1235 if (!cqp_request) 1236 return -ENOMEM; 1237 1238 cqp_info = &cqp_request->info; 1239 m_info = &cqp_info->in.u.qp_modify.info; 1240 memcpy(m_info, info, sizeof(*m_info)); 1241 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1242 cqp_info->post_sq = 1; 1243 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1244 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1245 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1246 m_info->reset_tcp_conn = true; 1247 irdma_handle_cqp_op(rf, cqp_request); 1248 irdma_put_cqp_request(&rf->cqp, cqp_request); 1249 } 1250 break; 1251 case IRDMA_QP_STATE_ERROR: 1252 default: 1253 break; 1254 } 1255 } 1256 1257 return status; 1258 } 1259 1260 /** 1261 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1262 * @dev: device pointer 1263 * @cq: pointer to cq 1264 */ 1265 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1266 { 1267 struct irdma_pci_f *rf = dev_to_rf(dev); 1268 1269 irdma_cq_wq_destroy(rf, cq); 1270 } 1271 1272 /** 1273 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1274 * @dev: device pointer 1275 * @qp: pointer to qp 1276 */ 1277 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1278 { 1279 struct irdma_pci_f *rf = dev_to_rf(dev); 1280 struct irdma_cqp *iwcqp = &rf->cqp; 1281 struct irdma_cqp_request *cqp_request; 1282 struct cqp_cmds_info *cqp_info; 1283 int status; 1284 1285 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1286 if (!cqp_request) 1287 return -ENOMEM; 1288 1289 cqp_info = &cqp_request->info; 1290 memset(cqp_info, 0, sizeof(*cqp_info)); 1291 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1292 cqp_info->post_sq = 1; 1293 cqp_info->in.u.qp_destroy.qp = qp; 1294 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1295 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1296 1297 status = irdma_handle_cqp_op(rf, cqp_request); 1298 irdma_put_cqp_request(&rf->cqp, cqp_request); 1299 1300 return status; 1301 } 1302 1303 /** 1304 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1305 * @dev: hardware control device structure 1306 * @qp: hardware control qp 1307 */ 1308 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1309 { 1310 struct irdma_gen_ae_info info = {}; 1311 struct irdma_pci_f *rf = dev_to_rf(dev); 1312 1313 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); 1314 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1315 info.ae_src = IRDMA_AE_SOURCE_RQ; 1316 irdma_gen_ae(rf, qp, &info, false); 1317 } 1318 1319 /** 1320 * irdma_init_hash_desc - initialize hash for crc calculation 1321 * @desc: cryption type 1322 */ 1323 int irdma_init_hash_desc(struct shash_desc **desc) 1324 { 1325 struct crypto_shash *tfm; 1326 struct shash_desc *tdesc; 1327 1328 tfm = crypto_alloc_shash("crc32c", 0, 0); 1329 if (IS_ERR(tfm)) 1330 return -EINVAL; 1331 1332 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm), 1333 GFP_KERNEL); 1334 if (!tdesc) { 1335 crypto_free_shash(tfm); 1336 return -EINVAL; 1337 } 1338 1339 tdesc->tfm = tfm; 1340 *desc = tdesc; 1341 1342 return 0; 1343 } 1344 1345 /** 1346 * irdma_free_hash_desc - free hash desc 1347 * @desc: to be freed 1348 */ 1349 void irdma_free_hash_desc(struct shash_desc *desc) 1350 { 1351 if (desc) { 1352 crypto_free_shash(desc->tfm); 1353 kfree(desc); 1354 } 1355 } 1356 1357 /** 1358 * irdma_ieq_check_mpacrc - check if mpa crc is OK 1359 * @desc: desc for hash 1360 * @addr: address of buffer for crc 1361 * @len: length of buffer 1362 * @val: value to be compared 1363 */ 1364 int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len, 1365 u32 val) 1366 { 1367 u32 crc = 0; 1368 int ret; 1369 int ret_code = 0; 1370 1371 crypto_shash_init(desc); 1372 ret = crypto_shash_update(desc, addr, len); 1373 if (!ret) 1374 crypto_shash_final(desc, (u8 *)&crc); 1375 if (crc != val) 1376 ret_code = -EINVAL; 1377 1378 return ret_code; 1379 } 1380 1381 /** 1382 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1383 * @dev: hardware control device structure 1384 * @buf: receive puda buffer on exception q 1385 */ 1386 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1387 struct irdma_puda_buf *buf) 1388 { 1389 struct irdma_qp *iwqp; 1390 struct irdma_cm_node *cm_node; 1391 struct irdma_device *iwdev = buf->vsi->back_vsi; 1392 u32 loc_addr[4] = {}; 1393 u32 rem_addr[4] = {}; 1394 u16 loc_port, rem_port; 1395 struct ipv6hdr *ip6h; 1396 struct iphdr *iph = (struct iphdr *)buf->iph; 1397 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1398 1399 if (iph->version == 4) { 1400 loc_addr[0] = ntohl(iph->daddr); 1401 rem_addr[0] = ntohl(iph->saddr); 1402 } else { 1403 ip6h = (struct ipv6hdr *)buf->iph; 1404 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); 1405 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); 1406 } 1407 loc_port = ntohs(tcph->dest); 1408 rem_port = ntohs(tcph->source); 1409 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1410 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1411 if (!cm_node) 1412 return NULL; 1413 1414 iwqp = cm_node->iwqp; 1415 irdma_rem_ref_cm_node(cm_node); 1416 1417 return &iwqp->sc_qp; 1418 } 1419 1420 /** 1421 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1422 * @qp: qp ptr 1423 */ 1424 void irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1425 { 1426 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1427 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1428 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1429 1430 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1431 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); 1432 1433 irdma_send_ack(cm_node); 1434 } 1435 1436 /** 1437 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1438 * @qp: qp pointer 1439 * @ah_info: AH info pointer 1440 */ 1441 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1442 struct irdma_ah_info *ah_info) 1443 { 1444 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1445 struct iphdr *iph; 1446 struct ipv6hdr *ip6h; 1447 1448 memset(ah_info, 0, sizeof(*ah_info)); 1449 ah_info->do_lpbk = true; 1450 ah_info->vlan_tag = buf->vlan_id; 1451 ah_info->insert_vlan_tag = buf->vlan_valid; 1452 ah_info->ipv4_valid = buf->ipv4; 1453 ah_info->vsi = qp->vsi; 1454 1455 if (buf->smac_valid) 1456 ether_addr_copy(ah_info->mac_addr, buf->smac); 1457 1458 if (buf->ipv4) { 1459 ah_info->ipv4_valid = true; 1460 iph = (struct iphdr *)buf->iph; 1461 ah_info->hop_ttl = iph->ttl; 1462 ah_info->tc_tos = iph->tos; 1463 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); 1464 ah_info->src_ip_addr[0] = ntohl(iph->saddr); 1465 } else { 1466 ip6h = (struct ipv6hdr *)buf->iph; 1467 ah_info->hop_ttl = ip6h->hop_limit; 1468 ah_info->tc_tos = ip6h->priority; 1469 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1470 ip6h->daddr.in6_u.u6_addr32); 1471 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1472 ip6h->saddr.in6_u.u6_addr32); 1473 } 1474 1475 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1476 ah_info->dest_ip_addr, 1477 ah_info->ipv4_valid, 1478 NULL, IRDMA_ARP_RESOLVE); 1479 } 1480 1481 /** 1482 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1483 * @buf: puda to update 1484 * @len: length of buffer 1485 * @seqnum: seq number for tcp 1486 */ 1487 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1488 u16 len, u32 seqnum) 1489 { 1490 struct tcphdr *tcph; 1491 struct iphdr *iph; 1492 u16 iphlen; 1493 u16 pktsize; 1494 u8 *addr = buf->mem.va; 1495 1496 iphlen = (buf->ipv4) ? 20 : 40; 1497 iph = (struct iphdr *)(addr + buf->maclen); 1498 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1499 pktsize = len + buf->tcphlen + iphlen; 1500 iph->tot_len = htons(pktsize); 1501 tcph->seq = htonl(seqnum); 1502 } 1503 1504 /** 1505 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1506 * @buf: puda to update 1507 * @len: length of buffer 1508 * @seqnum: seq number for tcp 1509 */ 1510 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1511 u32 seqnum) 1512 { 1513 struct tcphdr *tcph; 1514 u8 *addr; 1515 1516 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1517 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1518 1519 addr = buf->mem.va; 1520 tcph = (struct tcphdr *)addr; 1521 tcph->seq = htonl(seqnum); 1522 } 1523 1524 /** 1525 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1526 * buffer 1527 * @info: to get information 1528 * @buf: puda buffer 1529 */ 1530 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1531 struct irdma_puda_buf *buf) 1532 { 1533 struct iphdr *iph; 1534 struct ipv6hdr *ip6h; 1535 struct tcphdr *tcph; 1536 u16 iphlen; 1537 u16 pkt_len; 1538 u8 *mem = buf->mem.va; 1539 struct ethhdr *ethh = buf->mem.va; 1540 1541 if (ethh->h_proto == htons(0x8100)) { 1542 info->vlan_valid = true; 1543 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & 1544 VLAN_VID_MASK; 1545 } 1546 1547 buf->maclen = (info->vlan_valid) ? 18 : 14; 1548 iphlen = (info->l3proto) ? 40 : 20; 1549 buf->ipv4 = (info->l3proto) ? false : true; 1550 buf->iph = mem + buf->maclen; 1551 iph = (struct iphdr *)buf->iph; 1552 buf->tcph = buf->iph + iphlen; 1553 tcph = (struct tcphdr *)buf->tcph; 1554 1555 if (buf->ipv4) { 1556 pkt_len = ntohs(iph->tot_len); 1557 } else { 1558 ip6h = (struct ipv6hdr *)buf->iph; 1559 pkt_len = ntohs(ip6h->payload_len) + iphlen; 1560 } 1561 1562 buf->totallen = pkt_len + buf->maclen; 1563 1564 if (info->payload_len < buf->totallen) { 1565 ibdev_dbg(to_ibdev(buf->vsi->dev), 1566 "ERR: payload_len = 0x%x totallen expected0x%x\n", 1567 info->payload_len, buf->totallen); 1568 return -EINVAL; 1569 } 1570 1571 buf->tcphlen = tcph->doff << 2; 1572 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1573 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1574 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1575 buf->seqnum = ntohl(tcph->seq); 1576 1577 return 0; 1578 } 1579 1580 /** 1581 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1582 * @info: to get information 1583 * @buf: puda buffer 1584 */ 1585 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1586 struct irdma_puda_buf *buf) 1587 { 1588 struct tcphdr *tcph; 1589 u32 pkt_len; 1590 u8 *mem; 1591 1592 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1593 return irdma_gen1_puda_get_tcpip_info(info, buf); 1594 1595 mem = buf->mem.va; 1596 buf->vlan_valid = info->vlan_valid; 1597 if (info->vlan_valid) 1598 buf->vlan_id = info->vlan; 1599 1600 buf->ipv4 = info->ipv4; 1601 if (buf->ipv4) 1602 buf->iph = mem + IRDMA_IPV4_PAD; 1603 else 1604 buf->iph = mem; 1605 1606 buf->tcph = mem + IRDMA_TCP_OFFSET; 1607 tcph = (struct tcphdr *)buf->tcph; 1608 pkt_len = info->payload_len; 1609 buf->totallen = pkt_len; 1610 buf->tcphlen = tcph->doff << 2; 1611 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1612 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1613 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1614 buf->seqnum = ntohl(tcph->seq); 1615 1616 if (info->smac_valid) { 1617 ether_addr_copy(buf->smac, info->smac); 1618 buf->smac_valid = true; 1619 } 1620 1621 return 0; 1622 } 1623 1624 /** 1625 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1626 * @t: timer_list pointer 1627 */ 1628 static void irdma_hw_stats_timeout(struct timer_list *t) 1629 { 1630 struct irdma_vsi_pestat *pf_devstat = 1631 from_timer(pf_devstat, t, stats_timer); 1632 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1633 1634 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1635 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); 1636 else 1637 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1638 1639 mod_timer(&pf_devstat->stats_timer, 1640 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1641 } 1642 1643 /** 1644 * irdma_hw_stats_start_timer - Start periodic stats timer 1645 * @vsi: vsi structure pointer 1646 */ 1647 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1648 { 1649 struct irdma_vsi_pestat *devstat = vsi->pestat; 1650 1651 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1652 mod_timer(&devstat->stats_timer, 1653 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1654 } 1655 1656 /** 1657 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1658 * @vsi: pointer to vsi structure 1659 */ 1660 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1661 { 1662 struct irdma_vsi_pestat *devstat = vsi->pestat; 1663 1664 del_timer_sync(&devstat->stats_timer); 1665 } 1666 1667 /** 1668 * irdma_process_stats - Checking for wrap and update stats 1669 * @pestat: stats structure pointer 1670 */ 1671 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) 1672 { 1673 sc_vsi_update_stats(pestat->vsi); 1674 } 1675 1676 /** 1677 * irdma_cqp_gather_stats_gen1 - Gather stats 1678 * @dev: pointer to device structure 1679 * @pestat: statistics structure 1680 */ 1681 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, 1682 struct irdma_vsi_pestat *pestat) 1683 { 1684 struct irdma_gather_stats *gather_stats = 1685 pestat->gather_info.gather_stats_va; 1686 u32 stats_inst_offset_32; 1687 u32 stats_inst_offset_64; 1688 1689 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? 1690 pestat->gather_info.stats_inst_index : 1691 pestat->hw->hmc.hmc_fn_id; 1692 stats_inst_offset_32 *= 4; 1693 stats_inst_offset_64 = stats_inst_offset_32 * 2; 1694 1695 gather_stats->rxvlanerr = 1696 rd32(dev->hw, 1697 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR] 1698 + stats_inst_offset_32); 1699 gather_stats->ip4rxdiscard = 1700 rd32(dev->hw, 1701 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] 1702 + stats_inst_offset_32); 1703 gather_stats->ip4rxtrunc = 1704 rd32(dev->hw, 1705 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] 1706 + stats_inst_offset_32); 1707 gather_stats->ip4txnoroute = 1708 rd32(dev->hw, 1709 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] 1710 + stats_inst_offset_32); 1711 gather_stats->ip6rxdiscard = 1712 rd32(dev->hw, 1713 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] 1714 + stats_inst_offset_32); 1715 gather_stats->ip6rxtrunc = 1716 rd32(dev->hw, 1717 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] 1718 + stats_inst_offset_32); 1719 gather_stats->ip6txnoroute = 1720 rd32(dev->hw, 1721 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] 1722 + stats_inst_offset_32); 1723 gather_stats->tcprtxseg = 1724 rd32(dev->hw, 1725 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG] 1726 + stats_inst_offset_32); 1727 gather_stats->tcprxopterr = 1728 rd32(dev->hw, 1729 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] 1730 + stats_inst_offset_32); 1731 1732 gather_stats->ip4rxocts = 1733 rd64(dev->hw, 1734 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS] 1735 + stats_inst_offset_64); 1736 gather_stats->ip4rxpkts = 1737 rd64(dev->hw, 1738 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS] 1739 + stats_inst_offset_64); 1740 gather_stats->ip4txfrag = 1741 rd64(dev->hw, 1742 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] 1743 + stats_inst_offset_64); 1744 gather_stats->ip4rxmcpkts = 1745 rd64(dev->hw, 1746 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] 1747 + stats_inst_offset_64); 1748 gather_stats->ip4txocts = 1749 rd64(dev->hw, 1750 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS] 1751 + stats_inst_offset_64); 1752 gather_stats->ip4txpkts = 1753 rd64(dev->hw, 1754 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS] 1755 + stats_inst_offset_64); 1756 gather_stats->ip4txfrag = 1757 rd64(dev->hw, 1758 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] 1759 + stats_inst_offset_64); 1760 gather_stats->ip4txmcpkts = 1761 rd64(dev->hw, 1762 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] 1763 + stats_inst_offset_64); 1764 gather_stats->ip6rxocts = 1765 rd64(dev->hw, 1766 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS] 1767 + stats_inst_offset_64); 1768 gather_stats->ip6rxpkts = 1769 rd64(dev->hw, 1770 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS] 1771 + stats_inst_offset_64); 1772 gather_stats->ip6txfrags = 1773 rd64(dev->hw, 1774 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] 1775 + stats_inst_offset_64); 1776 gather_stats->ip6rxmcpkts = 1777 rd64(dev->hw, 1778 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] 1779 + stats_inst_offset_64); 1780 gather_stats->ip6txocts = 1781 rd64(dev->hw, 1782 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS] 1783 + stats_inst_offset_64); 1784 gather_stats->ip6txpkts = 1785 rd64(dev->hw, 1786 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS] 1787 + stats_inst_offset_64); 1788 gather_stats->ip6txfrags = 1789 rd64(dev->hw, 1790 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] 1791 + stats_inst_offset_64); 1792 gather_stats->ip6txmcpkts = 1793 rd64(dev->hw, 1794 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] 1795 + stats_inst_offset_64); 1796 gather_stats->tcprxsegs = 1797 rd64(dev->hw, 1798 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS] 1799 + stats_inst_offset_64); 1800 gather_stats->tcptxsegs = 1801 rd64(dev->hw, 1802 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG] 1803 + stats_inst_offset_64); 1804 gather_stats->rdmarxrds = 1805 rd64(dev->hw, 1806 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS] 1807 + stats_inst_offset_64); 1808 gather_stats->rdmarxsnds = 1809 rd64(dev->hw, 1810 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS] 1811 + stats_inst_offset_64); 1812 gather_stats->rdmarxwrs = 1813 rd64(dev->hw, 1814 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS] 1815 + stats_inst_offset_64); 1816 gather_stats->rdmatxrds = 1817 rd64(dev->hw, 1818 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS] 1819 + stats_inst_offset_64); 1820 gather_stats->rdmatxsnds = 1821 rd64(dev->hw, 1822 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS] 1823 + stats_inst_offset_64); 1824 gather_stats->rdmatxwrs = 1825 rd64(dev->hw, 1826 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS] 1827 + stats_inst_offset_64); 1828 gather_stats->rdmavbn = 1829 rd64(dev->hw, 1830 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND] 1831 + stats_inst_offset_64); 1832 gather_stats->rdmavinv = 1833 rd64(dev->hw, 1834 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV] 1835 + stats_inst_offset_64); 1836 gather_stats->udprxpkts = 1837 rd64(dev->hw, 1838 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS] 1839 + stats_inst_offset_64); 1840 gather_stats->udptxpkts = 1841 rd64(dev->hw, 1842 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS] 1843 + stats_inst_offset_64); 1844 1845 irdma_process_stats(pestat); 1846 } 1847 1848 /** 1849 * irdma_process_cqp_stats - Checking for wrap and update stats 1850 * @cqp_request: cqp_request structure pointer 1851 */ 1852 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1853 { 1854 struct irdma_vsi_pestat *pestat = cqp_request->param; 1855 1856 irdma_process_stats(pestat); 1857 } 1858 1859 /** 1860 * irdma_cqp_gather_stats_cmd - Gather stats 1861 * @dev: pointer to device structure 1862 * @pestat: pointer to stats info 1863 * @wait: flag to wait or not wait for stats 1864 */ 1865 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1866 struct irdma_vsi_pestat *pestat, bool wait) 1867 1868 { 1869 struct irdma_pci_f *rf = dev_to_rf(dev); 1870 struct irdma_cqp *iwcqp = &rf->cqp; 1871 struct irdma_cqp_request *cqp_request; 1872 struct cqp_cmds_info *cqp_info; 1873 int status; 1874 1875 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1876 if (!cqp_request) 1877 return -ENOMEM; 1878 1879 cqp_info = &cqp_request->info; 1880 memset(cqp_info, 0, sizeof(*cqp_info)); 1881 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1882 cqp_info->post_sq = 1; 1883 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1884 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1885 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1886 cqp_request->param = pestat; 1887 if (!wait) 1888 cqp_request->callback_fcn = irdma_process_cqp_stats; 1889 status = irdma_handle_cqp_op(rf, cqp_request); 1890 if (wait) 1891 irdma_process_stats(pestat); 1892 irdma_put_cqp_request(&rf->cqp, cqp_request); 1893 1894 return status; 1895 } 1896 1897 /** 1898 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1899 * @vsi: pointer to vsi structure 1900 * @cmd: command to allocate or free 1901 * @stats_info: pointer to allocate stats info 1902 */ 1903 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1904 struct irdma_stats_inst_info *stats_info) 1905 { 1906 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1907 struct irdma_cqp *iwcqp = &rf->cqp; 1908 struct irdma_cqp_request *cqp_request; 1909 struct cqp_cmds_info *cqp_info; 1910 int status; 1911 bool wait = false; 1912 1913 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1914 wait = true; 1915 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1916 if (!cqp_request) 1917 return -ENOMEM; 1918 1919 cqp_info = &cqp_request->info; 1920 memset(cqp_info, 0, sizeof(*cqp_info)); 1921 cqp_info->cqp_cmd = cmd; 1922 cqp_info->post_sq = 1; 1923 cqp_info->in.u.stats_manage.info = *stats_info; 1924 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1925 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1926 status = irdma_handle_cqp_op(rf, cqp_request); 1927 if (wait) 1928 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1929 irdma_put_cqp_request(iwcqp, cqp_request); 1930 1931 return status; 1932 } 1933 1934 /** 1935 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1936 * @dev: pointer to device info 1937 * @sc_ceq: pointer to ceq structure 1938 * @op: Create or Destroy 1939 */ 1940 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1941 u8 op) 1942 { 1943 struct irdma_cqp_request *cqp_request; 1944 struct cqp_cmds_info *cqp_info; 1945 struct irdma_pci_f *rf = dev_to_rf(dev); 1946 int status; 1947 1948 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1949 if (!cqp_request) 1950 return -ENOMEM; 1951 1952 cqp_info = &cqp_request->info; 1953 cqp_info->post_sq = 1; 1954 cqp_info->cqp_cmd = op; 1955 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1956 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1957 1958 status = irdma_handle_cqp_op(rf, cqp_request); 1959 irdma_put_cqp_request(&rf->cqp, cqp_request); 1960 1961 return status; 1962 } 1963 1964 /** 1965 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1966 * @dev: pointer to device info 1967 * @sc_aeq: pointer to aeq structure 1968 * @op: Create or Destroy 1969 */ 1970 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1971 u8 op) 1972 { 1973 struct irdma_cqp_request *cqp_request; 1974 struct cqp_cmds_info *cqp_info; 1975 struct irdma_pci_f *rf = dev_to_rf(dev); 1976 int status; 1977 1978 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1979 if (!cqp_request) 1980 return -ENOMEM; 1981 1982 cqp_info = &cqp_request->info; 1983 cqp_info->post_sq = 1; 1984 cqp_info->cqp_cmd = op; 1985 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1986 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1987 1988 status = irdma_handle_cqp_op(rf, cqp_request); 1989 irdma_put_cqp_request(&rf->cqp, cqp_request); 1990 1991 return status; 1992 } 1993 1994 /** 1995 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1996 * @dev: pointer to device structure 1997 * @cmd: Add, modify or delete 1998 * @node_info: pointer to ws node info 1999 */ 2000 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 2001 struct irdma_ws_node_info *node_info) 2002 { 2003 struct irdma_pci_f *rf = dev_to_rf(dev); 2004 struct irdma_cqp *iwcqp = &rf->cqp; 2005 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 2006 struct irdma_cqp_request *cqp_request; 2007 struct cqp_cmds_info *cqp_info; 2008 int status; 2009 bool poll; 2010 2011 if (!rf->sc_dev.ceq_valid) 2012 poll = true; 2013 else 2014 poll = false; 2015 2016 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 2017 if (!cqp_request) 2018 return -ENOMEM; 2019 2020 cqp_info = &cqp_request->info; 2021 memset(cqp_info, 0, sizeof(*cqp_info)); 2022 cqp_info->cqp_cmd = cmd; 2023 cqp_info->post_sq = 1; 2024 cqp_info->in.u.ws_node.info = *node_info; 2025 cqp_info->in.u.ws_node.cqp = cqp; 2026 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 2027 status = irdma_handle_cqp_op(rf, cqp_request); 2028 if (status) 2029 goto exit; 2030 2031 if (poll) { 2032 struct irdma_ccq_cqe_info compl_info; 2033 2034 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 2035 &compl_info); 2036 node_info->qs_handle = compl_info.op_ret_val; 2037 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", 2038 compl_info.op_code, compl_info.op_ret_val); 2039 } else { 2040 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 2041 } 2042 2043 exit: 2044 irdma_put_cqp_request(&rf->cqp, cqp_request); 2045 2046 return status; 2047 } 2048 2049 /** 2050 * irdma_ah_cqp_op - perform an AH cqp operation 2051 * @rf: RDMA PCI function 2052 * @sc_ah: address handle 2053 * @cmd: AH operation 2054 * @wait: wait if true 2055 * @callback_fcn: Callback function on CQP op completion 2056 * @cb_param: parameter for callback function 2057 * 2058 * returns errno 2059 */ 2060 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 2061 bool wait, 2062 void (*callback_fcn)(struct irdma_cqp_request *), 2063 void *cb_param) 2064 { 2065 struct irdma_cqp_request *cqp_request; 2066 struct cqp_cmds_info *cqp_info; 2067 int status; 2068 2069 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 2070 return -EINVAL; 2071 2072 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 2073 if (!cqp_request) 2074 return -ENOMEM; 2075 2076 cqp_info = &cqp_request->info; 2077 cqp_info->cqp_cmd = cmd; 2078 cqp_info->post_sq = 1; 2079 if (cmd == IRDMA_OP_AH_CREATE) { 2080 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 2081 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 2082 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 2083 } else if (cmd == IRDMA_OP_AH_DESTROY) { 2084 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 2085 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 2086 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 2087 } 2088 2089 if (!wait) { 2090 cqp_request->callback_fcn = callback_fcn; 2091 cqp_request->param = cb_param; 2092 } 2093 status = irdma_handle_cqp_op(rf, cqp_request); 2094 irdma_put_cqp_request(&rf->cqp, cqp_request); 2095 2096 if (status) 2097 return -ENOMEM; 2098 2099 if (wait) 2100 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); 2101 2102 return 0; 2103 } 2104 2105 /** 2106 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 2107 * @cqp_request: pointer to cqp_request of create AH 2108 */ 2109 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 2110 { 2111 struct irdma_sc_qp *qp = cqp_request->param; 2112 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 2113 unsigned long flags; 2114 2115 spin_lock_irqsave(&qp->pfpdu.lock, flags); 2116 if (!cqp_request->compl_info.op_ret_val) { 2117 sc_ah->ah_info.ah_valid = true; 2118 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 2119 } else { 2120 sc_ah->ah_info.ah_valid = false; 2121 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 2122 } 2123 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 2124 } 2125 2126 /** 2127 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 2128 * @cqp_request: pointer to cqp_request of create AH 2129 */ 2130 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 2131 { 2132 struct irdma_cm_node *cm_node = cqp_request->param; 2133 struct irdma_sc_ah *sc_ah = cm_node->ah; 2134 2135 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 2136 irdma_add_conn_est_qh(cm_node); 2137 } 2138 2139 /** 2140 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 2141 * @dev: device pointer 2142 * @ah_info: Address handle info 2143 * @wait: When true will wait for operation to complete 2144 * @type: ILQ/IEQ 2145 * @cb_param: Callback param when not waiting 2146 * @ah_ret: Returned pointer to address handle if created 2147 * 2148 */ 2149 int irdma_puda_create_ah(struct irdma_sc_dev *dev, 2150 struct irdma_ah_info *ah_info, bool wait, 2151 enum puda_rsrc_type type, void *cb_param, 2152 struct irdma_sc_ah **ah_ret) 2153 { 2154 struct irdma_sc_ah *ah; 2155 struct irdma_pci_f *rf = dev_to_rf(dev); 2156 int err; 2157 2158 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2159 *ah_ret = ah; 2160 if (!ah) 2161 return -ENOMEM; 2162 2163 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 2164 &ah_info->ah_idx, &rf->next_ah); 2165 if (err) 2166 goto err_free; 2167 2168 ah->dev = dev; 2169 ah->ah_info = *ah_info; 2170 2171 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 2172 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2173 irdma_ilq_ah_cb, cb_param); 2174 else 2175 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2176 irdma_ieq_ah_cb, cb_param); 2177 2178 if (err) 2179 goto error; 2180 return 0; 2181 2182 error: 2183 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2184 err_free: 2185 kfree(ah); 2186 *ah_ret = NULL; 2187 return -ENOMEM; 2188 } 2189 2190 /** 2191 * irdma_puda_free_ah - free a puda address handle 2192 * @dev: device pointer 2193 * @ah: The address handle to free 2194 */ 2195 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2196 { 2197 struct irdma_pci_f *rf = dev_to_rf(dev); 2198 2199 if (!ah) 2200 return; 2201 2202 if (ah->ah_info.ah_valid) { 2203 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2204 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2205 } 2206 2207 kfree(ah); 2208 } 2209 2210 /** 2211 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 2212 * @cqp_request: pointer to cqp_request of create AH 2213 */ 2214 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 2215 { 2216 struct irdma_sc_ah *sc_ah = cqp_request->param; 2217 2218 if (!cqp_request->compl_info.op_ret_val) 2219 sc_ah->ah_info.ah_valid = true; 2220 else 2221 sc_ah->ah_info.ah_valid = false; 2222 } 2223 2224 /** 2225 * irdma_prm_add_pble_mem - add moemory to pble resources 2226 * @pprm: pble resource manager 2227 * @pchunk: chunk of memory to add 2228 */ 2229 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2230 struct irdma_chunk *pchunk) 2231 { 2232 u64 sizeofbitmap; 2233 2234 if (pchunk->size & 0xfff) 2235 return -EINVAL; 2236 2237 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2238 2239 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 2240 if (!pchunk->bitmapbuf) 2241 return -ENOMEM; 2242 2243 pchunk->sizeofbitmap = sizeofbitmap; 2244 /* each pble is 8 bytes hence shift by 3 */ 2245 pprm->total_pble_alloc += pchunk->size >> 3; 2246 pprm->free_pble_cnt += pchunk->size >> 3; 2247 2248 return 0; 2249 } 2250 2251 /** 2252 * irdma_prm_get_pbles - get pble's from prm 2253 * @pprm: pble resource manager 2254 * @chunkinfo: nformation about chunk where pble's were acquired 2255 * @mem_size: size of pble memory needed 2256 * @vaddr: returns virtual address of pble memory 2257 * @fpm_addr: returns fpm address of pble memory 2258 */ 2259 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2260 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2261 u64 **vaddr, u64 *fpm_addr) 2262 { 2263 u64 bits_needed; 2264 u64 bit_idx = PBLE_INVALID_IDX; 2265 struct irdma_chunk *pchunk = NULL; 2266 struct list_head *chunk_entry = pprm->clist.next; 2267 u32 offset; 2268 unsigned long flags; 2269 *vaddr = NULL; 2270 *fpm_addr = 0; 2271 2272 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2273 2274 spin_lock_irqsave(&pprm->prm_lock, flags); 2275 while (chunk_entry != &pprm->clist) { 2276 pchunk = (struct irdma_chunk *)chunk_entry; 2277 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2278 pchunk->sizeofbitmap, 0, 2279 bits_needed, 0); 2280 if (bit_idx < pchunk->sizeofbitmap) 2281 break; 2282 2283 /* list.next used macro */ 2284 chunk_entry = pchunk->list.next; 2285 } 2286 2287 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2288 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2289 return -ENOMEM; 2290 } 2291 2292 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2293 offset = bit_idx << pprm->pble_shift; 2294 *vaddr = pchunk->vaddr + offset; 2295 *fpm_addr = pchunk->fpm_addr + offset; 2296 2297 chunkinfo->pchunk = pchunk; 2298 chunkinfo->bit_idx = bit_idx; 2299 chunkinfo->bits_used = bits_needed; 2300 /* 3 is sizeof pble divide */ 2301 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2302 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2303 2304 return 0; 2305 } 2306 2307 /** 2308 * irdma_prm_return_pbles - return pbles back to prm 2309 * @pprm: pble resource manager 2310 * @chunkinfo: chunk where pble's were acquired and to be freed 2311 */ 2312 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2313 struct irdma_pble_chunkinfo *chunkinfo) 2314 { 2315 unsigned long flags; 2316 2317 spin_lock_irqsave(&pprm->prm_lock, flags); 2318 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2319 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2320 chunkinfo->bits_used); 2321 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2322 } 2323 2324 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma, 2325 u32 pg_cnt) 2326 { 2327 struct page *vm_page; 2328 int i; 2329 u8 *addr; 2330 2331 addr = (u8 *)(uintptr_t)va; 2332 for (i = 0; i < pg_cnt; i++) { 2333 vm_page = vmalloc_to_page(addr); 2334 if (!vm_page) 2335 goto err; 2336 2337 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, 2338 DMA_BIDIRECTIONAL); 2339 if (dma_mapping_error(hw->device, pg_dma[i])) 2340 goto err; 2341 2342 addr += PAGE_SIZE; 2343 } 2344 2345 return 0; 2346 2347 err: 2348 irdma_unmap_vm_page_list(hw, pg_dma, i); 2349 return -ENOMEM; 2350 } 2351 2352 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) 2353 { 2354 int i; 2355 2356 for (i = 0; i < pg_cnt; i++) 2357 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2358 } 2359 2360 /** 2361 * irdma_pble_free_paged_mem - free virtual paged memory 2362 * @chunk: chunk to free with paged memory 2363 */ 2364 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2365 { 2366 if (!chunk->pg_cnt) 2367 goto done; 2368 2369 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2370 chunk->pg_cnt); 2371 2372 done: 2373 kfree(chunk->dmainfo.dmaaddrs); 2374 chunk->dmainfo.dmaaddrs = NULL; 2375 vfree(chunk->vaddr); 2376 chunk->vaddr = NULL; 2377 chunk->type = 0; 2378 } 2379 2380 /** 2381 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2382 * @chunk: chunk to add for paged memory 2383 * @pg_cnt: number of pages needed 2384 */ 2385 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2386 { 2387 u32 size; 2388 void *va; 2389 2390 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2391 if (!chunk->dmainfo.dmaaddrs) 2392 return -ENOMEM; 2393 2394 size = PAGE_SIZE * pg_cnt; 2395 va = vmalloc(size); 2396 if (!va) 2397 goto err; 2398 2399 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2400 pg_cnt)) { 2401 vfree(va); 2402 goto err; 2403 } 2404 chunk->vaddr = va; 2405 chunk->size = size; 2406 chunk->pg_cnt = pg_cnt; 2407 chunk->type = PBLE_SD_PAGED; 2408 2409 return 0; 2410 err: 2411 kfree(chunk->dmainfo.dmaaddrs); 2412 chunk->dmainfo.dmaaddrs = NULL; 2413 2414 return -ENOMEM; 2415 } 2416 2417 /** 2418 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2419 * @dev: device pointer 2420 */ 2421 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2422 { 2423 struct irdma_pci_f *rf = dev_to_rf(dev); 2424 u32 next = 1; 2425 u32 node_id; 2426 2427 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2428 &node_id, &next)) 2429 return IRDMA_WS_NODE_INVALID; 2430 2431 return (u16)node_id; 2432 } 2433 2434 /** 2435 * irdma_free_ws_node_id - Free a tx scheduler node ID 2436 * @dev: device pointer 2437 * @node_id: Work scheduler node ID 2438 */ 2439 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2440 { 2441 struct irdma_pci_f *rf = dev_to_rf(dev); 2442 2443 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2444 } 2445 2446 /** 2447 * irdma_modify_qp_to_err - Modify a QP to error 2448 * @sc_qp: qp structure 2449 */ 2450 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2451 { 2452 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2453 struct ib_qp_attr attr; 2454 2455 if (qp->iwdev->rf->reset) 2456 return; 2457 attr.qp_state = IB_QPS_ERR; 2458 2459 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2460 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2461 else 2462 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2463 } 2464 2465 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2466 { 2467 struct ib_event ibevent; 2468 2469 if (!iwqp->ibqp.event_handler) 2470 return; 2471 2472 switch (event) { 2473 case IRDMA_QP_EVENT_CATASTROPHIC: 2474 ibevent.event = IB_EVENT_QP_FATAL; 2475 break; 2476 case IRDMA_QP_EVENT_ACCESS_ERR: 2477 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2478 break; 2479 } 2480 ibevent.device = iwqp->ibqp.device; 2481 ibevent.element.qp = &iwqp->ibqp; 2482 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2483 } 2484 2485 bool irdma_cq_empty(struct irdma_cq *iwcq) 2486 { 2487 struct irdma_cq_uk *ukcq; 2488 u64 qword3; 2489 __le64 *cqe; 2490 u8 polarity; 2491 2492 ukcq = &iwcq->sc_cq.cq_uk; 2493 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2494 get_64bit_val(cqe, 24, &qword3); 2495 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2496 2497 return polarity != ukcq->polarity; 2498 } 2499 2500 void irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2501 { 2502 struct irdma_cmpl_gen *cmpl_node; 2503 struct list_head *tmp_node, *list_node; 2504 2505 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { 2506 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2507 list_del(&cmpl_node->list); 2508 kfree(cmpl_node); 2509 } 2510 } 2511 2512 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2513 { 2514 struct irdma_cmpl_gen *cmpl; 2515 2516 if (list_empty(&iwcq->cmpl_generated)) 2517 return -ENOENT; 2518 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2519 list_del(&cmpl->list); 2520 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2521 kfree(cmpl); 2522 2523 ibdev_dbg(iwcq->ibcq.device, 2524 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n", 2525 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2526 cq_poll_info->wr_id); 2527 2528 return 0; 2529 } 2530 2531 /** 2532 * irdma_set_cpi_common_values - fill in values for polling info struct 2533 * @cpi: resulting structure of cq_poll_info type 2534 * @qp: QPair 2535 * @qp_num: id of the QP 2536 */ 2537 static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2538 struct irdma_qp_uk *qp, u32 qp_num) 2539 { 2540 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2541 cpi->error = true; 2542 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2543 cpi->minor_err = FLUSH_GENERAL_ERR; 2544 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; 2545 cpi->qp_id = qp_num; 2546 } 2547 2548 static inline void irdma_comp_handler(struct irdma_cq *cq) 2549 { 2550 if (!cq->ibcq.comp_handler) 2551 return; 2552 if (atomic_cmpxchg(&cq->armed, 1, 0)) 2553 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2554 } 2555 2556 void irdma_generate_flush_completions(struct irdma_qp *iwqp) 2557 { 2558 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2559 struct irdma_ring *sq_ring = &qp->sq_ring; 2560 struct irdma_ring *rq_ring = &qp->rq_ring; 2561 struct irdma_cmpl_gen *cmpl; 2562 __le64 *sw_wqe; 2563 u64 wqe_qword; 2564 u32 wqe_idx; 2565 bool compl_generated = false; 2566 unsigned long flags1; 2567 2568 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2569 if (irdma_cq_empty(iwqp->iwscq)) { 2570 unsigned long flags2; 2571 2572 spin_lock_irqsave(&iwqp->lock, flags2); 2573 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2574 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2575 if (!cmpl) { 2576 spin_unlock_irqrestore(&iwqp->lock, flags2); 2577 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2578 return; 2579 } 2580 2581 wqe_idx = sq_ring->tail; 2582 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2583 2584 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2585 sw_wqe = qp->sq_base[wqe_idx].elem; 2586 get_64bit_val(sw_wqe, 24, &wqe_qword); 2587 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); 2588 /* remove the SQ WR by moving SQ tail*/ 2589 IRDMA_RING_SET_TAIL(*sq_ring, 2590 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2591 2592 ibdev_dbg(iwqp->iwscq->ibcq.device, 2593 "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n", 2594 __func__, cmpl->cpi.wr_id, qp->qp_id); 2595 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2596 compl_generated = true; 2597 } 2598 spin_unlock_irqrestore(&iwqp->lock, flags2); 2599 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2600 if (compl_generated) 2601 irdma_comp_handler(iwqp->iwrcq); 2602 } else { 2603 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2604 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2605 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2606 } 2607 2608 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2609 if (irdma_cq_empty(iwqp->iwrcq)) { 2610 unsigned long flags2; 2611 2612 spin_lock_irqsave(&iwqp->lock, flags2); 2613 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2614 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2615 if (!cmpl) { 2616 spin_unlock_irqrestore(&iwqp->lock, flags2); 2617 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2618 return; 2619 } 2620 2621 wqe_idx = rq_ring->tail; 2622 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2623 2624 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2625 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2626 /* remove the RQ WR by moving RQ tail */ 2627 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2628 ibdev_dbg(iwqp->iwrcq->ibcq.device, 2629 "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2630 __func__, cmpl->cpi.wr_id, qp->qp_id, 2631 wqe_idx); 2632 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2633 2634 compl_generated = true; 2635 } 2636 spin_unlock_irqrestore(&iwqp->lock, flags2); 2637 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2638 if (compl_generated) 2639 irdma_comp_handler(iwqp->iwrcq); 2640 } else { 2641 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2642 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2643 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2644 } 2645 } 2646