1 /* 2 * Copyright(c) 2015-2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/spinlock.h> 49 #include <linux/pci.h> 50 #include <linux/io.h> 51 #include <linux/delay.h> 52 #include <linux/netdevice.h> 53 #include <linux/vmalloc.h> 54 #include <linux/module.h> 55 #include <linux/prefetch.h> 56 #include <rdma/ib_verbs.h> 57 58 #include "hfi.h" 59 #include "trace.h" 60 #include "qp.h" 61 #include "sdma.h" 62 #include "debugfs.h" 63 #include "vnic.h" 64 65 #undef pr_fmt 66 #define pr_fmt(fmt) DRIVER_NAME ": " fmt 67 68 /* 69 * The size has to be longer than this string, so we can append 70 * board/chip information to it in the initialization code. 71 */ 72 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n"; 73 74 DEFINE_SPINLOCK(hfi1_devs_lock); 75 LIST_HEAD(hfi1_dev_list); 76 DEFINE_MUTEX(hfi1_mutex); /* general driver use */ 77 78 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU; 79 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO); 80 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify( 81 HFI1_DEFAULT_MAX_MTU)); 82 83 unsigned int hfi1_cu = 1; 84 module_param_named(cu, hfi1_cu, uint, S_IRUGO); 85 MODULE_PARM_DESC(cu, "Credit return units"); 86 87 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; 88 static int hfi1_caps_set(const char *val, const struct kernel_param *kp); 89 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp); 90 static const struct kernel_param_ops cap_ops = { 91 .set = hfi1_caps_set, 92 .get = hfi1_caps_get 93 }; 94 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO); 95 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features"); 96 97 MODULE_LICENSE("Dual BSD/GPL"); 98 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver"); 99 MODULE_VERSION(HFI1_DRIVER_VERSION); 100 101 /* 102 * MAX_PKT_RCV is the max # if packets processed per receive interrupt. 103 */ 104 #define MAX_PKT_RECV 64 105 /* 106 * MAX_PKT_THREAD_RCV is the max # of packets processed before 107 * the qp_wait_list queue is flushed. 108 */ 109 #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4) 110 #define EGR_HEAD_UPDATE_THRESHOLD 16 111 112 struct hfi1_ib_stats hfi1_stats; 113 114 static int hfi1_caps_set(const char *val, const struct kernel_param *kp) 115 { 116 int ret = 0; 117 unsigned long *cap_mask_ptr = (unsigned long *)kp->arg, 118 cap_mask = *cap_mask_ptr, value, diff, 119 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) | 120 HFI1_CAP_WRITABLE_MASK); 121 122 ret = kstrtoul(val, 0, &value); 123 if (ret) { 124 pr_warn("Invalid module parameter value for 'cap_mask'\n"); 125 goto done; 126 } 127 /* Get the changed bits (except the locked bit) */ 128 diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK); 129 130 /* Remove any bits that are not allowed to change after driver load */ 131 if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) { 132 pr_warn("Ignoring non-writable capability bits %#lx\n", 133 diff & ~write_mask); 134 diff &= write_mask; 135 } 136 137 /* Mask off any reserved bits */ 138 diff &= ~HFI1_CAP_RESERVED_MASK; 139 /* Clear any previously set and changing bits */ 140 cap_mask &= ~diff; 141 /* Update the bits with the new capability */ 142 cap_mask |= (value & diff); 143 /* Check for any kernel/user restrictions */ 144 diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^ 145 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT); 146 cap_mask &= ~diff; 147 /* Set the bitmask to the final set */ 148 *cap_mask_ptr = cap_mask; 149 done: 150 return ret; 151 } 152 153 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) 154 { 155 unsigned long cap_mask = *(unsigned long *)kp->arg; 156 157 cap_mask &= ~HFI1_CAP_LOCKED_SMASK; 158 cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT); 159 160 return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); 161 } 162 163 const char *get_unit_name(int unit) 164 { 165 static char iname[16]; 166 167 snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit); 168 return iname; 169 } 170 171 const char *get_card_name(struct rvt_dev_info *rdi) 172 { 173 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 174 struct hfi1_devdata *dd = container_of(ibdev, 175 struct hfi1_devdata, verbs_dev); 176 return get_unit_name(dd->unit); 177 } 178 179 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) 180 { 181 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 182 struct hfi1_devdata *dd = container_of(ibdev, 183 struct hfi1_devdata, verbs_dev); 184 return dd->pcidev; 185 } 186 187 /* 188 * Return count of units with at least one port ACTIVE. 189 */ 190 int hfi1_count_active_units(void) 191 { 192 struct hfi1_devdata *dd; 193 struct hfi1_pportdata *ppd; 194 unsigned long flags; 195 int pidx, nunits_active = 0; 196 197 spin_lock_irqsave(&hfi1_devs_lock, flags); 198 list_for_each_entry(dd, &hfi1_dev_list, list) { 199 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase) 200 continue; 201 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 202 ppd = dd->pport + pidx; 203 if (ppd->lid && ppd->linkup) { 204 nunits_active++; 205 break; 206 } 207 } 208 } 209 spin_unlock_irqrestore(&hfi1_devs_lock, flags); 210 return nunits_active; 211 } 212 213 /* 214 * Get address of eager buffer from it's index (allocated in chunks, not 215 * contiguous). 216 */ 217 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, 218 u8 *update) 219 { 220 u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf); 221 222 *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset; 223 return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) + 224 (offset * RCV_BUF_BLOCK_SIZE)); 225 } 226 227 /* 228 * Validate and encode the a given RcvArray Buffer size. 229 * The function will check whether the given size falls within 230 * allowed size ranges for the respective type and, optionally, 231 * return the proper encoding. 232 */ 233 int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded) 234 { 235 if (unlikely(!PAGE_ALIGNED(size))) 236 return 0; 237 if (unlikely(size < MIN_EAGER_BUFFER)) 238 return 0; 239 if (size > 240 (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER)) 241 return 0; 242 if (encoded) 243 *encoded = ilog2(size / PAGE_SIZE) + 1; 244 return 1; 245 } 246 247 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, 248 struct hfi1_packet *packet) 249 { 250 struct ib_header *rhdr = packet->hdr; 251 u32 rte = rhf_rcv_type_err(packet->rhf); 252 int lnh = ib_get_lnh(rhdr); 253 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 254 struct hfi1_devdata *dd = ppd->dd; 255 struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 256 257 if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) 258 return; 259 260 if (packet->rhf & RHF_TID_ERR) { 261 /* For TIDERR and RC QPs preemptively schedule a NAK */ 262 struct ib_other_headers *ohdr = NULL; 263 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 264 u16 lid = ib_get_dlid(rhdr); 265 u32 qp_num; 266 u32 rcv_flags = 0; 267 268 /* Sanity check packet */ 269 if (tlen < 24) 270 goto drop; 271 272 /* Check for GRH */ 273 if (lnh == HFI1_LRH_BTH) { 274 ohdr = &rhdr->u.oth; 275 } else if (lnh == HFI1_LRH_GRH) { 276 u32 vtf; 277 278 ohdr = &rhdr->u.l.oth; 279 if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) 280 goto drop; 281 vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow); 282 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 283 goto drop; 284 rcv_flags |= HFI1_HAS_GRH; 285 } else { 286 goto drop; 287 } 288 /* Get the destination QP number. */ 289 qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; 290 if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) { 291 struct rvt_qp *qp; 292 unsigned long flags; 293 294 rcu_read_lock(); 295 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 296 if (!qp) { 297 rcu_read_unlock(); 298 goto drop; 299 } 300 301 /* 302 * Handle only RC QPs - for other QP types drop error 303 * packet. 304 */ 305 spin_lock_irqsave(&qp->r_lock, flags); 306 307 /* Check for valid receive state. */ 308 if (!(ib_rvt_state_ops[qp->state] & 309 RVT_PROCESS_RECV_OK)) { 310 ibp->rvp.n_pkt_drops++; 311 } 312 313 switch (qp->ibqp.qp_type) { 314 case IB_QPT_RC: 315 hfi1_rc_hdrerr( 316 rcd, 317 rhdr, 318 rcv_flags, 319 qp); 320 break; 321 default: 322 /* For now don't handle any other QP types */ 323 break; 324 } 325 326 spin_unlock_irqrestore(&qp->r_lock, flags); 327 rcu_read_unlock(); 328 } /* Unicast QP */ 329 } /* Valid packet with TIDErr */ 330 331 /* handle "RcvTypeErr" flags */ 332 switch (rte) { 333 case RHF_RTE_ERROR_OP_CODE_ERR: 334 { 335 u32 opcode; 336 void *ebuf = NULL; 337 __be32 *bth = NULL; 338 339 if (rhf_use_egr_bfr(packet->rhf)) 340 ebuf = packet->ebuf; 341 342 if (!ebuf) 343 goto drop; /* this should never happen */ 344 345 if (lnh == HFI1_LRH_BTH) 346 bth = (__be32 *)ebuf; 347 else if (lnh == HFI1_LRH_GRH) 348 bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh)); 349 else 350 goto drop; 351 352 opcode = be32_to_cpu(bth[0]) >> 24; 353 opcode &= 0xff; 354 355 if (opcode == IB_OPCODE_CNP) { 356 /* 357 * Only in pre-B0 h/w is the CNP_OPCODE handled 358 * via this code path. 359 */ 360 struct rvt_qp *qp = NULL; 361 u32 lqpn, rqpn; 362 u16 rlid; 363 u8 svc_type, sl, sc5; 364 365 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); 366 sl = ibp->sc_to_sl[sc5]; 367 368 lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK; 369 rcu_read_lock(); 370 qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); 371 if (!qp) { 372 rcu_read_unlock(); 373 goto drop; 374 } 375 376 switch (qp->ibqp.qp_type) { 377 case IB_QPT_UD: 378 rlid = 0; 379 rqpn = 0; 380 svc_type = IB_CC_SVCTYPE_UD; 381 break; 382 case IB_QPT_UC: 383 rlid = ib_get_slid(rhdr); 384 rqpn = qp->remote_qpn; 385 svc_type = IB_CC_SVCTYPE_UC; 386 break; 387 default: 388 goto drop; 389 } 390 391 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 392 rcu_read_unlock(); 393 } 394 395 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; 396 break; 397 } 398 default: 399 break; 400 } 401 402 drop: 403 return; 404 } 405 406 static inline void init_packet(struct hfi1_ctxtdata *rcd, 407 struct hfi1_packet *packet) 408 { 409 packet->rsize = rcd->rcvhdrqentsize; /* words */ 410 packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ 411 packet->rcd = rcd; 412 packet->updegr = 0; 413 packet->etail = -1; 414 packet->rhf_addr = get_rhf_addr(rcd); 415 packet->rhf = rhf_to_cpu(packet->rhf_addr); 416 packet->rhqoff = rcd->head; 417 packet->numpkt = 0; 418 packet->rcv_flags = 0; 419 } 420 421 void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, 422 bool do_cnp) 423 { 424 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 425 struct ib_header *hdr = pkt->hdr; 426 struct ib_other_headers *ohdr = pkt->ohdr; 427 struct ib_grh *grh = NULL; 428 u32 rqpn = 0, bth1; 429 u16 rlid, dlid = ib_get_dlid(hdr); 430 u8 sc, svc_type; 431 bool is_mcast = false; 432 433 if (pkt->rcv_flags & HFI1_HAS_GRH) 434 grh = &hdr->u.l.grh; 435 436 switch (qp->ibqp.qp_type) { 437 case IB_QPT_SMI: 438 case IB_QPT_GSI: 439 case IB_QPT_UD: 440 rlid = ib_get_slid(hdr); 441 rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK; 442 svc_type = IB_CC_SVCTYPE_UD; 443 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) && 444 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)); 445 break; 446 case IB_QPT_UC: 447 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 448 rqpn = qp->remote_qpn; 449 svc_type = IB_CC_SVCTYPE_UC; 450 break; 451 case IB_QPT_RC: 452 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 453 rqpn = qp->remote_qpn; 454 svc_type = IB_CC_SVCTYPE_RC; 455 break; 456 default: 457 return; 458 } 459 460 sc = hfi1_9B_get_sc5(hdr, pkt->rhf); 461 462 bth1 = be32_to_cpu(ohdr->bth[1]); 463 if (do_cnp && (bth1 & IB_FECN_SMASK)) { 464 u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]); 465 466 return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh); 467 } 468 469 if (!is_mcast && (bth1 & IB_BECN_SMASK)) { 470 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 471 u32 lqpn = bth1 & RVT_QPN_MASK; 472 u8 sl = ibp->sc_to_sl[sc]; 473 474 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 475 } 476 477 } 478 479 struct ps_mdata { 480 struct hfi1_ctxtdata *rcd; 481 u32 rsize; 482 u32 maxcnt; 483 u32 ps_head; 484 u32 ps_tail; 485 u32 ps_seq; 486 }; 487 488 static inline void init_ps_mdata(struct ps_mdata *mdata, 489 struct hfi1_packet *packet) 490 { 491 struct hfi1_ctxtdata *rcd = packet->rcd; 492 493 mdata->rcd = rcd; 494 mdata->rsize = packet->rsize; 495 mdata->maxcnt = packet->maxcnt; 496 mdata->ps_head = packet->rhqoff; 497 498 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 499 mdata->ps_tail = get_rcvhdrtail(rcd); 500 if (rcd->ctxt == HFI1_CTRL_CTXT) 501 mdata->ps_seq = rcd->seq_cnt; 502 else 503 mdata->ps_seq = 0; /* not used with DMA_RTAIL */ 504 } else { 505 mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ 506 mdata->ps_seq = rcd->seq_cnt; 507 } 508 } 509 510 static inline int ps_done(struct ps_mdata *mdata, u64 rhf, 511 struct hfi1_ctxtdata *rcd) 512 { 513 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 514 return mdata->ps_head == mdata->ps_tail; 515 return mdata->ps_seq != rhf_rcv_seq(rhf); 516 } 517 518 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, 519 struct hfi1_ctxtdata *rcd) 520 { 521 /* 522 * Control context can potentially receive an invalid rhf. 523 * Drop such packets. 524 */ 525 if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) 526 return mdata->ps_seq != rhf_rcv_seq(rhf); 527 528 return 0; 529 } 530 531 static inline void update_ps_mdata(struct ps_mdata *mdata, 532 struct hfi1_ctxtdata *rcd) 533 { 534 mdata->ps_head += mdata->rsize; 535 if (mdata->ps_head >= mdata->maxcnt) 536 mdata->ps_head = 0; 537 538 /* Control context must do seq counting */ 539 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || 540 (rcd->ctxt == HFI1_CTRL_CTXT)) { 541 if (++mdata->ps_seq > 13) 542 mdata->ps_seq = 1; 543 } 544 } 545 546 /* 547 * prescan_rxq - search through the receive queue looking for packets 548 * containing Excplicit Congestion Notifications (FECNs, or BECNs). 549 * When an ECN is found, process the Congestion Notification, and toggle 550 * it off. 551 * This is declared as a macro to allow quick checking of the port to avoid 552 * the overhead of a function call if not enabled. 553 */ 554 #define prescan_rxq(rcd, packet) \ 555 do { \ 556 if (rcd->ppd->cc_prescan) \ 557 __prescan_rxq(packet); \ 558 } while (0) 559 static void __prescan_rxq(struct hfi1_packet *packet) 560 { 561 struct hfi1_ctxtdata *rcd = packet->rcd; 562 struct ps_mdata mdata; 563 564 init_ps_mdata(&mdata, packet); 565 566 while (1) { 567 struct hfi1_devdata *dd = rcd->dd; 568 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 569 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 570 dd->rhf_offset; 571 struct rvt_qp *qp; 572 struct ib_header *hdr; 573 struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 574 u64 rhf = rhf_to_cpu(rhf_addr); 575 u32 etype = rhf_rcv_type(rhf), qpn, bth1; 576 int is_ecn = 0; 577 u8 lnh; 578 579 if (ps_done(&mdata, rhf, rcd)) 580 break; 581 582 if (ps_skip(&mdata, rhf, rcd)) 583 goto next; 584 585 if (etype != RHF_RCV_TYPE_IB) 586 goto next; 587 588 packet->hdr = hfi1_get_msgheader(dd, rhf_addr); 589 hdr = packet->hdr; 590 lnh = ib_get_lnh(hdr); 591 592 if (lnh == HFI1_LRH_BTH) { 593 packet->ohdr = &hdr->u.oth; 594 } else if (lnh == HFI1_LRH_GRH) { 595 packet->ohdr = &hdr->u.l.oth; 596 packet->rcv_flags |= HFI1_HAS_GRH; 597 } else { 598 goto next; /* just in case */ 599 } 600 601 bth1 = be32_to_cpu(packet->ohdr->bth[1]); 602 is_ecn = !!(bth1 & (IB_FECN_SMASK | IB_BECN_SMASK)); 603 604 if (!is_ecn) 605 goto next; 606 607 qpn = bth1 & RVT_QPN_MASK; 608 rcu_read_lock(); 609 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); 610 611 if (!qp) { 612 rcu_read_unlock(); 613 goto next; 614 } 615 616 process_ecn(qp, packet, true); 617 rcu_read_unlock(); 618 619 /* turn off BECN, FECN */ 620 bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK); 621 packet->ohdr->bth[1] = cpu_to_be32(bth1); 622 next: 623 update_ps_mdata(&mdata, rcd); 624 } 625 } 626 627 static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd) 628 { 629 struct rvt_qp *qp, *nqp; 630 631 /* 632 * Iterate over all QPs waiting to respond. 633 * The list won't change since the IRQ is only run on one CPU. 634 */ 635 list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { 636 list_del_init(&qp->rspwait); 637 if (qp->r_flags & RVT_R_RSP_NAK) { 638 qp->r_flags &= ~RVT_R_RSP_NAK; 639 hfi1_send_rc_ack(rcd, qp, 0); 640 } 641 if (qp->r_flags & RVT_R_RSP_SEND) { 642 unsigned long flags; 643 644 qp->r_flags &= ~RVT_R_RSP_SEND; 645 spin_lock_irqsave(&qp->s_lock, flags); 646 if (ib_rvt_state_ops[qp->state] & 647 RVT_PROCESS_OR_FLUSH_SEND) 648 hfi1_schedule_send(qp); 649 spin_unlock_irqrestore(&qp->s_lock, flags); 650 } 651 rvt_put_qp(qp); 652 } 653 } 654 655 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) 656 { 657 if (thread) { 658 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) 659 /* allow defered processing */ 660 process_rcv_qp_work(packet->rcd); 661 cond_resched(); 662 return RCV_PKT_OK; 663 } else { 664 this_cpu_inc(*packet->rcd->dd->rcv_limit); 665 return RCV_PKT_LIMIT; 666 } 667 } 668 669 static inline int check_max_packet(struct hfi1_packet *packet, int thread) 670 { 671 int ret = RCV_PKT_OK; 672 673 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) 674 ret = max_packet_exceeded(packet, thread); 675 return ret; 676 } 677 678 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) 679 { 680 int ret; 681 682 /* Set up for the next packet */ 683 packet->rhqoff += packet->rsize; 684 if (packet->rhqoff >= packet->maxcnt) 685 packet->rhqoff = 0; 686 687 packet->numpkt++; 688 ret = check_max_packet(packet, thread); 689 690 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 691 packet->rcd->dd->rhf_offset; 692 packet->rhf = rhf_to_cpu(packet->rhf_addr); 693 694 return ret; 695 } 696 697 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) 698 { 699 int ret; 700 701 packet->hdr = hfi1_get_msgheader(packet->rcd->dd, 702 packet->rhf_addr); 703 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; 704 packet->etype = rhf_rcv_type(packet->rhf); 705 /* total length */ 706 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 707 /* retrieve eager buffer details */ 708 packet->ebuf = NULL; 709 if (rhf_use_egr_bfr(packet->rhf)) { 710 packet->etail = rhf_egr_index(packet->rhf); 711 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, 712 &packet->updegr); 713 /* 714 * Prefetch the contents of the eager buffer. It is 715 * OK to send a negative length to prefetch_range(). 716 * The +2 is the size of the RHF. 717 */ 718 prefetch_range(packet->ebuf, 719 packet->tlen - ((packet->rcd->rcvhdrqentsize - 720 (rhf_hdrq_offset(packet->rhf) 721 + 2)) * 4)); 722 } 723 724 /* 725 * Call a type specific handler for the packet. We 726 * should be able to trust that etype won't be beyond 727 * the range of valid indexes. If so something is really 728 * wrong and we can probably just let things come 729 * crashing down. There is no need to eat another 730 * comparison in this performance critical code. 731 */ 732 packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet); 733 packet->numpkt++; 734 735 /* Set up for the next packet */ 736 packet->rhqoff += packet->rsize; 737 if (packet->rhqoff >= packet->maxcnt) 738 packet->rhqoff = 0; 739 740 ret = check_max_packet(packet, thread); 741 742 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 743 packet->rcd->dd->rhf_offset; 744 packet->rhf = rhf_to_cpu(packet->rhf_addr); 745 746 return ret; 747 } 748 749 static inline void process_rcv_update(int last, struct hfi1_packet *packet) 750 { 751 /* 752 * Update head regs etc., every 16 packets, if not last pkt, 753 * to help prevent rcvhdrq overflows, when many packets 754 * are processed and queue is nearly full. 755 * Don't request an interrupt for intermediate updates. 756 */ 757 if (!last && !(packet->numpkt & 0xf)) { 758 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, 759 packet->etail, 0, 0); 760 packet->updegr = 0; 761 } 762 packet->rcv_flags = 0; 763 } 764 765 static inline void finish_packet(struct hfi1_packet *packet) 766 { 767 /* 768 * Nothing we need to free for the packet. 769 * 770 * The only thing we need to do is a final update and call for an 771 * interrupt 772 */ 773 update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, 774 packet->etail, rcv_intr_dynamic, packet->numpkt); 775 } 776 777 /* 778 * Handle receive interrupts when using the no dma rtail option. 779 */ 780 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) 781 { 782 u32 seq; 783 int last = RCV_PKT_OK; 784 struct hfi1_packet packet; 785 786 init_packet(rcd, &packet); 787 seq = rhf_rcv_seq(packet.rhf); 788 if (seq != rcd->seq_cnt) { 789 last = RCV_PKT_DONE; 790 goto bail; 791 } 792 793 prescan_rxq(rcd, &packet); 794 795 while (last == RCV_PKT_OK) { 796 last = process_rcv_packet(&packet, thread); 797 seq = rhf_rcv_seq(packet.rhf); 798 if (++rcd->seq_cnt > 13) 799 rcd->seq_cnt = 1; 800 if (seq != rcd->seq_cnt) 801 last = RCV_PKT_DONE; 802 process_rcv_update(last, &packet); 803 } 804 process_rcv_qp_work(rcd); 805 rcd->head = packet.rhqoff; 806 bail: 807 finish_packet(&packet); 808 return last; 809 } 810 811 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) 812 { 813 u32 hdrqtail; 814 int last = RCV_PKT_OK; 815 struct hfi1_packet packet; 816 817 init_packet(rcd, &packet); 818 hdrqtail = get_rcvhdrtail(rcd); 819 if (packet.rhqoff == hdrqtail) { 820 last = RCV_PKT_DONE; 821 goto bail; 822 } 823 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 824 825 prescan_rxq(rcd, &packet); 826 827 while (last == RCV_PKT_OK) { 828 last = process_rcv_packet(&packet, thread); 829 if (packet.rhqoff == hdrqtail) 830 last = RCV_PKT_DONE; 831 process_rcv_update(last, &packet); 832 } 833 process_rcv_qp_work(rcd); 834 rcd->head = packet.rhqoff; 835 bail: 836 finish_packet(&packet); 837 return last; 838 } 839 840 static inline void set_nodma_rtail(struct hfi1_devdata *dd, u8 ctxt) 841 { 842 int i; 843 844 /* 845 * For dynamically allocated kernel contexts (like vnic) switch 846 * interrupt handler only for that context. Otherwise, switch 847 * interrupt handler for all statically allocated kernel contexts. 848 */ 849 if (ctxt >= dd->first_dyn_alloc_ctxt) { 850 dd->rcd[ctxt]->do_interrupt = 851 &handle_receive_interrupt_nodma_rtail; 852 return; 853 } 854 855 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) 856 dd->rcd[i]->do_interrupt = 857 &handle_receive_interrupt_nodma_rtail; 858 } 859 860 static inline void set_dma_rtail(struct hfi1_devdata *dd, u8 ctxt) 861 { 862 int i; 863 864 /* 865 * For dynamically allocated kernel contexts (like vnic) switch 866 * interrupt handler only for that context. Otherwise, switch 867 * interrupt handler for all statically allocated kernel contexts. 868 */ 869 if (ctxt >= dd->first_dyn_alloc_ctxt) { 870 dd->rcd[ctxt]->do_interrupt = 871 &handle_receive_interrupt_dma_rtail; 872 return; 873 } 874 875 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) 876 dd->rcd[i]->do_interrupt = 877 &handle_receive_interrupt_dma_rtail; 878 } 879 880 void set_all_slowpath(struct hfi1_devdata *dd) 881 { 882 int i; 883 884 /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */ 885 for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) { 886 struct hfi1_ctxtdata *rcd = dd->rcd[i]; 887 888 if ((i < dd->first_dyn_alloc_ctxt) || 889 (rcd && rcd->sc && (rcd->sc->type == SC_KERNEL))) 890 rcd->do_interrupt = &handle_receive_interrupt; 891 } 892 } 893 894 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, 895 struct hfi1_packet *packet, 896 struct hfi1_devdata *dd) 897 { 898 struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; 899 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd, 900 packet->rhf_addr); 901 u8 etype = rhf_rcv_type(packet->rhf); 902 903 if (etype == RHF_RCV_TYPE_IB && 904 hfi1_9B_get_sc5(hdr, packet->rhf) != 0xf) { 905 int hwstate = read_logical_state(dd); 906 907 if (hwstate != LSTATE_ACTIVE) { 908 dd_dev_info(dd, "Unexpected link state %d\n", hwstate); 909 return 0; 910 } 911 912 queue_work(rcd->ppd->hfi1_wq, lsaw); 913 return 1; 914 } 915 return 0; 916 } 917 918 /* 919 * handle_receive_interrupt - receive a packet 920 * @rcd: the context 921 * 922 * Called from interrupt handler for errors or receive interrupt. 923 * This is the slow path interrupt handler. 924 */ 925 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) 926 { 927 struct hfi1_devdata *dd = rcd->dd; 928 u32 hdrqtail; 929 int needset, last = RCV_PKT_OK; 930 struct hfi1_packet packet; 931 int skip_pkt = 0; 932 933 /* Control context will always use the slow path interrupt handler */ 934 needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; 935 936 init_packet(rcd, &packet); 937 938 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 939 u32 seq = rhf_rcv_seq(packet.rhf); 940 941 if (seq != rcd->seq_cnt) { 942 last = RCV_PKT_DONE; 943 goto bail; 944 } 945 hdrqtail = 0; 946 } else { 947 hdrqtail = get_rcvhdrtail(rcd); 948 if (packet.rhqoff == hdrqtail) { 949 last = RCV_PKT_DONE; 950 goto bail; 951 } 952 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 953 954 /* 955 * Control context can potentially receive an invalid 956 * rhf. Drop such packets. 957 */ 958 if (rcd->ctxt == HFI1_CTRL_CTXT) { 959 u32 seq = rhf_rcv_seq(packet.rhf); 960 961 if (seq != rcd->seq_cnt) 962 skip_pkt = 1; 963 } 964 } 965 966 prescan_rxq(rcd, &packet); 967 968 while (last == RCV_PKT_OK) { 969 if (unlikely(dd->do_drop && 970 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == 971 DROP_PACKET_ON)) { 972 dd->do_drop = 0; 973 974 /* On to the next packet */ 975 packet.rhqoff += packet.rsize; 976 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + 977 packet.rhqoff + 978 dd->rhf_offset; 979 packet.rhf = rhf_to_cpu(packet.rhf_addr); 980 981 } else if (skip_pkt) { 982 last = skip_rcv_packet(&packet, thread); 983 skip_pkt = 0; 984 } else { 985 /* Auto activate link on non-SC15 packet receive */ 986 if (unlikely(rcd->ppd->host_link_state == 987 HLS_UP_ARMED) && 988 set_armed_to_active(rcd, &packet, dd)) 989 goto bail; 990 last = process_rcv_packet(&packet, thread); 991 } 992 993 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 994 u32 seq = rhf_rcv_seq(packet.rhf); 995 996 if (++rcd->seq_cnt > 13) 997 rcd->seq_cnt = 1; 998 if (seq != rcd->seq_cnt) 999 last = RCV_PKT_DONE; 1000 if (needset) { 1001 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); 1002 set_nodma_rtail(dd, rcd->ctxt); 1003 needset = 0; 1004 } 1005 } else { 1006 if (packet.rhqoff == hdrqtail) 1007 last = RCV_PKT_DONE; 1008 /* 1009 * Control context can potentially receive an invalid 1010 * rhf. Drop such packets. 1011 */ 1012 if (rcd->ctxt == HFI1_CTRL_CTXT) { 1013 u32 seq = rhf_rcv_seq(packet.rhf); 1014 1015 if (++rcd->seq_cnt > 13) 1016 rcd->seq_cnt = 1; 1017 if (!last && (seq != rcd->seq_cnt)) 1018 skip_pkt = 1; 1019 } 1020 1021 if (needset) { 1022 dd_dev_info(dd, 1023 "Switching to DMA_RTAIL\n"); 1024 set_dma_rtail(dd, rcd->ctxt); 1025 needset = 0; 1026 } 1027 } 1028 1029 process_rcv_update(last, &packet); 1030 } 1031 1032 process_rcv_qp_work(rcd); 1033 rcd->head = packet.rhqoff; 1034 1035 bail: 1036 /* 1037 * Always write head at end, and setup rcv interrupt, even 1038 * if no packets were processed. 1039 */ 1040 finish_packet(&packet); 1041 return last; 1042 } 1043 1044 /* 1045 * We may discover in the interrupt that the hardware link state has 1046 * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet), 1047 * and we need to update the driver's notion of the link state. We cannot 1048 * run set_link_state from interrupt context, so we queue this function on 1049 * a workqueue. 1050 * 1051 * We delay the regular interrupt processing until after the state changes 1052 * so that the link will be in the correct state by the time any application 1053 * we wake up attempts to send a reply to any message it received. 1054 * (Subsequent receive interrupts may possibly force the wakeup before we 1055 * update the link state.) 1056 * 1057 * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes 1058 * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues, 1059 * so we're safe from use-after-free of the rcd. 1060 */ 1061 void receive_interrupt_work(struct work_struct *work) 1062 { 1063 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 1064 linkstate_active_work); 1065 struct hfi1_devdata *dd = ppd->dd; 1066 int i; 1067 1068 /* Received non-SC15 packet implies neighbor_normal */ 1069 ppd->neighbor_normal = 1; 1070 set_link_state(ppd, HLS_UP_ACTIVE); 1071 1072 /* 1073 * Interrupt all statically allocated kernel contexts that could 1074 * have had an interrupt during auto activation. 1075 */ 1076 for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) 1077 force_recv_intr(dd->rcd[i]); 1078 } 1079 1080 /* 1081 * Convert a given MTU size to the on-wire MAD packet enumeration. 1082 * Return -1 if the size is invalid. 1083 */ 1084 int mtu_to_enum(u32 mtu, int default_if_bad) 1085 { 1086 switch (mtu) { 1087 case 0: return OPA_MTU_0; 1088 case 256: return OPA_MTU_256; 1089 case 512: return OPA_MTU_512; 1090 case 1024: return OPA_MTU_1024; 1091 case 2048: return OPA_MTU_2048; 1092 case 4096: return OPA_MTU_4096; 1093 case 8192: return OPA_MTU_8192; 1094 case 10240: return OPA_MTU_10240; 1095 } 1096 return default_if_bad; 1097 } 1098 1099 u16 enum_to_mtu(int mtu) 1100 { 1101 switch (mtu) { 1102 case OPA_MTU_0: return 0; 1103 case OPA_MTU_256: return 256; 1104 case OPA_MTU_512: return 512; 1105 case OPA_MTU_1024: return 1024; 1106 case OPA_MTU_2048: return 2048; 1107 case OPA_MTU_4096: return 4096; 1108 case OPA_MTU_8192: return 8192; 1109 case OPA_MTU_10240: return 10240; 1110 default: return 0xffff; 1111 } 1112 } 1113 1114 /* 1115 * set_mtu - set the MTU 1116 * @ppd: the per port data 1117 * 1118 * We can handle "any" incoming size, the issue here is whether we 1119 * need to restrict our outgoing size. We do not deal with what happens 1120 * to programs that are already running when the size changes. 1121 */ 1122 int set_mtu(struct hfi1_pportdata *ppd) 1123 { 1124 struct hfi1_devdata *dd = ppd->dd; 1125 int i, drain, ret = 0, is_up = 0; 1126 1127 ppd->ibmtu = 0; 1128 for (i = 0; i < ppd->vls_supported; i++) 1129 if (ppd->ibmtu < dd->vld[i].mtu) 1130 ppd->ibmtu = dd->vld[i].mtu; 1131 ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd); 1132 1133 mutex_lock(&ppd->hls_lock); 1134 if (ppd->host_link_state == HLS_UP_INIT || 1135 ppd->host_link_state == HLS_UP_ARMED || 1136 ppd->host_link_state == HLS_UP_ACTIVE) 1137 is_up = 1; 1138 1139 drain = !is_ax(dd) && is_up; 1140 1141 if (drain) 1142 /* 1143 * MTU is specified per-VL. To ensure that no packet gets 1144 * stuck (due, e.g., to the MTU for the packet's VL being 1145 * reduced), empty the per-VL FIFOs before adjusting MTU. 1146 */ 1147 ret = stop_drain_data_vls(dd); 1148 1149 if (ret) { 1150 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n", 1151 __func__); 1152 goto err; 1153 } 1154 1155 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0); 1156 1157 if (drain) 1158 open_fill_data_vls(dd); /* reopen all VLs */ 1159 1160 err: 1161 mutex_unlock(&ppd->hls_lock); 1162 1163 return ret; 1164 } 1165 1166 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc) 1167 { 1168 struct hfi1_devdata *dd = ppd->dd; 1169 1170 ppd->lid = lid; 1171 ppd->lmc = lmc; 1172 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0); 1173 1174 dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid); 1175 1176 return 0; 1177 } 1178 1179 void shutdown_led_override(struct hfi1_pportdata *ppd) 1180 { 1181 struct hfi1_devdata *dd = ppd->dd; 1182 1183 /* 1184 * This pairs with the memory barrier in hfi1_start_led_override to 1185 * ensure that we read the correct state of LED beaconing represented 1186 * by led_override_timer_active 1187 */ 1188 smp_rmb(); 1189 if (atomic_read(&ppd->led_override_timer_active)) { 1190 del_timer_sync(&ppd->led_override_timer); 1191 atomic_set(&ppd->led_override_timer_active, 0); 1192 /* Ensure the atomic_set is visible to all CPUs */ 1193 smp_wmb(); 1194 } 1195 1196 /* Hand control of the LED to the DC for normal operation */ 1197 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 1198 } 1199 1200 static void run_led_override(unsigned long opaque) 1201 { 1202 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)opaque; 1203 struct hfi1_devdata *dd = ppd->dd; 1204 unsigned long timeout; 1205 int phase_idx; 1206 1207 if (!(dd->flags & HFI1_INITTED)) 1208 return; 1209 1210 phase_idx = ppd->led_override_phase & 1; 1211 1212 setextled(dd, phase_idx); 1213 1214 timeout = ppd->led_override_vals[phase_idx]; 1215 1216 /* Set up for next phase */ 1217 ppd->led_override_phase = !ppd->led_override_phase; 1218 1219 mod_timer(&ppd->led_override_timer, jiffies + timeout); 1220 } 1221 1222 /* 1223 * To have the LED blink in a particular pattern, provide timeon and timeoff 1224 * in milliseconds. 1225 * To turn off custom blinking and return to normal operation, use 1226 * shutdown_led_override() 1227 */ 1228 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, 1229 unsigned int timeoff) 1230 { 1231 if (!(ppd->dd->flags & HFI1_INITTED)) 1232 return; 1233 1234 /* Convert to jiffies for direct use in timer */ 1235 ppd->led_override_vals[0] = msecs_to_jiffies(timeoff); 1236 ppd->led_override_vals[1] = msecs_to_jiffies(timeon); 1237 1238 /* Arbitrarily start from LED on phase */ 1239 ppd->led_override_phase = 1; 1240 1241 /* 1242 * If the timer has not already been started, do so. Use a "quick" 1243 * timeout so the handler will be called soon to look at our request. 1244 */ 1245 if (!timer_pending(&ppd->led_override_timer)) { 1246 setup_timer(&ppd->led_override_timer, run_led_override, 1247 (unsigned long)ppd); 1248 ppd->led_override_timer.expires = jiffies + 1; 1249 add_timer(&ppd->led_override_timer); 1250 atomic_set(&ppd->led_override_timer_active, 1); 1251 /* Ensure the atomic_set is visible to all CPUs */ 1252 smp_wmb(); 1253 } 1254 } 1255 1256 /** 1257 * hfi1_reset_device - reset the chip if possible 1258 * @unit: the device to reset 1259 * 1260 * Whether or not reset is successful, we attempt to re-initialize the chip 1261 * (that is, much like a driver unload/reload). We clear the INITTED flag 1262 * so that the various entry points will fail until we reinitialize. For 1263 * now, we only allow this if no user contexts are open that use chip resources 1264 */ 1265 int hfi1_reset_device(int unit) 1266 { 1267 int ret, i; 1268 struct hfi1_devdata *dd = hfi1_lookup(unit); 1269 struct hfi1_pportdata *ppd; 1270 unsigned long flags; 1271 int pidx; 1272 1273 if (!dd) { 1274 ret = -ENODEV; 1275 goto bail; 1276 } 1277 1278 dd_dev_info(dd, "Reset on unit %u requested\n", unit); 1279 1280 if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) { 1281 dd_dev_info(dd, 1282 "Invalid unit number %u or not initialized or not present\n", 1283 unit); 1284 ret = -ENXIO; 1285 goto bail; 1286 } 1287 1288 spin_lock_irqsave(&dd->uctxt_lock, flags); 1289 if (dd->rcd) 1290 for (i = dd->first_dyn_alloc_ctxt; 1291 i < dd->num_rcv_contexts; i++) { 1292 if (!dd->rcd[i]) 1293 continue; 1294 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1295 ret = -EBUSY; 1296 goto bail; 1297 } 1298 spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1299 1300 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1301 ppd = dd->pport + pidx; 1302 1303 shutdown_led_override(ppd); 1304 } 1305 if (dd->flags & HFI1_HAS_SEND_DMA) 1306 sdma_exit(dd); 1307 1308 hfi1_reset_cpu_counters(dd); 1309 1310 ret = hfi1_init(dd, 1); 1311 1312 if (ret) 1313 dd_dev_err(dd, 1314 "Reinitialize unit %u after reset failed with %d\n", 1315 unit, ret); 1316 else 1317 dd_dev_info(dd, "Reinitialized unit %u after resetting\n", 1318 unit); 1319 1320 bail: 1321 return ret; 1322 } 1323 1324 void handle_eflags(struct hfi1_packet *packet) 1325 { 1326 struct hfi1_ctxtdata *rcd = packet->rcd; 1327 u32 rte = rhf_rcv_type_err(packet->rhf); 1328 1329 rcv_hdrerr(rcd, rcd->ppd, packet); 1330 if (rhf_err_flags(packet->rhf)) 1331 dd_dev_err(rcd->dd, 1332 "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", 1333 rcd->ctxt, packet->rhf, 1334 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", 1335 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", 1336 packet->rhf & RHF_DC_ERR ? "dc " : "", 1337 packet->rhf & RHF_TID_ERR ? "tid " : "", 1338 packet->rhf & RHF_LEN_ERR ? "len " : "", 1339 packet->rhf & RHF_ECC_ERR ? "ecc " : "", 1340 packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", 1341 packet->rhf & RHF_ICRC_ERR ? "icrc " : "", 1342 rte); 1343 } 1344 1345 /* 1346 * The following functions are called by the interrupt handler. They are type 1347 * specific handlers for each packet type. 1348 */ 1349 int process_receive_ib(struct hfi1_packet *packet) 1350 { 1351 if (unlikely(hfi1_dbg_fault_packet(packet))) 1352 return RHF_RCV_CONTINUE; 1353 1354 trace_hfi1_rcvhdr(packet->rcd->ppd->dd, 1355 packet->rcd->ctxt, 1356 rhf_err_flags(packet->rhf), 1357 RHF_RCV_TYPE_IB, 1358 packet->hlen, 1359 packet->tlen, 1360 packet->updegr, 1361 rhf_egr_index(packet->rhf)); 1362 1363 if (unlikely( 1364 (hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && 1365 (packet->rhf & RHF_DC_ERR)))) 1366 return RHF_RCV_CONTINUE; 1367 1368 if (unlikely(rhf_err_flags(packet->rhf))) { 1369 handle_eflags(packet); 1370 return RHF_RCV_CONTINUE; 1371 } 1372 1373 hfi1_ib_rcv(packet); 1374 return RHF_RCV_CONTINUE; 1375 } 1376 1377 static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) 1378 { 1379 /* Packet received in VNIC context via RSM */ 1380 if (packet->rcd->is_vnic) 1381 return true; 1382 1383 if ((HFI1_GET_L2_TYPE(packet->ebuf) == OPA_VNIC_L2_TYPE) && 1384 (HFI1_GET_L4_TYPE(packet->ebuf) == OPA_VNIC_L4_ETHR)) 1385 return true; 1386 1387 return false; 1388 } 1389 1390 int process_receive_bypass(struct hfi1_packet *packet) 1391 { 1392 struct hfi1_devdata *dd = packet->rcd->dd; 1393 1394 if (unlikely(rhf_err_flags(packet->rhf))) { 1395 handle_eflags(packet); 1396 } else if (hfi1_is_vnic_packet(packet)) { 1397 hfi1_vnic_bypass_rcv(packet); 1398 return RHF_RCV_CONTINUE; 1399 } 1400 1401 dd_dev_err(dd, "Unsupported bypass packet. Dropping\n"); 1402 incr_cntr64(&dd->sw_rcv_bypass_packet_errors); 1403 if (!(dd->err_info_rcvport.status_and_code & OPA_EI_STATUS_SMASK)) { 1404 u64 *flits = packet->ebuf; 1405 1406 if (flits && !(packet->rhf & RHF_LEN_ERR)) { 1407 dd->err_info_rcvport.packet_flit1 = flits[0]; 1408 dd->err_info_rcvport.packet_flit2 = 1409 packet->tlen > sizeof(flits[0]) ? flits[1] : 0; 1410 } 1411 dd->err_info_rcvport.status_and_code |= 1412 (OPA_EI_STATUS_SMASK | BAD_L2_ERR); 1413 } 1414 return RHF_RCV_CONTINUE; 1415 } 1416 1417 int process_receive_error(struct hfi1_packet *packet) 1418 { 1419 /* KHdrHCRCErr -- KDETH packet with a bad HCRC */ 1420 if (unlikely( 1421 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && 1422 rhf_rcv_type_err(packet->rhf) == 3)) 1423 return RHF_RCV_CONTINUE; 1424 1425 handle_eflags(packet); 1426 1427 if (unlikely(rhf_err_flags(packet->rhf))) 1428 dd_dev_err(packet->rcd->dd, 1429 "Unhandled error packet received. Dropping.\n"); 1430 1431 return RHF_RCV_CONTINUE; 1432 } 1433 1434 int kdeth_process_expected(struct hfi1_packet *packet) 1435 { 1436 if (unlikely(hfi1_dbg_fault_packet(packet))) 1437 return RHF_RCV_CONTINUE; 1438 if (unlikely(rhf_err_flags(packet->rhf))) 1439 handle_eflags(packet); 1440 1441 dd_dev_err(packet->rcd->dd, 1442 "Unhandled expected packet received. Dropping.\n"); 1443 return RHF_RCV_CONTINUE; 1444 } 1445 1446 int kdeth_process_eager(struct hfi1_packet *packet) 1447 { 1448 if (unlikely(rhf_err_flags(packet->rhf))) 1449 handle_eflags(packet); 1450 if (unlikely(hfi1_dbg_fault_packet(packet))) 1451 return RHF_RCV_CONTINUE; 1452 1453 dd_dev_err(packet->rcd->dd, 1454 "Unhandled eager packet received. Dropping.\n"); 1455 return RHF_RCV_CONTINUE; 1456 } 1457 1458 int process_receive_invalid(struct hfi1_packet *packet) 1459 { 1460 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", 1461 rhf_rcv_type(packet->rhf)); 1462 return RHF_RCV_CONTINUE; 1463 } 1464