1 /* 2 * Copyright(c) 2015-2018 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/spinlock.h> 49 #include <linux/pci.h> 50 #include <linux/io.h> 51 #include <linux/delay.h> 52 #include <linux/netdevice.h> 53 #include <linux/vmalloc.h> 54 #include <linux/module.h> 55 #include <linux/prefetch.h> 56 #include <rdma/ib_verbs.h> 57 58 #include "hfi.h" 59 #include "trace.h" 60 #include "qp.h" 61 #include "sdma.h" 62 #include "debugfs.h" 63 #include "vnic.h" 64 #include "fault.h" 65 66 #undef pr_fmt 67 #define pr_fmt(fmt) DRIVER_NAME ": " fmt 68 69 /* 70 * The size has to be longer than this string, so we can append 71 * board/chip information to it in the initialization code. 72 */ 73 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n"; 74 75 DEFINE_SPINLOCK(hfi1_devs_lock); 76 LIST_HEAD(hfi1_dev_list); 77 DEFINE_MUTEX(hfi1_mutex); /* general driver use */ 78 79 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU; 80 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO); 81 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify( 82 HFI1_DEFAULT_MAX_MTU)); 83 84 unsigned int hfi1_cu = 1; 85 module_param_named(cu, hfi1_cu, uint, S_IRUGO); 86 MODULE_PARM_DESC(cu, "Credit return units"); 87 88 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; 89 static int hfi1_caps_set(const char *val, const struct kernel_param *kp); 90 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp); 91 static const struct kernel_param_ops cap_ops = { 92 .set = hfi1_caps_set, 93 .get = hfi1_caps_get 94 }; 95 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO); 96 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features"); 97 98 MODULE_LICENSE("Dual BSD/GPL"); 99 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver"); 100 101 /* 102 * MAX_PKT_RCV is the max # if packets processed per receive interrupt. 103 */ 104 #define MAX_PKT_RECV 64 105 /* 106 * MAX_PKT_THREAD_RCV is the max # of packets processed before 107 * the qp_wait_list queue is flushed. 108 */ 109 #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4) 110 #define EGR_HEAD_UPDATE_THRESHOLD 16 111 112 struct hfi1_ib_stats hfi1_stats; 113 114 static int hfi1_caps_set(const char *val, const struct kernel_param *kp) 115 { 116 int ret = 0; 117 unsigned long *cap_mask_ptr = (unsigned long *)kp->arg, 118 cap_mask = *cap_mask_ptr, value, diff, 119 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) | 120 HFI1_CAP_WRITABLE_MASK); 121 122 ret = kstrtoul(val, 0, &value); 123 if (ret) { 124 pr_warn("Invalid module parameter value for 'cap_mask'\n"); 125 goto done; 126 } 127 /* Get the changed bits (except the locked bit) */ 128 diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK); 129 130 /* Remove any bits that are not allowed to change after driver load */ 131 if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) { 132 pr_warn("Ignoring non-writable capability bits %#lx\n", 133 diff & ~write_mask); 134 diff &= write_mask; 135 } 136 137 /* Mask off any reserved bits */ 138 diff &= ~HFI1_CAP_RESERVED_MASK; 139 /* Clear any previously set and changing bits */ 140 cap_mask &= ~diff; 141 /* Update the bits with the new capability */ 142 cap_mask |= (value & diff); 143 /* Check for any kernel/user restrictions */ 144 diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^ 145 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT); 146 cap_mask &= ~diff; 147 /* Set the bitmask to the final set */ 148 *cap_mask_ptr = cap_mask; 149 done: 150 return ret; 151 } 152 153 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) 154 { 155 unsigned long cap_mask = *(unsigned long *)kp->arg; 156 157 cap_mask &= ~HFI1_CAP_LOCKED_SMASK; 158 cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT); 159 160 return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); 161 } 162 163 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) 164 { 165 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 166 struct hfi1_devdata *dd = container_of(ibdev, 167 struct hfi1_devdata, verbs_dev); 168 return dd->pcidev; 169 } 170 171 /* 172 * Return count of units with at least one port ACTIVE. 173 */ 174 int hfi1_count_active_units(void) 175 { 176 struct hfi1_devdata *dd; 177 struct hfi1_pportdata *ppd; 178 unsigned long flags; 179 int pidx, nunits_active = 0; 180 181 spin_lock_irqsave(&hfi1_devs_lock, flags); 182 list_for_each_entry(dd, &hfi1_dev_list, list) { 183 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1) 184 continue; 185 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 186 ppd = dd->pport + pidx; 187 if (ppd->lid && ppd->linkup) { 188 nunits_active++; 189 break; 190 } 191 } 192 } 193 spin_unlock_irqrestore(&hfi1_devs_lock, flags); 194 return nunits_active; 195 } 196 197 /* 198 * Get address of eager buffer from it's index (allocated in chunks, not 199 * contiguous). 200 */ 201 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, 202 u8 *update) 203 { 204 u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf); 205 206 *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset; 207 return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) + 208 (offset * RCV_BUF_BLOCK_SIZE)); 209 } 210 211 static inline void *hfi1_get_header(struct hfi1_devdata *dd, 212 __le32 *rhf_addr) 213 { 214 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr)); 215 216 return (void *)(rhf_addr - dd->rhf_offset + offset); 217 } 218 219 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_devdata *dd, 220 __le32 *rhf_addr) 221 { 222 return (struct ib_header *)hfi1_get_header(dd, rhf_addr); 223 } 224 225 static inline struct hfi1_16b_header 226 *hfi1_get_16B_header(struct hfi1_devdata *dd, 227 __le32 *rhf_addr) 228 { 229 return (struct hfi1_16b_header *)hfi1_get_header(dd, rhf_addr); 230 } 231 232 /* 233 * Validate and encode the a given RcvArray Buffer size. 234 * The function will check whether the given size falls within 235 * allowed size ranges for the respective type and, optionally, 236 * return the proper encoding. 237 */ 238 int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded) 239 { 240 if (unlikely(!PAGE_ALIGNED(size))) 241 return 0; 242 if (unlikely(size < MIN_EAGER_BUFFER)) 243 return 0; 244 if (size > 245 (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER)) 246 return 0; 247 if (encoded) 248 *encoded = ilog2(size / PAGE_SIZE) + 1; 249 return 1; 250 } 251 252 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, 253 struct hfi1_packet *packet) 254 { 255 struct ib_header *rhdr = packet->hdr; 256 u32 rte = rhf_rcv_type_err(packet->rhf); 257 u32 mlid_base; 258 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 259 struct hfi1_devdata *dd = ppd->dd; 260 struct hfi1_ibdev *verbs_dev = &dd->verbs_dev; 261 struct rvt_dev_info *rdi = &verbs_dev->rdi; 262 263 if ((packet->rhf & RHF_DC_ERR) && 264 hfi1_dbg_fault_suppress_err(verbs_dev)) 265 return; 266 267 if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) 268 return; 269 270 if (packet->etype == RHF_RCV_TYPE_BYPASS) { 271 goto drop; 272 } else { 273 u8 lnh = ib_get_lnh(rhdr); 274 275 mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE); 276 if (lnh == HFI1_LRH_BTH) { 277 packet->ohdr = &rhdr->u.oth; 278 } else if (lnh == HFI1_LRH_GRH) { 279 packet->ohdr = &rhdr->u.l.oth; 280 packet->grh = &rhdr->u.l.grh; 281 } else { 282 goto drop; 283 } 284 } 285 286 if (packet->rhf & RHF_TID_ERR) { 287 /* For TIDERR and RC QPs preemptively schedule a NAK */ 288 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 289 u32 dlid = ib_get_dlid(rhdr); 290 u32 qp_num; 291 292 /* Sanity check packet */ 293 if (tlen < 24) 294 goto drop; 295 296 /* Check for GRH */ 297 if (packet->grh) { 298 u32 vtf; 299 struct ib_grh *grh = packet->grh; 300 301 if (grh->next_hdr != IB_GRH_NEXT_HDR) 302 goto drop; 303 vtf = be32_to_cpu(grh->version_tclass_flow); 304 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 305 goto drop; 306 } 307 308 /* Get the destination QP number. */ 309 qp_num = ib_bth_get_qpn(packet->ohdr); 310 if (dlid < mlid_base) { 311 struct rvt_qp *qp; 312 unsigned long flags; 313 314 rcu_read_lock(); 315 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 316 if (!qp) { 317 rcu_read_unlock(); 318 goto drop; 319 } 320 321 /* 322 * Handle only RC QPs - for other QP types drop error 323 * packet. 324 */ 325 spin_lock_irqsave(&qp->r_lock, flags); 326 327 /* Check for valid receive state. */ 328 if (!(ib_rvt_state_ops[qp->state] & 329 RVT_PROCESS_RECV_OK)) { 330 ibp->rvp.n_pkt_drops++; 331 } 332 333 switch (qp->ibqp.qp_type) { 334 case IB_QPT_RC: 335 hfi1_rc_hdrerr(rcd, packet, qp); 336 break; 337 default: 338 /* For now don't handle any other QP types */ 339 break; 340 } 341 342 spin_unlock_irqrestore(&qp->r_lock, flags); 343 rcu_read_unlock(); 344 } /* Unicast QP */ 345 } /* Valid packet with TIDErr */ 346 347 /* handle "RcvTypeErr" flags */ 348 switch (rte) { 349 case RHF_RTE_ERROR_OP_CODE_ERR: 350 { 351 void *ebuf = NULL; 352 u8 opcode; 353 354 if (rhf_use_egr_bfr(packet->rhf)) 355 ebuf = packet->ebuf; 356 357 if (!ebuf) 358 goto drop; /* this should never happen */ 359 360 opcode = ib_bth_get_opcode(packet->ohdr); 361 if (opcode == IB_OPCODE_CNP) { 362 /* 363 * Only in pre-B0 h/w is the CNP_OPCODE handled 364 * via this code path. 365 */ 366 struct rvt_qp *qp = NULL; 367 u32 lqpn, rqpn; 368 u16 rlid; 369 u8 svc_type, sl, sc5; 370 371 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); 372 sl = ibp->sc_to_sl[sc5]; 373 374 lqpn = ib_bth_get_qpn(packet->ohdr); 375 rcu_read_lock(); 376 qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); 377 if (!qp) { 378 rcu_read_unlock(); 379 goto drop; 380 } 381 382 switch (qp->ibqp.qp_type) { 383 case IB_QPT_UD: 384 rlid = 0; 385 rqpn = 0; 386 svc_type = IB_CC_SVCTYPE_UD; 387 break; 388 case IB_QPT_UC: 389 rlid = ib_get_slid(rhdr); 390 rqpn = qp->remote_qpn; 391 svc_type = IB_CC_SVCTYPE_UC; 392 break; 393 default: 394 rcu_read_unlock(); 395 goto drop; 396 } 397 398 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 399 rcu_read_unlock(); 400 } 401 402 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; 403 break; 404 } 405 default: 406 break; 407 } 408 409 drop: 410 return; 411 } 412 413 static inline void init_packet(struct hfi1_ctxtdata *rcd, 414 struct hfi1_packet *packet) 415 { 416 packet->rsize = rcd->rcvhdrqentsize; /* words */ 417 packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ 418 packet->rcd = rcd; 419 packet->updegr = 0; 420 packet->etail = -1; 421 packet->rhf_addr = get_rhf_addr(rcd); 422 packet->rhf = rhf_to_cpu(packet->rhf_addr); 423 packet->rhqoff = rcd->head; 424 packet->numpkt = 0; 425 } 426 427 /* We support only two types - 9B and 16B for now */ 428 static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = { 429 [HFI1_PKT_TYPE_9B] = &return_cnp, 430 [HFI1_PKT_TYPE_16B] = &return_cnp_16B 431 }; 432 433 void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, 434 bool do_cnp) 435 { 436 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 437 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 438 struct ib_other_headers *ohdr = pkt->ohdr; 439 struct ib_grh *grh = pkt->grh; 440 u32 rqpn = 0, bth1; 441 u16 pkey; 442 u32 rlid, slid, dlid = 0; 443 u8 hdr_type, sc, svc_type; 444 bool is_mcast = false; 445 446 /* can be called from prescan */ 447 if (pkt->etype == RHF_RCV_TYPE_BYPASS) { 448 is_mcast = hfi1_is_16B_mcast(dlid); 449 pkey = hfi1_16B_get_pkey(pkt->hdr); 450 sc = hfi1_16B_get_sc(pkt->hdr); 451 dlid = hfi1_16B_get_dlid(pkt->hdr); 452 slid = hfi1_16B_get_slid(pkt->hdr); 453 hdr_type = HFI1_PKT_TYPE_16B; 454 } else { 455 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) && 456 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)); 457 pkey = ib_bth_get_pkey(ohdr); 458 sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf); 459 dlid = ib_get_dlid(pkt->hdr); 460 slid = ib_get_slid(pkt->hdr); 461 hdr_type = HFI1_PKT_TYPE_9B; 462 } 463 464 switch (qp->ibqp.qp_type) { 465 case IB_QPT_UD: 466 dlid = ppd->lid; 467 rlid = slid; 468 rqpn = ib_get_sqpn(pkt->ohdr); 469 svc_type = IB_CC_SVCTYPE_UD; 470 break; 471 case IB_QPT_SMI: 472 case IB_QPT_GSI: 473 rlid = slid; 474 rqpn = ib_get_sqpn(pkt->ohdr); 475 svc_type = IB_CC_SVCTYPE_UD; 476 break; 477 case IB_QPT_UC: 478 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 479 rqpn = qp->remote_qpn; 480 svc_type = IB_CC_SVCTYPE_UC; 481 break; 482 case IB_QPT_RC: 483 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 484 rqpn = qp->remote_qpn; 485 svc_type = IB_CC_SVCTYPE_RC; 486 break; 487 default: 488 return; 489 } 490 491 bth1 = be32_to_cpu(ohdr->bth[1]); 492 /* Call appropriate CNP handler */ 493 if (do_cnp && (bth1 & IB_FECN_SMASK)) 494 hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey, 495 dlid, rlid, sc, grh); 496 497 if (!is_mcast && (bth1 & IB_BECN_SMASK)) { 498 u32 lqpn = bth1 & RVT_QPN_MASK; 499 u8 sl = ibp->sc_to_sl[sc]; 500 501 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 502 } 503 504 } 505 506 struct ps_mdata { 507 struct hfi1_ctxtdata *rcd; 508 u32 rsize; 509 u32 maxcnt; 510 u32 ps_head; 511 u32 ps_tail; 512 u32 ps_seq; 513 }; 514 515 static inline void init_ps_mdata(struct ps_mdata *mdata, 516 struct hfi1_packet *packet) 517 { 518 struct hfi1_ctxtdata *rcd = packet->rcd; 519 520 mdata->rcd = rcd; 521 mdata->rsize = packet->rsize; 522 mdata->maxcnt = packet->maxcnt; 523 mdata->ps_head = packet->rhqoff; 524 525 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 526 mdata->ps_tail = get_rcvhdrtail(rcd); 527 if (rcd->ctxt == HFI1_CTRL_CTXT) 528 mdata->ps_seq = rcd->seq_cnt; 529 else 530 mdata->ps_seq = 0; /* not used with DMA_RTAIL */ 531 } else { 532 mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ 533 mdata->ps_seq = rcd->seq_cnt; 534 } 535 } 536 537 static inline int ps_done(struct ps_mdata *mdata, u64 rhf, 538 struct hfi1_ctxtdata *rcd) 539 { 540 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 541 return mdata->ps_head == mdata->ps_tail; 542 return mdata->ps_seq != rhf_rcv_seq(rhf); 543 } 544 545 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, 546 struct hfi1_ctxtdata *rcd) 547 { 548 /* 549 * Control context can potentially receive an invalid rhf. 550 * Drop such packets. 551 */ 552 if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) 553 return mdata->ps_seq != rhf_rcv_seq(rhf); 554 555 return 0; 556 } 557 558 static inline void update_ps_mdata(struct ps_mdata *mdata, 559 struct hfi1_ctxtdata *rcd) 560 { 561 mdata->ps_head += mdata->rsize; 562 if (mdata->ps_head >= mdata->maxcnt) 563 mdata->ps_head = 0; 564 565 /* Control context must do seq counting */ 566 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || 567 (rcd->ctxt == HFI1_CTRL_CTXT)) { 568 if (++mdata->ps_seq > 13) 569 mdata->ps_seq = 1; 570 } 571 } 572 573 /* 574 * prescan_rxq - search through the receive queue looking for packets 575 * containing Excplicit Congestion Notifications (FECNs, or BECNs). 576 * When an ECN is found, process the Congestion Notification, and toggle 577 * it off. 578 * This is declared as a macro to allow quick checking of the port to avoid 579 * the overhead of a function call if not enabled. 580 */ 581 #define prescan_rxq(rcd, packet) \ 582 do { \ 583 if (rcd->ppd->cc_prescan) \ 584 __prescan_rxq(packet); \ 585 } while (0) 586 static void __prescan_rxq(struct hfi1_packet *packet) 587 { 588 struct hfi1_ctxtdata *rcd = packet->rcd; 589 struct ps_mdata mdata; 590 591 init_ps_mdata(&mdata, packet); 592 593 while (1) { 594 struct hfi1_devdata *dd = rcd->dd; 595 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 596 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 597 dd->rhf_offset; 598 struct rvt_qp *qp; 599 struct ib_header *hdr; 600 struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 601 u64 rhf = rhf_to_cpu(rhf_addr); 602 u32 etype = rhf_rcv_type(rhf), qpn, bth1; 603 int is_ecn = 0; 604 u8 lnh; 605 606 if (ps_done(&mdata, rhf, rcd)) 607 break; 608 609 if (ps_skip(&mdata, rhf, rcd)) 610 goto next; 611 612 if (etype != RHF_RCV_TYPE_IB) 613 goto next; 614 615 packet->hdr = hfi1_get_msgheader(dd, rhf_addr); 616 hdr = packet->hdr; 617 lnh = ib_get_lnh(hdr); 618 619 if (lnh == HFI1_LRH_BTH) { 620 packet->ohdr = &hdr->u.oth; 621 packet->grh = NULL; 622 } else if (lnh == HFI1_LRH_GRH) { 623 packet->ohdr = &hdr->u.l.oth; 624 packet->grh = &hdr->u.l.grh; 625 } else { 626 goto next; /* just in case */ 627 } 628 629 bth1 = be32_to_cpu(packet->ohdr->bth[1]); 630 is_ecn = !!(bth1 & (IB_FECN_SMASK | IB_BECN_SMASK)); 631 632 if (!is_ecn) 633 goto next; 634 635 qpn = bth1 & RVT_QPN_MASK; 636 rcu_read_lock(); 637 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); 638 639 if (!qp) { 640 rcu_read_unlock(); 641 goto next; 642 } 643 644 process_ecn(qp, packet, true); 645 rcu_read_unlock(); 646 647 /* turn off BECN, FECN */ 648 bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK); 649 packet->ohdr->bth[1] = cpu_to_be32(bth1); 650 next: 651 update_ps_mdata(&mdata, rcd); 652 } 653 } 654 655 static void process_rcv_qp_work(struct hfi1_packet *packet) 656 { 657 struct rvt_qp *qp, *nqp; 658 struct hfi1_ctxtdata *rcd = packet->rcd; 659 660 /* 661 * Iterate over all QPs waiting to respond. 662 * The list won't change since the IRQ is only run on one CPU. 663 */ 664 list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { 665 list_del_init(&qp->rspwait); 666 if (qp->r_flags & RVT_R_RSP_NAK) { 667 qp->r_flags &= ~RVT_R_RSP_NAK; 668 packet->qp = qp; 669 hfi1_send_rc_ack(packet, 0); 670 } 671 if (qp->r_flags & RVT_R_RSP_SEND) { 672 unsigned long flags; 673 674 qp->r_flags &= ~RVT_R_RSP_SEND; 675 spin_lock_irqsave(&qp->s_lock, flags); 676 if (ib_rvt_state_ops[qp->state] & 677 RVT_PROCESS_OR_FLUSH_SEND) 678 hfi1_schedule_send(qp); 679 spin_unlock_irqrestore(&qp->s_lock, flags); 680 } 681 rvt_put_qp(qp); 682 } 683 } 684 685 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) 686 { 687 if (thread) { 688 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) 689 /* allow defered processing */ 690 process_rcv_qp_work(packet); 691 cond_resched(); 692 return RCV_PKT_OK; 693 } else { 694 this_cpu_inc(*packet->rcd->dd->rcv_limit); 695 return RCV_PKT_LIMIT; 696 } 697 } 698 699 static inline int check_max_packet(struct hfi1_packet *packet, int thread) 700 { 701 int ret = RCV_PKT_OK; 702 703 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) 704 ret = max_packet_exceeded(packet, thread); 705 return ret; 706 } 707 708 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) 709 { 710 int ret; 711 712 /* Set up for the next packet */ 713 packet->rhqoff += packet->rsize; 714 if (packet->rhqoff >= packet->maxcnt) 715 packet->rhqoff = 0; 716 717 packet->numpkt++; 718 ret = check_max_packet(packet, thread); 719 720 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 721 packet->rcd->dd->rhf_offset; 722 packet->rhf = rhf_to_cpu(packet->rhf_addr); 723 724 return ret; 725 } 726 727 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) 728 { 729 int ret; 730 731 packet->etype = rhf_rcv_type(packet->rhf); 732 733 /* total length */ 734 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 735 /* retrieve eager buffer details */ 736 packet->ebuf = NULL; 737 if (rhf_use_egr_bfr(packet->rhf)) { 738 packet->etail = rhf_egr_index(packet->rhf); 739 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, 740 &packet->updegr); 741 /* 742 * Prefetch the contents of the eager buffer. It is 743 * OK to send a negative length to prefetch_range(). 744 * The +2 is the size of the RHF. 745 */ 746 prefetch_range(packet->ebuf, 747 packet->tlen - ((packet->rcd->rcvhdrqentsize - 748 (rhf_hdrq_offset(packet->rhf) 749 + 2)) * 4)); 750 } 751 752 /* 753 * Call a type specific handler for the packet. We 754 * should be able to trust that etype won't be beyond 755 * the range of valid indexes. If so something is really 756 * wrong and we can probably just let things come 757 * crashing down. There is no need to eat another 758 * comparison in this performance critical code. 759 */ 760 packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet); 761 packet->numpkt++; 762 763 /* Set up for the next packet */ 764 packet->rhqoff += packet->rsize; 765 if (packet->rhqoff >= packet->maxcnt) 766 packet->rhqoff = 0; 767 768 ret = check_max_packet(packet, thread); 769 770 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 771 packet->rcd->dd->rhf_offset; 772 packet->rhf = rhf_to_cpu(packet->rhf_addr); 773 774 return ret; 775 } 776 777 static inline void process_rcv_update(int last, struct hfi1_packet *packet) 778 { 779 /* 780 * Update head regs etc., every 16 packets, if not last pkt, 781 * to help prevent rcvhdrq overflows, when many packets 782 * are processed and queue is nearly full. 783 * Don't request an interrupt for intermediate updates. 784 */ 785 if (!last && !(packet->numpkt & 0xf)) { 786 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, 787 packet->etail, 0, 0); 788 packet->updegr = 0; 789 } 790 packet->grh = NULL; 791 } 792 793 static inline void finish_packet(struct hfi1_packet *packet) 794 { 795 /* 796 * Nothing we need to free for the packet. 797 * 798 * The only thing we need to do is a final update and call for an 799 * interrupt 800 */ 801 update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, 802 packet->etail, rcv_intr_dynamic, packet->numpkt); 803 } 804 805 /* 806 * Handle receive interrupts when using the no dma rtail option. 807 */ 808 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) 809 { 810 u32 seq; 811 int last = RCV_PKT_OK; 812 struct hfi1_packet packet; 813 814 init_packet(rcd, &packet); 815 seq = rhf_rcv_seq(packet.rhf); 816 if (seq != rcd->seq_cnt) { 817 last = RCV_PKT_DONE; 818 goto bail; 819 } 820 821 prescan_rxq(rcd, &packet); 822 823 while (last == RCV_PKT_OK) { 824 last = process_rcv_packet(&packet, thread); 825 seq = rhf_rcv_seq(packet.rhf); 826 if (++rcd->seq_cnt > 13) 827 rcd->seq_cnt = 1; 828 if (seq != rcd->seq_cnt) 829 last = RCV_PKT_DONE; 830 process_rcv_update(last, &packet); 831 } 832 process_rcv_qp_work(&packet); 833 rcd->head = packet.rhqoff; 834 bail: 835 finish_packet(&packet); 836 return last; 837 } 838 839 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) 840 { 841 u32 hdrqtail; 842 int last = RCV_PKT_OK; 843 struct hfi1_packet packet; 844 845 init_packet(rcd, &packet); 846 hdrqtail = get_rcvhdrtail(rcd); 847 if (packet.rhqoff == hdrqtail) { 848 last = RCV_PKT_DONE; 849 goto bail; 850 } 851 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 852 853 prescan_rxq(rcd, &packet); 854 855 while (last == RCV_PKT_OK) { 856 last = process_rcv_packet(&packet, thread); 857 if (packet.rhqoff == hdrqtail) 858 last = RCV_PKT_DONE; 859 process_rcv_update(last, &packet); 860 } 861 process_rcv_qp_work(&packet); 862 rcd->head = packet.rhqoff; 863 bail: 864 finish_packet(&packet); 865 return last; 866 } 867 868 static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt) 869 { 870 struct hfi1_ctxtdata *rcd; 871 u16 i; 872 873 /* 874 * For dynamically allocated kernel contexts (like vnic) switch 875 * interrupt handler only for that context. Otherwise, switch 876 * interrupt handler for all statically allocated kernel contexts. 877 */ 878 if (ctxt >= dd->first_dyn_alloc_ctxt) { 879 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 880 if (rcd) { 881 rcd->do_interrupt = 882 &handle_receive_interrupt_nodma_rtail; 883 hfi1_rcd_put(rcd); 884 } 885 return; 886 } 887 888 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 889 rcd = hfi1_rcd_get_by_index(dd, i); 890 if (rcd) 891 rcd->do_interrupt = 892 &handle_receive_interrupt_nodma_rtail; 893 hfi1_rcd_put(rcd); 894 } 895 } 896 897 static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt) 898 { 899 struct hfi1_ctxtdata *rcd; 900 u16 i; 901 902 /* 903 * For dynamically allocated kernel contexts (like vnic) switch 904 * interrupt handler only for that context. Otherwise, switch 905 * interrupt handler for all statically allocated kernel contexts. 906 */ 907 if (ctxt >= dd->first_dyn_alloc_ctxt) { 908 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 909 if (rcd) { 910 rcd->do_interrupt = 911 &handle_receive_interrupt_dma_rtail; 912 hfi1_rcd_put(rcd); 913 } 914 return; 915 } 916 917 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 918 rcd = hfi1_rcd_get_by_index(dd, i); 919 if (rcd) 920 rcd->do_interrupt = 921 &handle_receive_interrupt_dma_rtail; 922 hfi1_rcd_put(rcd); 923 } 924 } 925 926 void set_all_slowpath(struct hfi1_devdata *dd) 927 { 928 struct hfi1_ctxtdata *rcd; 929 u16 i; 930 931 /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */ 932 for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) { 933 rcd = hfi1_rcd_get_by_index(dd, i); 934 if (!rcd) 935 continue; 936 if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic) 937 rcd->do_interrupt = &handle_receive_interrupt; 938 939 hfi1_rcd_put(rcd); 940 } 941 } 942 943 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, 944 struct hfi1_packet *packet, 945 struct hfi1_devdata *dd) 946 { 947 struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; 948 u8 etype = rhf_rcv_type(packet->rhf); 949 u8 sc = SC15_PACKET; 950 951 if (etype == RHF_RCV_TYPE_IB) { 952 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd, 953 packet->rhf_addr); 954 sc = hfi1_9B_get_sc5(hdr, packet->rhf); 955 } else if (etype == RHF_RCV_TYPE_BYPASS) { 956 struct hfi1_16b_header *hdr = hfi1_get_16B_header( 957 packet->rcd->dd, 958 packet->rhf_addr); 959 sc = hfi1_16B_get_sc(hdr); 960 } 961 if (sc != SC15_PACKET) { 962 int hwstate = driver_lstate(rcd->ppd); 963 964 if (hwstate != IB_PORT_ACTIVE) { 965 dd_dev_info(dd, 966 "Unexpected link state %s\n", 967 opa_lstate_name(hwstate)); 968 return 0; 969 } 970 971 queue_work(rcd->ppd->link_wq, lsaw); 972 return 1; 973 } 974 return 0; 975 } 976 977 /* 978 * handle_receive_interrupt - receive a packet 979 * @rcd: the context 980 * 981 * Called from interrupt handler for errors or receive interrupt. 982 * This is the slow path interrupt handler. 983 */ 984 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) 985 { 986 struct hfi1_devdata *dd = rcd->dd; 987 u32 hdrqtail; 988 int needset, last = RCV_PKT_OK; 989 struct hfi1_packet packet; 990 int skip_pkt = 0; 991 992 /* Control context will always use the slow path interrupt handler */ 993 needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; 994 995 init_packet(rcd, &packet); 996 997 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 998 u32 seq = rhf_rcv_seq(packet.rhf); 999 1000 if (seq != rcd->seq_cnt) { 1001 last = RCV_PKT_DONE; 1002 goto bail; 1003 } 1004 hdrqtail = 0; 1005 } else { 1006 hdrqtail = get_rcvhdrtail(rcd); 1007 if (packet.rhqoff == hdrqtail) { 1008 last = RCV_PKT_DONE; 1009 goto bail; 1010 } 1011 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 1012 1013 /* 1014 * Control context can potentially receive an invalid 1015 * rhf. Drop such packets. 1016 */ 1017 if (rcd->ctxt == HFI1_CTRL_CTXT) { 1018 u32 seq = rhf_rcv_seq(packet.rhf); 1019 1020 if (seq != rcd->seq_cnt) 1021 skip_pkt = 1; 1022 } 1023 } 1024 1025 prescan_rxq(rcd, &packet); 1026 1027 while (last == RCV_PKT_OK) { 1028 if (unlikely(dd->do_drop && 1029 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == 1030 DROP_PACKET_ON)) { 1031 dd->do_drop = 0; 1032 1033 /* On to the next packet */ 1034 packet.rhqoff += packet.rsize; 1035 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + 1036 packet.rhqoff + 1037 dd->rhf_offset; 1038 packet.rhf = rhf_to_cpu(packet.rhf_addr); 1039 1040 } else if (skip_pkt) { 1041 last = skip_rcv_packet(&packet, thread); 1042 skip_pkt = 0; 1043 } else { 1044 /* Auto activate link on non-SC15 packet receive */ 1045 if (unlikely(rcd->ppd->host_link_state == 1046 HLS_UP_ARMED) && 1047 set_armed_to_active(rcd, &packet, dd)) 1048 goto bail; 1049 last = process_rcv_packet(&packet, thread); 1050 } 1051 1052 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 1053 u32 seq = rhf_rcv_seq(packet.rhf); 1054 1055 if (++rcd->seq_cnt > 13) 1056 rcd->seq_cnt = 1; 1057 if (seq != rcd->seq_cnt) 1058 last = RCV_PKT_DONE; 1059 if (needset) { 1060 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); 1061 set_nodma_rtail(dd, rcd->ctxt); 1062 needset = 0; 1063 } 1064 } else { 1065 if (packet.rhqoff == hdrqtail) 1066 last = RCV_PKT_DONE; 1067 /* 1068 * Control context can potentially receive an invalid 1069 * rhf. Drop such packets. 1070 */ 1071 if (rcd->ctxt == HFI1_CTRL_CTXT) { 1072 u32 seq = rhf_rcv_seq(packet.rhf); 1073 1074 if (++rcd->seq_cnt > 13) 1075 rcd->seq_cnt = 1; 1076 if (!last && (seq != rcd->seq_cnt)) 1077 skip_pkt = 1; 1078 } 1079 1080 if (needset) { 1081 dd_dev_info(dd, 1082 "Switching to DMA_RTAIL\n"); 1083 set_dma_rtail(dd, rcd->ctxt); 1084 needset = 0; 1085 } 1086 } 1087 1088 process_rcv_update(last, &packet); 1089 } 1090 1091 process_rcv_qp_work(&packet); 1092 rcd->head = packet.rhqoff; 1093 1094 bail: 1095 /* 1096 * Always write head at end, and setup rcv interrupt, even 1097 * if no packets were processed. 1098 */ 1099 finish_packet(&packet); 1100 return last; 1101 } 1102 1103 /* 1104 * We may discover in the interrupt that the hardware link state has 1105 * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet), 1106 * and we need to update the driver's notion of the link state. We cannot 1107 * run set_link_state from interrupt context, so we queue this function on 1108 * a workqueue. 1109 * 1110 * We delay the regular interrupt processing until after the state changes 1111 * so that the link will be in the correct state by the time any application 1112 * we wake up attempts to send a reply to any message it received. 1113 * (Subsequent receive interrupts may possibly force the wakeup before we 1114 * update the link state.) 1115 * 1116 * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes 1117 * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues, 1118 * so we're safe from use-after-free of the rcd. 1119 */ 1120 void receive_interrupt_work(struct work_struct *work) 1121 { 1122 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 1123 linkstate_active_work); 1124 struct hfi1_devdata *dd = ppd->dd; 1125 struct hfi1_ctxtdata *rcd; 1126 u16 i; 1127 1128 /* Received non-SC15 packet implies neighbor_normal */ 1129 ppd->neighbor_normal = 1; 1130 set_link_state(ppd, HLS_UP_ACTIVE); 1131 1132 /* 1133 * Interrupt all statically allocated kernel contexts that could 1134 * have had an interrupt during auto activation. 1135 */ 1136 for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) { 1137 rcd = hfi1_rcd_get_by_index(dd, i); 1138 if (rcd) 1139 force_recv_intr(rcd); 1140 hfi1_rcd_put(rcd); 1141 } 1142 } 1143 1144 /* 1145 * Convert a given MTU size to the on-wire MAD packet enumeration. 1146 * Return -1 if the size is invalid. 1147 */ 1148 int mtu_to_enum(u32 mtu, int default_if_bad) 1149 { 1150 switch (mtu) { 1151 case 0: return OPA_MTU_0; 1152 case 256: return OPA_MTU_256; 1153 case 512: return OPA_MTU_512; 1154 case 1024: return OPA_MTU_1024; 1155 case 2048: return OPA_MTU_2048; 1156 case 4096: return OPA_MTU_4096; 1157 case 8192: return OPA_MTU_8192; 1158 case 10240: return OPA_MTU_10240; 1159 } 1160 return default_if_bad; 1161 } 1162 1163 u16 enum_to_mtu(int mtu) 1164 { 1165 switch (mtu) { 1166 case OPA_MTU_0: return 0; 1167 case OPA_MTU_256: return 256; 1168 case OPA_MTU_512: return 512; 1169 case OPA_MTU_1024: return 1024; 1170 case OPA_MTU_2048: return 2048; 1171 case OPA_MTU_4096: return 4096; 1172 case OPA_MTU_8192: return 8192; 1173 case OPA_MTU_10240: return 10240; 1174 default: return 0xffff; 1175 } 1176 } 1177 1178 /* 1179 * set_mtu - set the MTU 1180 * @ppd: the per port data 1181 * 1182 * We can handle "any" incoming size, the issue here is whether we 1183 * need to restrict our outgoing size. We do not deal with what happens 1184 * to programs that are already running when the size changes. 1185 */ 1186 int set_mtu(struct hfi1_pportdata *ppd) 1187 { 1188 struct hfi1_devdata *dd = ppd->dd; 1189 int i, drain, ret = 0, is_up = 0; 1190 1191 ppd->ibmtu = 0; 1192 for (i = 0; i < ppd->vls_supported; i++) 1193 if (ppd->ibmtu < dd->vld[i].mtu) 1194 ppd->ibmtu = dd->vld[i].mtu; 1195 ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd); 1196 1197 mutex_lock(&ppd->hls_lock); 1198 if (ppd->host_link_state == HLS_UP_INIT || 1199 ppd->host_link_state == HLS_UP_ARMED || 1200 ppd->host_link_state == HLS_UP_ACTIVE) 1201 is_up = 1; 1202 1203 drain = !is_ax(dd) && is_up; 1204 1205 if (drain) 1206 /* 1207 * MTU is specified per-VL. To ensure that no packet gets 1208 * stuck (due, e.g., to the MTU for the packet's VL being 1209 * reduced), empty the per-VL FIFOs before adjusting MTU. 1210 */ 1211 ret = stop_drain_data_vls(dd); 1212 1213 if (ret) { 1214 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n", 1215 __func__); 1216 goto err; 1217 } 1218 1219 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0); 1220 1221 if (drain) 1222 open_fill_data_vls(dd); /* reopen all VLs */ 1223 1224 err: 1225 mutex_unlock(&ppd->hls_lock); 1226 1227 return ret; 1228 } 1229 1230 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc) 1231 { 1232 struct hfi1_devdata *dd = ppd->dd; 1233 1234 ppd->lid = lid; 1235 ppd->lmc = lmc; 1236 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0); 1237 1238 dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid); 1239 1240 return 0; 1241 } 1242 1243 void shutdown_led_override(struct hfi1_pportdata *ppd) 1244 { 1245 struct hfi1_devdata *dd = ppd->dd; 1246 1247 /* 1248 * This pairs with the memory barrier in hfi1_start_led_override to 1249 * ensure that we read the correct state of LED beaconing represented 1250 * by led_override_timer_active 1251 */ 1252 smp_rmb(); 1253 if (atomic_read(&ppd->led_override_timer_active)) { 1254 del_timer_sync(&ppd->led_override_timer); 1255 atomic_set(&ppd->led_override_timer_active, 0); 1256 /* Ensure the atomic_set is visible to all CPUs */ 1257 smp_wmb(); 1258 } 1259 1260 /* Hand control of the LED to the DC for normal operation */ 1261 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 1262 } 1263 1264 static void run_led_override(struct timer_list *t) 1265 { 1266 struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer); 1267 struct hfi1_devdata *dd = ppd->dd; 1268 unsigned long timeout; 1269 int phase_idx; 1270 1271 if (!(dd->flags & HFI1_INITTED)) 1272 return; 1273 1274 phase_idx = ppd->led_override_phase & 1; 1275 1276 setextled(dd, phase_idx); 1277 1278 timeout = ppd->led_override_vals[phase_idx]; 1279 1280 /* Set up for next phase */ 1281 ppd->led_override_phase = !ppd->led_override_phase; 1282 1283 mod_timer(&ppd->led_override_timer, jiffies + timeout); 1284 } 1285 1286 /* 1287 * To have the LED blink in a particular pattern, provide timeon and timeoff 1288 * in milliseconds. 1289 * To turn off custom blinking and return to normal operation, use 1290 * shutdown_led_override() 1291 */ 1292 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, 1293 unsigned int timeoff) 1294 { 1295 if (!(ppd->dd->flags & HFI1_INITTED)) 1296 return; 1297 1298 /* Convert to jiffies for direct use in timer */ 1299 ppd->led_override_vals[0] = msecs_to_jiffies(timeoff); 1300 ppd->led_override_vals[1] = msecs_to_jiffies(timeon); 1301 1302 /* Arbitrarily start from LED on phase */ 1303 ppd->led_override_phase = 1; 1304 1305 /* 1306 * If the timer has not already been started, do so. Use a "quick" 1307 * timeout so the handler will be called soon to look at our request. 1308 */ 1309 if (!timer_pending(&ppd->led_override_timer)) { 1310 timer_setup(&ppd->led_override_timer, run_led_override, 0); 1311 ppd->led_override_timer.expires = jiffies + 1; 1312 add_timer(&ppd->led_override_timer); 1313 atomic_set(&ppd->led_override_timer_active, 1); 1314 /* Ensure the atomic_set is visible to all CPUs */ 1315 smp_wmb(); 1316 } 1317 } 1318 1319 /** 1320 * hfi1_reset_device - reset the chip if possible 1321 * @unit: the device to reset 1322 * 1323 * Whether or not reset is successful, we attempt to re-initialize the chip 1324 * (that is, much like a driver unload/reload). We clear the INITTED flag 1325 * so that the various entry points will fail until we reinitialize. For 1326 * now, we only allow this if no user contexts are open that use chip resources 1327 */ 1328 int hfi1_reset_device(int unit) 1329 { 1330 int ret; 1331 struct hfi1_devdata *dd = hfi1_lookup(unit); 1332 struct hfi1_pportdata *ppd; 1333 int pidx; 1334 1335 if (!dd) { 1336 ret = -ENODEV; 1337 goto bail; 1338 } 1339 1340 dd_dev_info(dd, "Reset on unit %u requested\n", unit); 1341 1342 if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) { 1343 dd_dev_info(dd, 1344 "Invalid unit number %u or not initialized or not present\n", 1345 unit); 1346 ret = -ENXIO; 1347 goto bail; 1348 } 1349 1350 /* If there are any user/vnic contexts, we cannot reset */ 1351 mutex_lock(&hfi1_mutex); 1352 if (dd->rcd) 1353 if (hfi1_stats.sps_ctxts) { 1354 mutex_unlock(&hfi1_mutex); 1355 ret = -EBUSY; 1356 goto bail; 1357 } 1358 mutex_unlock(&hfi1_mutex); 1359 1360 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1361 ppd = dd->pport + pidx; 1362 1363 shutdown_led_override(ppd); 1364 } 1365 if (dd->flags & HFI1_HAS_SEND_DMA) 1366 sdma_exit(dd); 1367 1368 hfi1_reset_cpu_counters(dd); 1369 1370 ret = hfi1_init(dd, 1); 1371 1372 if (ret) 1373 dd_dev_err(dd, 1374 "Reinitialize unit %u after reset failed with %d\n", 1375 unit, ret); 1376 else 1377 dd_dev_info(dd, "Reinitialized unit %u after resetting\n", 1378 unit); 1379 1380 bail: 1381 return ret; 1382 } 1383 1384 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) 1385 { 1386 packet->hdr = (struct hfi1_ib_message_header *) 1387 hfi1_get_msgheader(packet->rcd->dd, 1388 packet->rhf_addr); 1389 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; 1390 } 1391 1392 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet) 1393 { 1394 struct hfi1_pportdata *ppd = packet->rcd->ppd; 1395 1396 /* slid and dlid cannot be 0 */ 1397 if ((!packet->slid) || (!packet->dlid)) 1398 return -EINVAL; 1399 1400 /* Compare port lid with incoming packet dlid */ 1401 if ((!(hfi1_is_16B_mcast(packet->dlid))) && 1402 (packet->dlid != 1403 opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) { 1404 if (packet->dlid != ppd->lid) 1405 return -EINVAL; 1406 } 1407 1408 /* No multicast packets with SC15 */ 1409 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF)) 1410 return -EINVAL; 1411 1412 /* Packets with permissive DLID always on SC15 */ 1413 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 1414 16B)) && 1415 (packet->sc != 0xF)) 1416 return -EINVAL; 1417 1418 return 0; 1419 } 1420 1421 static int hfi1_setup_9B_packet(struct hfi1_packet *packet) 1422 { 1423 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); 1424 struct ib_header *hdr; 1425 u8 lnh; 1426 1427 hfi1_setup_ib_header(packet); 1428 hdr = packet->hdr; 1429 1430 lnh = ib_get_lnh(hdr); 1431 if (lnh == HFI1_LRH_BTH) { 1432 packet->ohdr = &hdr->u.oth; 1433 packet->grh = NULL; 1434 } else if (lnh == HFI1_LRH_GRH) { 1435 u32 vtf; 1436 1437 packet->ohdr = &hdr->u.l.oth; 1438 packet->grh = &hdr->u.l.grh; 1439 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 1440 goto drop; 1441 vtf = be32_to_cpu(packet->grh->version_tclass_flow); 1442 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 1443 goto drop; 1444 } else { 1445 goto drop; 1446 } 1447 1448 /* Query commonly used fields from packet header */ 1449 packet->payload = packet->ebuf; 1450 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1451 packet->slid = ib_get_slid(hdr); 1452 packet->dlid = ib_get_dlid(hdr); 1453 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && 1454 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE)))) 1455 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 1456 be16_to_cpu(IB_MULTICAST_LID_BASE); 1457 packet->sl = ib_get_sl(hdr); 1458 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); 1459 packet->pad = ib_bth_get_pad(packet->ohdr); 1460 packet->extra_byte = 0; 1461 packet->pkey = ib_bth_get_pkey(packet->ohdr); 1462 packet->migrated = ib_bth_is_migration(packet->ohdr); 1463 1464 return 0; 1465 drop: 1466 ibp->rvp.n_pkt_drops++; 1467 return -EINVAL; 1468 } 1469 1470 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) 1471 { 1472 /* 1473 * Bypass packets have a different header/payload split 1474 * compared to an IB packet. 1475 * Current split is set such that 16 bytes of the actual 1476 * header is in the header buffer and the remining is in 1477 * the eager buffer. We chose 16 since hfi1 driver only 1478 * supports 16B bypass packets and we will be able to 1479 * receive the entire LRH with such a split. 1480 */ 1481 1482 struct hfi1_ctxtdata *rcd = packet->rcd; 1483 struct hfi1_pportdata *ppd = rcd->ppd; 1484 struct hfi1_ibport *ibp = &ppd->ibport_data; 1485 u8 l4; 1486 1487 packet->hdr = (struct hfi1_16b_header *) 1488 hfi1_get_16B_header(packet->rcd->dd, 1489 packet->rhf_addr); 1490 l4 = hfi1_16B_get_l4(packet->hdr); 1491 if (l4 == OPA_16B_L4_IB_LOCAL) { 1492 packet->ohdr = packet->ebuf; 1493 packet->grh = NULL; 1494 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1495 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 1496 /* hdr_len_by_opcode already has an IB LRH factored in */ 1497 packet->hlen = hdr_len_by_opcode[packet->opcode] + 1498 (LRH_16B_BYTES - LRH_9B_BYTES); 1499 packet->migrated = opa_bth_is_migration(packet->ohdr); 1500 } else if (l4 == OPA_16B_L4_IB_GLOBAL) { 1501 u32 vtf; 1502 u8 grh_len = sizeof(struct ib_grh); 1503 1504 packet->ohdr = packet->ebuf + grh_len; 1505 packet->grh = packet->ebuf; 1506 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1507 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 1508 /* hdr_len_by_opcode already has an IB LRH factored in */ 1509 packet->hlen = hdr_len_by_opcode[packet->opcode] + 1510 (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len; 1511 packet->migrated = opa_bth_is_migration(packet->ohdr); 1512 1513 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 1514 goto drop; 1515 vtf = be32_to_cpu(packet->grh->version_tclass_flow); 1516 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 1517 goto drop; 1518 } else if (l4 == OPA_16B_L4_FM) { 1519 packet->mgmt = packet->ebuf; 1520 packet->ohdr = NULL; 1521 packet->grh = NULL; 1522 packet->opcode = IB_OPCODE_UD_SEND_ONLY; 1523 packet->pad = OPA_16B_L4_FM_PAD; 1524 packet->hlen = OPA_16B_L4_FM_HLEN; 1525 packet->migrated = false; 1526 } else { 1527 goto drop; 1528 } 1529 1530 /* Query commonly used fields from packet header */ 1531 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; 1532 packet->slid = hfi1_16B_get_slid(packet->hdr); 1533 packet->dlid = hfi1_16B_get_dlid(packet->hdr); 1534 if (unlikely(hfi1_is_16B_mcast(packet->dlid))) 1535 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 1536 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 1537 16B); 1538 packet->sc = hfi1_16B_get_sc(packet->hdr); 1539 packet->sl = ibp->sc_to_sl[packet->sc]; 1540 packet->extra_byte = SIZE_OF_LT; 1541 packet->pkey = hfi1_16B_get_pkey(packet->hdr); 1542 1543 if (hfi1_bypass_ingress_pkt_check(packet)) 1544 goto drop; 1545 1546 return 0; 1547 drop: 1548 hfi1_cdbg(PKT, "%s: packet dropped\n", __func__); 1549 ibp->rvp.n_pkt_drops++; 1550 return -EINVAL; 1551 } 1552 1553 void handle_eflags(struct hfi1_packet *packet) 1554 { 1555 struct hfi1_ctxtdata *rcd = packet->rcd; 1556 u32 rte = rhf_rcv_type_err(packet->rhf); 1557 1558 rcv_hdrerr(rcd, rcd->ppd, packet); 1559 if (rhf_err_flags(packet->rhf)) 1560 dd_dev_err(rcd->dd, 1561 "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", 1562 rcd->ctxt, packet->rhf, 1563 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", 1564 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", 1565 packet->rhf & RHF_DC_ERR ? "dc " : "", 1566 packet->rhf & RHF_TID_ERR ? "tid " : "", 1567 packet->rhf & RHF_LEN_ERR ? "len " : "", 1568 packet->rhf & RHF_ECC_ERR ? "ecc " : "", 1569 packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", 1570 packet->rhf & RHF_ICRC_ERR ? "icrc " : "", 1571 rte); 1572 } 1573 1574 /* 1575 * The following functions are called by the interrupt handler. They are type 1576 * specific handlers for each packet type. 1577 */ 1578 int process_receive_ib(struct hfi1_packet *packet) 1579 { 1580 if (hfi1_setup_9B_packet(packet)) 1581 return RHF_RCV_CONTINUE; 1582 1583 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1584 return RHF_RCV_CONTINUE; 1585 1586 trace_hfi1_rcvhdr(packet); 1587 1588 if (unlikely(rhf_err_flags(packet->rhf))) { 1589 handle_eflags(packet); 1590 return RHF_RCV_CONTINUE; 1591 } 1592 1593 hfi1_ib_rcv(packet); 1594 return RHF_RCV_CONTINUE; 1595 } 1596 1597 static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) 1598 { 1599 /* Packet received in VNIC context via RSM */ 1600 if (packet->rcd->is_vnic) 1601 return true; 1602 1603 if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) && 1604 (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR)) 1605 return true; 1606 1607 return false; 1608 } 1609 1610 int process_receive_bypass(struct hfi1_packet *packet) 1611 { 1612 struct hfi1_devdata *dd = packet->rcd->dd; 1613 1614 if (hfi1_is_vnic_packet(packet)) { 1615 hfi1_vnic_bypass_rcv(packet); 1616 return RHF_RCV_CONTINUE; 1617 } 1618 1619 if (hfi1_setup_bypass_packet(packet)) 1620 return RHF_RCV_CONTINUE; 1621 1622 trace_hfi1_rcvhdr(packet); 1623 1624 if (unlikely(rhf_err_flags(packet->rhf))) { 1625 handle_eflags(packet); 1626 return RHF_RCV_CONTINUE; 1627 } 1628 1629 if (hfi1_16B_get_l2(packet->hdr) == 0x2) { 1630 hfi1_16B_rcv(packet); 1631 } else { 1632 dd_dev_err(dd, 1633 "Bypass packets other than 16B are not supported in normal operation. Dropping\n"); 1634 incr_cntr64(&dd->sw_rcv_bypass_packet_errors); 1635 if (!(dd->err_info_rcvport.status_and_code & 1636 OPA_EI_STATUS_SMASK)) { 1637 u64 *flits = packet->ebuf; 1638 1639 if (flits && !(packet->rhf & RHF_LEN_ERR)) { 1640 dd->err_info_rcvport.packet_flit1 = flits[0]; 1641 dd->err_info_rcvport.packet_flit2 = 1642 packet->tlen > sizeof(flits[0]) ? 1643 flits[1] : 0; 1644 } 1645 dd->err_info_rcvport.status_and_code |= 1646 (OPA_EI_STATUS_SMASK | BAD_L2_ERR); 1647 } 1648 } 1649 return RHF_RCV_CONTINUE; 1650 } 1651 1652 int process_receive_error(struct hfi1_packet *packet) 1653 { 1654 /* KHdrHCRCErr -- KDETH packet with a bad HCRC */ 1655 if (unlikely( 1656 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && 1657 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || 1658 packet->rhf & RHF_DC_ERR))) 1659 return RHF_RCV_CONTINUE; 1660 1661 hfi1_setup_ib_header(packet); 1662 handle_eflags(packet); 1663 1664 if (unlikely(rhf_err_flags(packet->rhf))) 1665 dd_dev_err(packet->rcd->dd, 1666 "Unhandled error packet received. Dropping.\n"); 1667 1668 return RHF_RCV_CONTINUE; 1669 } 1670 1671 int kdeth_process_expected(struct hfi1_packet *packet) 1672 { 1673 hfi1_setup_9B_packet(packet); 1674 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1675 return RHF_RCV_CONTINUE; 1676 1677 if (unlikely(rhf_err_flags(packet->rhf))) 1678 handle_eflags(packet); 1679 1680 dd_dev_err(packet->rcd->dd, 1681 "Unhandled expected packet received. Dropping.\n"); 1682 return RHF_RCV_CONTINUE; 1683 } 1684 1685 int kdeth_process_eager(struct hfi1_packet *packet) 1686 { 1687 hfi1_setup_9B_packet(packet); 1688 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1689 return RHF_RCV_CONTINUE; 1690 if (unlikely(rhf_err_flags(packet->rhf))) 1691 handle_eflags(packet); 1692 1693 dd_dev_err(packet->rcd->dd, 1694 "Unhandled eager packet received. Dropping.\n"); 1695 return RHF_RCV_CONTINUE; 1696 } 1697 1698 int process_receive_invalid(struct hfi1_packet *packet) 1699 { 1700 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", 1701 rhf_rcv_type(packet->rhf)); 1702 return RHF_RCV_CONTINUE; 1703 } 1704 1705 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) 1706 { 1707 struct hfi1_packet packet; 1708 struct ps_mdata mdata; 1709 1710 seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n", 1711 rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize, 1712 HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? 1713 "dma_rtail" : "nodma_rtail", 1714 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) & 1715 RCV_HDR_HEAD_HEAD_MASK, 1716 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL)); 1717 1718 init_packet(rcd, &packet); 1719 init_ps_mdata(&mdata, &packet); 1720 1721 while (1) { 1722 struct hfi1_devdata *dd = rcd->dd; 1723 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 1724 dd->rhf_offset; 1725 struct ib_header *hdr; 1726 u64 rhf = rhf_to_cpu(rhf_addr); 1727 u32 etype = rhf_rcv_type(rhf), qpn; 1728 u8 opcode; 1729 u32 psn; 1730 u8 lnh; 1731 1732 if (ps_done(&mdata, rhf, rcd)) 1733 break; 1734 1735 if (ps_skip(&mdata, rhf, rcd)) 1736 goto next; 1737 1738 if (etype > RHF_RCV_TYPE_IB) 1739 goto next; 1740 1741 packet.hdr = hfi1_get_msgheader(dd, rhf_addr); 1742 hdr = packet.hdr; 1743 1744 lnh = be16_to_cpu(hdr->lrh[0]) & 3; 1745 1746 if (lnh == HFI1_LRH_BTH) 1747 packet.ohdr = &hdr->u.oth; 1748 else if (lnh == HFI1_LRH_GRH) 1749 packet.ohdr = &hdr->u.l.oth; 1750 else 1751 goto next; /* just in case */ 1752 1753 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24); 1754 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK; 1755 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2])); 1756 1757 seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n", 1758 mdata.ps_head, opcode, qpn, psn); 1759 next: 1760 update_ps_mdata(&mdata, rcd); 1761 } 1762 } 1763