1 /* 2 * Copyright(c) 2015-2018 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/spinlock.h> 49 #include <linux/pci.h> 50 #include <linux/io.h> 51 #include <linux/delay.h> 52 #include <linux/netdevice.h> 53 #include <linux/vmalloc.h> 54 #include <linux/module.h> 55 #include <linux/prefetch.h> 56 #include <rdma/ib_verbs.h> 57 58 #include "hfi.h" 59 #include "trace.h" 60 #include "qp.h" 61 #include "sdma.h" 62 #include "debugfs.h" 63 #include "vnic.h" 64 #include "fault.h" 65 66 #undef pr_fmt 67 #define pr_fmt(fmt) DRIVER_NAME ": " fmt 68 69 /* 70 * The size has to be longer than this string, so we can append 71 * board/chip information to it in the initialization code. 72 */ 73 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n"; 74 75 DEFINE_MUTEX(hfi1_mutex); /* general driver use */ 76 77 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU; 78 module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO); 79 MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify( 80 HFI1_DEFAULT_MAX_MTU)); 81 82 unsigned int hfi1_cu = 1; 83 module_param_named(cu, hfi1_cu, uint, S_IRUGO); 84 MODULE_PARM_DESC(cu, "Credit return units"); 85 86 unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; 87 static int hfi1_caps_set(const char *val, const struct kernel_param *kp); 88 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp); 89 static const struct kernel_param_ops cap_ops = { 90 .set = hfi1_caps_set, 91 .get = hfi1_caps_get 92 }; 93 module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO); 94 MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features"); 95 96 MODULE_LICENSE("Dual BSD/GPL"); 97 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver"); 98 99 /* 100 * MAX_PKT_RCV is the max # if packets processed per receive interrupt. 101 */ 102 #define MAX_PKT_RECV 64 103 /* 104 * MAX_PKT_THREAD_RCV is the max # of packets processed before 105 * the qp_wait_list queue is flushed. 106 */ 107 #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4) 108 #define EGR_HEAD_UPDATE_THRESHOLD 16 109 110 struct hfi1_ib_stats hfi1_stats; 111 112 static int hfi1_caps_set(const char *val, const struct kernel_param *kp) 113 { 114 int ret = 0; 115 unsigned long *cap_mask_ptr = (unsigned long *)kp->arg, 116 cap_mask = *cap_mask_ptr, value, diff, 117 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) | 118 HFI1_CAP_WRITABLE_MASK); 119 120 ret = kstrtoul(val, 0, &value); 121 if (ret) { 122 pr_warn("Invalid module parameter value for 'cap_mask'\n"); 123 goto done; 124 } 125 /* Get the changed bits (except the locked bit) */ 126 diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK); 127 128 /* Remove any bits that are not allowed to change after driver load */ 129 if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) { 130 pr_warn("Ignoring non-writable capability bits %#lx\n", 131 diff & ~write_mask); 132 diff &= write_mask; 133 } 134 135 /* Mask off any reserved bits */ 136 diff &= ~HFI1_CAP_RESERVED_MASK; 137 /* Clear any previously set and changing bits */ 138 cap_mask &= ~diff; 139 /* Update the bits with the new capability */ 140 cap_mask |= (value & diff); 141 /* Check for any kernel/user restrictions */ 142 diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^ 143 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT); 144 cap_mask &= ~diff; 145 /* Set the bitmask to the final set */ 146 *cap_mask_ptr = cap_mask; 147 done: 148 return ret; 149 } 150 151 static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) 152 { 153 unsigned long cap_mask = *(unsigned long *)kp->arg; 154 155 cap_mask &= ~HFI1_CAP_LOCKED_SMASK; 156 cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT); 157 158 return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); 159 } 160 161 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) 162 { 163 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 164 struct hfi1_devdata *dd = container_of(ibdev, 165 struct hfi1_devdata, verbs_dev); 166 return dd->pcidev; 167 } 168 169 /* 170 * Return count of units with at least one port ACTIVE. 171 */ 172 int hfi1_count_active_units(void) 173 { 174 struct hfi1_devdata *dd; 175 struct hfi1_pportdata *ppd; 176 unsigned long index, flags; 177 int pidx, nunits_active = 0; 178 179 xa_lock_irqsave(&hfi1_dev_table, flags); 180 xa_for_each(&hfi1_dev_table, index, dd) { 181 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1) 182 continue; 183 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 184 ppd = dd->pport + pidx; 185 if (ppd->lid && ppd->linkup) { 186 nunits_active++; 187 break; 188 } 189 } 190 } 191 xa_unlock_irqrestore(&hfi1_dev_table, flags); 192 return nunits_active; 193 } 194 195 /* 196 * Get address of eager buffer from it's index (allocated in chunks, not 197 * contiguous). 198 */ 199 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, 200 u8 *update) 201 { 202 u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf); 203 204 *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset; 205 return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) + 206 (offset * RCV_BUF_BLOCK_SIZE)); 207 } 208 209 static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd, 210 __le32 *rhf_addr) 211 { 212 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr)); 213 214 return (void *)(rhf_addr - rcd->rhf_offset + offset); 215 } 216 217 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd, 218 __le32 *rhf_addr) 219 { 220 return (struct ib_header *)hfi1_get_header(rcd, rhf_addr); 221 } 222 223 static inline struct hfi1_16b_header 224 *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd, 225 __le32 *rhf_addr) 226 { 227 return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr); 228 } 229 230 /* 231 * Validate and encode the a given RcvArray Buffer size. 232 * The function will check whether the given size falls within 233 * allowed size ranges for the respective type and, optionally, 234 * return the proper encoding. 235 */ 236 int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded) 237 { 238 if (unlikely(!PAGE_ALIGNED(size))) 239 return 0; 240 if (unlikely(size < MIN_EAGER_BUFFER)) 241 return 0; 242 if (size > 243 (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER)) 244 return 0; 245 if (encoded) 246 *encoded = ilog2(size / PAGE_SIZE) + 1; 247 return 1; 248 } 249 250 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, 251 struct hfi1_packet *packet) 252 { 253 struct ib_header *rhdr = packet->hdr; 254 u32 rte = rhf_rcv_type_err(packet->rhf); 255 u32 mlid_base; 256 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 257 struct hfi1_devdata *dd = ppd->dd; 258 struct hfi1_ibdev *verbs_dev = &dd->verbs_dev; 259 struct rvt_dev_info *rdi = &verbs_dev->rdi; 260 261 if ((packet->rhf & RHF_DC_ERR) && 262 hfi1_dbg_fault_suppress_err(verbs_dev)) 263 return; 264 265 if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) 266 return; 267 268 if (packet->etype == RHF_RCV_TYPE_BYPASS) { 269 goto drop; 270 } else { 271 u8 lnh = ib_get_lnh(rhdr); 272 273 mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE); 274 if (lnh == HFI1_LRH_BTH) { 275 packet->ohdr = &rhdr->u.oth; 276 } else if (lnh == HFI1_LRH_GRH) { 277 packet->ohdr = &rhdr->u.l.oth; 278 packet->grh = &rhdr->u.l.grh; 279 } else { 280 goto drop; 281 } 282 } 283 284 if (packet->rhf & RHF_TID_ERR) { 285 /* For TIDERR and RC QPs preemptively schedule a NAK */ 286 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 287 u32 dlid = ib_get_dlid(rhdr); 288 u32 qp_num; 289 290 /* Sanity check packet */ 291 if (tlen < 24) 292 goto drop; 293 294 /* Check for GRH */ 295 if (packet->grh) { 296 u32 vtf; 297 struct ib_grh *grh = packet->grh; 298 299 if (grh->next_hdr != IB_GRH_NEXT_HDR) 300 goto drop; 301 vtf = be32_to_cpu(grh->version_tclass_flow); 302 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 303 goto drop; 304 } 305 306 /* Get the destination QP number. */ 307 qp_num = ib_bth_get_qpn(packet->ohdr); 308 if (dlid < mlid_base) { 309 struct rvt_qp *qp; 310 unsigned long flags; 311 312 rcu_read_lock(); 313 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 314 if (!qp) { 315 rcu_read_unlock(); 316 goto drop; 317 } 318 319 /* 320 * Handle only RC QPs - for other QP types drop error 321 * packet. 322 */ 323 spin_lock_irqsave(&qp->r_lock, flags); 324 325 /* Check for valid receive state. */ 326 if (!(ib_rvt_state_ops[qp->state] & 327 RVT_PROCESS_RECV_OK)) { 328 ibp->rvp.n_pkt_drops++; 329 } 330 331 switch (qp->ibqp.qp_type) { 332 case IB_QPT_RC: 333 hfi1_rc_hdrerr(rcd, packet, qp); 334 break; 335 default: 336 /* For now don't handle any other QP types */ 337 break; 338 } 339 340 spin_unlock_irqrestore(&qp->r_lock, flags); 341 rcu_read_unlock(); 342 } /* Unicast QP */ 343 } /* Valid packet with TIDErr */ 344 345 /* handle "RcvTypeErr" flags */ 346 switch (rte) { 347 case RHF_RTE_ERROR_OP_CODE_ERR: 348 { 349 void *ebuf = NULL; 350 u8 opcode; 351 352 if (rhf_use_egr_bfr(packet->rhf)) 353 ebuf = packet->ebuf; 354 355 if (!ebuf) 356 goto drop; /* this should never happen */ 357 358 opcode = ib_bth_get_opcode(packet->ohdr); 359 if (opcode == IB_OPCODE_CNP) { 360 /* 361 * Only in pre-B0 h/w is the CNP_OPCODE handled 362 * via this code path. 363 */ 364 struct rvt_qp *qp = NULL; 365 u32 lqpn, rqpn; 366 u16 rlid; 367 u8 svc_type, sl, sc5; 368 369 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); 370 sl = ibp->sc_to_sl[sc5]; 371 372 lqpn = ib_bth_get_qpn(packet->ohdr); 373 rcu_read_lock(); 374 qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); 375 if (!qp) { 376 rcu_read_unlock(); 377 goto drop; 378 } 379 380 switch (qp->ibqp.qp_type) { 381 case IB_QPT_UD: 382 rlid = 0; 383 rqpn = 0; 384 svc_type = IB_CC_SVCTYPE_UD; 385 break; 386 case IB_QPT_UC: 387 rlid = ib_get_slid(rhdr); 388 rqpn = qp->remote_qpn; 389 svc_type = IB_CC_SVCTYPE_UC; 390 break; 391 default: 392 rcu_read_unlock(); 393 goto drop; 394 } 395 396 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 397 rcu_read_unlock(); 398 } 399 400 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; 401 break; 402 } 403 default: 404 break; 405 } 406 407 drop: 408 return; 409 } 410 411 static inline void init_packet(struct hfi1_ctxtdata *rcd, 412 struct hfi1_packet *packet) 413 { 414 packet->rsize = rcd->rcvhdrqentsize; /* words */ 415 packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ 416 packet->rcd = rcd; 417 packet->updegr = 0; 418 packet->etail = -1; 419 packet->rhf_addr = get_rhf_addr(rcd); 420 packet->rhf = rhf_to_cpu(packet->rhf_addr); 421 packet->rhqoff = rcd->head; 422 packet->numpkt = 0; 423 } 424 425 /* We support only two types - 9B and 16B for now */ 426 static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = { 427 [HFI1_PKT_TYPE_9B] = &return_cnp, 428 [HFI1_PKT_TYPE_16B] = &return_cnp_16B 429 }; 430 431 /** 432 * hfi1_process_ecn_slowpath - Process FECN or BECN bits 433 * @qp: The packet's destination QP 434 * @pkt: The packet itself. 435 * @prescan: Is the caller the RXQ prescan 436 * 437 * Process the packet's FECN or BECN bits. By now, the packet 438 * has already been evaluated whether processing of those bit should 439 * be done. 440 * The significance of the @prescan argument is that if the caller 441 * is the RXQ prescan, a CNP will be send out instead of waiting for the 442 * normal packet processing to send an ACK with BECN set (or a CNP). 443 */ 444 bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, 445 bool prescan) 446 { 447 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 448 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 449 struct ib_other_headers *ohdr = pkt->ohdr; 450 struct ib_grh *grh = pkt->grh; 451 u32 rqpn = 0; 452 u16 pkey; 453 u32 rlid, slid, dlid = 0; 454 u8 hdr_type, sc, svc_type, opcode; 455 bool is_mcast = false, ignore_fecn = false, do_cnp = false, 456 fecn, becn; 457 458 /* can be called from prescan */ 459 if (pkt->etype == RHF_RCV_TYPE_BYPASS) { 460 pkey = hfi1_16B_get_pkey(pkt->hdr); 461 sc = hfi1_16B_get_sc(pkt->hdr); 462 dlid = hfi1_16B_get_dlid(pkt->hdr); 463 slid = hfi1_16B_get_slid(pkt->hdr); 464 is_mcast = hfi1_is_16B_mcast(dlid); 465 opcode = ib_bth_get_opcode(ohdr); 466 hdr_type = HFI1_PKT_TYPE_16B; 467 fecn = hfi1_16B_get_fecn(pkt->hdr); 468 becn = hfi1_16B_get_becn(pkt->hdr); 469 } else { 470 pkey = ib_bth_get_pkey(ohdr); 471 sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf); 472 dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) : 473 ppd->lid; 474 slid = ib_get_slid(pkt->hdr); 475 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) && 476 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)); 477 opcode = ib_bth_get_opcode(ohdr); 478 hdr_type = HFI1_PKT_TYPE_9B; 479 fecn = ib_bth_get_fecn(ohdr); 480 becn = ib_bth_get_becn(ohdr); 481 } 482 483 switch (qp->ibqp.qp_type) { 484 case IB_QPT_UD: 485 rlid = slid; 486 rqpn = ib_get_sqpn(pkt->ohdr); 487 svc_type = IB_CC_SVCTYPE_UD; 488 break; 489 case IB_QPT_SMI: 490 case IB_QPT_GSI: 491 rlid = slid; 492 rqpn = ib_get_sqpn(pkt->ohdr); 493 svc_type = IB_CC_SVCTYPE_UD; 494 break; 495 case IB_QPT_UC: 496 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 497 rqpn = qp->remote_qpn; 498 svc_type = IB_CC_SVCTYPE_UC; 499 break; 500 case IB_QPT_RC: 501 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 502 rqpn = qp->remote_qpn; 503 svc_type = IB_CC_SVCTYPE_RC; 504 break; 505 default: 506 return false; 507 } 508 509 ignore_fecn = is_mcast || (opcode == IB_OPCODE_CNP) || 510 (opcode == IB_OPCODE_RC_ACKNOWLEDGE); 511 /* 512 * ACKNOWLEDGE packets do not get a CNP but this will be 513 * guarded by ignore_fecn above. 514 */ 515 do_cnp = prescan || 516 (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST && 517 opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE); 518 519 /* Call appropriate CNP handler */ 520 if (!ignore_fecn && do_cnp && fecn) 521 hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey, 522 dlid, rlid, sc, grh); 523 524 if (becn) { 525 u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; 526 u8 sl = ibp->sc_to_sl[sc]; 527 528 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 529 } 530 return !ignore_fecn && fecn; 531 } 532 533 struct ps_mdata { 534 struct hfi1_ctxtdata *rcd; 535 u32 rsize; 536 u32 maxcnt; 537 u32 ps_head; 538 u32 ps_tail; 539 u32 ps_seq; 540 }; 541 542 static inline void init_ps_mdata(struct ps_mdata *mdata, 543 struct hfi1_packet *packet) 544 { 545 struct hfi1_ctxtdata *rcd = packet->rcd; 546 547 mdata->rcd = rcd; 548 mdata->rsize = packet->rsize; 549 mdata->maxcnt = packet->maxcnt; 550 mdata->ps_head = packet->rhqoff; 551 552 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 553 mdata->ps_tail = get_rcvhdrtail(rcd); 554 if (rcd->ctxt == HFI1_CTRL_CTXT) 555 mdata->ps_seq = rcd->seq_cnt; 556 else 557 mdata->ps_seq = 0; /* not used with DMA_RTAIL */ 558 } else { 559 mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ 560 mdata->ps_seq = rcd->seq_cnt; 561 } 562 } 563 564 static inline int ps_done(struct ps_mdata *mdata, u64 rhf, 565 struct hfi1_ctxtdata *rcd) 566 { 567 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 568 return mdata->ps_head == mdata->ps_tail; 569 return mdata->ps_seq != rhf_rcv_seq(rhf); 570 } 571 572 static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, 573 struct hfi1_ctxtdata *rcd) 574 { 575 /* 576 * Control context can potentially receive an invalid rhf. 577 * Drop such packets. 578 */ 579 if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) 580 return mdata->ps_seq != rhf_rcv_seq(rhf); 581 582 return 0; 583 } 584 585 static inline void update_ps_mdata(struct ps_mdata *mdata, 586 struct hfi1_ctxtdata *rcd) 587 { 588 mdata->ps_head += mdata->rsize; 589 if (mdata->ps_head >= mdata->maxcnt) 590 mdata->ps_head = 0; 591 592 /* Control context must do seq counting */ 593 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || 594 (rcd->ctxt == HFI1_CTRL_CTXT)) { 595 if (++mdata->ps_seq > 13) 596 mdata->ps_seq = 1; 597 } 598 } 599 600 /* 601 * prescan_rxq - search through the receive queue looking for packets 602 * containing Excplicit Congestion Notifications (FECNs, or BECNs). 603 * When an ECN is found, process the Congestion Notification, and toggle 604 * it off. 605 * This is declared as a macro to allow quick checking of the port to avoid 606 * the overhead of a function call if not enabled. 607 */ 608 #define prescan_rxq(rcd, packet) \ 609 do { \ 610 if (rcd->ppd->cc_prescan) \ 611 __prescan_rxq(packet); \ 612 } while (0) 613 static void __prescan_rxq(struct hfi1_packet *packet) 614 { 615 struct hfi1_ctxtdata *rcd = packet->rcd; 616 struct ps_mdata mdata; 617 618 init_ps_mdata(&mdata, packet); 619 620 while (1) { 621 struct hfi1_ibport *ibp = rcd_to_iport(rcd); 622 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 623 packet->rcd->rhf_offset; 624 struct rvt_qp *qp; 625 struct ib_header *hdr; 626 struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi; 627 u64 rhf = rhf_to_cpu(rhf_addr); 628 u32 etype = rhf_rcv_type(rhf), qpn, bth1; 629 u8 lnh; 630 631 if (ps_done(&mdata, rhf, rcd)) 632 break; 633 634 if (ps_skip(&mdata, rhf, rcd)) 635 goto next; 636 637 if (etype != RHF_RCV_TYPE_IB) 638 goto next; 639 640 packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr); 641 hdr = packet->hdr; 642 lnh = ib_get_lnh(hdr); 643 644 if (lnh == HFI1_LRH_BTH) { 645 packet->ohdr = &hdr->u.oth; 646 packet->grh = NULL; 647 } else if (lnh == HFI1_LRH_GRH) { 648 packet->ohdr = &hdr->u.l.oth; 649 packet->grh = &hdr->u.l.grh; 650 } else { 651 goto next; /* just in case */ 652 } 653 654 if (!hfi1_may_ecn(packet)) 655 goto next; 656 657 bth1 = be32_to_cpu(packet->ohdr->bth[1]); 658 qpn = bth1 & RVT_QPN_MASK; 659 rcu_read_lock(); 660 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); 661 662 if (!qp) { 663 rcu_read_unlock(); 664 goto next; 665 } 666 667 hfi1_process_ecn_slowpath(qp, packet, true); 668 rcu_read_unlock(); 669 670 /* turn off BECN, FECN */ 671 bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK); 672 packet->ohdr->bth[1] = cpu_to_be32(bth1); 673 next: 674 update_ps_mdata(&mdata, rcd); 675 } 676 } 677 678 static void process_rcv_qp_work(struct hfi1_packet *packet) 679 { 680 struct rvt_qp *qp, *nqp; 681 struct hfi1_ctxtdata *rcd = packet->rcd; 682 683 /* 684 * Iterate over all QPs waiting to respond. 685 * The list won't change since the IRQ is only run on one CPU. 686 */ 687 list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { 688 list_del_init(&qp->rspwait); 689 if (qp->r_flags & RVT_R_RSP_NAK) { 690 qp->r_flags &= ~RVT_R_RSP_NAK; 691 packet->qp = qp; 692 hfi1_send_rc_ack(packet, 0); 693 } 694 if (qp->r_flags & RVT_R_RSP_SEND) { 695 unsigned long flags; 696 697 qp->r_flags &= ~RVT_R_RSP_SEND; 698 spin_lock_irqsave(&qp->s_lock, flags); 699 if (ib_rvt_state_ops[qp->state] & 700 RVT_PROCESS_OR_FLUSH_SEND) 701 hfi1_schedule_send(qp); 702 spin_unlock_irqrestore(&qp->s_lock, flags); 703 } 704 rvt_put_qp(qp); 705 } 706 } 707 708 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) 709 { 710 if (thread) { 711 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) 712 /* allow defered processing */ 713 process_rcv_qp_work(packet); 714 cond_resched(); 715 return RCV_PKT_OK; 716 } else { 717 this_cpu_inc(*packet->rcd->dd->rcv_limit); 718 return RCV_PKT_LIMIT; 719 } 720 } 721 722 static inline int check_max_packet(struct hfi1_packet *packet, int thread) 723 { 724 int ret = RCV_PKT_OK; 725 726 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) 727 ret = max_packet_exceeded(packet, thread); 728 return ret; 729 } 730 731 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) 732 { 733 int ret; 734 735 /* Set up for the next packet */ 736 packet->rhqoff += packet->rsize; 737 if (packet->rhqoff >= packet->maxcnt) 738 packet->rhqoff = 0; 739 740 packet->numpkt++; 741 ret = check_max_packet(packet, thread); 742 743 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 744 packet->rcd->rhf_offset; 745 packet->rhf = rhf_to_cpu(packet->rhf_addr); 746 747 return ret; 748 } 749 750 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) 751 { 752 int ret; 753 754 packet->etype = rhf_rcv_type(packet->rhf); 755 756 /* total length */ 757 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 758 /* retrieve eager buffer details */ 759 packet->ebuf = NULL; 760 if (rhf_use_egr_bfr(packet->rhf)) { 761 packet->etail = rhf_egr_index(packet->rhf); 762 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, 763 &packet->updegr); 764 /* 765 * Prefetch the contents of the eager buffer. It is 766 * OK to send a negative length to prefetch_range(). 767 * The +2 is the size of the RHF. 768 */ 769 prefetch_range(packet->ebuf, 770 packet->tlen - ((packet->rcd->rcvhdrqentsize - 771 (rhf_hdrq_offset(packet->rhf) 772 + 2)) * 4)); 773 } 774 775 /* 776 * Call a type specific handler for the packet. We 777 * should be able to trust that etype won't be beyond 778 * the range of valid indexes. If so something is really 779 * wrong and we can probably just let things come 780 * crashing down. There is no need to eat another 781 * comparison in this performance critical code. 782 */ 783 packet->rcd->rhf_rcv_function_map[packet->etype](packet); 784 packet->numpkt++; 785 786 /* Set up for the next packet */ 787 packet->rhqoff += packet->rsize; 788 if (packet->rhqoff >= packet->maxcnt) 789 packet->rhqoff = 0; 790 791 ret = check_max_packet(packet, thread); 792 793 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 794 packet->rcd->rhf_offset; 795 packet->rhf = rhf_to_cpu(packet->rhf_addr); 796 797 return ret; 798 } 799 800 static inline void process_rcv_update(int last, struct hfi1_packet *packet) 801 { 802 /* 803 * Update head regs etc., every 16 packets, if not last pkt, 804 * to help prevent rcvhdrq overflows, when many packets 805 * are processed and queue is nearly full. 806 * Don't request an interrupt for intermediate updates. 807 */ 808 if (!last && !(packet->numpkt & 0xf)) { 809 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, 810 packet->etail, 0, 0); 811 packet->updegr = 0; 812 } 813 packet->grh = NULL; 814 } 815 816 static inline void finish_packet(struct hfi1_packet *packet) 817 { 818 /* 819 * Nothing we need to free for the packet. 820 * 821 * The only thing we need to do is a final update and call for an 822 * interrupt 823 */ 824 update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, 825 packet->etail, rcv_intr_dynamic, packet->numpkt); 826 } 827 828 /* 829 * Handle receive interrupts when using the no dma rtail option. 830 */ 831 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) 832 { 833 u32 seq; 834 int last = RCV_PKT_OK; 835 struct hfi1_packet packet; 836 837 init_packet(rcd, &packet); 838 seq = rhf_rcv_seq(packet.rhf); 839 if (seq != rcd->seq_cnt) { 840 last = RCV_PKT_DONE; 841 goto bail; 842 } 843 844 prescan_rxq(rcd, &packet); 845 846 while (last == RCV_PKT_OK) { 847 last = process_rcv_packet(&packet, thread); 848 seq = rhf_rcv_seq(packet.rhf); 849 if (++rcd->seq_cnt > 13) 850 rcd->seq_cnt = 1; 851 if (seq != rcd->seq_cnt) 852 last = RCV_PKT_DONE; 853 process_rcv_update(last, &packet); 854 } 855 process_rcv_qp_work(&packet); 856 rcd->head = packet.rhqoff; 857 bail: 858 finish_packet(&packet); 859 return last; 860 } 861 862 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) 863 { 864 u32 hdrqtail; 865 int last = RCV_PKT_OK; 866 struct hfi1_packet packet; 867 868 init_packet(rcd, &packet); 869 hdrqtail = get_rcvhdrtail(rcd); 870 if (packet.rhqoff == hdrqtail) { 871 last = RCV_PKT_DONE; 872 goto bail; 873 } 874 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 875 876 prescan_rxq(rcd, &packet); 877 878 while (last == RCV_PKT_OK) { 879 last = process_rcv_packet(&packet, thread); 880 if (packet.rhqoff == hdrqtail) 881 last = RCV_PKT_DONE; 882 process_rcv_update(last, &packet); 883 } 884 process_rcv_qp_work(&packet); 885 rcd->head = packet.rhqoff; 886 bail: 887 finish_packet(&packet); 888 return last; 889 } 890 891 static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt) 892 { 893 struct hfi1_ctxtdata *rcd; 894 u16 i; 895 896 /* 897 * For dynamically allocated kernel contexts (like vnic) switch 898 * interrupt handler only for that context. Otherwise, switch 899 * interrupt handler for all statically allocated kernel contexts. 900 */ 901 if (ctxt >= dd->first_dyn_alloc_ctxt) { 902 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 903 if (rcd) { 904 rcd->do_interrupt = 905 &handle_receive_interrupt_nodma_rtail; 906 hfi1_rcd_put(rcd); 907 } 908 return; 909 } 910 911 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 912 rcd = hfi1_rcd_get_by_index(dd, i); 913 if (rcd) 914 rcd->do_interrupt = 915 &handle_receive_interrupt_nodma_rtail; 916 hfi1_rcd_put(rcd); 917 } 918 } 919 920 static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt) 921 { 922 struct hfi1_ctxtdata *rcd; 923 u16 i; 924 925 /* 926 * For dynamically allocated kernel contexts (like vnic) switch 927 * interrupt handler only for that context. Otherwise, switch 928 * interrupt handler for all statically allocated kernel contexts. 929 */ 930 if (ctxt >= dd->first_dyn_alloc_ctxt) { 931 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 932 if (rcd) { 933 rcd->do_interrupt = 934 &handle_receive_interrupt_dma_rtail; 935 hfi1_rcd_put(rcd); 936 } 937 return; 938 } 939 940 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 941 rcd = hfi1_rcd_get_by_index(dd, i); 942 if (rcd) 943 rcd->do_interrupt = 944 &handle_receive_interrupt_dma_rtail; 945 hfi1_rcd_put(rcd); 946 } 947 } 948 949 void set_all_slowpath(struct hfi1_devdata *dd) 950 { 951 struct hfi1_ctxtdata *rcd; 952 u16 i; 953 954 /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */ 955 for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) { 956 rcd = hfi1_rcd_get_by_index(dd, i); 957 if (!rcd) 958 continue; 959 if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic) 960 rcd->do_interrupt = &handle_receive_interrupt; 961 962 hfi1_rcd_put(rcd); 963 } 964 } 965 966 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, 967 struct hfi1_packet *packet, 968 struct hfi1_devdata *dd) 969 { 970 struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; 971 u8 etype = rhf_rcv_type(packet->rhf); 972 u8 sc = SC15_PACKET; 973 974 if (etype == RHF_RCV_TYPE_IB) { 975 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd, 976 packet->rhf_addr); 977 sc = hfi1_9B_get_sc5(hdr, packet->rhf); 978 } else if (etype == RHF_RCV_TYPE_BYPASS) { 979 struct hfi1_16b_header *hdr = hfi1_get_16B_header( 980 packet->rcd, 981 packet->rhf_addr); 982 sc = hfi1_16B_get_sc(hdr); 983 } 984 if (sc != SC15_PACKET) { 985 int hwstate = driver_lstate(rcd->ppd); 986 987 if (hwstate != IB_PORT_ACTIVE) { 988 dd_dev_info(dd, 989 "Unexpected link state %s\n", 990 opa_lstate_name(hwstate)); 991 return 0; 992 } 993 994 queue_work(rcd->ppd->link_wq, lsaw); 995 return 1; 996 } 997 return 0; 998 } 999 1000 /* 1001 * handle_receive_interrupt - receive a packet 1002 * @rcd: the context 1003 * 1004 * Called from interrupt handler for errors or receive interrupt. 1005 * This is the slow path interrupt handler. 1006 */ 1007 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) 1008 { 1009 struct hfi1_devdata *dd = rcd->dd; 1010 u32 hdrqtail; 1011 int needset, last = RCV_PKT_OK; 1012 struct hfi1_packet packet; 1013 int skip_pkt = 0; 1014 1015 /* Control context will always use the slow path interrupt handler */ 1016 needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; 1017 1018 init_packet(rcd, &packet); 1019 1020 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 1021 u32 seq = rhf_rcv_seq(packet.rhf); 1022 1023 if (seq != rcd->seq_cnt) { 1024 last = RCV_PKT_DONE; 1025 goto bail; 1026 } 1027 hdrqtail = 0; 1028 } else { 1029 hdrqtail = get_rcvhdrtail(rcd); 1030 if (packet.rhqoff == hdrqtail) { 1031 last = RCV_PKT_DONE; 1032 goto bail; 1033 } 1034 smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 1035 1036 /* 1037 * Control context can potentially receive an invalid 1038 * rhf. Drop such packets. 1039 */ 1040 if (rcd->ctxt == HFI1_CTRL_CTXT) { 1041 u32 seq = rhf_rcv_seq(packet.rhf); 1042 1043 if (seq != rcd->seq_cnt) 1044 skip_pkt = 1; 1045 } 1046 } 1047 1048 prescan_rxq(rcd, &packet); 1049 1050 while (last == RCV_PKT_OK) { 1051 if (unlikely(dd->do_drop && 1052 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == 1053 DROP_PACKET_ON)) { 1054 dd->do_drop = 0; 1055 1056 /* On to the next packet */ 1057 packet.rhqoff += packet.rsize; 1058 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + 1059 packet.rhqoff + 1060 rcd->rhf_offset; 1061 packet.rhf = rhf_to_cpu(packet.rhf_addr); 1062 1063 } else if (skip_pkt) { 1064 last = skip_rcv_packet(&packet, thread); 1065 skip_pkt = 0; 1066 } else { 1067 /* Auto activate link on non-SC15 packet receive */ 1068 if (unlikely(rcd->ppd->host_link_state == 1069 HLS_UP_ARMED) && 1070 set_armed_to_active(rcd, &packet, dd)) 1071 goto bail; 1072 last = process_rcv_packet(&packet, thread); 1073 } 1074 1075 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 1076 u32 seq = rhf_rcv_seq(packet.rhf); 1077 1078 if (++rcd->seq_cnt > 13) 1079 rcd->seq_cnt = 1; 1080 if (seq != rcd->seq_cnt) 1081 last = RCV_PKT_DONE; 1082 if (needset) { 1083 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); 1084 set_nodma_rtail(dd, rcd->ctxt); 1085 needset = 0; 1086 } 1087 } else { 1088 if (packet.rhqoff == hdrqtail) 1089 last = RCV_PKT_DONE; 1090 /* 1091 * Control context can potentially receive an invalid 1092 * rhf. Drop such packets. 1093 */ 1094 if (rcd->ctxt == HFI1_CTRL_CTXT) { 1095 u32 seq = rhf_rcv_seq(packet.rhf); 1096 1097 if (++rcd->seq_cnt > 13) 1098 rcd->seq_cnt = 1; 1099 if (!last && (seq != rcd->seq_cnt)) 1100 skip_pkt = 1; 1101 } 1102 1103 if (needset) { 1104 dd_dev_info(dd, 1105 "Switching to DMA_RTAIL\n"); 1106 set_dma_rtail(dd, rcd->ctxt); 1107 needset = 0; 1108 } 1109 } 1110 1111 process_rcv_update(last, &packet); 1112 } 1113 1114 process_rcv_qp_work(&packet); 1115 rcd->head = packet.rhqoff; 1116 1117 bail: 1118 /* 1119 * Always write head at end, and setup rcv interrupt, even 1120 * if no packets were processed. 1121 */ 1122 finish_packet(&packet); 1123 return last; 1124 } 1125 1126 /* 1127 * We may discover in the interrupt that the hardware link state has 1128 * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet), 1129 * and we need to update the driver's notion of the link state. We cannot 1130 * run set_link_state from interrupt context, so we queue this function on 1131 * a workqueue. 1132 * 1133 * We delay the regular interrupt processing until after the state changes 1134 * so that the link will be in the correct state by the time any application 1135 * we wake up attempts to send a reply to any message it received. 1136 * (Subsequent receive interrupts may possibly force the wakeup before we 1137 * update the link state.) 1138 * 1139 * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes 1140 * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues, 1141 * so we're safe from use-after-free of the rcd. 1142 */ 1143 void receive_interrupt_work(struct work_struct *work) 1144 { 1145 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 1146 linkstate_active_work); 1147 struct hfi1_devdata *dd = ppd->dd; 1148 struct hfi1_ctxtdata *rcd; 1149 u16 i; 1150 1151 /* Received non-SC15 packet implies neighbor_normal */ 1152 ppd->neighbor_normal = 1; 1153 set_link_state(ppd, HLS_UP_ACTIVE); 1154 1155 /* 1156 * Interrupt all statically allocated kernel contexts that could 1157 * have had an interrupt during auto activation. 1158 */ 1159 for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) { 1160 rcd = hfi1_rcd_get_by_index(dd, i); 1161 if (rcd) 1162 force_recv_intr(rcd); 1163 hfi1_rcd_put(rcd); 1164 } 1165 } 1166 1167 /* 1168 * Convert a given MTU size to the on-wire MAD packet enumeration. 1169 * Return -1 if the size is invalid. 1170 */ 1171 int mtu_to_enum(u32 mtu, int default_if_bad) 1172 { 1173 switch (mtu) { 1174 case 0: return OPA_MTU_0; 1175 case 256: return OPA_MTU_256; 1176 case 512: return OPA_MTU_512; 1177 case 1024: return OPA_MTU_1024; 1178 case 2048: return OPA_MTU_2048; 1179 case 4096: return OPA_MTU_4096; 1180 case 8192: return OPA_MTU_8192; 1181 case 10240: return OPA_MTU_10240; 1182 } 1183 return default_if_bad; 1184 } 1185 1186 u16 enum_to_mtu(int mtu) 1187 { 1188 switch (mtu) { 1189 case OPA_MTU_0: return 0; 1190 case OPA_MTU_256: return 256; 1191 case OPA_MTU_512: return 512; 1192 case OPA_MTU_1024: return 1024; 1193 case OPA_MTU_2048: return 2048; 1194 case OPA_MTU_4096: return 4096; 1195 case OPA_MTU_8192: return 8192; 1196 case OPA_MTU_10240: return 10240; 1197 default: return 0xffff; 1198 } 1199 } 1200 1201 /* 1202 * set_mtu - set the MTU 1203 * @ppd: the per port data 1204 * 1205 * We can handle "any" incoming size, the issue here is whether we 1206 * need to restrict our outgoing size. We do not deal with what happens 1207 * to programs that are already running when the size changes. 1208 */ 1209 int set_mtu(struct hfi1_pportdata *ppd) 1210 { 1211 struct hfi1_devdata *dd = ppd->dd; 1212 int i, drain, ret = 0, is_up = 0; 1213 1214 ppd->ibmtu = 0; 1215 for (i = 0; i < ppd->vls_supported; i++) 1216 if (ppd->ibmtu < dd->vld[i].mtu) 1217 ppd->ibmtu = dd->vld[i].mtu; 1218 ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd); 1219 1220 mutex_lock(&ppd->hls_lock); 1221 if (ppd->host_link_state == HLS_UP_INIT || 1222 ppd->host_link_state == HLS_UP_ARMED || 1223 ppd->host_link_state == HLS_UP_ACTIVE) 1224 is_up = 1; 1225 1226 drain = !is_ax(dd) && is_up; 1227 1228 if (drain) 1229 /* 1230 * MTU is specified per-VL. To ensure that no packet gets 1231 * stuck (due, e.g., to the MTU for the packet's VL being 1232 * reduced), empty the per-VL FIFOs before adjusting MTU. 1233 */ 1234 ret = stop_drain_data_vls(dd); 1235 1236 if (ret) { 1237 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n", 1238 __func__); 1239 goto err; 1240 } 1241 1242 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0); 1243 1244 if (drain) 1245 open_fill_data_vls(dd); /* reopen all VLs */ 1246 1247 err: 1248 mutex_unlock(&ppd->hls_lock); 1249 1250 return ret; 1251 } 1252 1253 int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc) 1254 { 1255 struct hfi1_devdata *dd = ppd->dd; 1256 1257 ppd->lid = lid; 1258 ppd->lmc = lmc; 1259 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0); 1260 1261 dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid); 1262 1263 return 0; 1264 } 1265 1266 void shutdown_led_override(struct hfi1_pportdata *ppd) 1267 { 1268 struct hfi1_devdata *dd = ppd->dd; 1269 1270 /* 1271 * This pairs with the memory barrier in hfi1_start_led_override to 1272 * ensure that we read the correct state of LED beaconing represented 1273 * by led_override_timer_active 1274 */ 1275 smp_rmb(); 1276 if (atomic_read(&ppd->led_override_timer_active)) { 1277 del_timer_sync(&ppd->led_override_timer); 1278 atomic_set(&ppd->led_override_timer_active, 0); 1279 /* Ensure the atomic_set is visible to all CPUs */ 1280 smp_wmb(); 1281 } 1282 1283 /* Hand control of the LED to the DC for normal operation */ 1284 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 1285 } 1286 1287 static void run_led_override(struct timer_list *t) 1288 { 1289 struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer); 1290 struct hfi1_devdata *dd = ppd->dd; 1291 unsigned long timeout; 1292 int phase_idx; 1293 1294 if (!(dd->flags & HFI1_INITTED)) 1295 return; 1296 1297 phase_idx = ppd->led_override_phase & 1; 1298 1299 setextled(dd, phase_idx); 1300 1301 timeout = ppd->led_override_vals[phase_idx]; 1302 1303 /* Set up for next phase */ 1304 ppd->led_override_phase = !ppd->led_override_phase; 1305 1306 mod_timer(&ppd->led_override_timer, jiffies + timeout); 1307 } 1308 1309 /* 1310 * To have the LED blink in a particular pattern, provide timeon and timeoff 1311 * in milliseconds. 1312 * To turn off custom blinking and return to normal operation, use 1313 * shutdown_led_override() 1314 */ 1315 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, 1316 unsigned int timeoff) 1317 { 1318 if (!(ppd->dd->flags & HFI1_INITTED)) 1319 return; 1320 1321 /* Convert to jiffies for direct use in timer */ 1322 ppd->led_override_vals[0] = msecs_to_jiffies(timeoff); 1323 ppd->led_override_vals[1] = msecs_to_jiffies(timeon); 1324 1325 /* Arbitrarily start from LED on phase */ 1326 ppd->led_override_phase = 1; 1327 1328 /* 1329 * If the timer has not already been started, do so. Use a "quick" 1330 * timeout so the handler will be called soon to look at our request. 1331 */ 1332 if (!timer_pending(&ppd->led_override_timer)) { 1333 timer_setup(&ppd->led_override_timer, run_led_override, 0); 1334 ppd->led_override_timer.expires = jiffies + 1; 1335 add_timer(&ppd->led_override_timer); 1336 atomic_set(&ppd->led_override_timer_active, 1); 1337 /* Ensure the atomic_set is visible to all CPUs */ 1338 smp_wmb(); 1339 } 1340 } 1341 1342 /** 1343 * hfi1_reset_device - reset the chip if possible 1344 * @unit: the device to reset 1345 * 1346 * Whether or not reset is successful, we attempt to re-initialize the chip 1347 * (that is, much like a driver unload/reload). We clear the INITTED flag 1348 * so that the various entry points will fail until we reinitialize. For 1349 * now, we only allow this if no user contexts are open that use chip resources 1350 */ 1351 int hfi1_reset_device(int unit) 1352 { 1353 int ret; 1354 struct hfi1_devdata *dd = hfi1_lookup(unit); 1355 struct hfi1_pportdata *ppd; 1356 int pidx; 1357 1358 if (!dd) { 1359 ret = -ENODEV; 1360 goto bail; 1361 } 1362 1363 dd_dev_info(dd, "Reset on unit %u requested\n", unit); 1364 1365 if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) { 1366 dd_dev_info(dd, 1367 "Invalid unit number %u or not initialized or not present\n", 1368 unit); 1369 ret = -ENXIO; 1370 goto bail; 1371 } 1372 1373 /* If there are any user/vnic contexts, we cannot reset */ 1374 mutex_lock(&hfi1_mutex); 1375 if (dd->rcd) 1376 if (hfi1_stats.sps_ctxts) { 1377 mutex_unlock(&hfi1_mutex); 1378 ret = -EBUSY; 1379 goto bail; 1380 } 1381 mutex_unlock(&hfi1_mutex); 1382 1383 for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1384 ppd = dd->pport + pidx; 1385 1386 shutdown_led_override(ppd); 1387 } 1388 if (dd->flags & HFI1_HAS_SEND_DMA) 1389 sdma_exit(dd); 1390 1391 hfi1_reset_cpu_counters(dd); 1392 1393 ret = hfi1_init(dd, 1); 1394 1395 if (ret) 1396 dd_dev_err(dd, 1397 "Reinitialize unit %u after reset failed with %d\n", 1398 unit, ret); 1399 else 1400 dd_dev_info(dd, "Reinitialized unit %u after resetting\n", 1401 unit); 1402 1403 bail: 1404 return ret; 1405 } 1406 1407 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) 1408 { 1409 packet->hdr = (struct hfi1_ib_message_header *) 1410 hfi1_get_msgheader(packet->rcd, 1411 packet->rhf_addr); 1412 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; 1413 } 1414 1415 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet) 1416 { 1417 struct hfi1_pportdata *ppd = packet->rcd->ppd; 1418 1419 /* slid and dlid cannot be 0 */ 1420 if ((!packet->slid) || (!packet->dlid)) 1421 return -EINVAL; 1422 1423 /* Compare port lid with incoming packet dlid */ 1424 if ((!(hfi1_is_16B_mcast(packet->dlid))) && 1425 (packet->dlid != 1426 opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) { 1427 if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid) 1428 return -EINVAL; 1429 } 1430 1431 /* No multicast packets with SC15 */ 1432 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF)) 1433 return -EINVAL; 1434 1435 /* Packets with permissive DLID always on SC15 */ 1436 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 1437 16B)) && 1438 (packet->sc != 0xF)) 1439 return -EINVAL; 1440 1441 return 0; 1442 } 1443 1444 static int hfi1_setup_9B_packet(struct hfi1_packet *packet) 1445 { 1446 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); 1447 struct ib_header *hdr; 1448 u8 lnh; 1449 1450 hfi1_setup_ib_header(packet); 1451 hdr = packet->hdr; 1452 1453 lnh = ib_get_lnh(hdr); 1454 if (lnh == HFI1_LRH_BTH) { 1455 packet->ohdr = &hdr->u.oth; 1456 packet->grh = NULL; 1457 } else if (lnh == HFI1_LRH_GRH) { 1458 u32 vtf; 1459 1460 packet->ohdr = &hdr->u.l.oth; 1461 packet->grh = &hdr->u.l.grh; 1462 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 1463 goto drop; 1464 vtf = be32_to_cpu(packet->grh->version_tclass_flow); 1465 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 1466 goto drop; 1467 } else { 1468 goto drop; 1469 } 1470 1471 /* Query commonly used fields from packet header */ 1472 packet->payload = packet->ebuf; 1473 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1474 packet->slid = ib_get_slid(hdr); 1475 packet->dlid = ib_get_dlid(hdr); 1476 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && 1477 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE)))) 1478 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 1479 be16_to_cpu(IB_MULTICAST_LID_BASE); 1480 packet->sl = ib_get_sl(hdr); 1481 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); 1482 packet->pad = ib_bth_get_pad(packet->ohdr); 1483 packet->extra_byte = 0; 1484 packet->pkey = ib_bth_get_pkey(packet->ohdr); 1485 packet->migrated = ib_bth_is_migration(packet->ohdr); 1486 1487 return 0; 1488 drop: 1489 ibp->rvp.n_pkt_drops++; 1490 return -EINVAL; 1491 } 1492 1493 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) 1494 { 1495 /* 1496 * Bypass packets have a different header/payload split 1497 * compared to an IB packet. 1498 * Current split is set such that 16 bytes of the actual 1499 * header is in the header buffer and the remining is in 1500 * the eager buffer. We chose 16 since hfi1 driver only 1501 * supports 16B bypass packets and we will be able to 1502 * receive the entire LRH with such a split. 1503 */ 1504 1505 struct hfi1_ctxtdata *rcd = packet->rcd; 1506 struct hfi1_pportdata *ppd = rcd->ppd; 1507 struct hfi1_ibport *ibp = &ppd->ibport_data; 1508 u8 l4; 1509 1510 packet->hdr = (struct hfi1_16b_header *) 1511 hfi1_get_16B_header(packet->rcd, 1512 packet->rhf_addr); 1513 l4 = hfi1_16B_get_l4(packet->hdr); 1514 if (l4 == OPA_16B_L4_IB_LOCAL) { 1515 packet->ohdr = packet->ebuf; 1516 packet->grh = NULL; 1517 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1518 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 1519 /* hdr_len_by_opcode already has an IB LRH factored in */ 1520 packet->hlen = hdr_len_by_opcode[packet->opcode] + 1521 (LRH_16B_BYTES - LRH_9B_BYTES); 1522 packet->migrated = opa_bth_is_migration(packet->ohdr); 1523 } else if (l4 == OPA_16B_L4_IB_GLOBAL) { 1524 u32 vtf; 1525 u8 grh_len = sizeof(struct ib_grh); 1526 1527 packet->ohdr = packet->ebuf + grh_len; 1528 packet->grh = packet->ebuf; 1529 packet->opcode = ib_bth_get_opcode(packet->ohdr); 1530 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 1531 /* hdr_len_by_opcode already has an IB LRH factored in */ 1532 packet->hlen = hdr_len_by_opcode[packet->opcode] + 1533 (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len; 1534 packet->migrated = opa_bth_is_migration(packet->ohdr); 1535 1536 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 1537 goto drop; 1538 vtf = be32_to_cpu(packet->grh->version_tclass_flow); 1539 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 1540 goto drop; 1541 } else if (l4 == OPA_16B_L4_FM) { 1542 packet->mgmt = packet->ebuf; 1543 packet->ohdr = NULL; 1544 packet->grh = NULL; 1545 packet->opcode = IB_OPCODE_UD_SEND_ONLY; 1546 packet->pad = OPA_16B_L4_FM_PAD; 1547 packet->hlen = OPA_16B_L4_FM_HLEN; 1548 packet->migrated = false; 1549 } else { 1550 goto drop; 1551 } 1552 1553 /* Query commonly used fields from packet header */ 1554 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; 1555 packet->slid = hfi1_16B_get_slid(packet->hdr); 1556 packet->dlid = hfi1_16B_get_dlid(packet->hdr); 1557 if (unlikely(hfi1_is_16B_mcast(packet->dlid))) 1558 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 1559 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 1560 16B); 1561 packet->sc = hfi1_16B_get_sc(packet->hdr); 1562 packet->sl = ibp->sc_to_sl[packet->sc]; 1563 packet->extra_byte = SIZE_OF_LT; 1564 packet->pkey = hfi1_16B_get_pkey(packet->hdr); 1565 1566 if (hfi1_bypass_ingress_pkt_check(packet)) 1567 goto drop; 1568 1569 return 0; 1570 drop: 1571 hfi1_cdbg(PKT, "%s: packet dropped\n", __func__); 1572 ibp->rvp.n_pkt_drops++; 1573 return -EINVAL; 1574 } 1575 1576 static void show_eflags_errs(struct hfi1_packet *packet) 1577 { 1578 struct hfi1_ctxtdata *rcd = packet->rcd; 1579 u32 rte = rhf_rcv_type_err(packet->rhf); 1580 1581 dd_dev_err(rcd->dd, 1582 "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", 1583 rcd->ctxt, packet->rhf, 1584 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", 1585 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", 1586 packet->rhf & RHF_DC_ERR ? "dc " : "", 1587 packet->rhf & RHF_TID_ERR ? "tid " : "", 1588 packet->rhf & RHF_LEN_ERR ? "len " : "", 1589 packet->rhf & RHF_ECC_ERR ? "ecc " : "", 1590 packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", 1591 packet->rhf & RHF_ICRC_ERR ? "icrc " : "", 1592 rte); 1593 } 1594 1595 void handle_eflags(struct hfi1_packet *packet) 1596 { 1597 struct hfi1_ctxtdata *rcd = packet->rcd; 1598 1599 rcv_hdrerr(rcd, rcd->ppd, packet); 1600 if (rhf_err_flags(packet->rhf)) 1601 show_eflags_errs(packet); 1602 } 1603 1604 /* 1605 * The following functions are called by the interrupt handler. They are type 1606 * specific handlers for each packet type. 1607 */ 1608 static int process_receive_ib(struct hfi1_packet *packet) 1609 { 1610 if (hfi1_setup_9B_packet(packet)) 1611 return RHF_RCV_CONTINUE; 1612 1613 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1614 return RHF_RCV_CONTINUE; 1615 1616 trace_hfi1_rcvhdr(packet); 1617 1618 if (unlikely(rhf_err_flags(packet->rhf))) { 1619 handle_eflags(packet); 1620 return RHF_RCV_CONTINUE; 1621 } 1622 1623 hfi1_ib_rcv(packet); 1624 return RHF_RCV_CONTINUE; 1625 } 1626 1627 static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) 1628 { 1629 /* Packet received in VNIC context via RSM */ 1630 if (packet->rcd->is_vnic) 1631 return true; 1632 1633 if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) && 1634 (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR)) 1635 return true; 1636 1637 return false; 1638 } 1639 1640 static int process_receive_bypass(struct hfi1_packet *packet) 1641 { 1642 struct hfi1_devdata *dd = packet->rcd->dd; 1643 1644 if (hfi1_is_vnic_packet(packet)) { 1645 hfi1_vnic_bypass_rcv(packet); 1646 return RHF_RCV_CONTINUE; 1647 } 1648 1649 if (hfi1_setup_bypass_packet(packet)) 1650 return RHF_RCV_CONTINUE; 1651 1652 trace_hfi1_rcvhdr(packet); 1653 1654 if (unlikely(rhf_err_flags(packet->rhf))) { 1655 handle_eflags(packet); 1656 return RHF_RCV_CONTINUE; 1657 } 1658 1659 if (hfi1_16B_get_l2(packet->hdr) == 0x2) { 1660 hfi1_16B_rcv(packet); 1661 } else { 1662 dd_dev_err(dd, 1663 "Bypass packets other than 16B are not supported in normal operation. Dropping\n"); 1664 incr_cntr64(&dd->sw_rcv_bypass_packet_errors); 1665 if (!(dd->err_info_rcvport.status_and_code & 1666 OPA_EI_STATUS_SMASK)) { 1667 u64 *flits = packet->ebuf; 1668 1669 if (flits && !(packet->rhf & RHF_LEN_ERR)) { 1670 dd->err_info_rcvport.packet_flit1 = flits[0]; 1671 dd->err_info_rcvport.packet_flit2 = 1672 packet->tlen > sizeof(flits[0]) ? 1673 flits[1] : 0; 1674 } 1675 dd->err_info_rcvport.status_and_code |= 1676 (OPA_EI_STATUS_SMASK | BAD_L2_ERR); 1677 } 1678 } 1679 return RHF_RCV_CONTINUE; 1680 } 1681 1682 static int process_receive_error(struct hfi1_packet *packet) 1683 { 1684 /* KHdrHCRCErr -- KDETH packet with a bad HCRC */ 1685 if (unlikely( 1686 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && 1687 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || 1688 packet->rhf & RHF_DC_ERR))) 1689 return RHF_RCV_CONTINUE; 1690 1691 hfi1_setup_ib_header(packet); 1692 handle_eflags(packet); 1693 1694 if (unlikely(rhf_err_flags(packet->rhf))) 1695 dd_dev_err(packet->rcd->dd, 1696 "Unhandled error packet received. Dropping.\n"); 1697 1698 return RHF_RCV_CONTINUE; 1699 } 1700 1701 static int kdeth_process_expected(struct hfi1_packet *packet) 1702 { 1703 hfi1_setup_9B_packet(packet); 1704 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1705 return RHF_RCV_CONTINUE; 1706 1707 if (unlikely(rhf_err_flags(packet->rhf))) { 1708 struct hfi1_ctxtdata *rcd = packet->rcd; 1709 1710 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) 1711 return RHF_RCV_CONTINUE; 1712 } 1713 1714 hfi1_kdeth_expected_rcv(packet); 1715 return RHF_RCV_CONTINUE; 1716 } 1717 1718 static int kdeth_process_eager(struct hfi1_packet *packet) 1719 { 1720 hfi1_setup_9B_packet(packet); 1721 if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1722 return RHF_RCV_CONTINUE; 1723 1724 trace_hfi1_rcvhdr(packet); 1725 if (unlikely(rhf_err_flags(packet->rhf))) { 1726 struct hfi1_ctxtdata *rcd = packet->rcd; 1727 1728 show_eflags_errs(packet); 1729 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) 1730 return RHF_RCV_CONTINUE; 1731 } 1732 1733 hfi1_kdeth_eager_rcv(packet); 1734 return RHF_RCV_CONTINUE; 1735 } 1736 1737 static int process_receive_invalid(struct hfi1_packet *packet) 1738 { 1739 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", 1740 rhf_rcv_type(packet->rhf)); 1741 return RHF_RCV_CONTINUE; 1742 } 1743 1744 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) 1745 { 1746 struct hfi1_packet packet; 1747 struct ps_mdata mdata; 1748 1749 seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n", 1750 rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize, 1751 HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? 1752 "dma_rtail" : "nodma_rtail", 1753 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) & 1754 RCV_HDR_HEAD_HEAD_MASK, 1755 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL)); 1756 1757 init_packet(rcd, &packet); 1758 init_ps_mdata(&mdata, &packet); 1759 1760 while (1) { 1761 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 1762 rcd->rhf_offset; 1763 struct ib_header *hdr; 1764 u64 rhf = rhf_to_cpu(rhf_addr); 1765 u32 etype = rhf_rcv_type(rhf), qpn; 1766 u8 opcode; 1767 u32 psn; 1768 u8 lnh; 1769 1770 if (ps_done(&mdata, rhf, rcd)) 1771 break; 1772 1773 if (ps_skip(&mdata, rhf, rcd)) 1774 goto next; 1775 1776 if (etype > RHF_RCV_TYPE_IB) 1777 goto next; 1778 1779 packet.hdr = hfi1_get_msgheader(rcd, rhf_addr); 1780 hdr = packet.hdr; 1781 1782 lnh = be16_to_cpu(hdr->lrh[0]) & 3; 1783 1784 if (lnh == HFI1_LRH_BTH) 1785 packet.ohdr = &hdr->u.oth; 1786 else if (lnh == HFI1_LRH_GRH) 1787 packet.ohdr = &hdr->u.l.oth; 1788 else 1789 goto next; /* just in case */ 1790 1791 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24); 1792 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK; 1793 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2])); 1794 1795 seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n", 1796 mdata.ps_head, opcode, qpn, psn); 1797 next: 1798 update_ps_mdata(&mdata, rcd); 1799 } 1800 } 1801 1802 const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = { 1803 [RHF_RCV_TYPE_EXPECTED] = kdeth_process_expected, 1804 [RHF_RCV_TYPE_EAGER] = kdeth_process_eager, 1805 [RHF_RCV_TYPE_IB] = process_receive_ib, 1806 [RHF_RCV_TYPE_ERROR] = process_receive_error, 1807 [RHF_RCV_TYPE_BYPASS] = process_receive_bypass, 1808 [RHF_RCV_TYPE_INVALID5] = process_receive_invalid, 1809 [RHF_RCV_TYPE_INVALID6] = process_receive_invalid, 1810 [RHF_RCV_TYPE_INVALID7] = process_receive_invalid, 1811 }; 1812