verbs.c (bec7c79cd8f764ba84c8ec6d8c402b8a7cd3a54f) verbs.c (8e959601996dc645f4ed7004482a1667c27deb39)
1/*
2 * Copyright(c) 2015 - 2017 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *

--- 494 unchanged lines hidden (view full) ---

503 length = 8;
504 goto again;
505 }
506}
507
508/*
509 * Make sure the QP is ready and able to accept the given opcode.
510 */
1/*
2 * Copyright(c) 2015 - 2017 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *

--- 494 unchanged lines hidden (view full) ---

503 length = 8;
504 goto again;
505 }
506}
507
508/*
509 * Make sure the QP is ready and able to accept the given opcode.
510 */
511static inline opcode_handler qp_ok(struct hfi1_packet *packet)
511static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet)
512{
513 if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
514 return NULL;
512{
513 if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
514 return NULL;
515 if (((packet->opcode & RVT_OPCODE_QP_MASK) ==
516 packet->qp->allowed_ops) ||
517 (packet->opcode == IB_OPCODE_CNP))
518 return opcode_handler_tbl[packet->opcode];
515 if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) ||
516 (opcode == IB_OPCODE_CNP))
517 return opcode_handler_tbl[opcode];
519
520 return NULL;
521}
522
523static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc)
524{
525#ifdef CONFIG_FAULT_INJECTION
526 if ((opcode & IB_OPCODE_MSP) == IB_OPCODE_MSP)

--- 17 unchanged lines hidden (view full) ---

544 * a 'late ebp error' will be
545 * triggered and will be dropped.
546 */
547 pbc |= PBC_TEST_EBP;
548#endif
549 return pbc;
550}
551
518
519 return NULL;
520}
521
522static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc)
523{
524#ifdef CONFIG_FAULT_INJECTION
525 if ((opcode & IB_OPCODE_MSP) == IB_OPCODE_MSP)

--- 17 unchanged lines hidden (view full) ---

543 * a 'late ebp error' will be
544 * triggered and will be dropped.
545 */
546 pbc |= PBC_TEST_EBP;
547#endif
548 return pbc;
549}
550
552static inline void hfi1_handle_packet(struct hfi1_packet *packet,
553 bool is_mcast)
551/**
552 * hfi1_ib_rcv - process an incoming packet
553 * @packet: data packet information
554 *
555 * This is called to process an incoming packet at interrupt level.
556 *
557 * Tlen is the length of the header + data + CRC in bytes.
558 */
559void hfi1_ib_rcv(struct hfi1_packet *packet)
554{
560{
555 u32 qp_num;
556 struct hfi1_ctxtdata *rcd = packet->rcd;
561 struct hfi1_ctxtdata *rcd = packet->rcd;
562 struct ib_header *hdr = packet->hdr;
563 u32 tlen = packet->tlen;
557 struct hfi1_pportdata *ppd = rcd->ppd;
558 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
559 struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
560 opcode_handler packet_handler;
561 unsigned long flags;
564 struct hfi1_pportdata *ppd = rcd->ppd;
565 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
566 struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
567 opcode_handler packet_handler;
568 unsigned long flags;
569 u32 qp_num;
570 int lnh;
571 u8 opcode;
572 u16 lid;
562
573
563 inc_opstats(packet->tlen, &rcd->opstats->stats[packet->opcode]);
574 /* Check for GRH */
575 lnh = ib_get_lnh(hdr);
576 if (lnh == HFI1_LRH_BTH) {
577 packet->ohdr = &hdr->u.oth;
578 } else if (lnh == HFI1_LRH_GRH) {
579 u32 vtf;
564
580
565 if (unlikely(is_mcast)) {
581 packet->ohdr = &hdr->u.l.oth;
582 if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
583 goto drop;
584 vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
585 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
586 goto drop;
587 packet->rcv_flags |= HFI1_HAS_GRH;
588 } else {
589 goto drop;
590 }
591
592 trace_input_ibhdr(rcd->dd, hdr);
593
594 opcode = ib_bth_get_opcode(packet->ohdr);
595 inc_opstats(tlen, &rcd->opstats->stats[opcode]);
596
597 /* Get the destination QP number. */
598 qp_num = be32_to_cpu(packet->ohdr->bth[1]) & RVT_QPN_MASK;
599 lid = ib_get_dlid(hdr);
600 if (unlikely((lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
601 (lid != be16_to_cpu(IB_LID_PERMISSIVE)))) {
566 struct rvt_mcast *mcast;
567 struct rvt_mcast_qp *p;
568
602 struct rvt_mcast *mcast;
603 struct rvt_mcast_qp *p;
604
569 if (!packet->grh)
605 if (lnh != HFI1_LRH_GRH)
570 goto drop;
606 goto drop;
571 mcast = rvt_mcast_find(&ibp->rvp,
572 &packet->grh->dgid,
573 packet->dlid);
607 mcast = rvt_mcast_find(&ibp->rvp, &hdr->u.l.grh.dgid, lid);
574 if (!mcast)
575 goto drop;
576 list_for_each_entry_rcu(p, &mcast->qp_list, list) {
577 packet->qp = p->qp;
578 spin_lock_irqsave(&packet->qp->r_lock, flags);
608 if (!mcast)
609 goto drop;
610 list_for_each_entry_rcu(p, &mcast->qp_list, list) {
611 packet->qp = p->qp;
612 spin_lock_irqsave(&packet->qp->r_lock, flags);
579 packet_handler = qp_ok(packet);
613 packet_handler = qp_ok(opcode, packet);
580 if (likely(packet_handler))
581 packet_handler(packet);
582 else
583 ibp->rvp.n_pkt_drops++;
584 spin_unlock_irqrestore(&packet->qp->r_lock, flags);
585 }
586 /*
587 * Notify rvt_multicast_detach() if it is waiting for us
588 * to finish.
589 */
590 if (atomic_dec_return(&mcast->refcount) <= 1)
591 wake_up(&mcast->wait);
592 } else {
614 if (likely(packet_handler))
615 packet_handler(packet);
616 else
617 ibp->rvp.n_pkt_drops++;
618 spin_unlock_irqrestore(&packet->qp->r_lock, flags);
619 }
620 /*
621 * Notify rvt_multicast_detach() if it is waiting for us
622 * to finish.
623 */
624 if (atomic_dec_return(&mcast->refcount) <= 1)
625 wake_up(&mcast->wait);
626 } else {
593 /* Get the destination QP number. */
594 qp_num = ib_bth_get_qpn(packet->ohdr);
595 rcu_read_lock();
596 packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
597 if (!packet->qp) {
598 rcu_read_unlock();
599 goto drop;
600 }
627 rcu_read_lock();
628 packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
629 if (!packet->qp) {
630 rcu_read_unlock();
631 goto drop;
632 }
601 if (unlikely(hfi1_dbg_fault_opcode(packet->qp, packet->opcode,
633 if (unlikely(hfi1_dbg_fault_opcode(packet->qp, opcode,
602 true))) {
603 rcu_read_unlock();
604 goto drop;
605 }
606 spin_lock_irqsave(&packet->qp->r_lock, flags);
634 true))) {
635 rcu_read_unlock();
636 goto drop;
637 }
638 spin_lock_irqsave(&packet->qp->r_lock, flags);
607 packet_handler = qp_ok(packet);
639 packet_handler = qp_ok(opcode, packet);
608 if (likely(packet_handler))
609 packet_handler(packet);
610 else
611 ibp->rvp.n_pkt_drops++;
612 spin_unlock_irqrestore(&packet->qp->r_lock, flags);
613 rcu_read_unlock();
614 }
615 return;
640 if (likely(packet_handler))
641 packet_handler(packet);
642 else
643 ibp->rvp.n_pkt_drops++;
644 spin_unlock_irqrestore(&packet->qp->r_lock, flags);
645 rcu_read_unlock();
646 }
647 return;
648
616drop:
617 ibp->rvp.n_pkt_drops++;
618}
619
649drop:
650 ibp->rvp.n_pkt_drops++;
651}
652
620/**
621 * hfi1_ib_rcv - process an incoming packet
622 * @packet: data packet information
623 *
624 * This is called to process an incoming packet at interrupt level.
625 */
626void hfi1_ib_rcv(struct hfi1_packet *packet)
627{
628 struct hfi1_ctxtdata *rcd = packet->rcd;
629 bool is_mcast = false;
630
631 if (unlikely(hfi1_check_mcast(packet->dlid)))
632 is_mcast = true;
633
634 trace_input_ibhdr(rcd->dd, packet,
635 !!(packet->rhf & RHF_DC_INFO_SMASK));
636 hfi1_handle_packet(packet, is_mcast);
637}
638
639/*
640 * This is called from a timer to check for QPs
641 * which need kernel memory in order to send a packet.
642 */
643static void mem_timer(unsigned long data)
644{
645 struct hfi1_ibdev *dev = (struct hfi1_ibdev *)data;
646 struct list_head *list = &dev->memwait;

--- 197 unchanged lines hidden (view full) ---

844 tx = ps->s_txreq;
845 if (!sdma_txreq_built(&tx->txreq)) {
846 if (likely(pbc == 0)) {
847 u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
848 u8 opcode = get_opcode(&tx->phdr.hdr);
849
850 /* No vl15 here */
851 /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
653/*
654 * This is called from a timer to check for QPs
655 * which need kernel memory in order to send a packet.
656 */
657static void mem_timer(unsigned long data)
658{
659 struct hfi1_ibdev *dev = (struct hfi1_ibdev *)data;
660 struct list_head *list = &dev->memwait;

--- 197 unchanged lines hidden (view full) ---

858 tx = ps->s_txreq;
859 if (!sdma_txreq_built(&tx->txreq)) {
860 if (likely(pbc == 0)) {
861 u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
862 u8 opcode = get_opcode(&tx->phdr.hdr);
863
864 /* No vl15 here */
865 /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
852 pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
866 pbc |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT;
853
854 if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
855 pbc = hfi1_fault_tx(qp, opcode, pbc);
856 pbc = create_pbc(ppd,
857 pbc,
858 qp->srate_mbps,
859 vl,
860 plen);

--- 5 unchanged lines hidden (view full) ---

866 }
867 ret = sdma_send_txreq(tx->sde, &priv->s_iowait, &tx->txreq);
868 if (unlikely(ret < 0)) {
869 if (ret == -ECOMM)
870 goto bail_ecomm;
871 return ret;
872 }
873 trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
867
868 if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
869 pbc = hfi1_fault_tx(qp, opcode, pbc);
870 pbc = create_pbc(ppd,
871 pbc,
872 qp->srate_mbps,
873 vl,
874 plen);

--- 5 unchanged lines hidden (view full) ---

880 }
881 ret = sdma_send_txreq(tx->sde, &priv->s_iowait, &tx->txreq);
882 if (unlikely(ret < 0)) {
883 if (ret == -ECOMM)
884 goto bail_ecomm;
885 return ret;
886 }
887 trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
874 &ps->s_txreq->phdr.hdr, ib_is_sc5(sc5));
888 &ps->s_txreq->phdr.hdr);
875 return ret;
876
877bail_ecomm:
878 /* The current one got "sent" */
879 return 0;
880bail_build:
881 ret = wait_kmem(dev, qp, ps);
882 if (!ret) {

--- 97 unchanged lines hidden (view full) ---

980 sc = ps->s_txreq->psc;
981
982 if (likely(pbc == 0)) {
983 u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
984 struct verbs_txreq *tx = ps->s_txreq;
985 u8 opcode = get_opcode(&tx->phdr.hdr);
986
987 /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
889 return ret;
890
891bail_ecomm:
892 /* The current one got "sent" */
893 return 0;
894bail_build:
895 ret = wait_kmem(dev, qp, ps);
896 if (!ret) {

--- 97 unchanged lines hidden (view full) ---

994 sc = ps->s_txreq->psc;
995
996 if (likely(pbc == 0)) {
997 u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
998 struct verbs_txreq *tx = ps->s_txreq;
999 u8 opcode = get_opcode(&tx->phdr.hdr);
1000
1001 /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
988 pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
1002 pbc |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT;
989 if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
990 pbc = hfi1_fault_tx(qp, opcode, pbc);
991 pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen);
992 }
993 if (cb)
994 iowait_pio_inc(&priv->s_iowait);
995 pbuf = sc_buffer_alloc(sc, plen, cb, qp);
996 if (unlikely(!pbuf)) {

--- 42 unchanged lines hidden (view full) ---

1039 seg_pio_copy_mid(pbuf, addr, slen);
1040 len -= slen;
1041 }
1042 seg_pio_copy_end(pbuf);
1043 }
1044 }
1045
1046 trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
1003 if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
1004 pbc = hfi1_fault_tx(qp, opcode, pbc);
1005 pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen);
1006 }
1007 if (cb)
1008 iowait_pio_inc(&priv->s_iowait);
1009 pbuf = sc_buffer_alloc(sc, plen, cb, qp);
1010 if (unlikely(!pbuf)) {

--- 42 unchanged lines hidden (view full) ---

1053 seg_pio_copy_mid(pbuf, addr, slen);
1054 len -= slen;
1055 }
1056 seg_pio_copy_end(pbuf);
1057 }
1058 }
1059
1060 trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
1047 &ps->s_txreq->phdr.hdr, ib_is_sc5(sc5));
1061 &ps->s_txreq->phdr.hdr);
1048
1049pio_bail:
1050 if (qp->s_wqe) {
1051 spin_lock_irqsave(&qp->s_lock, flags);
1052 hfi1_send_complete(qp, qp->s_wqe, wc_status);
1053 spin_unlock_irqrestore(&qp->s_lock, flags);
1054 } else if (qp->ibqp.qp_type == IB_QPT_RC) {
1055 spin_lock_irqsave(&qp->s_lock, flags);

--- 293 unchanged lines hidden (view full) ---

1349 struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];
1350 u16 lid = ppd->lid;
1351
1352 /* props being zeroed by the caller, avoid zeroing it here */
1353 props->lid = lid ? lid : 0;
1354 props->lmc = ppd->lmc;
1355 /* OPA logical states match IB logical states */
1356 props->state = driver_lstate(ppd);
1062
1063pio_bail:
1064 if (qp->s_wqe) {
1065 spin_lock_irqsave(&qp->s_lock, flags);
1066 hfi1_send_complete(qp, qp->s_wqe, wc_status);
1067 spin_unlock_irqrestore(&qp->s_lock, flags);
1068 } else if (qp->ibqp.qp_type == IB_QPT_RC) {
1069 spin_lock_irqsave(&qp->s_lock, flags);

--- 293 unchanged lines hidden (view full) ---

1363 struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];
1364 u16 lid = ppd->lid;
1365
1366 /* props being zeroed by the caller, avoid zeroing it here */
1367 props->lid = lid ? lid : 0;
1368 props->lmc = ppd->lmc;
1369 /* OPA logical states match IB logical states */
1370 props->state = driver_lstate(ppd);
1357 props->phys_state = driver_pstate(ppd);
1371 props->phys_state = hfi1_ibphys_portstate(ppd);
1358 props->gid_tbl_len = HFI1_GUIDS_PER_PORT;
1359 props->active_width = (u8)opa_width_to_ib(ppd->link_width_active);
1360 /* see rate_show() in ib core/sysfs.c */
1361 props->active_speed = (u8)opa_speed_to_ib(ppd->link_speed_active);
1362 props->max_vl_num = ppd->vls_supported;
1363
1364 /* Once we are a "first class" citizen and have added the OPA MTUs to
1365 * the core we can advertise the larger MTU enum to the ULPs, for now

--- 166 unchanged lines hidden (view full) ---

1532 ibp->sl_to_sc[i] = i;
1533 ibp->sc_to_sl[i] = i;
1534 }
1535
1536 spin_lock_init(&ibp->rvp.lock);
1537 /* Set the prefix to the default value (see ch. 4.1.1) */
1538 ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX;
1539 ibp->rvp.sm_lid = 0;
1372 props->gid_tbl_len = HFI1_GUIDS_PER_PORT;
1373 props->active_width = (u8)opa_width_to_ib(ppd->link_width_active);
1374 /* see rate_show() in ib core/sysfs.c */
1375 props->active_speed = (u8)opa_speed_to_ib(ppd->link_speed_active);
1376 props->max_vl_num = ppd->vls_supported;
1377
1378 /* Once we are a "first class" citizen and have added the OPA MTUs to
1379 * the core we can advertise the larger MTU enum to the ULPs, for now

--- 166 unchanged lines hidden (view full) ---

1546 ibp->sl_to_sc[i] = i;
1547 ibp->sc_to_sl[i] = i;
1548 }
1549
1550 spin_lock_init(&ibp->rvp.lock);
1551 /* Set the prefix to the default value (see ch. 4.1.1) */
1552 ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX;
1553 ibp->rvp.sm_lid = 0;
1540 /*
1541 * Below should only set bits defined in OPA PortInfo.CapabilityMask
1542 * and PortInfo.CapabilityMask3
1543 */
1554 /* Below should only set bits defined in OPA PortInfo.CapabilityMask */
1544 ibp->rvp.port_cap_flags = IB_PORT_AUTO_MIGR_SUP |
1545 IB_PORT_CAP_MASK_NOTICE_SUP;
1555 ibp->rvp.port_cap_flags = IB_PORT_AUTO_MIGR_SUP |
1556 IB_PORT_CAP_MASK_NOTICE_SUP;
1546 ibp->rvp.port_cap3_flags = OPA_CAP_MASK3_IsSharedSpaceSupported;
1547 ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
1548 ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
1549 ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
1550 ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS;
1551 ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT;
1552
1553 RCU_INIT_POINTER(ibp->rvp.qp[0], NULL);
1554 RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);

--- 209 unchanged lines hidden (view full) ---

1764 strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz);
1765 ibdev->owner = THIS_MODULE;
1766 ibdev->phys_port_cnt = dd->num_pports;
1767 ibdev->dev.parent = &dd->pcidev->dev;
1768 ibdev->modify_device = modify_device;
1769 ibdev->alloc_hw_stats = alloc_hw_stats;
1770 ibdev->get_hw_stats = get_hw_stats;
1771 ibdev->alloc_rdma_netdev = hfi1_vnic_alloc_rn;
1557 ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
1558 ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
1559 ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
1560 ibp->rvp.pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS;
1561 ibp->rvp.pma_counter_select[4] = IB_PMA_PORT_XMIT_WAIT;
1562
1563 RCU_INIT_POINTER(ibp->rvp.qp[0], NULL);
1564 RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);

--- 209 unchanged lines hidden (view full) ---

1774 strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz);
1775 ibdev->owner = THIS_MODULE;
1776 ibdev->phys_port_cnt = dd->num_pports;
1777 ibdev->dev.parent = &dd->pcidev->dev;
1778 ibdev->modify_device = modify_device;
1779 ibdev->alloc_hw_stats = alloc_hw_stats;
1780 ibdev->get_hw_stats = get_hw_stats;
1781 ibdev->alloc_rdma_netdev = hfi1_vnic_alloc_rn;
1772 ibdev->free_rdma_netdev = hfi1_vnic_free_rn;
1773
1774 /* keep process mad in the driver */
1775 ibdev->process_mad = hfi1_process_mad;
1776 ibdev->get_dev_fw_str = hfi1_get_dev_fw_str;
1777
1778 strncpy(ibdev->node_desc, init_utsname()->nodename,
1779 sizeof(ibdev->node_desc));
1780

--- 154 unchanged lines hidden --- 		1782
1783 /* keep process mad in the driver */
1784 ibdev->process_mad = hfi1_process_mad;
1785 ibdev->get_dev_fw_str = hfi1_get_dev_fw_str;
1786
1787 strncpy(ibdev->node_desc, init_utsname()->nodename,
1788 sizeof(ibdev->node_desc));
1789

--- 154 unchanged lines hidden ---