Lines Matching refs:packet
208 struct hfi1_packet *packet) in rcv_hdrerr() argument
210 struct ib_header *rhdr = packet->hdr; in rcv_hdrerr()
211 u32 rte = rhf_rcv_type_err(packet->rhf); in rcv_hdrerr()
218 if ((packet->rhf & RHF_DC_ERR) && in rcv_hdrerr()
222 if (packet->rhf & RHF_ICRC_ERR) in rcv_hdrerr()
225 if (packet->etype == RHF_RCV_TYPE_BYPASS) { in rcv_hdrerr()
232 packet->ohdr = &rhdr->u.oth; in rcv_hdrerr()
234 packet->ohdr = &rhdr->u.l.oth; in rcv_hdrerr()
235 packet->grh = &rhdr->u.l.grh; in rcv_hdrerr()
241 if (packet->rhf & RHF_TID_ERR) { in rcv_hdrerr()
243 u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in rcv_hdrerr()
252 if (packet->grh) { in rcv_hdrerr()
254 struct ib_grh *grh = packet->grh; in rcv_hdrerr()
264 qp_num = ib_bth_get_qpn(packet->ohdr); in rcv_hdrerr()
290 hfi1_rc_hdrerr(rcd, packet, qp); in rcv_hdrerr()
309 if (rhf_use_egr_bfr(packet->rhf)) in rcv_hdrerr()
310 ebuf = packet->ebuf; in rcv_hdrerr()
315 opcode = ib_bth_get_opcode(packet->ohdr); in rcv_hdrerr()
326 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); in rcv_hdrerr()
329 lqpn = ib_bth_get_qpn(packet->ohdr); in rcv_hdrerr()
357 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; in rcv_hdrerr()
369 struct hfi1_packet *packet) in init_packet() argument
371 packet->rsize = get_hdrqentsize(rcd); /* words */ in init_packet()
372 packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */ in init_packet()
373 packet->rcd = rcd; in init_packet()
374 packet->updegr = 0; in init_packet()
375 packet->etail = -1; in init_packet()
376 packet->rhf_addr = get_rhf_addr(rcd); in init_packet()
377 packet->rhf = rhf_to_cpu(packet->rhf_addr); in init_packet()
378 packet->rhqoff = hfi1_rcd_head(rcd); in init_packet()
379 packet->numpkt = 0; in init_packet()
502 struct hfi1_packet *packet) in init_ps_mdata() argument
504 struct hfi1_ctxtdata *rcd = packet->rcd; in init_ps_mdata()
507 mdata->rsize = packet->rsize; in init_ps_mdata()
508 mdata->maxcnt = packet->maxcnt; in init_ps_mdata()
509 mdata->ps_head = packet->rhqoff; in init_ps_mdata()
565 #define prescan_rxq(rcd, packet) \ argument
568 __prescan_rxq(packet); \
570 static void __prescan_rxq(struct hfi1_packet *packet) in __prescan_rxq() argument
572 struct hfi1_ctxtdata *rcd = packet->rcd; in __prescan_rxq()
575 init_ps_mdata(&mdata, packet); in __prescan_rxq()
580 packet->rcd->rhf_offset; in __prescan_rxq()
597 packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr); in __prescan_rxq()
598 hdr = packet->hdr; in __prescan_rxq()
602 packet->ohdr = &hdr->u.oth; in __prescan_rxq()
603 packet->grh = NULL; in __prescan_rxq()
605 packet->ohdr = &hdr->u.l.oth; in __prescan_rxq()
606 packet->grh = &hdr->u.l.grh; in __prescan_rxq()
611 if (!hfi1_may_ecn(packet)) in __prescan_rxq()
614 bth1 = be32_to_cpu(packet->ohdr->bth[1]); in __prescan_rxq()
624 hfi1_process_ecn_slowpath(qp, packet, true); in __prescan_rxq()
629 packet->ohdr->bth[1] = cpu_to_be32(bth1); in __prescan_rxq()
635 static void process_rcv_qp_work(struct hfi1_packet *packet) in process_rcv_qp_work() argument
638 struct hfi1_ctxtdata *rcd = packet->rcd; in process_rcv_qp_work()
648 packet->qp = qp; in process_rcv_qp_work()
649 hfi1_send_rc_ack(packet, 0); in process_rcv_qp_work()
665 static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) in max_packet_exceeded() argument
668 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) in max_packet_exceeded()
670 process_rcv_qp_work(packet); in max_packet_exceeded()
674 this_cpu_inc(*packet->rcd->dd->rcv_limit); in max_packet_exceeded()
679 static inline int check_max_packet(struct hfi1_packet *packet, int thread) in check_max_packet() argument
683 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) in check_max_packet()
684 ret = max_packet_exceeded(packet, thread); in check_max_packet()
688 static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) in skip_rcv_packet() argument
692 packet->rcd->dd->ctx0_seq_drop++; in skip_rcv_packet()
694 packet->rhqoff += packet->rsize; in skip_rcv_packet()
695 if (packet->rhqoff >= packet->maxcnt) in skip_rcv_packet()
696 packet->rhqoff = 0; in skip_rcv_packet()
698 packet->numpkt++; in skip_rcv_packet()
699 ret = check_max_packet(packet, thread); in skip_rcv_packet()
701 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in skip_rcv_packet()
702 packet->rcd->rhf_offset; in skip_rcv_packet()
703 packet->rhf = rhf_to_cpu(packet->rhf_addr); in skip_rcv_packet()
708 static void process_rcv_packet_napi(struct hfi1_packet *packet) in process_rcv_packet_napi() argument
710 packet->etype = rhf_rcv_type(packet->rhf); in process_rcv_packet_napi()
713 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in process_rcv_packet_napi()
715 packet->etail = rhf_egr_index(packet->rhf); in process_rcv_packet_napi()
716 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, in process_rcv_packet_napi()
717 &packet->updegr); in process_rcv_packet_napi()
723 prefetch_range(packet->ebuf, in process_rcv_packet_napi()
724 packet->tlen - ((packet->rcd->rcvhdrqentsize - in process_rcv_packet_napi()
725 (rhf_hdrq_offset(packet->rhf) in process_rcv_packet_napi()
728 packet->rcd->rhf_rcv_function_map[packet->etype](packet); in process_rcv_packet_napi()
729 packet->numpkt++; in process_rcv_packet_napi()
732 packet->rhqoff += packet->rsize; in process_rcv_packet_napi()
733 if (packet->rhqoff >= packet->maxcnt) in process_rcv_packet_napi()
734 packet->rhqoff = 0; in process_rcv_packet_napi()
736 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in process_rcv_packet_napi()
737 packet->rcd->rhf_offset; in process_rcv_packet_napi()
738 packet->rhf = rhf_to_cpu(packet->rhf_addr); in process_rcv_packet_napi()
741 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) in process_rcv_packet() argument
745 packet->etype = rhf_rcv_type(packet->rhf); in process_rcv_packet()
748 packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ in process_rcv_packet()
750 packet->ebuf = NULL; in process_rcv_packet()
751 if (rhf_use_egr_bfr(packet->rhf)) { in process_rcv_packet()
752 packet->etail = rhf_egr_index(packet->rhf); in process_rcv_packet()
753 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, in process_rcv_packet()
754 &packet->updegr); in process_rcv_packet()
760 prefetch_range(packet->ebuf, in process_rcv_packet()
761 packet->tlen - ((get_hdrqentsize(packet->rcd) - in process_rcv_packet()
762 (rhf_hdrq_offset(packet->rhf) in process_rcv_packet()
774 packet->rcd->rhf_rcv_function_map[packet->etype](packet); in process_rcv_packet()
775 packet->numpkt++; in process_rcv_packet()
778 packet->rhqoff += packet->rsize; in process_rcv_packet()
779 if (packet->rhqoff >= packet->maxcnt) in process_rcv_packet()
780 packet->rhqoff = 0; in process_rcv_packet()
782 ret = check_max_packet(packet, thread); in process_rcv_packet()
784 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + in process_rcv_packet()
785 packet->rcd->rhf_offset; in process_rcv_packet()
786 packet->rhf = rhf_to_cpu(packet->rhf_addr); in process_rcv_packet()
791 static inline void process_rcv_update(int last, struct hfi1_packet *packet) in process_rcv_update() argument
799 if (!last && !(packet->numpkt & 0xf)) { in process_rcv_update()
800 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, in process_rcv_update()
801 packet->etail, 0, 0); in process_rcv_update()
802 packet->updegr = 0; in process_rcv_update()
804 packet->grh = NULL; in process_rcv_update()
807 static inline void finish_packet(struct hfi1_packet *packet) in finish_packet() argument
815 update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr, in finish_packet()
816 packet->etail, rcv_intr_dynamic, packet->numpkt); in finish_packet()
830 struct hfi1_packet packet; in handle_receive_interrupt_napi_fp() local
832 init_packet(rcd, &packet); in handle_receive_interrupt_napi_fp()
833 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_fp()
836 while (packet.numpkt < budget) { in handle_receive_interrupt_napi_fp()
837 process_rcv_packet_napi(&packet); in handle_receive_interrupt_napi_fp()
838 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_fp()
841 process_rcv_update(0, &packet); in handle_receive_interrupt_napi_fp()
843 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_napi_fp()
845 finish_packet(&packet); in handle_receive_interrupt_napi_fp()
846 return packet.numpkt; in handle_receive_interrupt_napi_fp()
855 struct hfi1_packet packet; in handle_receive_interrupt_nodma_rtail() local
857 init_packet(rcd, &packet); in handle_receive_interrupt_nodma_rtail()
858 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { in handle_receive_interrupt_nodma_rtail()
863 prescan_rxq(rcd, &packet); in handle_receive_interrupt_nodma_rtail()
866 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt_nodma_rtail()
867 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_nodma_rtail()
869 process_rcv_update(last, &packet); in handle_receive_interrupt_nodma_rtail()
871 process_rcv_qp_work(&packet); in handle_receive_interrupt_nodma_rtail()
872 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_nodma_rtail()
874 finish_packet(&packet); in handle_receive_interrupt_nodma_rtail()
882 struct hfi1_packet packet; in handle_receive_interrupt_dma_rtail() local
884 init_packet(rcd, &packet); in handle_receive_interrupt_dma_rtail()
886 if (packet.rhqoff == hdrqtail) { in handle_receive_interrupt_dma_rtail()
892 prescan_rxq(rcd, &packet); in handle_receive_interrupt_dma_rtail()
895 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt_dma_rtail()
896 if (packet.rhqoff == hdrqtail) in handle_receive_interrupt_dma_rtail()
898 process_rcv_update(last, &packet); in handle_receive_interrupt_dma_rtail()
900 process_rcv_qp_work(&packet); in handle_receive_interrupt_dma_rtail()
901 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_dma_rtail()
903 finish_packet(&packet); in handle_receive_interrupt_dma_rtail()
948 static bool __set_armed_to_active(struct hfi1_packet *packet) in __set_armed_to_active() argument
950 u8 etype = rhf_rcv_type(packet->rhf); in __set_armed_to_active()
954 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd, in __set_armed_to_active()
955 packet->rhf_addr); in __set_armed_to_active()
956 sc = hfi1_9B_get_sc5(hdr, packet->rhf); in __set_armed_to_active()
959 packet->rcd, in __set_armed_to_active()
960 packet->rhf_addr); in __set_armed_to_active()
964 int hwstate = driver_lstate(packet->rcd->ppd); in __set_armed_to_active()
966 &packet->rcd->ppd->linkstate_active_work; in __set_armed_to_active()
969 dd_dev_info(packet->rcd->dd, in __set_armed_to_active()
975 queue_work(packet->rcd->ppd->link_wq, lsaw); in __set_armed_to_active()
987 static bool set_armed_to_active(struct hfi1_packet *packet) in set_armed_to_active() argument
989 if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED)) in set_armed_to_active()
991 return __set_armed_to_active(packet); in set_armed_to_active()
1006 struct hfi1_packet packet; in handle_receive_interrupt() local
1014 init_packet(rcd, &packet); in handle_receive_interrupt()
1017 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) { in handle_receive_interrupt()
1024 if (packet.rhqoff == hdrqtail) { in handle_receive_interrupt()
1035 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt()
1039 prescan_rxq(rcd, &packet); in handle_receive_interrupt()
1044 packet.rhqoff += packet.rsize; in handle_receive_interrupt()
1045 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + in handle_receive_interrupt()
1046 packet.rhqoff + in handle_receive_interrupt()
1048 packet.rhf = rhf_to_cpu(packet.rhf_addr); in handle_receive_interrupt()
1051 last = skip_rcv_packet(&packet, thread); in handle_receive_interrupt()
1054 if (set_armed_to_active(&packet)) in handle_receive_interrupt()
1056 last = process_rcv_packet(&packet, thread); in handle_receive_interrupt()
1060 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt()
1063 if (packet.rhqoff == hdrqtail) in handle_receive_interrupt()
1073 rhf_rcv_seq(packet.rhf)); in handle_receive_interrupt()
1083 process_rcv_update(last, &packet); in handle_receive_interrupt()
1086 process_rcv_qp_work(&packet); in handle_receive_interrupt()
1087 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt()
1094 finish_packet(&packet); in handle_receive_interrupt()
1112 struct hfi1_packet packet; in handle_receive_interrupt_napi_sp() local
1114 init_packet(rcd, &packet); in handle_receive_interrupt_napi_sp()
1115 if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_sp()
1118 while (last != RCV_PKT_DONE && packet.numpkt < budget) { in handle_receive_interrupt_napi_sp()
1121 packet.rhqoff += packet.rsize; in handle_receive_interrupt_napi_sp()
1122 packet.rhf_addr = (__le32 *)rcd->rcvhdrq + in handle_receive_interrupt_napi_sp()
1123 packet.rhqoff + in handle_receive_interrupt_napi_sp()
1125 packet.rhf = rhf_to_cpu(packet.rhf_addr); in handle_receive_interrupt_napi_sp()
1128 if (set_armed_to_active(&packet)) in handle_receive_interrupt_napi_sp()
1130 process_rcv_packet_napi(&packet); in handle_receive_interrupt_napi_sp()
1133 if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf))) in handle_receive_interrupt_napi_sp()
1141 process_rcv_update(last, &packet); in handle_receive_interrupt_napi_sp()
1144 hfi1_set_rcd_head(rcd, packet.rhqoff); in handle_receive_interrupt_napi_sp()
1151 finish_packet(&packet); in handle_receive_interrupt_napi_sp()
1152 return packet.numpkt; in handle_receive_interrupt_napi_sp()
1436 static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) in hfi1_setup_ib_header() argument
1438 packet->hdr = (struct hfi1_ib_message_header *) in hfi1_setup_ib_header()
1439 hfi1_get_msgheader(packet->rcd, in hfi1_setup_ib_header()
1440 packet->rhf_addr); in hfi1_setup_ib_header()
1441 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; in hfi1_setup_ib_header()
1444 static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet) in hfi1_bypass_ingress_pkt_check() argument
1446 struct hfi1_pportdata *ppd = packet->rcd->ppd; in hfi1_bypass_ingress_pkt_check()
1449 if ((!packet->slid) || (!packet->dlid)) in hfi1_bypass_ingress_pkt_check()
1453 if ((!(hfi1_is_16B_mcast(packet->dlid))) && in hfi1_bypass_ingress_pkt_check()
1454 (packet->dlid != in hfi1_bypass_ingress_pkt_check()
1456 if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid) in hfi1_bypass_ingress_pkt_check()
1461 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF)) in hfi1_bypass_ingress_pkt_check()
1465 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), in hfi1_bypass_ingress_pkt_check()
1467 (packet->sc != 0xF)) in hfi1_bypass_ingress_pkt_check()
1473 static int hfi1_setup_9B_packet(struct hfi1_packet *packet) in hfi1_setup_9B_packet() argument
1475 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); in hfi1_setup_9B_packet()
1479 hfi1_setup_ib_header(packet); in hfi1_setup_9B_packet()
1480 hdr = packet->hdr; in hfi1_setup_9B_packet()
1484 packet->ohdr = &hdr->u.oth; in hfi1_setup_9B_packet()
1485 packet->grh = NULL; in hfi1_setup_9B_packet()
1489 packet->ohdr = &hdr->u.l.oth; in hfi1_setup_9B_packet()
1490 packet->grh = &hdr->u.l.grh; in hfi1_setup_9B_packet()
1491 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) in hfi1_setup_9B_packet()
1493 vtf = be32_to_cpu(packet->grh->version_tclass_flow); in hfi1_setup_9B_packet()
1501 packet->payload = packet->ebuf; in hfi1_setup_9B_packet()
1502 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_9B_packet()
1503 packet->slid = ib_get_slid(hdr); in hfi1_setup_9B_packet()
1504 packet->dlid = ib_get_dlid(hdr); in hfi1_setup_9B_packet()
1505 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && in hfi1_setup_9B_packet()
1506 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE)))) in hfi1_setup_9B_packet()
1507 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - in hfi1_setup_9B_packet()
1509 packet->sl = ib_get_sl(hdr); in hfi1_setup_9B_packet()
1510 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); in hfi1_setup_9B_packet()
1511 packet->pad = ib_bth_get_pad(packet->ohdr); in hfi1_setup_9B_packet()
1512 packet->extra_byte = 0; in hfi1_setup_9B_packet()
1513 packet->pkey = ib_bth_get_pkey(packet->ohdr); in hfi1_setup_9B_packet()
1514 packet->migrated = ib_bth_is_migration(packet->ohdr); in hfi1_setup_9B_packet()
1522 static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) in hfi1_setup_bypass_packet() argument
1534 struct hfi1_ctxtdata *rcd = packet->rcd; in hfi1_setup_bypass_packet()
1539 packet->hdr = (struct hfi1_16b_header *) in hfi1_setup_bypass_packet()
1540 hfi1_get_16B_header(packet->rcd, in hfi1_setup_bypass_packet()
1541 packet->rhf_addr); in hfi1_setup_bypass_packet()
1542 l4 = hfi1_16B_get_l4(packet->hdr); in hfi1_setup_bypass_packet()
1544 packet->ohdr = packet->ebuf; in hfi1_setup_bypass_packet()
1545 packet->grh = NULL; in hfi1_setup_bypass_packet()
1546 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_bypass_packet()
1547 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); in hfi1_setup_bypass_packet()
1549 packet->hlen = hdr_len_by_opcode[packet->opcode] + in hfi1_setup_bypass_packet()
1551 packet->migrated = opa_bth_is_migration(packet->ohdr); in hfi1_setup_bypass_packet()
1556 packet->ohdr = packet->ebuf + grh_len; in hfi1_setup_bypass_packet()
1557 packet->grh = packet->ebuf; in hfi1_setup_bypass_packet()
1558 packet->opcode = ib_bth_get_opcode(packet->ohdr); in hfi1_setup_bypass_packet()
1559 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); in hfi1_setup_bypass_packet()
1561 packet->hlen = hdr_len_by_opcode[packet->opcode] + in hfi1_setup_bypass_packet()
1563 packet->migrated = opa_bth_is_migration(packet->ohdr); in hfi1_setup_bypass_packet()
1565 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) in hfi1_setup_bypass_packet()
1567 vtf = be32_to_cpu(packet->grh->version_tclass_flow); in hfi1_setup_bypass_packet()
1571 packet->mgmt = packet->ebuf; in hfi1_setup_bypass_packet()
1572 packet->ohdr = NULL; in hfi1_setup_bypass_packet()
1573 packet->grh = NULL; in hfi1_setup_bypass_packet()
1574 packet->opcode = IB_OPCODE_UD_SEND_ONLY; in hfi1_setup_bypass_packet()
1575 packet->pad = OPA_16B_L4_FM_PAD; in hfi1_setup_bypass_packet()
1576 packet->hlen = OPA_16B_L4_FM_HLEN; in hfi1_setup_bypass_packet()
1577 packet->migrated = false; in hfi1_setup_bypass_packet()
1583 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; in hfi1_setup_bypass_packet()
1584 packet->slid = hfi1_16B_get_slid(packet->hdr); in hfi1_setup_bypass_packet()
1585 packet->dlid = hfi1_16B_get_dlid(packet->hdr); in hfi1_setup_bypass_packet()
1586 if (unlikely(hfi1_is_16B_mcast(packet->dlid))) in hfi1_setup_bypass_packet()
1587 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - in hfi1_setup_bypass_packet()
1590 packet->sc = hfi1_16B_get_sc(packet->hdr); in hfi1_setup_bypass_packet()
1591 packet->sl = ibp->sc_to_sl[packet->sc]; in hfi1_setup_bypass_packet()
1592 packet->extra_byte = SIZE_OF_LT; in hfi1_setup_bypass_packet()
1593 packet->pkey = hfi1_16B_get_pkey(packet->hdr); in hfi1_setup_bypass_packet()
1595 if (hfi1_bypass_ingress_pkt_check(packet)) in hfi1_setup_bypass_packet()
1605 static void show_eflags_errs(struct hfi1_packet *packet) in show_eflags_errs() argument
1607 struct hfi1_ctxtdata *rcd = packet->rcd; in show_eflags_errs()
1608 u32 rte = rhf_rcv_type_err(packet->rhf); in show_eflags_errs()
1612 rcd->ctxt, packet->rhf, in show_eflags_errs()
1613 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", in show_eflags_errs()
1614 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", in show_eflags_errs()
1615 packet->rhf & RHF_DC_ERR ? "dc " : "", in show_eflags_errs()
1616 packet->rhf & RHF_TID_ERR ? "tid " : "", in show_eflags_errs()
1617 packet->rhf & RHF_LEN_ERR ? "len " : "", in show_eflags_errs()
1618 packet->rhf & RHF_ECC_ERR ? "ecc " : "", in show_eflags_errs()
1619 packet->rhf & RHF_ICRC_ERR ? "icrc " : "", in show_eflags_errs()
1623 void handle_eflags(struct hfi1_packet *packet) in handle_eflags() argument
1625 struct hfi1_ctxtdata *rcd = packet->rcd; in handle_eflags()
1627 rcv_hdrerr(rcd, rcd->ppd, packet); in handle_eflags()
1628 if (rhf_err_flags(packet->rhf)) in handle_eflags()
1629 show_eflags_errs(packet); in handle_eflags()
1632 static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet) in hfi1_ipoib_ib_rcv() argument
1636 struct hfi1_ctxtdata *rcd = packet->rcd; in hfi1_ipoib_ib_rcv()
1645 trace_hfi1_rcvhdr(packet); in hfi1_ipoib_ib_rcv()
1647 hfi1_setup_ib_header(packet); in hfi1_ipoib_ib_rcv()
1649 packet->ohdr = &((struct ib_header *)packet->hdr)->u.oth; in hfi1_ipoib_ib_rcv()
1650 packet->grh = NULL; in hfi1_ipoib_ib_rcv()
1652 if (unlikely(rhf_err_flags(packet->rhf))) { in hfi1_ipoib_ib_rcv()
1653 handle_eflags(packet); in hfi1_ipoib_ib_rcv()
1657 qpnum = ib_bth_get_qpn(packet->ohdr); in hfi1_ipoib_ib_rcv()
1662 trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf))); in hfi1_ipoib_ib_rcv()
1666 do_work = hfi1_may_ecn(packet); in hfi1_ipoib_ib_rcv()
1668 do_cnp = (packet->opcode != IB_OPCODE_CNP); in hfi1_ipoib_ib_rcv()
1670 packet, do_cnp); in hfi1_ipoib_ib_rcv()
1679 tlen = packet->tlen; in hfi1_ipoib_ib_rcv()
1680 extra_bytes = ib_bth_get_pad(packet->ohdr) + (SIZE_OF_CRC << 2) + in hfi1_ipoib_ib_rcv()
1681 packet->hlen; in hfi1_ipoib_ib_rcv()
1687 skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet->ebuf); in hfi1_ipoib_ib_rcv()
1702 ibp = rcd_to_iport(packet->rcd); in hfi1_ipoib_ib_rcv()
1710 static void process_receive_ib(struct hfi1_packet *packet) in process_receive_ib() argument
1712 if (hfi1_setup_9B_packet(packet)) in process_receive_ib()
1715 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in process_receive_ib()
1718 trace_hfi1_rcvhdr(packet); in process_receive_ib()
1720 if (unlikely(rhf_err_flags(packet->rhf))) { in process_receive_ib()
1721 handle_eflags(packet); in process_receive_ib()
1725 hfi1_ib_rcv(packet); in process_receive_ib()
1728 static void process_receive_bypass(struct hfi1_packet *packet) in process_receive_bypass() argument
1730 struct hfi1_devdata *dd = packet->rcd->dd; in process_receive_bypass()
1732 if (hfi1_setup_bypass_packet(packet)) in process_receive_bypass()
1735 trace_hfi1_rcvhdr(packet); in process_receive_bypass()
1737 if (unlikely(rhf_err_flags(packet->rhf))) { in process_receive_bypass()
1738 handle_eflags(packet); in process_receive_bypass()
1742 if (hfi1_16B_get_l2(packet->hdr) == 0x2) { in process_receive_bypass()
1743 hfi1_16B_rcv(packet); in process_receive_bypass()
1750 u64 *flits = packet->ebuf; in process_receive_bypass()
1752 if (flits && !(packet->rhf & RHF_LEN_ERR)) { in process_receive_bypass()
1755 packet->tlen > sizeof(flits[0]) ? in process_receive_bypass()
1764 static void process_receive_error(struct hfi1_packet *packet) in process_receive_error() argument
1768 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && in process_receive_error()
1769 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || in process_receive_error()
1770 packet->rhf & RHF_DC_ERR))) in process_receive_error()
1773 hfi1_setup_ib_header(packet); in process_receive_error()
1774 handle_eflags(packet); in process_receive_error()
1776 if (unlikely(rhf_err_flags(packet->rhf))) in process_receive_error()
1777 dd_dev_err(packet->rcd->dd, in process_receive_error()
1781 static void kdeth_process_expected(struct hfi1_packet *packet) in kdeth_process_expected() argument
1783 hfi1_setup_9B_packet(packet); in kdeth_process_expected()
1784 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in kdeth_process_expected()
1787 if (unlikely(rhf_err_flags(packet->rhf))) { in kdeth_process_expected()
1788 struct hfi1_ctxtdata *rcd = packet->rcd; in kdeth_process_expected()
1790 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) in kdeth_process_expected()
1794 hfi1_kdeth_expected_rcv(packet); in kdeth_process_expected()
1797 static void kdeth_process_eager(struct hfi1_packet *packet) in kdeth_process_eager() argument
1799 hfi1_setup_9B_packet(packet); in kdeth_process_eager()
1800 if (unlikely(hfi1_dbg_should_fault_rx(packet))) in kdeth_process_eager()
1803 trace_hfi1_rcvhdr(packet); in kdeth_process_eager()
1804 if (unlikely(rhf_err_flags(packet->rhf))) { in kdeth_process_eager()
1805 struct hfi1_ctxtdata *rcd = packet->rcd; in kdeth_process_eager()
1807 show_eflags_errs(packet); in kdeth_process_eager()
1808 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) in kdeth_process_eager()
1812 hfi1_kdeth_eager_rcv(packet); in kdeth_process_eager()
1815 static void process_receive_invalid(struct hfi1_packet *packet) in process_receive_invalid() argument
1817 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", in process_receive_invalid()
1818 rhf_rcv_type(packet->rhf)); in process_receive_invalid()
1825 struct hfi1_packet packet; in seqfile_dump_rcd() local
1840 init_packet(rcd, &packet); in seqfile_dump_rcd()
1841 init_ps_mdata(&mdata, &packet); in seqfile_dump_rcd()
1862 packet.hdr = hfi1_get_msgheader(rcd, rhf_addr); in seqfile_dump_rcd()
1863 hdr = packet.hdr; in seqfile_dump_rcd()
1868 packet.ohdr = &hdr->u.oth; in seqfile_dump_rcd()
1870 packet.ohdr = &hdr->u.l.oth; in seqfile_dump_rcd()
1874 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24); in seqfile_dump_rcd()
1875 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK; in seqfile_dump_rcd()
1876 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2])); in seqfile_dump_rcd()