156 static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,  in get_egrbuf()  argument
161 	*update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;  in get_egrbuf()
162 	return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +  in get_egrbuf()
166 static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,  in hfi1_get_header()  argument
171 	return (void *)(rhf_addr - rcd->rhf_offset + offset);  in hfi1_get_header()
174 static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,  in hfi1_get_msgheader()  argument
177 	return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);  in hfi1_get_msgheader()
181 		*hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,  in hfi1_get_16B_header()  argument
184 	return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);  in hfi1_get_16B_header()
207 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,  in rcv_hdrerr()  argument
213 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);  in rcv_hdrerr()
290 				hfi1_rc_hdrerr(rcd, packet, qp);  in rcv_hdrerr()
368 static inline void init_packet(struct hfi1_ctxtdata *rcd,  in init_packet()  argument
371 	packet->rsize = get_hdrqentsize(rcd); /* words */  in init_packet()
372 	packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */  in init_packet()
373 	packet->rcd = rcd;  in init_packet()
376 	packet->rhf_addr = get_rhf_addr(rcd);  in init_packet()
378 	packet->rhqoff = hfi1_rcd_head(rcd);  in init_packet()
493 	struct hfi1_ctxtdata *rcd;  member
504 	struct hfi1_ctxtdata *rcd = packet->rcd;  in init_ps_mdata()  local
506 	mdata->rcd = rcd;  in init_ps_mdata()
511 	if (get_dma_rtail_setting(rcd)) {  in init_ps_mdata()
512 		mdata->ps_tail = get_rcvhdrtail(rcd);  in init_ps_mdata()
513 		if (rcd->ctxt == HFI1_CTRL_CTXT)  in init_ps_mdata()
514 			mdata->ps_seq = hfi1_seq_cnt(rcd);  in init_ps_mdata()
519 		mdata->ps_seq = hfi1_seq_cnt(rcd);  in init_ps_mdata()
524 			  struct hfi1_ctxtdata *rcd)  in ps_done()  argument
526 	if (get_dma_rtail_setting(rcd))  in ps_done()
532 			  struct hfi1_ctxtdata *rcd)  in ps_skip()  argument
538 	if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))  in ps_skip()
545 				   struct hfi1_ctxtdata *rcd)  in update_ps_mdata()  argument
552 	if (!get_dma_rtail_setting(rcd) ||  in update_ps_mdata()
553 	    rcd->ctxt == HFI1_CTRL_CTXT)  in update_ps_mdata()
565 #define prescan_rxq(rcd, packet) \  argument
567 		if (rcd->ppd->cc_prescan) \
572 	struct hfi1_ctxtdata *rcd = packet->rcd;  in __prescan_rxq()  local
578 		struct hfi1_ibport *ibp = rcd_to_iport(rcd);  in __prescan_rxq()
579 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +  in __prescan_rxq()
580 					 packet->rcd->rhf_offset;  in __prescan_rxq()
583 		struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;  in __prescan_rxq()
588 		if (ps_done(&mdata, rhf, rcd))  in __prescan_rxq()
591 		if (ps_skip(&mdata, rhf, rcd))  in __prescan_rxq()
597 		packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);  in __prescan_rxq()
631 		update_ps_mdata(&mdata, rcd);  in __prescan_rxq()
638 	struct hfi1_ctxtdata *rcd = packet->rcd;  in process_rcv_qp_work()  local
644 	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {  in process_rcv_qp_work()
674 		this_cpu_inc(*packet->rcd->dd->rcv_limit);  in max_packet_exceeded()
692 	packet->rcd->dd->ctx0_seq_drop++;  in skip_rcv_packet()
701 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in skip_rcv_packet()
702 				     packet->rcd->rhf_offset;  in skip_rcv_packet()
716 	packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,  in process_rcv_packet_napi()
724 		       packet->tlen - ((packet->rcd->rcvhdrqentsize -  in process_rcv_packet_napi()
728 	packet->rcd->rhf_rcv_function_map[packet->etype](packet);  in process_rcv_packet_napi()
736 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in process_rcv_packet_napi()
737 				      packet->rcd->rhf_offset;  in process_rcv_packet_napi()
753 		packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,  in process_rcv_packet()
761 			       packet->tlen - ((get_hdrqentsize(packet->rcd) -  in process_rcv_packet()
774 	packet->rcd->rhf_rcv_function_map[packet->etype](packet);  in process_rcv_packet()
784 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +  in process_rcv_packet()
785 				      packet->rcd->rhf_offset;  in process_rcv_packet()
800 		update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,  in process_rcv_update()
815 	update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr,  in finish_packet()
828 int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)  in handle_receive_interrupt_napi_fp()  argument
832 	init_packet(rcd, &packet);  in handle_receive_interrupt_napi_fp()
833 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_fp()
838 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_fp()
843 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_napi_fp()
852 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt_nodma_rtail()  argument
857 	init_packet(rcd, &packet);  in handle_receive_interrupt_nodma_rtail()
858 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {  in handle_receive_interrupt_nodma_rtail()
863 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt_nodma_rtail()
867 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_nodma_rtail()
872 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_nodma_rtail()
878 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt_dma_rtail()  argument
884 	init_packet(rcd, &packet);  in handle_receive_interrupt_dma_rtail()
885 	hdrqtail = get_rcvhdrtail(rcd);  in handle_receive_interrupt_dma_rtail()
892 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt_dma_rtail()
901 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_dma_rtail()
907 static void set_all_fastpath(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)  in set_all_fastpath()  argument
916 	if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) {  in set_all_fastpath()
917 		hfi1_rcd_get(rcd);  in set_all_fastpath()
918 		hfi1_set_fast(rcd);  in set_all_fastpath()
919 		hfi1_rcd_put(rcd);  in set_all_fastpath()
924 		rcd = hfi1_rcd_get_by_index(dd, i);  in set_all_fastpath()
925 		if (rcd && (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic))  in set_all_fastpath()
926 			hfi1_set_fast(rcd);  in set_all_fastpath()
927 		hfi1_rcd_put(rcd);  in set_all_fastpath()
933 	struct hfi1_ctxtdata *rcd;  in set_all_slowpath()  local
938 		rcd = hfi1_rcd_get_by_index(dd, i);  in set_all_slowpath()
939 		if (!rcd)  in set_all_slowpath()
941 		if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)  in set_all_slowpath()
942 			rcd->do_interrupt = rcd->slow_handler;  in set_all_slowpath()
944 		hfi1_rcd_put(rcd);  in set_all_slowpath()
954 		struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,  in __set_armed_to_active()
959 						packet->rcd,  in __set_armed_to_active()
964 		int hwstate = driver_lstate(packet->rcd->ppd);  in __set_armed_to_active()
966 				&packet->rcd->ppd->linkstate_active_work;  in __set_armed_to_active()
969 			dd_dev_info(packet->rcd->dd,  in __set_armed_to_active()
975 		queue_work(packet->rcd->ppd->link_wq, lsaw);  in __set_armed_to_active()
989 	if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED))  in set_armed_to_active()
1001 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)  in handle_receive_interrupt()  argument
1003 	struct hfi1_devdata *dd = rcd->dd;  in handle_receive_interrupt()
1009 	if (!rcd->rcvhdrq)  in handle_receive_interrupt()
1012 	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;  in handle_receive_interrupt()
1014 	init_packet(rcd, &packet);  in handle_receive_interrupt()
1016 	if (!get_dma_rtail_setting(rcd)) {  in handle_receive_interrupt()
1017 		if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {  in handle_receive_interrupt()
1023 		hdrqtail = get_rcvhdrtail(rcd);  in handle_receive_interrupt()
1034 		if (rcd->ctxt == HFI1_CTRL_CTXT)  in handle_receive_interrupt()
1035 			if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt()
1039 	prescan_rxq(rcd, &packet);  in handle_receive_interrupt()
1045 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +  in handle_receive_interrupt()
1047 					  rcd->rhf_offset;  in handle_receive_interrupt()
1059 		if (!get_dma_rtail_setting(rcd)) {  in handle_receive_interrupt()
1060 			if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt()
1069 			if (rcd->ctxt == HFI1_CTRL_CTXT) {  in handle_receive_interrupt()
1072 				lseq = hfi1_seq_incr(rcd,  in handle_receive_interrupt()
1081 			set_all_fastpath(dd, rcd);  in handle_receive_interrupt()
1087 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt()
1107 int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)  in handle_receive_interrupt_napi_sp()  argument
1109 	struct hfi1_devdata *dd = rcd->dd;  in handle_receive_interrupt_napi_sp()
1114 	init_packet(rcd, &packet);  in handle_receive_interrupt_napi_sp()
1115 	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_sp()
1122 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +  in handle_receive_interrupt_napi_sp()
1124 					  rcd->rhf_offset;  in handle_receive_interrupt_napi_sp()
1133 		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))  in handle_receive_interrupt_napi_sp()
1138 			set_all_fastpath(dd, rcd);  in handle_receive_interrupt_napi_sp()
1144 	hfi1_set_rcd_head(rcd, packet.rhqoff);  in handle_receive_interrupt_napi_sp()
1177 	struct hfi1_ctxtdata *rcd;  in receive_interrupt_work()  local
1189 		rcd = hfi1_rcd_get_by_index(dd, i);  in receive_interrupt_work()
1190 		if (rcd)  in receive_interrupt_work()
1191 			force_recv_intr(rcd);  in receive_interrupt_work()
1192 		hfi1_rcd_put(rcd);  in receive_interrupt_work()
1404 	if (dd->rcd)  in hfi1_reset_device()
1439 			hfi1_get_msgheader(packet->rcd,  in hfi1_setup_ib_header()
1446 	struct hfi1_pportdata *ppd = packet->rcd->ppd;  in hfi1_bypass_ingress_pkt_check()
1475 	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);  in hfi1_setup_9B_packet()
1534 	struct hfi1_ctxtdata *rcd = packet->rcd;  in hfi1_setup_bypass_packet()  local
1535 	struct hfi1_pportdata *ppd = rcd->ppd;  in hfi1_setup_bypass_packet()
1540 			hfi1_get_16B_header(packet->rcd,  in hfi1_setup_bypass_packet()
1607 	struct hfi1_ctxtdata *rcd = packet->rcd;  in show_eflags_errs()  local
1610 	dd_dev_err(rcd->dd,  in show_eflags_errs()
1612 		   rcd->ctxt, packet->rhf,  in show_eflags_errs()
1625 	struct hfi1_ctxtdata *rcd = packet->rcd;  in handle_eflags()  local
1627 	rcv_hdrerr(rcd, rcd->ppd, packet);  in handle_eflags()
1636 	struct hfi1_ctxtdata *rcd = packet->rcd;  in hfi1_ipoib_ib_rcv()  local
1637 	struct napi_struct *napi = rcd->napi;  in hfi1_ipoib_ib_rcv()
1658 	netdev = hfi1_netdev_get_data(rcd->dd, qpnum);  in hfi1_ipoib_ib_rcv()
1662 	trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));  in hfi1_ipoib_ib_rcv()
1663 	trace_ctxt_rsm_hist(rcd->ctxt);  in hfi1_ipoib_ib_rcv()
1702 	ibp = rcd_to_iport(packet->rcd);  in hfi1_ipoib_ib_rcv()
1730 	struct hfi1_devdata *dd = packet->rcd->dd;  in process_receive_bypass()
1768 		 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&  in process_receive_error()
1777 		dd_dev_err(packet->rcd->dd,  in process_receive_error()
1788 		struct hfi1_ctxtdata *rcd = packet->rcd;  in kdeth_process_expected()  local
1790 		if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))  in kdeth_process_expected()
1805 		struct hfi1_ctxtdata *rcd = packet->rcd;  in kdeth_process_eager()  local
1808 		if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))  in kdeth_process_eager()
1817 	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",  in process_receive_invalid()
1823 void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)  in seqfile_dump_rcd()  argument
1830 		   rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),  in seqfile_dump_rcd()
1831 		   get_dma_rtail_setting(rcd) ?  in seqfile_dump_rcd()
1833 		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL),  in seqfile_dump_rcd()
1834 		   read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS),  in seqfile_dump_rcd()
1835 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &  in seqfile_dump_rcd()
1837 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL),  in seqfile_dump_rcd()
1838 		   rcd->head);  in seqfile_dump_rcd()
1840 	init_packet(rcd, &packet);  in seqfile_dump_rcd()
1844 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +  in seqfile_dump_rcd()
1845 					 rcd->rhf_offset;  in seqfile_dump_rcd()
1853 		if (ps_done(&mdata, rhf, rcd))  in seqfile_dump_rcd()
1856 		if (ps_skip(&mdata, rhf, rcd))  in seqfile_dump_rcd()
1862 		packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);  in seqfile_dump_rcd()
1881 		update_ps_mdata(&mdata, rcd);  in seqfile_dump_rcd()