xref: /openbmc/linux/drivers/infiniband/hw/hfi1/tid_rdma.c (revision 8795a739)

// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/*
 * Copyright(c) 2018 Intel Corporation.
 *
 */

#include "hfi.h"
#include "qp.h"
#include "rc.h"
#include "verbs.h"
#include "tid_rdma.h"
#include "exp_rcv.h"
#include "trace.h"

/**
 * DOC: TID RDMA READ protocol
 *
 * This is an end-to-end protocol at the hfi1 level between two nodes that
 * improves performance by avoiding data copy on the requester side. It
 * converts a qualified RDMA READ request into a TID RDMA READ request on
 * the requester side and thereafter handles the request and response
 * differently. To be qualified, the RDMA READ request should meet the
 * following:
 * -- The total data length should be greater than 256K;
 * -- The total data length should be a multiple of 4K page size;
 * -- Each local scatter-gather entry should be 4K page aligned;
 * -- Each local scatter-gather entry should be a multiple of 4K page size;
 */

#define RCV_TID_FLOW_TABLE_CTRL_FLOW_VALID_SMASK BIT_ULL(32)
#define RCV_TID_FLOW_TABLE_CTRL_HDR_SUPP_EN_SMASK BIT_ULL(33)
#define RCV_TID_FLOW_TABLE_CTRL_KEEP_AFTER_SEQ_ERR_SMASK BIT_ULL(34)
#define RCV_TID_FLOW_TABLE_CTRL_KEEP_ON_GEN_ERR_SMASK BIT_ULL(35)
#define RCV_TID_FLOW_TABLE_STATUS_SEQ_MISMATCH_SMASK BIT_ULL(37)
#define RCV_TID_FLOW_TABLE_STATUS_GEN_MISMATCH_SMASK BIT_ULL(38)

/* Maximum number of packets within a flow generation. */
#define MAX_TID_FLOW_PSN BIT(HFI1_KDETH_BTH_SEQ_SHIFT)

#define GENERATION_MASK 0xFFFFF

static u32 mask_generation(u32 a)
{
	return a & GENERATION_MASK;
}

/* Reserved generation value to set to unused flows for kernel contexts */
#define KERN_GENERATION_RESERVED mask_generation(U32_MAX)

/*
 * J_KEY for kernel contexts when TID RDMA is used.
 * See generate_jkey() in hfi.h for more information.
 */
#define TID_RDMA_JKEY                   32
#define HFI1_KERNEL_MIN_JKEY HFI1_ADMIN_JKEY_RANGE
#define HFI1_KERNEL_MAX_JKEY (2 * HFI1_ADMIN_JKEY_RANGE - 1)

/* Maximum number of segments in flight per QP request. */
#define TID_RDMA_MAX_READ_SEGS_PER_REQ  6
#define TID_RDMA_MAX_WRITE_SEGS_PER_REQ 4
#define MAX_REQ max_t(u16, TID_RDMA_MAX_READ_SEGS_PER_REQ, \
			TID_RDMA_MAX_WRITE_SEGS_PER_REQ)
#define MAX_FLOWS roundup_pow_of_two(MAX_REQ + 1)

#define MAX_EXPECTED_PAGES     (MAX_EXPECTED_BUFFER / PAGE_SIZE)

#define TID_RDMA_DESTQP_FLOW_SHIFT      11
#define TID_RDMA_DESTQP_FLOW_MASK       0x1f

#define TID_OPFN_QP_CTXT_MASK 0xff
#define TID_OPFN_QP_CTXT_SHIFT 56
#define TID_OPFN_QP_KDETH_MASK 0xff
#define TID_OPFN_QP_KDETH_SHIFT 48
#define TID_OPFN_MAX_LEN_MASK 0x7ff
#define TID_OPFN_MAX_LEN_SHIFT 37
#define TID_OPFN_TIMEOUT_MASK 0x1f
#define TID_OPFN_TIMEOUT_SHIFT 32
#define TID_OPFN_RESERVED_MASK 0x3f
#define TID_OPFN_RESERVED_SHIFT 26
#define TID_OPFN_URG_MASK 0x1
#define TID_OPFN_URG_SHIFT 25
#define TID_OPFN_VER_MASK 0x7
#define TID_OPFN_VER_SHIFT 22
#define TID_OPFN_JKEY_MASK 0x3f
#define TID_OPFN_JKEY_SHIFT 16
#define TID_OPFN_MAX_READ_MASK 0x3f
#define TID_OPFN_MAX_READ_SHIFT 10
#define TID_OPFN_MAX_WRITE_MASK 0x3f
#define TID_OPFN_MAX_WRITE_SHIFT 4

/*
 * OPFN TID layout
 *
 * 63               47               31               15
 * NNNNNNNNKKKKKKKK MMMMMMMMMMMTTTTT DDDDDDUVVVJJJJJJ RRRRRRWWWWWWCCCC
 * 3210987654321098 7654321098765432 1098765432109876 5432109876543210
 * N - the context Number
 * K - the Kdeth_qp
 * M - Max_len
 * T - Timeout
 * D - reserveD
 * V - version
 * U - Urg capable
 * J - Jkey
 * R - max_Read
 * W - max_Write
 * C - Capcode
 */

static u32 tid_rdma_flow_wt;

static void tid_rdma_trigger_resume(struct work_struct *work);
static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req);
static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req,
					 gfp_t gfp);
static void hfi1_init_trdma_req(struct rvt_qp *qp,
				struct tid_rdma_request *req);
static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx);
static void hfi1_tid_timeout(struct timer_list *t);
static void hfi1_add_tid_reap_timer(struct rvt_qp *qp);
static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp);
static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp);
static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp);
static void hfi1_tid_retry_timeout(struct timer_list *t);
static int make_tid_rdma_ack(struct rvt_qp *qp,
			     struct ib_other_headers *ohdr,
			     struct hfi1_pkt_state *ps);
static void hfi1_do_tid_send(struct rvt_qp *qp);
static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx);
static void tid_rdma_rcv_err(struct hfi1_packet *packet,
			     struct ib_other_headers *ohdr,
			     struct rvt_qp *qp, u32 psn, int diff, bool fecn);
static void update_r_next_psn_fecn(struct hfi1_packet *packet,
				   struct hfi1_qp_priv *priv,
				   struct hfi1_ctxtdata *rcd,
				   struct tid_rdma_flow *flow,
				   bool fecn);

static u64 tid_rdma_opfn_encode(struct tid_rdma_params *p)
{
	return
		(((u64)p->qp & TID_OPFN_QP_CTXT_MASK) <<
			TID_OPFN_QP_CTXT_SHIFT) |
		((((u64)p->qp >> 16) & TID_OPFN_QP_KDETH_MASK) <<
			TID_OPFN_QP_KDETH_SHIFT) |
		(((u64)((p->max_len >> PAGE_SHIFT) - 1) &
			TID_OPFN_MAX_LEN_MASK) << TID_OPFN_MAX_LEN_SHIFT) |
		(((u64)p->timeout & TID_OPFN_TIMEOUT_MASK) <<
			TID_OPFN_TIMEOUT_SHIFT) |
		(((u64)p->urg & TID_OPFN_URG_MASK) << TID_OPFN_URG_SHIFT) |
		(((u64)p->jkey & TID_OPFN_JKEY_MASK) << TID_OPFN_JKEY_SHIFT) |
		(((u64)p->max_read & TID_OPFN_MAX_READ_MASK) <<
			TID_OPFN_MAX_READ_SHIFT) |
		(((u64)p->max_write & TID_OPFN_MAX_WRITE_MASK) <<
			TID_OPFN_MAX_WRITE_SHIFT);
}

static void tid_rdma_opfn_decode(struct tid_rdma_params *p, u64 data)
{
	p->max_len = (((data >> TID_OPFN_MAX_LEN_SHIFT) &
		TID_OPFN_MAX_LEN_MASK) + 1) << PAGE_SHIFT;
	p->jkey = (data >> TID_OPFN_JKEY_SHIFT) & TID_OPFN_JKEY_MASK;
	p->max_write = (data >> TID_OPFN_MAX_WRITE_SHIFT) &
		TID_OPFN_MAX_WRITE_MASK;
	p->max_read = (data >> TID_OPFN_MAX_READ_SHIFT) &
		TID_OPFN_MAX_READ_MASK;
	p->qp =
		((((data >> TID_OPFN_QP_KDETH_SHIFT) & TID_OPFN_QP_KDETH_MASK)
			<< 16) |
		((data >> TID_OPFN_QP_CTXT_SHIFT) & TID_OPFN_QP_CTXT_MASK));
	p->urg = (data >> TID_OPFN_URG_SHIFT) & TID_OPFN_URG_MASK;
	p->timeout = (data >> TID_OPFN_TIMEOUT_SHIFT) & TID_OPFN_TIMEOUT_MASK;
}

void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p)
{
	struct hfi1_qp_priv *priv = qp->priv;

	p->qp = (kdeth_qp << 16) | priv->rcd->ctxt;
	p->max_len = TID_RDMA_MAX_SEGMENT_SIZE;
	p->jkey = priv->rcd->jkey;
	p->max_read = TID_RDMA_MAX_READ_SEGS_PER_REQ;
	p->max_write = TID_RDMA_MAX_WRITE_SEGS_PER_REQ;
	p->timeout = qp->timeout;
	p->urg = is_urg_masked(priv->rcd);
}

bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data)
{
	struct hfi1_qp_priv *priv = qp->priv;

	*data = tid_rdma_opfn_encode(&priv->tid_rdma.local);
	return true;
}

bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct tid_rdma_params *remote, *old;
	bool ret = true;

	old = rcu_dereference_protected(priv->tid_rdma.remote,
					lockdep_is_held(&priv->opfn.lock));
	data &= ~0xfULL;
	/*
	 * If data passed in is zero, return true so as not to continue the
	 * negotiation process
	 */
	if (!data || !HFI1_CAP_IS_KSET(TID_RDMA))
		goto null;
	/*
	 * If kzalloc fails, return false. This will result in:
	 * * at the requester a new OPFN request being generated to retry
	 *   the negotiation
	 * * at the responder, 0 being returned to the requester so as to
	 *   disable TID RDMA at both the requester and the responder
	 */
	remote = kzalloc(sizeof(*remote), GFP_ATOMIC);
	if (!remote) {
		ret = false;
		goto null;
	}

	tid_rdma_opfn_decode(remote, data);
	priv->tid_timer_timeout_jiffies =
		usecs_to_jiffies((((4096UL * (1UL << remote->timeout)) /
				   1000UL) << 3) * 7);
	trace_hfi1_opfn_param(qp, 0, &priv->tid_rdma.local);
	trace_hfi1_opfn_param(qp, 1, remote);
	rcu_assign_pointer(priv->tid_rdma.remote, remote);
	/*
	 * A TID RDMA READ request's segment size is not equal to
	 * remote->max_len only when the request's data length is smaller
	 * than remote->max_len. In that case, there will be only one segment.
	 * Therefore, when priv->pkts_ps is used to calculate req->cur_seg
	 * during retry, it will lead to req->cur_seg = 0, which is exactly
	 * what is expected.
	 */
	priv->pkts_ps = (u16)rvt_div_mtu(qp, remote->max_len);
	priv->timeout_shift = ilog2(priv->pkts_ps - 1) + 1;
	goto free;
null:
	RCU_INIT_POINTER(priv->tid_rdma.remote, NULL);
	priv->timeout_shift = 0;
free:
	if (old)
		kfree_rcu(old, rcu_head);
	return ret;
}

bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data)
{
	bool ret;

	ret = tid_rdma_conn_reply(qp, *data);
	*data = 0;
	/*
	 * If tid_rdma_conn_reply() returns error, set *data as 0 to indicate
	 * TID RDMA could not be enabled. This will result in TID RDMA being
	 * disabled at the requester too.
	 */
	if (ret)
		(void)tid_rdma_conn_req(qp, data);
	return ret;
}

void tid_rdma_conn_error(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct tid_rdma_params *old;

	old = rcu_dereference_protected(priv->tid_rdma.remote,
					lockdep_is_held(&priv->opfn.lock));
	RCU_INIT_POINTER(priv->tid_rdma.remote, NULL);
	if (old)
		kfree_rcu(old, rcu_head);
}

/* This is called at context initialization time */
int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit)
{
	if (reinit)
		return 0;

	BUILD_BUG_ON(TID_RDMA_JKEY < HFI1_KERNEL_MIN_JKEY);
	BUILD_BUG_ON(TID_RDMA_JKEY > HFI1_KERNEL_MAX_JKEY);
	rcd->jkey = TID_RDMA_JKEY;
	hfi1_set_ctxt_jkey(rcd->dd, rcd, rcd->jkey);
	return hfi1_alloc_ctxt_rcv_groups(rcd);
}

/**
 * qp_to_rcd - determine the receive context used by a qp
 * @qp - the qp
 *
 * This routine returns the receive context associated
 * with a a qp's qpn.
 *
 * Returns the context.
 */
static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi,
				       struct rvt_qp *qp)
{
	struct hfi1_ibdev *verbs_dev = container_of(rdi,
						    struct hfi1_ibdev,
						    rdi);
	struct hfi1_devdata *dd = container_of(verbs_dev,
					       struct hfi1_devdata,
					       verbs_dev);
	unsigned int ctxt;

	if (qp->ibqp.qp_num == 0)
		ctxt = 0;
	else
		ctxt = hfi1_get_qp_map(dd, qp->ibqp.qp_num >> dd->qos_shift);
	return dd->rcd[ctxt];
}

int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp,
		      struct ib_qp_init_attr *init_attr)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	int i, ret;

	qpriv->rcd = qp_to_rcd(rdi, qp);

	spin_lock_init(&qpriv->opfn.lock);
	INIT_WORK(&qpriv->opfn.opfn_work, opfn_send_conn_request);
	INIT_WORK(&qpriv->tid_rdma.trigger_work, tid_rdma_trigger_resume);
	qpriv->flow_state.psn = 0;
	qpriv->flow_state.index = RXE_NUM_TID_FLOWS;
	qpriv->flow_state.last_index = RXE_NUM_TID_FLOWS;
	qpriv->flow_state.generation = KERN_GENERATION_RESERVED;
	qpriv->s_state = TID_OP(WRITE_RESP);
	qpriv->s_tid_cur = HFI1_QP_WQE_INVALID;
	qpriv->s_tid_head = HFI1_QP_WQE_INVALID;
	qpriv->s_tid_tail = HFI1_QP_WQE_INVALID;
	qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
	qpriv->r_tid_head = HFI1_QP_WQE_INVALID;
	qpriv->r_tid_tail = HFI1_QP_WQE_INVALID;
	qpriv->r_tid_ack = HFI1_QP_WQE_INVALID;
	qpriv->r_tid_alloc = HFI1_QP_WQE_INVALID;
	atomic_set(&qpriv->n_requests, 0);
	atomic_set(&qpriv->n_tid_requests, 0);
	timer_setup(&qpriv->s_tid_timer, hfi1_tid_timeout, 0);
	timer_setup(&qpriv->s_tid_retry_timer, hfi1_tid_retry_timeout, 0);
	INIT_LIST_HEAD(&qpriv->tid_wait);

	if (init_attr->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) {
		struct hfi1_devdata *dd = qpriv->rcd->dd;

		qpriv->pages = kzalloc_node(TID_RDMA_MAX_PAGES *
						sizeof(*qpriv->pages),
					    GFP_KERNEL, dd->node);
		if (!qpriv->pages)
			return -ENOMEM;
		for (i = 0; i < qp->s_size; i++) {
			struct hfi1_swqe_priv *priv;
			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i);

			priv = kzalloc_node(sizeof(*priv), GFP_KERNEL,
					    dd->node);
			if (!priv)
				return -ENOMEM;

			hfi1_init_trdma_req(qp, &priv->tid_req);
			priv->tid_req.e.swqe = wqe;
			wqe->priv = priv;
		}
		for (i = 0; i < rvt_max_atomic(rdi); i++) {
			struct hfi1_ack_priv *priv;

			priv = kzalloc_node(sizeof(*priv), GFP_KERNEL,
					    dd->node);
			if (!priv)
				return -ENOMEM;

			hfi1_init_trdma_req(qp, &priv->tid_req);
			priv->tid_req.e.ack = &qp->s_ack_queue[i];

			ret = hfi1_kern_exp_rcv_alloc_flows(&priv->tid_req,
							    GFP_KERNEL);
			if (ret) {
				kfree(priv);
				return ret;
			}
			qp->s_ack_queue[i].priv = priv;
		}
	}

	return 0;
}

void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct rvt_swqe *wqe;
	u32 i;

	if (qp->ibqp.qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) {
		for (i = 0; i < qp->s_size; i++) {
			wqe = rvt_get_swqe_ptr(qp, i);
			kfree(wqe->priv);
			wqe->priv = NULL;
		}
		for (i = 0; i < rvt_max_atomic(rdi); i++) {
			struct hfi1_ack_priv *priv = qp->s_ack_queue[i].priv;

			if (priv)
				hfi1_kern_exp_rcv_free_flows(&priv->tid_req);
			kfree(priv);
			qp->s_ack_queue[i].priv = NULL;
		}
		cancel_work_sync(&qpriv->opfn.opfn_work);
		kfree(qpriv->pages);
		qpriv->pages = NULL;
	}
}

/* Flow and tid waiter functions */
/**
 * DOC: lock ordering
 *
 * There are two locks involved with the queuing
 * routines: the qp s_lock and the exp_lock.
 *
 * Since the tid space allocation is called from
 * the send engine, the qp s_lock is already held.
 *
 * The allocation routines will get the exp_lock.
 *
 * The first_qp() call is provided to allow the head of
 * the rcd wait queue to be fetched under the exp_lock and
 * followed by a drop of the exp_lock.
 *
 * Any qp in the wait list will have the qp reference count held
 * to hold the qp in memory.
 */

/*
 * return head of rcd wait list
 *
 * Must hold the exp_lock.
 *
 * Get a reference to the QP to hold the QP in memory.
 *
 * The caller must release the reference when the local
 * is no longer being used.
 */
static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd,
			       struct tid_queue *queue)
	__must_hold(&rcd->exp_lock)
{
	struct hfi1_qp_priv *priv;

	lockdep_assert_held(&rcd->exp_lock);
	priv = list_first_entry_or_null(&queue->queue_head,
					struct hfi1_qp_priv,
					tid_wait);
	if (!priv)
		return NULL;
	rvt_get_qp(priv->owner);
	return priv->owner;
}

/**
 * kernel_tid_waiters - determine rcd wait
 * @rcd: the receive context
 * @qp: the head of the qp being processed
 *
 * This routine will return false IFF
 * the list is NULL or the head of the
 * list is the indicated qp.
 *
 * Must hold the qp s_lock and the exp_lock.
 *
 * Return:
 * false if either of the conditions below are satisfied:
 * 1. The list is empty or
 * 2. The indicated qp is at the head of the list and the
 *    HFI1_S_WAIT_TID_SPACE bit is set in qp->s_flags.
 * true is returned otherwise.
 */
static bool kernel_tid_waiters(struct hfi1_ctxtdata *rcd,
			       struct tid_queue *queue, struct rvt_qp *qp)
	__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock)
{
	struct rvt_qp *fqp;
	bool ret = true;

	lockdep_assert_held(&qp->s_lock);
	lockdep_assert_held(&rcd->exp_lock);
	fqp = first_qp(rcd, queue);
	if (!fqp || (fqp == qp && (qp->s_flags & HFI1_S_WAIT_TID_SPACE)))
		ret = false;
	rvt_put_qp(fqp);
	return ret;
}

/**
 * dequeue_tid_waiter - dequeue the qp from the list
 * @qp - the qp to remove the wait list
 *
 * This routine removes the indicated qp from the
 * wait list if it is there.
 *
 * This should be done after the hardware flow and
 * tid array resources have been allocated.
 *
 * Must hold the qp s_lock and the rcd exp_lock.
 *
 * It assumes the s_lock to protect the s_flags
 * field and to reliably test the HFI1_S_WAIT_TID_SPACE flag.
 */
static void dequeue_tid_waiter(struct hfi1_ctxtdata *rcd,
			       struct tid_queue *queue, struct rvt_qp *qp)
	__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv = qp->priv;

	lockdep_assert_held(&qp->s_lock);
	lockdep_assert_held(&rcd->exp_lock);
	if (list_empty(&priv->tid_wait))
		return;
	list_del_init(&priv->tid_wait);
	qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE;
	queue->dequeue++;
	rvt_put_qp(qp);
}

/**
 * queue_qp_for_tid_wait - suspend QP on tid space
 * @rcd: the receive context
 * @qp: the qp
 *
 * The qp is inserted at the tail of the rcd
 * wait queue and the HFI1_S_WAIT_TID_SPACE s_flag is set.
 *
 * Must hold the qp s_lock and the exp_lock.
 */
static void queue_qp_for_tid_wait(struct hfi1_ctxtdata *rcd,
				  struct tid_queue *queue, struct rvt_qp *qp)
	__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv = qp->priv;

	lockdep_assert_held(&qp->s_lock);
	lockdep_assert_held(&rcd->exp_lock);
	if (list_empty(&priv->tid_wait)) {
		qp->s_flags |= HFI1_S_WAIT_TID_SPACE;
		list_add_tail(&priv->tid_wait, &queue->queue_head);
		priv->tid_enqueue = ++queue->enqueue;
		rcd->dd->verbs_dev.n_tidwait++;
		trace_hfi1_qpsleep(qp, HFI1_S_WAIT_TID_SPACE);
		rvt_get_qp(qp);
	}
}

/**
 * __trigger_tid_waiter - trigger tid waiter
 * @qp: the qp
 *
 * This is a private entrance to schedule the qp
 * assuming the caller is holding the qp->s_lock.
 */
static void __trigger_tid_waiter(struct rvt_qp *qp)
	__must_hold(&qp->s_lock)
{
	lockdep_assert_held(&qp->s_lock);
	if (!(qp->s_flags & HFI1_S_WAIT_TID_SPACE))
		return;
	trace_hfi1_qpwakeup(qp, HFI1_S_WAIT_TID_SPACE);
	hfi1_schedule_send(qp);
}

/**
 * tid_rdma_schedule_tid_wakeup - schedule wakeup for a qp
 * @qp - the qp
 *
 * trigger a schedule or a waiting qp in a deadlock
 * safe manner.  The qp reference is held prior
 * to this call via first_qp().
 *
 * If the qp trigger was already scheduled (!rval)
 * the the reference is dropped, otherwise the resume
 * or the destroy cancel will dispatch the reference.
 */
static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv;
	struct hfi1_ibport *ibp;
	struct hfi1_pportdata *ppd;
	struct hfi1_devdata *dd;
	bool rval;

	if (!qp)
		return;

	priv = qp->priv;
	ibp = to_iport(qp->ibqp.device, qp->port_num);
	ppd = ppd_from_ibp(ibp);
	dd = dd_from_ibdev(qp->ibqp.device);

	rval = queue_work_on(priv->s_sde ?
			     priv->s_sde->cpu :
			     cpumask_first(cpumask_of_node(dd->node)),
			     ppd->hfi1_wq,
			     &priv->tid_rdma.trigger_work);
	if (!rval)
		rvt_put_qp(qp);
}

/**
 * tid_rdma_trigger_resume - field a trigger work request
 * @work - the work item
 *
 * Complete the off qp trigger processing by directly
 * calling the progress routine.
 */
static void tid_rdma_trigger_resume(struct work_struct *work)
{
	struct tid_rdma_qp_params *tr;
	struct hfi1_qp_priv *priv;
	struct rvt_qp *qp;

	tr = container_of(work, struct tid_rdma_qp_params, trigger_work);
	priv = container_of(tr, struct hfi1_qp_priv, tid_rdma);
	qp = priv->owner;
	spin_lock_irq(&qp->s_lock);
	if (qp->s_flags & HFI1_S_WAIT_TID_SPACE) {
		spin_unlock_irq(&qp->s_lock);
		hfi1_do_send(priv->owner, true);
	} else {
		spin_unlock_irq(&qp->s_lock);
	}
	rvt_put_qp(qp);
}

/**
 * tid_rdma_flush_wait - unwind any tid space wait
 *
 * This is called when resetting a qp to
 * allow a destroy or reset to get rid
 * of any tid space linkage and reference counts.
 */
static void _tid_rdma_flush_wait(struct rvt_qp *qp, struct tid_queue *queue)
	__must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv;

	if (!qp)
		return;
	lockdep_assert_held(&qp->s_lock);
	priv = qp->priv;
	qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE;
	spin_lock(&priv->rcd->exp_lock);
	if (!list_empty(&priv->tid_wait)) {
		list_del_init(&priv->tid_wait);
		qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE;
		queue->dequeue++;
		rvt_put_qp(qp);
	}
	spin_unlock(&priv->rcd->exp_lock);
}

void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp)
	__must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv = qp->priv;

	_tid_rdma_flush_wait(qp, &priv->rcd->flow_queue);
	_tid_rdma_flush_wait(qp, &priv->rcd->rarr_queue);
}

/* Flow functions */
/**
 * kern_reserve_flow - allocate a hardware flow
 * @rcd - the context to use for allocation
 * @last - the index of the preferred flow. Use RXE_NUM_TID_FLOWS to
 *         signify "don't care".
 *
 * Use a bit mask based allocation to reserve a hardware
 * flow for use in receiving KDETH data packets. If a preferred flow is
 * specified the function will attempt to reserve that flow again, if
 * available.
 *
 * The exp_lock must be held.
 *
 * Return:
 * On success: a value postive value between 0 and RXE_NUM_TID_FLOWS - 1
 * On failure: -EAGAIN
 */
static int kern_reserve_flow(struct hfi1_ctxtdata *rcd, int last)
	__must_hold(&rcd->exp_lock)
{
	int nr;

	/* Attempt to reserve the preferred flow index */
	if (last >= 0 && last < RXE_NUM_TID_FLOWS &&
	    !test_and_set_bit(last, &rcd->flow_mask))
		return last;

	nr = ffz(rcd->flow_mask);
	BUILD_BUG_ON(RXE_NUM_TID_FLOWS >=
		     (sizeof(rcd->flow_mask) * BITS_PER_BYTE));
	if (nr > (RXE_NUM_TID_FLOWS - 1))
		return -EAGAIN;
	set_bit(nr, &rcd->flow_mask);
	return nr;
}

static void kern_set_hw_flow(struct hfi1_ctxtdata *rcd, u32 generation,
			     u32 flow_idx)
{
	u64 reg;

	reg = ((u64)generation << HFI1_KDETH_BTH_SEQ_SHIFT) |
		RCV_TID_FLOW_TABLE_CTRL_FLOW_VALID_SMASK |
		RCV_TID_FLOW_TABLE_CTRL_KEEP_AFTER_SEQ_ERR_SMASK |
		RCV_TID_FLOW_TABLE_CTRL_KEEP_ON_GEN_ERR_SMASK |
		RCV_TID_FLOW_TABLE_STATUS_SEQ_MISMATCH_SMASK |
		RCV_TID_FLOW_TABLE_STATUS_GEN_MISMATCH_SMASK;

	if (generation != KERN_GENERATION_RESERVED)
		reg |= RCV_TID_FLOW_TABLE_CTRL_HDR_SUPP_EN_SMASK;

	write_uctxt_csr(rcd->dd, rcd->ctxt,
			RCV_TID_FLOW_TABLE + 8 * flow_idx, reg);
}

static u32 kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, u32 flow_idx)
	__must_hold(&rcd->exp_lock)
{
	u32 generation = rcd->flows[flow_idx].generation;

	kern_set_hw_flow(rcd, generation, flow_idx);
	return generation;
}

static u32 kern_flow_generation_next(u32 gen)
{
	u32 generation = mask_generation(gen + 1);

	if (generation == KERN_GENERATION_RESERVED)
		generation = mask_generation(generation + 1);
	return generation;
}

static void kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, u32 flow_idx)
	__must_hold(&rcd->exp_lock)
{
	rcd->flows[flow_idx].generation =
		kern_flow_generation_next(rcd->flows[flow_idx].generation);
	kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, flow_idx);
}

int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
	struct tid_flow_state *fs = &qpriv->flow_state;
	struct rvt_qp *fqp;
	unsigned long flags;
	int ret = 0;

	/* The QP already has an allocated flow */
	if (fs->index != RXE_NUM_TID_FLOWS)
		return ret;

	spin_lock_irqsave(&rcd->exp_lock, flags);
	if (kernel_tid_waiters(rcd, &rcd->flow_queue, qp))
		goto queue;

	ret = kern_reserve_flow(rcd, fs->last_index);
	if (ret < 0)
		goto queue;
	fs->index = ret;
	fs->last_index = fs->index;

	/* Generation received in a RESYNC overrides default flow generation */
	if (fs->generation != KERN_GENERATION_RESERVED)
		rcd->flows[fs->index].generation = fs->generation;
	fs->generation = kern_setup_hw_flow(rcd, fs->index);
	fs->psn = 0;
	dequeue_tid_waiter(rcd, &rcd->flow_queue, qp);
	/* get head before dropping lock */
	fqp = first_qp(rcd, &rcd->flow_queue);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);

	tid_rdma_schedule_tid_wakeup(fqp);
	return 0;
queue:
	queue_qp_for_tid_wait(rcd, &rcd->flow_queue, qp);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);
	return -EAGAIN;
}

void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
	struct tid_flow_state *fs = &qpriv->flow_state;
	struct rvt_qp *fqp;
	unsigned long flags;

	if (fs->index >= RXE_NUM_TID_FLOWS)
		return;
	spin_lock_irqsave(&rcd->exp_lock, flags);
	kern_clear_hw_flow(rcd, fs->index);
	clear_bit(fs->index, &rcd->flow_mask);
	fs->index = RXE_NUM_TID_FLOWS;
	fs->psn = 0;
	fs->generation = KERN_GENERATION_RESERVED;

	/* get head before dropping lock */
	fqp = first_qp(rcd, &rcd->flow_queue);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);

	if (fqp == qp) {
		__trigger_tid_waiter(fqp);
		rvt_put_qp(fqp);
	} else {
		tid_rdma_schedule_tid_wakeup(fqp);
	}
}

void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd)
{
	int i;

	for (i = 0; i < RXE_NUM_TID_FLOWS; i++) {
		rcd->flows[i].generation = mask_generation(prandom_u32());
		kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, i);
	}
}

/* TID allocation functions */
static u8 trdma_pset_order(struct tid_rdma_pageset *s)
{
	u8 count = s->count;

	return ilog2(count) + 1;
}

/**
 * tid_rdma_find_phys_blocks_4k - get groups base on mr info
 * @npages - number of pages
 * @pages - pointer to an array of page structs
 * @list - page set array to return
 *
 * This routine returns the number of groups associated with
 * the current sge information.  This implementation is based
 * on the expected receive find_phys_blocks() adjusted to
 * use the MR information vs. the pfn.
 *
 * Return:
 * the number of RcvArray entries
 */
static u32 tid_rdma_find_phys_blocks_4k(struct tid_rdma_flow *flow,
					struct page **pages,
					u32 npages,
					struct tid_rdma_pageset *list)
{
	u32 pagecount, pageidx, setcount = 0, i;
	void *vaddr, *this_vaddr;

	if (!npages)
		return 0;

	/*
	 * Look for sets of physically contiguous pages in the user buffer.
	 * This will allow us to optimize Expected RcvArray entry usage by
	 * using the bigger supported sizes.
	 */
	vaddr = page_address(pages[0]);
	trace_hfi1_tid_flow_page(flow->req->qp, flow, 0, 0, 0, vaddr);
	for (pageidx = 0, pagecount = 1, i = 1; i <= npages; i++) {
		this_vaddr = i < npages ? page_address(pages[i]) : NULL;
		trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 0, 0,
					 this_vaddr);
		/*
		 * If the vaddr's are not sequential, pages are not physically
		 * contiguous.
		 */
		if (this_vaddr != (vaddr + PAGE_SIZE)) {
			/*
			 * At this point we have to loop over the set of
			 * physically contiguous pages and break them down it
			 * sizes supported by the HW.
			 * There are two main constraints:
			 *     1. The max buffer size is MAX_EXPECTED_BUFFER.
			 *        If the total set size is bigger than that
			 *        program only a MAX_EXPECTED_BUFFER chunk.
			 *     2. The buffer size has to be a power of two. If
			 *        it is not, round down to the closes power of
			 *        2 and program that size.
			 */
			while (pagecount) {
				int maxpages = pagecount;
				u32 bufsize = pagecount * PAGE_SIZE;

				if (bufsize > MAX_EXPECTED_BUFFER)
					maxpages =
						MAX_EXPECTED_BUFFER >>
						PAGE_SHIFT;
				else if (!is_power_of_2(bufsize))
					maxpages =
						rounddown_pow_of_two(bufsize) >>
						PAGE_SHIFT;

				list[setcount].idx = pageidx;
				list[setcount].count = maxpages;
				trace_hfi1_tid_pageset(flow->req->qp, setcount,
						       list[setcount].idx,
						       list[setcount].count);
				pagecount -= maxpages;
				pageidx += maxpages;
				setcount++;
			}
			pageidx = i;
			pagecount = 1;
			vaddr = this_vaddr;
		} else {
			vaddr += PAGE_SIZE;
			pagecount++;
		}
	}
	/* insure we always return an even number of sets */
	if (setcount & 1)
		list[setcount++].count = 0;
	return setcount;
}

/**
 * tid_flush_pages - dump out pages into pagesets
 * @list - list of pagesets
 * @idx - pointer to current page index
 * @pages - number of pages to dump
 * @sets - current number of pagesset
 *
 * This routine flushes out accumuated pages.
 *
 * To insure an even number of sets the
 * code may add a filler.
 *
 * This can happen with when pages is not
 * a power of 2 or pages is a power of 2
 * less than the maximum pages.
 *
 * Return:
 * The new number of sets
 */

static u32 tid_flush_pages(struct tid_rdma_pageset *list,
			   u32 *idx, u32 pages, u32 sets)
{
	while (pages) {
		u32 maxpages = pages;

		if (maxpages > MAX_EXPECTED_PAGES)
			maxpages = MAX_EXPECTED_PAGES;
		else if (!is_power_of_2(maxpages))
			maxpages = rounddown_pow_of_two(maxpages);
		list[sets].idx = *idx;
		list[sets++].count = maxpages;
		*idx += maxpages;
		pages -= maxpages;
	}
	/* might need a filler */
	if (sets & 1)
		list[sets++].count = 0;
	return sets;
}

/**
 * tid_rdma_find_phys_blocks_8k - get groups base on mr info
 * @pages - pointer to an array of page structs
 * @npages - number of pages
 * @list - page set array to return
 *
 * This routine parses an array of pages to compute pagesets
 * in an 8k compatible way.
 *
 * pages are tested two at a time, i, i + 1 for contiguous
 * pages and i - 1 and i contiguous pages.
 *
 * If any condition is false, any accumlated pages are flushed and
 * v0,v1 are emitted as separate PAGE_SIZE pagesets
 *
 * Otherwise, the current 8k is totaled for a future flush.
 *
 * Return:
 * The number of pagesets
 * list set with the returned number of pagesets
 *
 */
static u32 tid_rdma_find_phys_blocks_8k(struct tid_rdma_flow *flow,
					struct page **pages,
					u32 npages,
					struct tid_rdma_pageset *list)
{
	u32 idx, sets = 0, i;
	u32 pagecnt = 0;
	void *v0, *v1, *vm1;

	if (!npages)
		return 0;
	for (idx = 0, i = 0, vm1 = NULL; i < npages; i += 2) {
		/* get a new v0 */
		v0 = page_address(pages[i]);
		trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 0, v0);
		v1 = i + 1 < npages ?
				page_address(pages[i + 1]) : NULL;
		trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 1, v1);
		/* compare i, i + 1 vaddr */
		if (v1 != (v0 + PAGE_SIZE)) {
			/* flush out pages */
			sets = tid_flush_pages(list, &idx, pagecnt, sets);
			/* output v0,v1 as two pagesets */
			list[sets].idx = idx++;
			list[sets++].count = 1;
			if (v1) {
				list[sets].count = 1;
				list[sets++].idx = idx++;
			} else {
				list[sets++].count = 0;
			}
			vm1 = NULL;
			pagecnt = 0;
			continue;
		}
		/* i,i+1 consecutive, look at i-1,i */
		if (vm1 && v0 != (vm1 + PAGE_SIZE)) {
			/* flush out pages */
			sets = tid_flush_pages(list, &idx, pagecnt, sets);
			pagecnt = 0;
		}
		/* pages will always be a multiple of 8k */
		pagecnt += 2;
		/* save i-1 */
		vm1 = v1;
		/* move to next pair */
	}
	/* dump residual pages at end */
	sets = tid_flush_pages(list, &idx, npages - idx, sets);
	/* by design cannot be odd sets */
	WARN_ON(sets & 1);
	return sets;
}

/**
 * Find pages for one segment of a sge array represented by @ss. The function
 * does not check the sge, the sge must have been checked for alignment with a
 * prior call to hfi1_kern_trdma_ok. Other sge checking is done as part of
 * rvt_lkey_ok and rvt_rkey_ok. Also, the function only modifies the local sge
 * copy maintained in @ss->sge, the original sge is not modified.
 *
 * Unlike IB RDMA WRITE, we can't decrement ss->num_sge here because we are not
 * releasing the MR reference count at the same time. Otherwise, we'll "leak"
 * references to the MR. This difference requires that we keep track of progress
 * into the sg_list. This is done by the cur_seg cursor in the tid_rdma_request
 * structure.
 */
static u32 kern_find_pages(struct tid_rdma_flow *flow,
			   struct page **pages,
			   struct rvt_sge_state *ss, bool *last)
{
	struct tid_rdma_request *req = flow->req;
	struct rvt_sge *sge = &ss->sge;
	u32 length = flow->req->seg_len;
	u32 len = PAGE_SIZE;
	u32 i = 0;

	while (length && req->isge < ss->num_sge) {
		pages[i++] = virt_to_page(sge->vaddr);

		sge->vaddr += len;
		sge->length -= len;
		sge->sge_length -= len;
		if (!sge->sge_length) {
			if (++req->isge < ss->num_sge)
				*sge = ss->sg_list[req->isge - 1];
		} else if (sge->length == 0 && sge->mr->lkey) {
			if (++sge->n >= RVT_SEGSZ) {
				++sge->m;
				sge->n = 0;
			}
			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
		}
		length -= len;
	}

	flow->length = flow->req->seg_len - length;
	*last = req->isge == ss->num_sge ? false : true;
	return i;
}

static void dma_unmap_flow(struct tid_rdma_flow *flow)
{
	struct hfi1_devdata *dd;
	int i;
	struct tid_rdma_pageset *pset;

	dd = flow->req->rcd->dd;
	for (i = 0, pset = &flow->pagesets[0]; i < flow->npagesets;
			i++, pset++) {
		if (pset->count && pset->addr) {
			dma_unmap_page(&dd->pcidev->dev,
				       pset->addr,
				       PAGE_SIZE * pset->count,
				       DMA_FROM_DEVICE);
			pset->mapped = 0;
		}
	}
}

static int dma_map_flow(struct tid_rdma_flow *flow, struct page **pages)
{
	int i;
	struct hfi1_devdata *dd = flow->req->rcd->dd;
	struct tid_rdma_pageset *pset;

	for (i = 0, pset = &flow->pagesets[0]; i < flow->npagesets;
			i++, pset++) {
		if (pset->count) {
			pset->addr = dma_map_page(&dd->pcidev->dev,
						  pages[pset->idx],
						  0,
						  PAGE_SIZE * pset->count,
						  DMA_FROM_DEVICE);

			if (dma_mapping_error(&dd->pcidev->dev, pset->addr)) {
				dma_unmap_flow(flow);
				return -ENOMEM;
			}
			pset->mapped = 1;
		}
	}
	return 0;
}

static inline bool dma_mapped(struct tid_rdma_flow *flow)
{
	return !!flow->pagesets[0].mapped;
}

/*
 * Get pages pointers and identify contiguous physical memory chunks for a
 * segment. All segments are of length flow->req->seg_len.
 */
static int kern_get_phys_blocks(struct tid_rdma_flow *flow,
				struct page **pages,
				struct rvt_sge_state *ss, bool *last)
{
	u8 npages;

	/* Reuse previously computed pagesets, if any */
	if (flow->npagesets) {
		trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head,
					  flow);
		if (!dma_mapped(flow))
			return dma_map_flow(flow, pages);
		return 0;
	}

	npages = kern_find_pages(flow, pages, ss, last);

	if (flow->req->qp->pmtu == enum_to_mtu(OPA_MTU_4096))
		flow->npagesets =
			tid_rdma_find_phys_blocks_4k(flow, pages, npages,
						     flow->pagesets);
	else
		flow->npagesets =
			tid_rdma_find_phys_blocks_8k(flow, pages, npages,
						     flow->pagesets);

	return dma_map_flow(flow, pages);
}

static inline void kern_add_tid_node(struct tid_rdma_flow *flow,
				     struct hfi1_ctxtdata *rcd, char *s,
				     struct tid_group *grp, u8 cnt)
{
	struct kern_tid_node *node = &flow->tnode[flow->tnode_cnt++];

	WARN_ON_ONCE(flow->tnode_cnt >=
		     (TID_RDMA_MAX_SEGMENT_SIZE >> PAGE_SHIFT));
	if (WARN_ON_ONCE(cnt & 1))
		dd_dev_err(rcd->dd,
			   "unexpected odd allocation cnt %u map 0x%x used %u",
			   cnt, grp->map, grp->used);

	node->grp = grp;
	node->map = grp->map;
	node->cnt = cnt;
	trace_hfi1_tid_node_add(flow->req->qp, s, flow->tnode_cnt - 1,
				grp->base, grp->map, grp->used, cnt);
}

/*
 * Try to allocate pageset_count TID's from TID groups for a context
 *
 * This function allocates TID's without moving groups between lists or
 * modifying grp->map. This is done as follows, being cogizant of the lists
 * between which the TID groups will move:
 * 1. First allocate complete groups of 8 TID's since this is more efficient,
 *    these groups will move from group->full without affecting used
 * 2. If more TID's are needed allocate from used (will move from used->full or
 *    stay in used)
 * 3. If we still don't have the required number of TID's go back and look again
 *    at a complete group (will move from group->used)
 */
static int kern_alloc_tids(struct tid_rdma_flow *flow)
{
	struct hfi1_ctxtdata *rcd = flow->req->rcd;
	struct hfi1_devdata *dd = rcd->dd;
	u32 ngroups, pageidx = 0;
	struct tid_group *group = NULL, *used;
	u8 use;

	flow->tnode_cnt = 0;
	ngroups = flow->npagesets / dd->rcv_entries.group_size;
	if (!ngroups)
		goto used_list;

	/* First look at complete groups */
	list_for_each_entry(group,  &rcd->tid_group_list.list, list) {
		kern_add_tid_node(flow, rcd, "complete groups", group,
				  group->size);

		pageidx += group->size;
		if (!--ngroups)
			break;
	}

	if (pageidx >= flow->npagesets)
		goto ok;

used_list:
	/* Now look at partially used groups */
	list_for_each_entry(used, &rcd->tid_used_list.list, list) {
		use = min_t(u32, flow->npagesets - pageidx,
			    used->size - used->used);
		kern_add_tid_node(flow, rcd, "used groups", used, use);

		pageidx += use;
		if (pageidx >= flow->npagesets)
			goto ok;
	}

	/*
	 * Look again at a complete group, continuing from where we left.
	 * However, if we are at the head, we have reached the end of the
	 * complete groups list from the first loop above
	 */
	if (group && &group->list == &rcd->tid_group_list.list)
		goto bail_eagain;
	group = list_prepare_entry(group, &rcd->tid_group_list.list,
				   list);
	if (list_is_last(&group->list, &rcd->tid_group_list.list))
		goto bail_eagain;
	group = list_next_entry(group, list);
	use = min_t(u32, flow->npagesets - pageidx, group->size);
	kern_add_tid_node(flow, rcd, "complete continue", group, use);
	pageidx += use;
	if (pageidx >= flow->npagesets)
		goto ok;
bail_eagain:
	trace_hfi1_msg_alloc_tids(flow->req->qp, " insufficient tids: needed ",
				  (u64)flow->npagesets);
	return -EAGAIN;
ok:
	return 0;
}

static void kern_program_rcv_group(struct tid_rdma_flow *flow, int grp_num,
				   u32 *pset_idx)
{
	struct hfi1_ctxtdata *rcd = flow->req->rcd;
	struct hfi1_devdata *dd = rcd->dd;
	struct kern_tid_node *node = &flow->tnode[grp_num];
	struct tid_group *grp = node->grp;
	struct tid_rdma_pageset *pset;
	u32 pmtu_pg = flow->req->qp->pmtu >> PAGE_SHIFT;
	u32 rcventry, npages = 0, pair = 0, tidctrl;
	u8 i, cnt = 0;

	for (i = 0; i < grp->size; i++) {
		rcventry = grp->base + i;

		if (node->map & BIT(i) || cnt >= node->cnt) {
			rcv_array_wc_fill(dd, rcventry);
			continue;
		}
		pset = &flow->pagesets[(*pset_idx)++];
		if (pset->count) {
			hfi1_put_tid(dd, rcventry, PT_EXPECTED,
				     pset->addr, trdma_pset_order(pset));
		} else {
			hfi1_put_tid(dd, rcventry, PT_INVALID, 0, 0);
		}
		npages += pset->count;

		rcventry -= rcd->expected_base;
		tidctrl = pair ? 0x3 : rcventry & 0x1 ? 0x2 : 0x1;
		/*
		 * A single TID entry will be used to use a rcvarr pair (with
		 * tidctrl 0x3), if ALL these are true (a) the bit pos is even
		 * (b) the group map shows current and the next bits as free
		 * indicating two consecutive rcvarry entries are available (c)
		 * we actually need 2 more entries
		 */
		pair = !(i & 0x1) && !((node->map >> i) & 0x3) &&
			node->cnt >= cnt + 2;
		if (!pair) {
			if (!pset->count)
				tidctrl = 0x1;
			flow->tid_entry[flow->tidcnt++] =
				EXP_TID_SET(IDX, rcventry >> 1) |
				EXP_TID_SET(CTRL, tidctrl) |
				EXP_TID_SET(LEN, npages);
			trace_hfi1_tid_entry_alloc(/* entry */
			   flow->req->qp, flow->tidcnt - 1,
			   flow->tid_entry[flow->tidcnt - 1]);

			/* Efficient DIV_ROUND_UP(npages, pmtu_pg) */
			flow->npkts += (npages + pmtu_pg - 1) >> ilog2(pmtu_pg);
			npages = 0;
		}

		if (grp->used == grp->size - 1)
			tid_group_move(grp, &rcd->tid_used_list,
				       &rcd->tid_full_list);
		else if (!grp->used)
			tid_group_move(grp, &rcd->tid_group_list,
				       &rcd->tid_used_list);

		grp->used++;
		grp->map |= BIT(i);
		cnt++;
	}
}

static void kern_unprogram_rcv_group(struct tid_rdma_flow *flow, int grp_num)
{
	struct hfi1_ctxtdata *rcd = flow->req->rcd;
	struct hfi1_devdata *dd = rcd->dd;
	struct kern_tid_node *node = &flow->tnode[grp_num];
	struct tid_group *grp = node->grp;
	u32 rcventry;
	u8 i, cnt = 0;

	for (i = 0; i < grp->size; i++) {
		rcventry = grp->base + i;

		if (node->map & BIT(i) || cnt >= node->cnt) {
			rcv_array_wc_fill(dd, rcventry);
			continue;
		}

		hfi1_put_tid(dd, rcventry, PT_INVALID, 0, 0);

		grp->used--;
		grp->map &= ~BIT(i);
		cnt++;

		if (grp->used == grp->size - 1)
			tid_group_move(grp, &rcd->tid_full_list,
				       &rcd->tid_used_list);
		else if (!grp->used)
			tid_group_move(grp, &rcd->tid_used_list,
				       &rcd->tid_group_list);
	}
	if (WARN_ON_ONCE(cnt & 1)) {
		struct hfi1_ctxtdata *rcd = flow->req->rcd;
		struct hfi1_devdata *dd = rcd->dd;

		dd_dev_err(dd, "unexpected odd free cnt %u map 0x%x used %u",
			   cnt, grp->map, grp->used);
	}
}

static void kern_program_rcvarray(struct tid_rdma_flow *flow)
{
	u32 pset_idx = 0;
	int i;

	flow->npkts = 0;
	flow->tidcnt = 0;
	for (i = 0; i < flow->tnode_cnt; i++)
		kern_program_rcv_group(flow, i, &pset_idx);
	trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, flow);
}

/**
 * hfi1_kern_exp_rcv_setup() - setup TID's and flow for one segment of a
 * TID RDMA request
 *
 * @req: TID RDMA request for which the segment/flow is being set up
 * @ss: sge state, maintains state across successive segments of a sge
 * @last: set to true after the last sge segment has been processed
 *
 * This function
 * (1) finds a free flow entry in the flow circular buffer
 * (2) finds pages and continuous physical chunks constituing one segment
 *     of an sge
 * (3) allocates TID group entries for those chunks
 * (4) programs rcvarray entries in the hardware corresponding to those
 *     TID's
 * (5) computes a tidarray with formatted TID entries which can be sent
 *     to the sender
 * (6) Reserves and programs HW flows.
 * (7) It also manages queing the QP when TID/flow resources are not
 *     available.
 *
 * @req points to struct tid_rdma_request of which the segments are a part. The
 * function uses qp, rcd and seg_len members of @req. In the absence of errors,
 * req->flow_idx is the index of the flow which has been prepared in this
 * invocation of function call. With flow = &req->flows[req->flow_idx],
 * flow->tid_entry contains the TID array which the sender can use for TID RDMA
 * sends and flow->npkts contains number of packets required to send the
 * segment.
 *
 * hfi1_check_sge_align should be called prior to calling this function and if
 * it signals error TID RDMA cannot be used for this sge and this function
 * should not be called.
 *
 * For the queuing, caller must hold the flow->req->qp s_lock from the send
 * engine and the function will procure the exp_lock.
 *
 * Return:
 * The function returns -EAGAIN if sufficient number of TID/flow resources to
 * map the segment could not be allocated. In this case the function should be
 * called again with previous arguments to retry the TID allocation. There are
 * no other error returns. The function returns 0 on success.
 */
int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req,
			    struct rvt_sge_state *ss, bool *last)
	__must_hold(&req->qp->s_lock)
{
	struct tid_rdma_flow *flow = &req->flows[req->setup_head];
	struct hfi1_ctxtdata *rcd = req->rcd;
	struct hfi1_qp_priv *qpriv = req->qp->priv;
	unsigned long flags;
	struct rvt_qp *fqp;
	u16 clear_tail = req->clear_tail;

	lockdep_assert_held(&req->qp->s_lock);
	/*
	 * We return error if either (a) we don't have space in the flow
	 * circular buffer, or (b) we already have max entries in the buffer.
	 * Max entries depend on the type of request we are processing and the
	 * negotiated TID RDMA parameters.
	 */
	if (!CIRC_SPACE(req->setup_head, clear_tail, MAX_FLOWS) ||
	    CIRC_CNT(req->setup_head, clear_tail, MAX_FLOWS) >=
	    req->n_flows)
		return -EINVAL;

	/*
	 * Get pages, identify contiguous physical memory chunks for the segment
	 * If we can not determine a DMA address mapping we will treat it just
	 * like if we ran out of space above.
	 */
	if (kern_get_phys_blocks(flow, qpriv->pages, ss, last)) {
		hfi1_wait_kmem(flow->req->qp);
		return -ENOMEM;
	}

	spin_lock_irqsave(&rcd->exp_lock, flags);
	if (kernel_tid_waiters(rcd, &rcd->rarr_queue, flow->req->qp))
		goto queue;

	/*
	 * At this point we know the number of pagesets and hence the number of
	 * TID's to map the segment. Allocate the TID's from the TID groups. If
	 * we cannot allocate the required number we exit and try again later
	 */
	if (kern_alloc_tids(flow))
		goto queue;
	/*
	 * Finally program the TID entries with the pagesets, compute the
	 * tidarray and enable the HW flow
	 */
	kern_program_rcvarray(flow);

	/*
	 * Setup the flow state with relevant information.
	 * This information is used for tracking the sequence of data packets
	 * for the segment.
	 * The flow is setup here as this is the most accurate time and place
	 * to do so. Doing at a later time runs the risk of the flow data in
	 * qpriv getting out of sync.
	 */
	memset(&flow->flow_state, 0x0, sizeof(flow->flow_state));
	flow->idx = qpriv->flow_state.index;
	flow->flow_state.generation = qpriv->flow_state.generation;
	flow->flow_state.spsn = qpriv->flow_state.psn;
	flow->flow_state.lpsn = flow->flow_state.spsn + flow->npkts - 1;
	flow->flow_state.r_next_psn =
		full_flow_psn(flow, flow->flow_state.spsn);
	qpriv->flow_state.psn += flow->npkts;

	dequeue_tid_waiter(rcd, &rcd->rarr_queue, flow->req->qp);
	/* get head before dropping lock */
	fqp = first_qp(rcd, &rcd->rarr_queue);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);
	tid_rdma_schedule_tid_wakeup(fqp);

	req->setup_head = (req->setup_head + 1) & (MAX_FLOWS - 1);
	return 0;
queue:
	queue_qp_for_tid_wait(rcd, &rcd->rarr_queue, flow->req->qp);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);
	return -EAGAIN;
}

static void hfi1_tid_rdma_reset_flow(struct tid_rdma_flow *flow)
{
	flow->npagesets = 0;
}

/*
 * This function is called after one segment has been successfully sent to
 * release the flow and TID HW/SW resources for that segment. The segments for a
 * TID RDMA request are setup and cleared in FIFO order which is managed using a
 * circular buffer.
 */
int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req)
	__must_hold(&req->qp->s_lock)
{
	struct tid_rdma_flow *flow = &req->flows[req->clear_tail];
	struct hfi1_ctxtdata *rcd = req->rcd;
	unsigned long flags;
	int i;
	struct rvt_qp *fqp;

	lockdep_assert_held(&req->qp->s_lock);
	/* Exit if we have nothing in the flow circular buffer */
	if (!CIRC_CNT(req->setup_head, req->clear_tail, MAX_FLOWS))
		return -EINVAL;

	spin_lock_irqsave(&rcd->exp_lock, flags);

	for (i = 0; i < flow->tnode_cnt; i++)
		kern_unprogram_rcv_group(flow, i);
	/* To prevent double unprogramming */
	flow->tnode_cnt = 0;
	/* get head before dropping lock */
	fqp = first_qp(rcd, &rcd->rarr_queue);
	spin_unlock_irqrestore(&rcd->exp_lock, flags);

	dma_unmap_flow(flow);

	hfi1_tid_rdma_reset_flow(flow);
	req->clear_tail = (req->clear_tail + 1) & (MAX_FLOWS - 1);

	if (fqp == req->qp) {
		__trigger_tid_waiter(fqp);
		rvt_put_qp(fqp);
	} else {
		tid_rdma_schedule_tid_wakeup(fqp);
	}

	return 0;
}

/*
 * This function is called to release all the tid entries for
 * a request.
 */
void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req)
	__must_hold(&req->qp->s_lock)
{
	/* Use memory barrier for proper ordering */
	while (CIRC_CNT(req->setup_head, req->clear_tail, MAX_FLOWS)) {
		if (hfi1_kern_exp_rcv_clear(req))
			break;
	}
}

/**
 * hfi1_kern_exp_rcv_free_flows - free priviously allocated flow information
 * @req - the tid rdma request to be cleaned
 */
static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req)
{
	kfree(req->flows);
	req->flows = NULL;
}

/**
 * __trdma_clean_swqe - clean up for large sized QPs
 * @qp: the queue patch
 * @wqe: the send wqe
 */
void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
{
	struct hfi1_swqe_priv *p = wqe->priv;

	hfi1_kern_exp_rcv_free_flows(&p->tid_req);
}

/*
 * This can be called at QP create time or in the data path.
 */
static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req,
					 gfp_t gfp)
{
	struct tid_rdma_flow *flows;
	int i;

	if (likely(req->flows))
		return 0;
	flows = kmalloc_node(MAX_FLOWS * sizeof(*flows), gfp,
			     req->rcd->numa_id);
	if (!flows)
		return -ENOMEM;
	/* mini init */
	for (i = 0; i < MAX_FLOWS; i++) {
		flows[i].req = req;
		flows[i].npagesets = 0;
		flows[i].pagesets[0].mapped =  0;
		flows[i].resync_npkts = 0;
	}
	req->flows = flows;
	return 0;
}

static void hfi1_init_trdma_req(struct rvt_qp *qp,
				struct tid_rdma_request *req)
{
	struct hfi1_qp_priv *qpriv = qp->priv;

	/*
	 * Initialize various TID RDMA request variables.
	 * These variables are "static", which is why they
	 * can be pre-initialized here before the WRs has
	 * even been submitted.
	 * However, non-NULL values for these variables do not
	 * imply that this WQE has been enabled for TID RDMA.
	 * Drivers should check the WQE's opcode to determine
	 * if a request is a TID RDMA one or not.
	 */
	req->qp = qp;
	req->rcd = qpriv->rcd;
}

u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry,
			    void *context, int vl, int mode, u64 data)
{
	struct hfi1_devdata *dd = context;

	return dd->verbs_dev.n_tidwait;
}

static struct tid_rdma_flow *find_flow_ib(struct tid_rdma_request *req,
					  u32 psn, u16 *fidx)
{
	u16 head, tail;
	struct tid_rdma_flow *flow;

	head = req->setup_head;
	tail = req->clear_tail;
	for ( ; CIRC_CNT(head, tail, MAX_FLOWS);
	     tail = CIRC_NEXT(tail, MAX_FLOWS)) {
		flow = &req->flows[tail];
		if (cmp_psn(psn, flow->flow_state.ib_spsn) >= 0 &&
		    cmp_psn(psn, flow->flow_state.ib_lpsn) <= 0) {
			if (fidx)
				*fidx = tail;
			return flow;
		}
	}
	return NULL;
}

/* TID RDMA READ functions */
u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe,
				    struct ib_other_headers *ohdr, u32 *bth1,
				    u32 *bth2, u32 *len)
{
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_flow *flow = &req->flows[req->flow_idx];
	struct rvt_qp *qp = req->qp;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct hfi1_swqe_priv *wpriv = wqe->priv;
	struct tid_rdma_read_req *rreq = &ohdr->u.tid_rdma.r_req;
	struct tid_rdma_params *remote;
	u32 req_len = 0;
	void *req_addr = NULL;

	/* This is the IB psn used to send the request */
	*bth2 = mask_psn(flow->flow_state.ib_spsn + flow->pkt);
	trace_hfi1_tid_flow_build_read_pkt(qp, req->flow_idx, flow);

	/* TID Entries for TID RDMA READ payload */
	req_addr = &flow->tid_entry[flow->tid_idx];
	req_len = sizeof(*flow->tid_entry) *
			(flow->tidcnt - flow->tid_idx);

	memset(&ohdr->u.tid_rdma.r_req, 0, sizeof(ohdr->u.tid_rdma.r_req));
	wpriv->ss.sge.vaddr = req_addr;
	wpriv->ss.sge.sge_length = req_len;
	wpriv->ss.sge.length = wpriv->ss.sge.sge_length;
	/*
	 * We can safely zero these out. Since the first SGE covers the
	 * entire packet, nothing else should even look at the MR.
	 */
	wpriv->ss.sge.mr = NULL;
	wpriv->ss.sge.m = 0;
	wpriv->ss.sge.n = 0;

	wpriv->ss.sg_list = NULL;
	wpriv->ss.total_len = wpriv->ss.sge.sge_length;
	wpriv->ss.num_sge = 1;

	/* Construct the TID RDMA READ REQ packet header */
	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);

	KDETH_RESET(rreq->kdeth0, KVER, 0x1);
	KDETH_RESET(rreq->kdeth1, JKEY, remote->jkey);
	rreq->reth.vaddr = cpu_to_be64(wqe->rdma_wr.remote_addr +
			   req->cur_seg * req->seg_len + flow->sent);
	rreq->reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey);
	rreq->reth.length = cpu_to_be32(*len);
	rreq->tid_flow_psn =
		cpu_to_be32((flow->flow_state.generation <<
			     HFI1_KDETH_BTH_SEQ_SHIFT) |
			    ((flow->flow_state.spsn + flow->pkt) &
			     HFI1_KDETH_BTH_SEQ_MASK));
	rreq->tid_flow_qp =
		cpu_to_be32(qpriv->tid_rdma.local.qp |
			    ((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) <<
			     TID_RDMA_DESTQP_FLOW_SHIFT) |
			    qpriv->rcd->ctxt);
	rreq->verbs_qp = cpu_to_be32(qp->remote_qpn);
	*bth1 &= ~RVT_QPN_MASK;
	*bth1 |= remote->qp;
	*bth2 |= IB_BTH_REQ_ACK;
	rcu_read_unlock();

	/* We are done with this segment */
	flow->sent += *len;
	req->cur_seg++;
	qp->s_state = TID_OP(READ_REQ);
	req->ack_pending++;
	req->flow_idx = (req->flow_idx + 1) & (MAX_FLOWS - 1);
	qpriv->pending_tid_r_segs++;
	qp->s_num_rd_atomic++;

	/* Set the TID RDMA READ request payload size */
	*len = req_len;

	return sizeof(ohdr->u.tid_rdma.r_req) / sizeof(u32);
}

/*
 * @len: contains the data length to read upon entry and the read request
 *       payload length upon exit.
 */
u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
				 struct ib_other_headers *ohdr, u32 *bth1,
				 u32 *bth2, u32 *len)
	__must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_flow *flow = NULL;
	u32 hdwords = 0;
	bool last;
	bool retry = true;
	u32 npkts = rvt_div_round_up_mtu(qp, *len);

	trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn,
					  wqe->lpsn, req);
	/*
	 * Check sync conditions. Make sure that there are no pending
	 * segments before freeing the flow.
	 */
sync_check:
	if (req->state == TID_REQUEST_SYNC) {
		if (qpriv->pending_tid_r_segs)
			goto done;

		hfi1_kern_clear_hw_flow(req->rcd, qp);
		qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
		req->state = TID_REQUEST_ACTIVE;
	}

	/*
	 * If the request for this segment is resent, the tid resources should
	 * have been allocated before. In this case, req->flow_idx should
	 * fall behind req->setup_head.
	 */
	if (req->flow_idx == req->setup_head) {
		retry = false;
		if (req->state == TID_REQUEST_RESEND) {
			/*
			 * This is the first new segment for a request whose
			 * earlier segments have been re-sent. We need to
			 * set up the sge pointer correctly.
			 */
			restart_sge(&qp->s_sge, wqe, req->s_next_psn,
				    qp->pmtu);
			req->isge = 0;
			req->state = TID_REQUEST_ACTIVE;
		}

		/*
		 * Check sync. The last PSN of each generation is reserved for
		 * RESYNC.
		 */
		if ((qpriv->flow_state.psn + npkts) > MAX_TID_FLOW_PSN - 1) {
			req->state = TID_REQUEST_SYNC;
			goto sync_check;
		}

		/* Allocate the flow if not yet */
		if (hfi1_kern_setup_hw_flow(qpriv->rcd, qp))
			goto done;

		/*
		 * The following call will advance req->setup_head after
		 * allocating the tid entries.
		 */
		if (hfi1_kern_exp_rcv_setup(req, &qp->s_sge, &last)) {
			req->state = TID_REQUEST_QUEUED;

			/*
			 * We don't have resources for this segment. The QP has
			 * already been queued.
			 */
			goto done;
		}
	}

	/* req->flow_idx should only be one slot behind req->setup_head */
	flow = &req->flows[req->flow_idx];
	flow->pkt = 0;
	flow->tid_idx = 0;
	flow->sent = 0;
	if (!retry) {
		/* Set the first and last IB PSN for the flow in use.*/
		flow->flow_state.ib_spsn = req->s_next_psn;
		flow->flow_state.ib_lpsn =
			flow->flow_state.ib_spsn + flow->npkts - 1;
	}

	/* Calculate the next segment start psn.*/
	req->s_next_psn += flow->npkts;

	/* Build the packet header */
	hdwords = hfi1_build_tid_rdma_read_packet(wqe, ohdr, bth1, bth2, len);
done:
	return hdwords;
}

/*
 * Validate and accept the TID RDMA READ request parameters.
 * Return 0 if the request is accepted successfully;
 * Return 1 otherwise.
 */
static int tid_rdma_rcv_read_request(struct rvt_qp *qp,
				     struct rvt_ack_entry *e,
				     struct hfi1_packet *packet,
				     struct ib_other_headers *ohdr,
				     u32 bth0, u32 psn, u64 vaddr, u32 len)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	u32 flow_psn, i, tidlen = 0, pktlen, tlen;

	req = ack_to_tid_req(e);

	/* Validate the payload first */
	flow = &req->flows[req->setup_head];

	/* payload length = packet length - (header length + ICRC length) */
	pktlen = packet->tlen - (packet->hlen + 4);
	if (pktlen > sizeof(flow->tid_entry))
		return 1;
	memcpy(flow->tid_entry, packet->ebuf, pktlen);
	flow->tidcnt = pktlen / sizeof(*flow->tid_entry);

	/*
	 * Walk the TID_ENTRY list to make sure we have enough space for a
	 * complete segment. Also calculate the number of required packets.
	 */
	flow->npkts = rvt_div_round_up_mtu(qp, len);
	for (i = 0; i < flow->tidcnt; i++) {
		trace_hfi1_tid_entry_rcv_read_req(qp, i,
						  flow->tid_entry[i]);
		tlen = EXP_TID_GET(flow->tid_entry[i], LEN);
		if (!tlen)
			return 1;

		/*
		 * For tid pair (tidctr == 3), the buffer size of the pair
		 * should be the sum of the buffer size described by each
		 * tid entry. However, only the first entry needs to be
		 * specified in the request (see WFR HAS Section 8.5.7.1).
		 */
		tidlen += tlen;
	}
	if (tidlen * PAGE_SIZE < len)
		return 1;

	/* Empty the flow array */
	req->clear_tail = req->setup_head;
	flow->pkt = 0;
	flow->tid_idx = 0;
	flow->tid_offset = 0;
	flow->sent = 0;
	flow->tid_qpn = be32_to_cpu(ohdr->u.tid_rdma.r_req.tid_flow_qp);
	flow->idx = (flow->tid_qpn >> TID_RDMA_DESTQP_FLOW_SHIFT) &
		    TID_RDMA_DESTQP_FLOW_MASK;
	flow_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.r_req.tid_flow_psn));
	flow->flow_state.generation = flow_psn >> HFI1_KDETH_BTH_SEQ_SHIFT;
	flow->flow_state.spsn = flow_psn & HFI1_KDETH_BTH_SEQ_MASK;
	flow->length = len;

	flow->flow_state.lpsn = flow->flow_state.spsn +
		flow->npkts - 1;
	flow->flow_state.ib_spsn = psn;
	flow->flow_state.ib_lpsn = flow->flow_state.ib_spsn + flow->npkts - 1;

	trace_hfi1_tid_flow_rcv_read_req(qp, req->setup_head, flow);
	/* Set the initial flow index to the current flow. */
	req->flow_idx = req->setup_head;

	/* advance circular buffer head */
	req->setup_head = (req->setup_head + 1) & (MAX_FLOWS - 1);

	/*
	 * Compute last PSN for request.
	 */
	e->opcode = (bth0 >> 24) & 0xff;
	e->psn = psn;
	e->lpsn = psn + flow->npkts - 1;
	e->sent = 0;

	req->n_flows = qpriv->tid_rdma.local.max_read;
	req->state = TID_REQUEST_ACTIVE;
	req->cur_seg = 0;
	req->comp_seg = 0;
	req->ack_seg = 0;
	req->isge = 0;
	req->seg_len = qpriv->tid_rdma.local.max_len;
	req->total_len = len;
	req->total_segs = 1;
	req->r_flow_psn = e->psn;

	trace_hfi1_tid_req_rcv_read_req(qp, 0, e->opcode, e->psn, e->lpsn,
					req);
	return 0;
}

static int tid_rdma_rcv_error(struct hfi1_packet *packet,
			      struct ib_other_headers *ohdr,
			      struct rvt_qp *qp, u32 psn, int diff)
{
	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
	struct hfi1_ctxtdata *rcd = ((struct hfi1_qp_priv *)qp->priv)->rcd;
	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct rvt_ack_entry *e;
	struct tid_rdma_request *req;
	unsigned long flags;
	u8 prev;
	bool old_req;

	trace_hfi1_rsp_tid_rcv_error(qp, psn);
	trace_hfi1_tid_rdma_rcv_err(qp, 0, psn, diff);
	if (diff > 0) {
		/* sequence error */
		if (!qp->r_nak_state) {
			ibp->rvp.n_rc_seqnak++;
			qp->r_nak_state = IB_NAK_PSN_ERROR;
			qp->r_ack_psn = qp->r_psn;
			rc_defered_ack(rcd, qp);
		}
		goto done;
	}

	ibp->rvp.n_rc_dupreq++;

	spin_lock_irqsave(&qp->s_lock, flags);
	e = find_prev_entry(qp, psn, &prev, NULL, &old_req);
	if (!e || (e->opcode != TID_OP(READ_REQ) &&
		   e->opcode != TID_OP(WRITE_REQ)))
		goto unlock;

	req = ack_to_tid_req(e);
	req->r_flow_psn = psn;
	trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req);
	if (e->opcode == TID_OP(READ_REQ)) {
		struct ib_reth *reth;
		u32 len;
		u32 rkey;
		u64 vaddr;
		int ok;
		u32 bth0;

		reth = &ohdr->u.tid_rdma.r_req.reth;
		/*
		 * The requester always restarts from the start of the original
		 * request.
		 */
		len = be32_to_cpu(reth->length);
		if (psn != e->psn || len != req->total_len)
			goto unlock;

		release_rdma_sge_mr(e);

		rkey = be32_to_cpu(reth->rkey);
		vaddr = get_ib_reth_vaddr(reth);

		qp->r_len = len;
		ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
				 IB_ACCESS_REMOTE_READ);
		if (unlikely(!ok))
			goto unlock;

		/*
		 * If all the response packets for the current request have
		 * been sent out and this request is complete (old_request
		 * == false) and the TID flow may be unusable (the
		 * req->clear_tail is advanced). However, when an earlier
		 * request is received, this request will not be complete any
		 * more (qp->s_tail_ack_queue is moved back, see below).
		 * Consequently, we need to update the TID flow info everytime
		 * a duplicate request is received.
		 */
		bth0 = be32_to_cpu(ohdr->bth[0]);
		if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn,
					      vaddr, len))
			goto unlock;

		/*
		 * True if the request is already scheduled (between
		 * qp->s_tail_ack_queue and qp->r_head_ack_queue);
		 */
		if (old_req)
			goto unlock;
	} else {
		struct flow_state *fstate;
		bool schedule = false;
		u8 i;

		if (req->state == TID_REQUEST_RESEND) {
			req->state = TID_REQUEST_RESEND_ACTIVE;
		} else if (req->state == TID_REQUEST_INIT_RESEND) {
			req->state = TID_REQUEST_INIT;
			schedule = true;
		}

		/*
		 * True if the request is already scheduled (between
		 * qp->s_tail_ack_queue and qp->r_head_ack_queue).
		 * Also, don't change requests, which are at the SYNC
		 * point and haven't generated any responses yet.
		 * There is nothing to retransmit for them yet.
		 */
		if (old_req || req->state == TID_REQUEST_INIT ||
		    (req->state == TID_REQUEST_SYNC && !req->cur_seg)) {
			for (i = prev + 1; ; i++) {
				if (i > rvt_size_atomic(&dev->rdi))
					i = 0;
				if (i == qp->r_head_ack_queue)
					break;
				e = &qp->s_ack_queue[i];
				req = ack_to_tid_req(e);
				if (e->opcode == TID_OP(WRITE_REQ) &&
				    req->state == TID_REQUEST_INIT)
					req->state = TID_REQUEST_INIT_RESEND;
			}
			/*
			 * If the state of the request has been changed,
			 * the first leg needs to get scheduled in order to
			 * pick up the change. Otherwise, normal response
			 * processing should take care of it.
			 */
			if (!schedule)
				goto unlock;
		}

		/*
		 * If there is no more allocated segment, just schedule the qp
		 * without changing any state.
		 */
		if (req->clear_tail == req->setup_head)
			goto schedule;
		/*
		 * If this request has sent responses for segments, which have
		 * not received data yet (flow_idx != clear_tail), the flow_idx
		 * pointer needs to be adjusted so the same responses can be
		 * re-sent.
		 */
		if (CIRC_CNT(req->flow_idx, req->clear_tail, MAX_FLOWS)) {
			fstate = &req->flows[req->clear_tail].flow_state;
			qpriv->pending_tid_w_segs -=
				CIRC_CNT(req->flow_idx, req->clear_tail,
					 MAX_FLOWS);
			req->flow_idx =
				CIRC_ADD(req->clear_tail,
					 delta_psn(psn, fstate->resp_ib_psn),
					 MAX_FLOWS);
			qpriv->pending_tid_w_segs +=
				delta_psn(psn, fstate->resp_ib_psn);
			/*
			 * When flow_idx == setup_head, we've gotten a duplicate
			 * request for a segment, which has not been allocated
			 * yet. In that case, don't adjust this request.
			 * However, we still want to go through the loop below
			 * to adjust all subsequent requests.
			 */
			if (CIRC_CNT(req->setup_head, req->flow_idx,
				     MAX_FLOWS)) {
				req->cur_seg = delta_psn(psn, e->psn);
				req->state = TID_REQUEST_RESEND_ACTIVE;
			}
		}

		for (i = prev + 1; ; i++) {
			/*
			 * Look at everything up to and including
			 * s_tail_ack_queue
			 */
			if (i > rvt_size_atomic(&dev->rdi))
				i = 0;
			if (i == qp->r_head_ack_queue)
				break;
			e = &qp->s_ack_queue[i];
			req = ack_to_tid_req(e);
			trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn,
						   e->lpsn, req);
			if (e->opcode != TID_OP(WRITE_REQ) ||
			    req->cur_seg == req->comp_seg ||
			    req->state == TID_REQUEST_INIT ||
			    req->state == TID_REQUEST_INIT_RESEND) {
				if (req->state == TID_REQUEST_INIT)
					req->state = TID_REQUEST_INIT_RESEND;
				continue;
			}
			qpriv->pending_tid_w_segs -=
				CIRC_CNT(req->flow_idx,
					 req->clear_tail,
					 MAX_FLOWS);
			req->flow_idx = req->clear_tail;
			req->state = TID_REQUEST_RESEND;
			req->cur_seg = req->comp_seg;
		}
		qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK;
	}
	/* Re-process old requests.*/
	if (qp->s_acked_ack_queue == qp->s_tail_ack_queue)
		qp->s_acked_ack_queue = prev;
	qp->s_tail_ack_queue = prev;
	/*
	 * Since the qp->s_tail_ack_queue is modified, the
	 * qp->s_ack_state must be changed to re-initialize
	 * qp->s_ack_rdma_sge; Otherwise, we will end up in
	 * wrong memory region.
	 */
	qp->s_ack_state = OP(ACKNOWLEDGE);
schedule:
	/*
	 * It's possible to receive a retry psn that is earlier than an RNRNAK
	 * psn. In this case, the rnrnak state should be cleared.
	 */
	if (qpriv->rnr_nak_state) {
		qp->s_nak_state = 0;
		qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
		qp->r_psn = e->lpsn + 1;
		hfi1_tid_write_alloc_resources(qp, true);
	}

	qp->r_state = e->opcode;
	qp->r_nak_state = 0;
	qp->s_flags |= RVT_S_RESP_PENDING;
	hfi1_schedule_send(qp);
unlock:
	spin_unlock_irqrestore(&qp->s_lock, flags);
done:
	return 1;
}

void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
{
	/* HANDLER FOR TID RDMA READ REQUEST packet (Responder side)*/

	/*
	 * 1. Verify TID RDMA READ REQ as per IB_OPCODE_RC_RDMA_READ
	 *    (see hfi1_rc_rcv())
	 * 2. Put TID RDMA READ REQ into the response queueu (s_ack_queue)
	 *     - Setup struct tid_rdma_req with request info
	 *     - Initialize struct tid_rdma_flow info;
	 *     - Copy TID entries;
	 * 3. Set the qp->s_ack_state.
	 * 4. Set RVT_S_RESP_PENDING in s_flags.
	 * 5. Kick the send engine (hfi1_schedule_send())
	 */
	struct hfi1_ctxtdata *rcd = packet->rcd;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_ack_entry *e;
	unsigned long flags;
	struct ib_reth *reth;
	struct hfi1_qp_priv *qpriv = qp->priv;
	u32 bth0, psn, len, rkey;
	bool fecn;
	u8 next;
	u64 vaddr;
	int diff;
	u8 nack_state = IB_NAK_INVALID_REQUEST;

	bth0 = be32_to_cpu(ohdr->bth[0]);
	if (hfi1_ruc_check_hdr(ibp, packet))
		return;

	fecn = process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	trace_hfi1_rsp_rcv_tid_read_req(qp, psn);

	if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
		rvt_comm_est(qp);

	if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
		goto nack_inv;

	reth = &ohdr->u.tid_rdma.r_req.reth;
	vaddr = be64_to_cpu(reth->vaddr);
	len = be32_to_cpu(reth->length);
	/* The length needs to be in multiples of PAGE_SIZE */
	if (!len || len & ~PAGE_MASK || len > qpriv->tid_rdma.local.max_len)
		goto nack_inv;

	diff = delta_psn(psn, qp->r_psn);
	if (unlikely(diff)) {
		tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn);
		return;
	}

	/* We've verified the request, insert it into the ack queue. */
	next = qp->r_head_ack_queue + 1;
	if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
		next = 0;
	spin_lock_irqsave(&qp->s_lock, flags);
	if (unlikely(next == qp->s_tail_ack_queue)) {
		if (!qp->s_ack_queue[next].sent) {
			nack_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
			goto nack_inv_unlock;
		}
		update_ack_queue(qp, next);
	}
	e = &qp->s_ack_queue[qp->r_head_ack_queue];
	release_rdma_sge_mr(e);

	rkey = be32_to_cpu(reth->rkey);
	qp->r_len = len;

	if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr,
				  rkey, IB_ACCESS_REMOTE_READ)))
		goto nack_acc;

	/* Accept the request parameters */
	if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, vaddr,
				      len))
		goto nack_inv_unlock;

	qp->r_state = e->opcode;
	qp->r_nak_state = 0;
	/*
	 * We need to increment the MSN here instead of when we
	 * finish sending the result since a duplicate request would
	 * increment it more than once.
	 */
	qp->r_msn++;
	qp->r_psn += e->lpsn - e->psn + 1;

	qp->r_head_ack_queue = next;

	/*
	 * For all requests other than TID WRITE which are added to the ack
	 * queue, qpriv->r_tid_alloc follows qp->r_head_ack_queue. It is ok to
	 * do this because of interlocks between these and TID WRITE
	 * requests. The same change has also been made in hfi1_rc_rcv().
	 */
	qpriv->r_tid_alloc = qp->r_head_ack_queue;

	/* Schedule the send tasklet. */
	qp->s_flags |= RVT_S_RESP_PENDING;
	if (fecn)
		qp->s_flags |= RVT_S_ECN;
	hfi1_schedule_send(qp);

	spin_unlock_irqrestore(&qp->s_lock, flags);
	return;

nack_inv_unlock:
	spin_unlock_irqrestore(&qp->s_lock, flags);
nack_inv:
	rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
	qp->r_nak_state = nack_state;
	qp->r_ack_psn = qp->r_psn;
	/* Queue NAK for later */
	rc_defered_ack(rcd, qp);
	return;
nack_acc:
	spin_unlock_irqrestore(&qp->s_lock, flags);
	rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
	qp->r_ack_psn = qp->r_psn;
}

u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
				  struct ib_other_headers *ohdr, u32 *bth0,
				  u32 *bth1, u32 *bth2, u32 *len, bool *last)
{
	struct hfi1_ack_priv *epriv = e->priv;
	struct tid_rdma_request *req = &epriv->tid_req;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_flow *flow = &req->flows[req->clear_tail];
	u32 tidentry = flow->tid_entry[flow->tid_idx];
	u32 tidlen = EXP_TID_GET(tidentry, LEN) << PAGE_SHIFT;
	struct tid_rdma_read_resp *resp = &ohdr->u.tid_rdma.r_rsp;
	u32 next_offset, om = KDETH_OM_LARGE;
	bool last_pkt;
	u32 hdwords = 0;
	struct tid_rdma_params *remote;

	*len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset);
	flow->sent += *len;
	next_offset = flow->tid_offset + *len;
	last_pkt = (flow->sent >= flow->length);

	trace_hfi1_tid_entry_build_read_resp(qp, flow->tid_idx, tidentry);
	trace_hfi1_tid_flow_build_read_resp(qp, req->clear_tail, flow);

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	if (!remote) {
		rcu_read_unlock();
		goto done;
	}
	KDETH_RESET(resp->kdeth0, KVER, 0x1);
	KDETH_SET(resp->kdeth0, SH, !last_pkt);
	KDETH_SET(resp->kdeth0, INTR, !!(!last_pkt && remote->urg));
	KDETH_SET(resp->kdeth0, TIDCTRL, EXP_TID_GET(tidentry, CTRL));
	KDETH_SET(resp->kdeth0, TID, EXP_TID_GET(tidentry, IDX));
	KDETH_SET(resp->kdeth0, OM, om == KDETH_OM_LARGE);
	KDETH_SET(resp->kdeth0, OFFSET, flow->tid_offset / om);
	KDETH_RESET(resp->kdeth1, JKEY, remote->jkey);
	resp->verbs_qp = cpu_to_be32(qp->remote_qpn);
	rcu_read_unlock();

	resp->aeth = rvt_compute_aeth(qp);
	resp->verbs_psn = cpu_to_be32(mask_psn(flow->flow_state.ib_spsn +
					       flow->pkt));

	*bth0 = TID_OP(READ_RESP) << 24;
	*bth1 = flow->tid_qpn;
	*bth2 = mask_psn(((flow->flow_state.spsn + flow->pkt++) &
			  HFI1_KDETH_BTH_SEQ_MASK) |
			 (flow->flow_state.generation <<
			  HFI1_KDETH_BTH_SEQ_SHIFT));
	*last = last_pkt;
	if (last_pkt)
		/* Advance to next flow */
		req->clear_tail = (req->clear_tail + 1) &
				  (MAX_FLOWS - 1);

	if (next_offset >= tidlen) {
		flow->tid_offset = 0;
		flow->tid_idx++;
	} else {
		flow->tid_offset = next_offset;
	}

	hdwords = sizeof(ohdr->u.tid_rdma.r_rsp) / sizeof(u32);

done:
	return hdwords;
}

static inline struct tid_rdma_request *
find_tid_request(struct rvt_qp *qp, u32 psn, enum ib_wr_opcode opcode)
	__must_hold(&qp->s_lock)
{
	struct rvt_swqe *wqe;
	struct tid_rdma_request *req = NULL;
	u32 i, end;

	end = qp->s_cur + 1;
	if (end == qp->s_size)
		end = 0;
	for (i = qp->s_acked; i != end;) {
		wqe = rvt_get_swqe_ptr(qp, i);
		if (cmp_psn(psn, wqe->psn) >= 0 &&
		    cmp_psn(psn, wqe->lpsn) <= 0) {
			if (wqe->wr.opcode == opcode)
				req = wqe_to_tid_req(wqe);
			break;
		}
		if (++i == qp->s_size)
			i = 0;
	}

	return req;
}

void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet)
{
	/* HANDLER FOR TID RDMA READ RESPONSE packet (Requestor side */

	/*
	 * 1. Find matching SWQE
	 * 2. Check that the entire segment has been read.
	 * 3. Remove HFI1_S_WAIT_TID_RESP from s_flags.
	 * 4. Free the TID flow resources.
	 * 5. Kick the send engine (hfi1_schedule_send())
	 */
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *priv = qp->priv;
	struct hfi1_ctxtdata *rcd = packet->rcd;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	u32 opcode, aeth;
	bool fecn;
	unsigned long flags;
	u32 kpsn, ipsn;

	trace_hfi1_sender_rcv_tid_read_resp(qp);
	fecn = process_ecn(qp, packet);
	kpsn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	aeth = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.aeth);
	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;

	spin_lock_irqsave(&qp->s_lock, flags);
	ipsn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_psn));
	req = find_tid_request(qp, ipsn, IB_WR_TID_RDMA_READ);
	if (unlikely(!req))
		goto ack_op_err;

	flow = &req->flows[req->clear_tail];
	/* When header suppression is disabled */
	if (cmp_psn(ipsn, flow->flow_state.ib_lpsn)) {
		update_r_next_psn_fecn(packet, priv, rcd, flow, fecn);

		if (cmp_psn(kpsn, flow->flow_state.r_next_psn))
			goto ack_done;
		flow->flow_state.r_next_psn = mask_psn(kpsn + 1);
		/*
		 * Copy the payload to destination buffer if this packet is
		 * delivered as an eager packet due to RSM rule and FECN.
		 * The RSM rule selects FECN bit in BTH and SH bit in
		 * KDETH header and therefore will not match the last
		 * packet of each segment that has SH bit cleared.
		 */
		if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) {
			struct rvt_sge_state ss;
			u32 len;
			u32 tlen = packet->tlen;
			u16 hdrsize = packet->hlen;
			u8 pad = packet->pad;
			u8 extra_bytes = pad + packet->extra_byte +
				(SIZE_OF_CRC << 2);
			u32 pmtu = qp->pmtu;

			if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
				goto ack_op_err;
			len = restart_sge(&ss, req->e.swqe, ipsn, pmtu);
			if (unlikely(len < pmtu))
				goto ack_op_err;
			rvt_copy_sge(qp, &ss, packet->payload, pmtu, false,
				     false);
			/* Raise the sw sequence check flag for next packet */
			priv->s_flags |= HFI1_R_TID_SW_PSN;
		}

		goto ack_done;
	}
	flow->flow_state.r_next_psn = mask_psn(kpsn + 1);
	req->ack_pending--;
	priv->pending_tid_r_segs--;
	qp->s_num_rd_atomic--;
	if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
	    !qp->s_num_rd_atomic) {
		qp->s_flags &= ~(RVT_S_WAIT_FENCE |
				 RVT_S_WAIT_ACK);
		hfi1_schedule_send(qp);
	}
	if (qp->s_flags & RVT_S_WAIT_RDMAR) {
		qp->s_flags &= ~(RVT_S_WAIT_RDMAR | RVT_S_WAIT_ACK);
		hfi1_schedule_send(qp);
	}

	trace_hfi1_ack(qp, ipsn);
	trace_hfi1_tid_req_rcv_read_resp(qp, 0, req->e.swqe->wr.opcode,
					 req->e.swqe->psn, req->e.swqe->lpsn,
					 req);
	trace_hfi1_tid_flow_rcv_read_resp(qp, req->clear_tail, flow);

	/* Release the tid resources */
	hfi1_kern_exp_rcv_clear(req);

	if (!do_rc_ack(qp, aeth, ipsn, opcode, 0, rcd))
		goto ack_done;

	/* If not done yet, build next read request */
	if (++req->comp_seg >= req->total_segs) {
		priv->tid_r_comp++;
		req->state = TID_REQUEST_COMPLETE;
	}

	/*
	 * Clear the hw flow under two conditions:
	 * 1. This request is a sync point and it is complete;
	 * 2. Current request is completed and there are no more requests.
	 */
	if ((req->state == TID_REQUEST_SYNC &&
	     req->comp_seg == req->cur_seg) ||
	    priv->tid_r_comp == priv->tid_r_reqs) {
		hfi1_kern_clear_hw_flow(priv->rcd, qp);
		priv->s_flags &= ~HFI1_R_TID_SW_PSN;
		if (req->state == TID_REQUEST_SYNC)
			req->state = TID_REQUEST_ACTIVE;
	}

	hfi1_schedule_send(qp);
	goto ack_done;

ack_op_err:
	/*
	 * The test indicates that the send engine has finished its cleanup
	 * after sending the request and it's now safe to put the QP into error
	 * state. However, if the wqe queue is empty (qp->s_acked == qp->s_tail
	 * == qp->s_head), it would be unsafe to complete the wqe pointed by
	 * qp->s_acked here. Putting the qp into error state will safely flush
	 * all remaining requests.
	 */
	if (qp->s_last == qp->s_acked)
		rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);

ack_done:
	spin_unlock_irqrestore(&qp->s_lock, flags);
}

void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp)
	__must_hold(&qp->s_lock)
{
	u32 n = qp->s_acked;
	struct rvt_swqe *wqe;
	struct tid_rdma_request *req;
	struct hfi1_qp_priv *priv = qp->priv;

	lockdep_assert_held(&qp->s_lock);
	/* Free any TID entries */
	while (n != qp->s_tail) {
		wqe = rvt_get_swqe_ptr(qp, n);
		if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
			req = wqe_to_tid_req(wqe);
			hfi1_kern_exp_rcv_clear_all(req);
		}

		if (++n == qp->s_size)
			n = 0;
	}
	/* Free flow */
	hfi1_kern_clear_hw_flow(priv->rcd, qp);
}

static bool tid_rdma_tid_err(struct hfi1_packet *packet, u8 rcv_type)
{
	struct rvt_qp *qp = packet->qp;

	if (rcv_type >= RHF_RCV_TYPE_IB)
		goto done;

	spin_lock(&qp->s_lock);

	/*
	 * We've ran out of space in the eager buffer.
	 * Eagerly received KDETH packets which require space in the
	 * Eager buffer (packet that have payload) are TID RDMA WRITE
	 * response packets. In this case, we have to re-transmit the
	 * TID RDMA WRITE request.
	 */
	if (rcv_type == RHF_RCV_TYPE_EAGER) {
		hfi1_restart_rc(qp, qp->s_last_psn + 1, 1);
		hfi1_schedule_send(qp);
	}

	/* Since no payload is delivered, just drop the packet */
	spin_unlock(&qp->s_lock);
done:
	return true;
}

static void restart_tid_rdma_read_req(struct hfi1_ctxtdata *rcd,
				      struct rvt_qp *qp, struct rvt_swqe *wqe)
{
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;

	/* Start from the right segment */
	qp->r_flags |= RVT_R_RDMAR_SEQ;
	req = wqe_to_tid_req(wqe);
	flow = &req->flows[req->clear_tail];
	hfi1_restart_rc(qp, flow->flow_state.ib_spsn, 0);
	if (list_empty(&qp->rspwait)) {
		qp->r_flags |= RVT_R_RSP_SEND;
		rvt_get_qp(qp);
		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
	}
}

/*
 * Handle the KDETH eflags for TID RDMA READ response.
 *
 * Return true if the last packet for a segment has been received and it is
 * time to process the response normally; otherwise, return true.
 *
 * The caller must hold the packet->qp->r_lock and the rcu_read_lock.
 */
static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,
				     struct hfi1_packet *packet, u8 rcv_type,
				     u8 rte, u32 psn, u32 ibpsn)
	__must_hold(&packet->qp->r_lock) __must_hold(RCU)
{
	struct hfi1_pportdata *ppd = rcd->ppd;
	struct hfi1_devdata *dd = ppd->dd;
	struct hfi1_ibport *ibp;
	struct rvt_swqe *wqe;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	u32 ack_psn;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *priv = qp->priv;
	bool ret = true;
	int diff = 0;
	u32 fpsn;

	lockdep_assert_held(&qp->r_lock);
	trace_hfi1_rsp_read_kdeth_eflags(qp, ibpsn);
	trace_hfi1_sender_read_kdeth_eflags(qp);
	trace_hfi1_tid_read_sender_kdeth_eflags(qp, 0);
	spin_lock(&qp->s_lock);
	/* If the psn is out of valid range, drop the packet */
	if (cmp_psn(ibpsn, qp->s_last_psn) < 0 ||
	    cmp_psn(ibpsn, qp->s_psn) > 0)
		goto s_unlock;

	/*
	 * Note that NAKs implicitly ACK outstanding SEND and RDMA write
	 * requests and implicitly NAK RDMA read and atomic requests issued
	 * before the NAK'ed request.
	 */
	ack_psn = ibpsn - 1;
	wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
	ibp = to_iport(qp->ibqp.device, qp->port_num);

	/* Complete WQEs that the PSN finishes. */
	while ((int)delta_psn(ack_psn, wqe->lpsn) >= 0) {
		/*
		 * If this request is a RDMA read or atomic, and the NACK is
		 * for a later operation, this NACK NAKs the RDMA read or
		 * atomic.
		 */
		if (wqe->wr.opcode == IB_WR_RDMA_READ ||
		    wqe->wr.opcode == IB_WR_TID_RDMA_READ ||
		    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
			/* Retry this request. */
			if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
				qp->r_flags |= RVT_R_RDMAR_SEQ;
				if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
					restart_tid_rdma_read_req(rcd, qp,
								  wqe);
				} else {
					hfi1_restart_rc(qp, qp->s_last_psn + 1,
							0);
					if (list_empty(&qp->rspwait)) {
						qp->r_flags |= RVT_R_RSP_SEND;
						rvt_get_qp(qp);
						list_add_tail(/* wait */
						   &qp->rspwait,
						   &rcd->qp_wait_list);
					}
				}
			}
			/*
			 * No need to process the NAK since we are
			 * restarting an earlier request.
			 */
			break;
		}

		wqe = do_rc_completion(qp, wqe, ibp);
		if (qp->s_acked == qp->s_tail)
			goto s_unlock;
	}

	if (qp->s_acked == qp->s_tail)
		goto s_unlock;

	/* Handle the eflags for the request */
	if (wqe->wr.opcode != IB_WR_TID_RDMA_READ)
		goto s_unlock;

	req = wqe_to_tid_req(wqe);
	trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn,
					     wqe->lpsn, req);
	switch (rcv_type) {
	case RHF_RCV_TYPE_EXPECTED:
		switch (rte) {
		case RHF_RTE_EXPECTED_FLOW_SEQ_ERR:
			/*
			 * On the first occurrence of a Flow Sequence error,
			 * the flag TID_FLOW_SW_PSN is set.
			 *
			 * After that, the flow is *not* reprogrammed and the
			 * protocol falls back to SW PSN checking. This is done
			 * to prevent continuous Flow Sequence errors for any
			 * packets that could be still in the fabric.
			 */
			flow = &req->flows[req->clear_tail];
			trace_hfi1_tid_flow_read_kdeth_eflags(qp,
							      req->clear_tail,
							      flow);
			if (priv->s_flags & HFI1_R_TID_SW_PSN) {
				diff = cmp_psn(psn,
					       flow->flow_state.r_next_psn);
				if (diff > 0) {
					if (!(qp->r_flags & RVT_R_RDMAR_SEQ))
						restart_tid_rdma_read_req(rcd,
									  qp,
									  wqe);

					/* Drop the packet.*/
					goto s_unlock;
				} else if (diff < 0) {
					/*
					 * If a response packet for a restarted
					 * request has come back, reset the
					 * restart flag.
					 */
					if (qp->r_flags & RVT_R_RDMAR_SEQ)
						qp->r_flags &=
							~RVT_R_RDMAR_SEQ;

					/* Drop the packet.*/
					goto s_unlock;
				}

				/*
				 * If SW PSN verification is successful and
				 * this is the last packet in the segment, tell
				 * the caller to process it as a normal packet.
				 */
				fpsn = full_flow_psn(flow,
						     flow->flow_state.lpsn);
				if (cmp_psn(fpsn, psn) == 0) {
					ret = false;
					if (qp->r_flags & RVT_R_RDMAR_SEQ)
						qp->r_flags &=
							~RVT_R_RDMAR_SEQ;
				}
				flow->flow_state.r_next_psn =
					mask_psn(psn + 1);
			} else {
				u32 last_psn;

				last_psn = read_r_next_psn(dd, rcd->ctxt,
							   flow->idx);
				flow->flow_state.r_next_psn = last_psn;
				priv->s_flags |= HFI1_R_TID_SW_PSN;
				/*
				 * If no request has been restarted yet,
				 * restart the current one.
				 */
				if (!(qp->r_flags & RVT_R_RDMAR_SEQ))
					restart_tid_rdma_read_req(rcd, qp,
								  wqe);
			}

			break;

		case RHF_RTE_EXPECTED_FLOW_GEN_ERR:
			/*
			 * Since the TID flow is able to ride through
			 * generation mismatch, drop this stale packet.
			 */
			break;

		default:
			break;
		}
		break;

	case RHF_RCV_TYPE_ERROR:
		switch (rte) {
		case RHF_RTE_ERROR_OP_CODE_ERR:
		case RHF_RTE_ERROR_KHDR_MIN_LEN_ERR:
		case RHF_RTE_ERROR_KHDR_HCRC_ERR:
		case RHF_RTE_ERROR_KHDR_KVER_ERR:
		case RHF_RTE_ERROR_CONTEXT_ERR:
		case RHF_RTE_ERROR_KHDR_TID_ERR:
		default:
			break;
		}
	default:
		break;
	}
s_unlock:
	spin_unlock(&qp->s_lock);
	return ret;
}

bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
			      struct hfi1_pportdata *ppd,
			      struct hfi1_packet *packet)
{
	struct hfi1_ibport *ibp = &ppd->ibport_data;
	struct hfi1_devdata *dd = ppd->dd;
	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
	u8 rcv_type = rhf_rcv_type(packet->rhf);
	u8 rte = rhf_rcv_type_err(packet->rhf);
	struct ib_header *hdr = packet->hdr;
	struct ib_other_headers *ohdr = NULL;
	int lnh = be16_to_cpu(hdr->lrh[0]) & 3;
	u16 lid  = be16_to_cpu(hdr->lrh[1]);
	u8 opcode;
	u32 qp_num, psn, ibpsn;
	struct rvt_qp *qp;
	struct hfi1_qp_priv *qpriv;
	unsigned long flags;
	bool ret = true;
	struct rvt_ack_entry *e;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	int diff = 0;

	trace_hfi1_msg_handle_kdeth_eflags(NULL, "Kdeth error: rhf ",
					   packet->rhf);
	if (packet->rhf & RHF_ICRC_ERR)
		return ret;

	packet->ohdr = &hdr->u.oth;
	ohdr = packet->ohdr;
	trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));

	/* Get the destination QP number. */
	qp_num = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_qp) &
		RVT_QPN_MASK;
	if (lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))
		goto drop;

	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;

	rcu_read_lock();
	qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
	if (!qp)
		goto rcu_unlock;

	packet->qp = qp;

	/* Check for valid receive state. */
	spin_lock_irqsave(&qp->r_lock, flags);
	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
		ibp->rvp.n_pkt_drops++;
		goto r_unlock;
	}

	if (packet->rhf & RHF_TID_ERR) {
		/* For TIDERR and RC QPs preemptively schedule a NAK */
		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */

		/* Sanity check packet */
		if (tlen < 24)
			goto r_unlock;

		/*
		 * Check for GRH. We should never get packets with GRH in this
		 * path.
		 */
		if (lnh == HFI1_LRH_GRH)
			goto r_unlock;

		if (tid_rdma_tid_err(packet, rcv_type))
			goto r_unlock;
	}

	/* handle TID RDMA READ */
	if (opcode == TID_OP(READ_RESP)) {
		ibpsn = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_psn);
		ibpsn = mask_psn(ibpsn);
		ret = handle_read_kdeth_eflags(rcd, packet, rcv_type, rte, psn,
					       ibpsn);
		goto r_unlock;
	}

	/*
	 * qp->s_tail_ack_queue points to the rvt_ack_entry currently being
	 * processed. These a completed sequentially so we can be sure that
	 * the pointer will not change until the entire request has completed.
	 */
	spin_lock(&qp->s_lock);
	qpriv = qp->priv;
	if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID ||
	    qpriv->r_tid_tail == qpriv->r_tid_head)
		goto unlock;
	e = &qp->s_ack_queue[qpriv->r_tid_tail];
	if (e->opcode != TID_OP(WRITE_REQ))
		goto unlock;
	req = ack_to_tid_req(e);
	if (req->comp_seg == req->cur_seg)
		goto unlock;
	flow = &req->flows[req->clear_tail];
	trace_hfi1_eflags_err_write(qp, rcv_type, rte, psn);
	trace_hfi1_rsp_handle_kdeth_eflags(qp, psn);
	trace_hfi1_tid_write_rsp_handle_kdeth_eflags(qp);
	trace_hfi1_tid_req_handle_kdeth_eflags(qp, 0, e->opcode, e->psn,
					       e->lpsn, req);
	trace_hfi1_tid_flow_handle_kdeth_eflags(qp, req->clear_tail, flow);

	switch (rcv_type) {
	case RHF_RCV_TYPE_EXPECTED:
		switch (rte) {
		case RHF_RTE_EXPECTED_FLOW_SEQ_ERR:
			if (!(qpriv->s_flags & HFI1_R_TID_SW_PSN)) {
				qpriv->s_flags |= HFI1_R_TID_SW_PSN;
				flow->flow_state.r_next_psn =
					read_r_next_psn(dd, rcd->ctxt,
							flow->idx);
				qpriv->r_next_psn_kdeth =
					flow->flow_state.r_next_psn;
				goto nak_psn;
			} else {
				/*
				 * If the received PSN does not match the next
				 * expected PSN, NAK the packet.
				 * However, only do that if we know that the a
				 * NAK has already been sent. Otherwise, this
				 * mismatch could be due to packets that were
				 * already in flight.
				 */
				diff = cmp_psn(psn,
					       flow->flow_state.r_next_psn);
				if (diff > 0)
					goto nak_psn;
				else if (diff < 0)
					break;

				qpriv->s_nak_state = 0;
				/*
				 * If SW PSN verification is successful and this
				 * is the last packet in the segment, tell the
				 * caller to process it as a normal packet.
				 */
				if (psn == full_flow_psn(flow,
							 flow->flow_state.lpsn))
					ret = false;
				flow->flow_state.r_next_psn =
					mask_psn(psn + 1);
				qpriv->r_next_psn_kdeth =
					flow->flow_state.r_next_psn;
			}
			break;

		case RHF_RTE_EXPECTED_FLOW_GEN_ERR:
			goto nak_psn;

		default:
			break;
		}
		break;

	case RHF_RCV_TYPE_ERROR:
		switch (rte) {
		case RHF_RTE_ERROR_OP_CODE_ERR:
		case RHF_RTE_ERROR_KHDR_MIN_LEN_ERR:
		case RHF_RTE_ERROR_KHDR_HCRC_ERR:
		case RHF_RTE_ERROR_KHDR_KVER_ERR:
		case RHF_RTE_ERROR_CONTEXT_ERR:
		case RHF_RTE_ERROR_KHDR_TID_ERR:
		default:
			break;
		}
	default:
		break;
	}

unlock:
	spin_unlock(&qp->s_lock);
r_unlock:
	spin_unlock_irqrestore(&qp->r_lock, flags);
rcu_unlock:
	rcu_read_unlock();
drop:
	return ret;
nak_psn:
	ibp->rvp.n_rc_seqnak++;
	if (!qpriv->s_nak_state) {
		qpriv->s_nak_state = IB_NAK_PSN_ERROR;
		/* We are NAK'ing the next expected PSN */
		qpriv->s_nak_psn = mask_psn(flow->flow_state.r_next_psn);
		qpriv->s_flags |= RVT_S_ACK_PENDING;
		if (qpriv->r_tid_ack == HFI1_QP_WQE_INVALID)
			qpriv->r_tid_ack = qpriv->r_tid_tail;
		hfi1_schedule_tid_send(qp);
	}
	goto unlock;
}

/*
 * "Rewind" the TID request information.
 * This means that we reset the state back to ACTIVE,
 * find the proper flow, set the flow index to that flow,
 * and reset the flow information.
 */
void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
			       u32 *bth2)
{
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_flow *flow;
	struct hfi1_qp_priv *qpriv = qp->priv;
	int diff, delta_pkts;
	u32 tididx = 0, i;
	u16 fidx;

	if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
		*bth2 = mask_psn(qp->s_psn);
		flow = find_flow_ib(req, *bth2, &fidx);
		if (!flow) {
			trace_hfi1_msg_tid_restart_req(/* msg */
			   qp, "!!!!!! Could not find flow to restart: bth2 ",
			   (u64)*bth2);
			trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode,
						       wqe->psn, wqe->lpsn,
						       req);
			return;
		}
	} else {
		fidx = req->acked_tail;
		flow = &req->flows[fidx];
		*bth2 = mask_psn(req->r_ack_psn);
	}

	if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
		delta_pkts = delta_psn(*bth2, flow->flow_state.ib_spsn);
	else
		delta_pkts = delta_psn(*bth2,
				       full_flow_psn(flow,
						     flow->flow_state.spsn));

	trace_hfi1_tid_flow_restart_req(qp, fidx, flow);
	diff = delta_pkts + flow->resync_npkts;

	flow->sent = 0;
	flow->pkt = 0;
	flow->tid_idx = 0;
	flow->tid_offset = 0;
	if (diff) {
		for (tididx = 0; tididx < flow->tidcnt; tididx++) {
			u32 tidentry = flow->tid_entry[tididx], tidlen,
				tidnpkts, npkts;

			flow->tid_offset = 0;
			tidlen = EXP_TID_GET(tidentry, LEN) * PAGE_SIZE;
			tidnpkts = rvt_div_round_up_mtu(qp, tidlen);
			npkts = min_t(u32, diff, tidnpkts);
			flow->pkt += npkts;
			flow->sent += (npkts == tidnpkts ? tidlen :
				       npkts * qp->pmtu);
			flow->tid_offset += npkts * qp->pmtu;
			diff -= npkts;
			if (!diff)
				break;
		}
	}
	if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) {
		rvt_skip_sge(&qpriv->tid_ss, (req->cur_seg * req->seg_len) +
			     flow->sent, 0);
		/*
		 * Packet PSN is based on flow_state.spsn + flow->pkt. However,
		 * during a RESYNC, the generation is incremented and the
		 * sequence is reset to 0. Since we've adjusted the npkts in the
		 * flow and the SGE has been sufficiently advanced, we have to
		 * adjust flow->pkt in order to calculate the correct PSN.
		 */
		flow->pkt -= flow->resync_npkts;
	}

	if (flow->tid_offset ==
	    EXP_TID_GET(flow->tid_entry[tididx], LEN) * PAGE_SIZE) {
		tididx++;
		flow->tid_offset = 0;
	}
	flow->tid_idx = tididx;
	if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
		/* Move flow_idx to correct index */
		req->flow_idx = fidx;
	else
		req->clear_tail = fidx;

	trace_hfi1_tid_flow_restart_req(qp, fidx, flow);
	trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn,
				       wqe->lpsn, req);
	req->state = TID_REQUEST_ACTIVE;
	if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) {
		/* Reset all the flows that we are going to resend */
		fidx = CIRC_NEXT(fidx, MAX_FLOWS);
		i = qpriv->s_tid_tail;
		do {
			for (; CIRC_CNT(req->setup_head, fidx, MAX_FLOWS);
			      fidx = CIRC_NEXT(fidx, MAX_FLOWS)) {
				req->flows[fidx].sent = 0;
				req->flows[fidx].pkt = 0;
				req->flows[fidx].tid_idx = 0;
				req->flows[fidx].tid_offset = 0;
				req->flows[fidx].resync_npkts = 0;
			}
			if (i == qpriv->s_tid_cur)
				break;
			do {
				i = (++i == qp->s_size ? 0 : i);
				wqe = rvt_get_swqe_ptr(qp, i);
			} while (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE);
			req = wqe_to_tid_req(wqe);
			req->cur_seg = req->ack_seg;
			fidx = req->acked_tail;
			/* Pull req->clear_tail back */
			req->clear_tail = fidx;
		} while (1);
	}
}

void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp)
{
	int i, ret;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_flow_state *fs;

	if (qp->ibqp.qp_type != IB_QPT_RC || !HFI1_CAP_IS_KSET(TID_RDMA))
		return;

	/*
	 * First, clear the flow to help prevent any delayed packets from
	 * being delivered.
	 */
	fs = &qpriv->flow_state;
	if (fs->index != RXE_NUM_TID_FLOWS)
		hfi1_kern_clear_hw_flow(qpriv->rcd, qp);

	for (i = qp->s_acked; i != qp->s_head;) {
		struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i);

		if (++i == qp->s_size)
			i = 0;
		/* Free only locally allocated TID entries */
		if (wqe->wr.opcode != IB_WR_TID_RDMA_READ)
			continue;
		do {
			struct hfi1_swqe_priv *priv = wqe->priv;

			ret = hfi1_kern_exp_rcv_clear(&priv->tid_req);
		} while (!ret);
	}
	for (i = qp->s_acked_ack_queue; i != qp->r_head_ack_queue;) {
		struct rvt_ack_entry *e = &qp->s_ack_queue[i];

		if (++i == rvt_max_atomic(ib_to_rvt(qp->ibqp.device)))
			i = 0;
		/* Free only locally allocated TID entries */
		if (e->opcode != TID_OP(WRITE_REQ))
			continue;
		do {
			struct hfi1_ack_priv *priv = e->priv;

			ret = hfi1_kern_exp_rcv_clear(&priv->tid_req);
		} while (!ret);
	}
}

bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe)
{
	struct rvt_swqe *prev;
	struct hfi1_qp_priv *priv = qp->priv;
	u32 s_prev;
	struct tid_rdma_request *req;

	s_prev = (qp->s_cur == 0 ? qp->s_size : qp->s_cur) - 1;
	prev = rvt_get_swqe_ptr(qp, s_prev);

	switch (wqe->wr.opcode) {
	case IB_WR_SEND:
	case IB_WR_SEND_WITH_IMM:
	case IB_WR_SEND_WITH_INV:
	case IB_WR_ATOMIC_CMP_AND_SWP:
	case IB_WR_ATOMIC_FETCH_AND_ADD:
	case IB_WR_RDMA_WRITE:
		switch (prev->wr.opcode) {
		case IB_WR_TID_RDMA_WRITE:
			req = wqe_to_tid_req(prev);
			if (req->ack_seg != req->total_segs)
				goto interlock;
		default:
			break;
		}
		break;
	case IB_WR_RDMA_READ:
		if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE)
			break;
		/* fall through */
	case IB_WR_TID_RDMA_READ:
		switch (prev->wr.opcode) {
		case IB_WR_RDMA_READ:
			if (qp->s_acked != qp->s_cur)
				goto interlock;
			break;
		case IB_WR_TID_RDMA_WRITE:
			req = wqe_to_tid_req(prev);
			if (req->ack_seg != req->total_segs)
				goto interlock;
		default:
			break;
		}
	default:
		break;
	}
	return false;

interlock:
	priv->s_flags |= HFI1_S_TID_WAIT_INTERLCK;
	return true;
}

/* Does @sge meet the alignment requirements for tid rdma? */
static inline bool hfi1_check_sge_align(struct rvt_qp *qp,
					struct rvt_sge *sge, int num_sge)
{
	int i;

	for (i = 0; i < num_sge; i++, sge++) {
		trace_hfi1_sge_check_align(qp, i, sge);
		if ((u64)sge->vaddr & ~PAGE_MASK ||
		    sge->sge_length & ~PAGE_MASK)
			return false;
	}
	return true;
}

void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
{
	struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv;
	struct hfi1_swqe_priv *priv = wqe->priv;
	struct tid_rdma_params *remote;
	enum ib_wr_opcode new_opcode;
	bool do_tid_rdma = false;
	struct hfi1_pportdata *ppd = qpriv->rcd->ppd;

	if ((rdma_ah_get_dlid(&qp->remote_ah_attr) & ~((1 << ppd->lmc) - 1)) ==
				ppd->lid)
		return;
	if (qpriv->hdr_type != HFI1_PKT_TYPE_9B)
		return;

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	/*
	 * If TID RDMA is disabled by the negotiation, don't
	 * use it.
	 */
	if (!remote)
		goto exit;

	if (wqe->wr.opcode == IB_WR_RDMA_READ) {
		if (hfi1_check_sge_align(qp, &wqe->sg_list[0],
					 wqe->wr.num_sge)) {
			new_opcode = IB_WR_TID_RDMA_READ;
			do_tid_rdma = true;
		}
	} else if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
		/*
		 * TID RDMA is enabled for this RDMA WRITE request iff:
		 *   1. The remote address is page-aligned,
		 *   2. The length is larger than the minimum segment size,
		 *   3. The length is page-multiple.
		 */
		if (!(wqe->rdma_wr.remote_addr & ~PAGE_MASK) &&
		    !(wqe->length & ~PAGE_MASK)) {
			new_opcode = IB_WR_TID_RDMA_WRITE;
			do_tid_rdma = true;
		}
	}

	if (do_tid_rdma) {
		if (hfi1_kern_exp_rcv_alloc_flows(&priv->tid_req, GFP_ATOMIC))
			goto exit;
		wqe->wr.opcode = new_opcode;
		priv->tid_req.seg_len =
			min_t(u32, remote->max_len, wqe->length);
		priv->tid_req.total_segs =
			DIV_ROUND_UP(wqe->length, priv->tid_req.seg_len);
		/* Compute the last PSN of the request */
		wqe->lpsn = wqe->psn;
		if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
			priv->tid_req.n_flows = remote->max_read;
			qpriv->tid_r_reqs++;
			wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1;
		} else {
			wqe->lpsn += priv->tid_req.total_segs - 1;
			atomic_inc(&qpriv->n_requests);
		}

		priv->tid_req.cur_seg = 0;
		priv->tid_req.comp_seg = 0;
		priv->tid_req.ack_seg = 0;
		priv->tid_req.state = TID_REQUEST_INACTIVE;
		/*
		 * Reset acked_tail.
		 * TID RDMA READ does not have ACKs so it does not
		 * update the pointer. We have to reset it so TID RDMA
		 * WRITE does not get confused.
		 */
		priv->tid_req.acked_tail = priv->tid_req.setup_head;
		trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode,
						 wqe->psn, wqe->lpsn,
						 &priv->tid_req);
	}
exit:
	rcu_read_unlock();
}

/* TID RDMA WRITE functions */

u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
				  struct ib_other_headers *ohdr,
				  u32 *bth1, u32 *bth2, u32 *len)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_params *remote;

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	/*
	 * Set the number of flow to be used based on negotiated
	 * parameters.
	 */
	req->n_flows = remote->max_write;
	req->state = TID_REQUEST_ACTIVE;

	KDETH_RESET(ohdr->u.tid_rdma.w_req.kdeth0, KVER, 0x1);
	KDETH_RESET(ohdr->u.tid_rdma.w_req.kdeth1, JKEY, remote->jkey);
	ohdr->u.tid_rdma.w_req.reth.vaddr =
		cpu_to_be64(wqe->rdma_wr.remote_addr + (wqe->length - *len));
	ohdr->u.tid_rdma.w_req.reth.rkey =
		cpu_to_be32(wqe->rdma_wr.rkey);
	ohdr->u.tid_rdma.w_req.reth.length = cpu_to_be32(*len);
	ohdr->u.tid_rdma.w_req.verbs_qp = cpu_to_be32(qp->remote_qpn);
	*bth1 &= ~RVT_QPN_MASK;
	*bth1 |= remote->qp;
	qp->s_state = TID_OP(WRITE_REQ);
	qp->s_flags |= HFI1_S_WAIT_TID_RESP;
	*bth2 |= IB_BTH_REQ_ACK;
	*len = 0;

	rcu_read_unlock();
	return sizeof(ohdr->u.tid_rdma.w_req) / sizeof(u32);
}

void hfi1_compute_tid_rdma_flow_wt(void)
{
	/*
	 * Heuristic for computing the RNR timeout when waiting on the flow
	 * queue. Rather than a computationaly expensive exact estimate of when
	 * a flow will be available, we assume that if a QP is at position N in
	 * the flow queue it has to wait approximately (N + 1) * (number of
	 * segments between two sync points), assuming PMTU of 4K. The rationale
	 * for this is that flows are released and recycled at each sync point.
	 */
	tid_rdma_flow_wt = MAX_TID_FLOW_PSN * enum_to_mtu(OPA_MTU_4096) /
		TID_RDMA_MAX_SEGMENT_SIZE;
}

static u32 position_in_queue(struct hfi1_qp_priv *qpriv,
			     struct tid_queue *queue)
{
	return qpriv->tid_enqueue - queue->dequeue;
}

/*
 * @qp: points to rvt_qp context.
 * @to_seg: desired RNR timeout in segments.
 * Return: index of the next highest timeout in the ib_hfi1_rnr_table[]
 */
static u32 hfi1_compute_tid_rnr_timeout(struct rvt_qp *qp, u32 to_seg)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	u64 timeout;
	u32 bytes_per_us;
	u8 i;

	bytes_per_us = active_egress_rate(qpriv->rcd->ppd) / 8;
	timeout = (to_seg * TID_RDMA_MAX_SEGMENT_SIZE) / bytes_per_us;
	/*
	 * Find the next highest value in the RNR table to the required
	 * timeout. This gives the responder some padding.
	 */
	for (i = 1; i <= IB_AETH_CREDIT_MASK; i++)
		if (rvt_rnr_tbl_to_usec(i) >= timeout)
			return i;
	return 0;
}

/**
 * Central place for resource allocation at TID write responder,
 * is called from write_req and write_data interrupt handlers as
 * well as the send thread when a queued QP is scheduled for
 * resource allocation.
 *
 * Iterates over (a) segments of a request and then (b) queued requests
 * themselves to allocate resources for up to local->max_write
 * segments across multiple requests. Stop allocating when we
 * hit a sync point, resume allocating after data packets at
 * sync point have been received.
 *
 * Resource allocation and sending of responses is decoupled. The
 * request/segment which are being allocated and sent are as follows.
 * Resources are allocated for:
 *     [request: qpriv->r_tid_alloc, segment: req->alloc_seg]
 * The send thread sends:
 *     [request: qp->s_tail_ack_queue, segment:req->cur_seg]
 */
static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)
{
	struct tid_rdma_request *req;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct hfi1_ctxtdata *rcd = qpriv->rcd;
	struct tid_rdma_params *local = &qpriv->tid_rdma.local;
	struct rvt_ack_entry *e;
	u32 npkts, to_seg;
	bool last;
	int ret = 0;

	lockdep_assert_held(&qp->s_lock);

	while (1) {
		trace_hfi1_rsp_tid_write_alloc_res(qp, 0);
		trace_hfi1_tid_write_rsp_alloc_res(qp);
		/*
		 * Don't allocate more segments if a RNR NAK has already been
		 * scheduled to avoid messing up qp->r_psn: the RNR NAK will
		 * be sent only when all allocated segments have been sent.
		 * However, if more segments are allocated before that, TID RDMA
		 * WRITE RESP packets will be sent out for these new segments
		 * before the RNR NAK packet. When the requester receives the
		 * RNR NAK packet, it will restart with qp->s_last_psn + 1,
		 * which does not match qp->r_psn and will be dropped.
		 * Consequently, the requester will exhaust its retries and
		 * put the qp into error state.
		 */
		if (qpriv->rnr_nak_state == TID_RNR_NAK_SEND)
			break;

		/* No requests left to process */
		if (qpriv->r_tid_alloc == qpriv->r_tid_head) {
			/* If all data has been received, clear the flow */
			if (qpriv->flow_state.index < RXE_NUM_TID_FLOWS &&
			    !qpriv->alloc_w_segs) {
				hfi1_kern_clear_hw_flow(rcd, qp);
				qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
			}
			break;
		}

		e = &qp->s_ack_queue[qpriv->r_tid_alloc];
		if (e->opcode != TID_OP(WRITE_REQ))
			goto next_req;
		req = ack_to_tid_req(e);
		trace_hfi1_tid_req_write_alloc_res(qp, 0, e->opcode, e->psn,
						   e->lpsn, req);
		/* Finished allocating for all segments of this request */
		if (req->alloc_seg >= req->total_segs)
			goto next_req;

		/* Can allocate only a maximum of local->max_write for a QP */
		if (qpriv->alloc_w_segs >= local->max_write)
			break;

		/* Don't allocate at a sync point with data packets pending */
		if (qpriv->sync_pt && qpriv->alloc_w_segs)
			break;

		/* All data received at the sync point, continue */
		if (qpriv->sync_pt && !qpriv->alloc_w_segs) {
			hfi1_kern_clear_hw_flow(rcd, qp);
			qpriv->sync_pt = false;
			qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
		}

		/* Allocate flow if we don't have one */
		if (qpriv->flow_state.index >= RXE_NUM_TID_FLOWS) {
			ret = hfi1_kern_setup_hw_flow(qpriv->rcd, qp);
			if (ret) {
				to_seg = tid_rdma_flow_wt *
					position_in_queue(qpriv,
							  &rcd->flow_queue);
				break;
			}
		}

		npkts = rvt_div_round_up_mtu(qp, req->seg_len);

		/*
		 * We are at a sync point if we run out of KDETH PSN space.
		 * Last PSN of every generation is reserved for RESYNC.
		 */
		if (qpriv->flow_state.psn + npkts > MAX_TID_FLOW_PSN - 1) {
			qpriv->sync_pt = true;
			break;
		}

		/*
		 * If overtaking req->acked_tail, send an RNR NAK. Because the
		 * QP is not queued in this case, and the issue can only be
		 * caused due a delay in scheduling the second leg which we
		 * cannot estimate, we use a rather arbitrary RNR timeout of
		 * (MAX_FLOWS / 2) segments
		 */
		if (!CIRC_SPACE(req->setup_head, req->acked_tail,
				MAX_FLOWS)) {
			ret = -EAGAIN;
			to_seg = MAX_FLOWS >> 1;
			qpriv->s_flags |= RVT_S_ACK_PENDING;
			hfi1_schedule_tid_send(qp);
			break;
		}

		/* Try to allocate rcv array / TID entries */
		ret = hfi1_kern_exp_rcv_setup(req, &req->ss, &last);
		if (ret == -EAGAIN)
			to_seg = position_in_queue(qpriv, &rcd->rarr_queue);
		if (ret)
			break;

		qpriv->alloc_w_segs++;
		req->alloc_seg++;
		continue;
next_req:
		/* Begin processing the next request */
		if (++qpriv->r_tid_alloc >
		    rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
			qpriv->r_tid_alloc = 0;
	}

	/*
	 * Schedule an RNR NAK to be sent if (a) flow or rcv array allocation
	 * has failed (b) we are called from the rcv handler interrupt context
	 * (c) an RNR NAK has not already been scheduled
	 */
	if (ret == -EAGAIN && intr_ctx && !qp->r_nak_state)
		goto send_rnr_nak;

	return;

send_rnr_nak:
	lockdep_assert_held(&qp->r_lock);

	/* Set r_nak_state to prevent unrelated events from generating NAK's */
	qp->r_nak_state = hfi1_compute_tid_rnr_timeout(qp, to_seg) | IB_RNR_NAK;

	/* Pull back r_psn to the segment being RNR NAK'd */
	qp->r_psn = e->psn + req->alloc_seg;
	qp->r_ack_psn = qp->r_psn;
	/*
	 * Pull back r_head_ack_queue to the ack entry following the request
	 * being RNR NAK'd. This allows resources to be allocated to the request
	 * if the queued QP is scheduled.
	 */
	qp->r_head_ack_queue = qpriv->r_tid_alloc + 1;
	if (qp->r_head_ack_queue > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
		qp->r_head_ack_queue = 0;
	qpriv->r_tid_head = qp->r_head_ack_queue;
	/*
	 * These send side fields are used in make_rc_ack(). They are set in
	 * hfi1_send_rc_ack() but must be set here before dropping qp->s_lock
	 * for consistency
	 */
	qp->s_nak_state = qp->r_nak_state;
	qp->s_ack_psn = qp->r_ack_psn;
	/*
	 * Clear the ACK PENDING flag to prevent unwanted ACK because we
	 * have modified qp->s_ack_psn here.
	 */
	qp->s_flags &= ~(RVT_S_ACK_PENDING);

	trace_hfi1_rsp_tid_write_alloc_res(qp, qp->r_psn);
	/*
	 * qpriv->rnr_nak_state is used to determine when the scheduled RNR NAK
	 * has actually been sent. qp->s_flags RVT_S_ACK_PENDING bit cannot be
	 * used for this because qp->s_lock is dropped before calling
	 * hfi1_send_rc_ack() leading to inconsistency between the receive
	 * interrupt handlers and the send thread in make_rc_ack()
	 */
	qpriv->rnr_nak_state = TID_RNR_NAK_SEND;

	/*
	 * Schedule RNR NAK to be sent. RNR NAK's are scheduled from the receive
	 * interrupt handlers but will be sent from the send engine behind any
	 * previous responses that may have been scheduled
	 */
	rc_defered_ack(rcd, qp);
}

void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)
{
	/* HANDLER FOR TID RDMA WRITE REQUEST packet (Responder side)*/

	/*
	 * 1. Verify TID RDMA WRITE REQ as per IB_OPCODE_RC_RDMA_WRITE_FIRST
	 *    (see hfi1_rc_rcv())
	 *     - Don't allow 0-length requests.
	 * 2. Put TID RDMA WRITE REQ into the response queueu (s_ack_queue)
	 *     - Setup struct tid_rdma_req with request info
	 *     - Prepare struct tid_rdma_flow array?
	 * 3. Set the qp->s_ack_state as state diagram in design doc.
	 * 4. Set RVT_S_RESP_PENDING in s_flags.
	 * 5. Kick the send engine (hfi1_schedule_send())
	 */
	struct hfi1_ctxtdata *rcd = packet->rcd;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_ack_entry *e;
	unsigned long flags;
	struct ib_reth *reth;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_request *req;
	u32 bth0, psn, len, rkey, num_segs;
	bool fecn;
	u8 next;
	u64 vaddr;
	int diff;

	bth0 = be32_to_cpu(ohdr->bth[0]);
	if (hfi1_ruc_check_hdr(ibp, packet))
		return;

	fecn = process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	trace_hfi1_rsp_rcv_tid_write_req(qp, psn);

	if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
		rvt_comm_est(qp);

	if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
		goto nack_inv;

	reth = &ohdr->u.tid_rdma.w_req.reth;
	vaddr = be64_to_cpu(reth->vaddr);
	len = be32_to_cpu(reth->length);

	num_segs = DIV_ROUND_UP(len, qpriv->tid_rdma.local.max_len);
	diff = delta_psn(psn, qp->r_psn);
	if (unlikely(diff)) {
		tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn);
		return;
	}

	/*
	 * The resent request which was previously RNR NAK'd is inserted at the
	 * location of the original request, which is one entry behind
	 * r_head_ack_queue
	 */
	if (qpriv->rnr_nak_state)
		qp->r_head_ack_queue = qp->r_head_ack_queue ?
			qp->r_head_ack_queue - 1 :
			rvt_size_atomic(ib_to_rvt(qp->ibqp.device));

	/* We've verified the request, insert it into the ack queue. */
	next = qp->r_head_ack_queue + 1;
	if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
		next = 0;
	spin_lock_irqsave(&qp->s_lock, flags);
	if (unlikely(next == qp->s_acked_ack_queue)) {
		if (!qp->s_ack_queue[next].sent)
			goto nack_inv_unlock;
		update_ack_queue(qp, next);
	}
	e = &qp->s_ack_queue[qp->r_head_ack_queue];
	req = ack_to_tid_req(e);

	/* Bring previously RNR NAK'd request back to life */
	if (qpriv->rnr_nak_state) {
		qp->r_nak_state = 0;
		qp->s_nak_state = 0;
		qpriv->rnr_nak_state = TID_RNR_NAK_INIT;
		qp->r_psn = e->lpsn + 1;
		req->state = TID_REQUEST_INIT;
		goto update_head;
	}

	release_rdma_sge_mr(e);

	/* The length needs to be in multiples of PAGE_SIZE */
	if (!len || len & ~PAGE_MASK)
		goto nack_inv_unlock;

	rkey = be32_to_cpu(reth->rkey);
	qp->r_len = len;

	if (e->opcode == TID_OP(WRITE_REQ) &&
	    (req->setup_head != req->clear_tail ||
	     req->clear_tail != req->acked_tail))
		goto nack_inv_unlock;

	if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr,
				  rkey, IB_ACCESS_REMOTE_WRITE)))
		goto nack_acc;

	qp->r_psn += num_segs - 1;

	e->opcode = (bth0 >> 24) & 0xff;
	e->psn = psn;
	e->lpsn = qp->r_psn;
	e->sent = 0;

	req->n_flows = min_t(u16, num_segs, qpriv->tid_rdma.local.max_write);
	req->state = TID_REQUEST_INIT;
	req->cur_seg = 0;
	req->comp_seg = 0;
	req->ack_seg = 0;
	req->alloc_seg = 0;
	req->isge = 0;
	req->seg_len = qpriv->tid_rdma.local.max_len;
	req->total_len = len;
	req->total_segs = num_segs;
	req->r_flow_psn = e->psn;
	req->ss.sge = e->rdma_sge;
	req->ss.num_sge = 1;

	req->flow_idx = req->setup_head;
	req->clear_tail = req->setup_head;
	req->acked_tail = req->setup_head;

	qp->r_state = e->opcode;
	qp->r_nak_state = 0;
	/*
	 * We need to increment the MSN here instead of when we
	 * finish sending the result since a duplicate request would
	 * increment it more than once.
	 */
	qp->r_msn++;
	qp->r_psn++;

	trace_hfi1_tid_req_rcv_write_req(qp, 0, e->opcode, e->psn, e->lpsn,
					 req);

	if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID) {
		qpriv->r_tid_tail = qp->r_head_ack_queue;
	} else if (qpriv->r_tid_tail == qpriv->r_tid_head) {
		struct tid_rdma_request *ptr;

		e = &qp->s_ack_queue[qpriv->r_tid_tail];
		ptr = ack_to_tid_req(e);

		if (e->opcode != TID_OP(WRITE_REQ) ||
		    ptr->comp_seg == ptr->total_segs) {
			if (qpriv->r_tid_tail == qpriv->r_tid_ack)
				qpriv->r_tid_ack = qp->r_head_ack_queue;
			qpriv->r_tid_tail = qp->r_head_ack_queue;
		}
	}
update_head:
	qp->r_head_ack_queue = next;
	qpriv->r_tid_head = qp->r_head_ack_queue;

	hfi1_tid_write_alloc_resources(qp, true);
	trace_hfi1_tid_write_rsp_rcv_req(qp);

	/* Schedule the send tasklet. */
	qp->s_flags |= RVT_S_RESP_PENDING;
	if (fecn)
		qp->s_flags |= RVT_S_ECN;
	hfi1_schedule_send(qp);

	spin_unlock_irqrestore(&qp->s_lock, flags);
	return;

nack_inv_unlock:
	spin_unlock_irqrestore(&qp->s_lock, flags);
nack_inv:
	rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
	qp->r_nak_state = IB_NAK_INVALID_REQUEST;
	qp->r_ack_psn = qp->r_psn;
	/* Queue NAK for later */
	rc_defered_ack(rcd, qp);
	return;
nack_acc:
	spin_unlock_irqrestore(&qp->s_lock, flags);
	rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
	qp->r_ack_psn = qp->r_psn;
}

u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
				   struct ib_other_headers *ohdr, u32 *bth1,
				   u32 bth2, u32 *len,
				   struct rvt_sge_state **ss)
{
	struct hfi1_ack_priv *epriv = e->priv;
	struct tid_rdma_request *req = &epriv->tid_req;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_flow *flow = NULL;
	u32 resp_len = 0, hdwords = 0;
	void *resp_addr = NULL;
	struct tid_rdma_params *remote;

	trace_hfi1_tid_req_build_write_resp(qp, 0, e->opcode, e->psn, e->lpsn,
					    req);
	trace_hfi1_tid_write_rsp_build_resp(qp);
	trace_hfi1_rsp_build_tid_write_resp(qp, bth2);
	flow = &req->flows[req->flow_idx];
	switch (req->state) {
	default:
		/*
		 * Try to allocate resources here in case QP was queued and was
		 * later scheduled when resources became available
		 */
		hfi1_tid_write_alloc_resources(qp, false);

		/* We've already sent everything which is ready */
		if (req->cur_seg >= req->alloc_seg)
			goto done;

		/*
		 * Resources can be assigned but responses cannot be sent in
		 * rnr_nak state, till the resent request is received
		 */
		if (qpriv->rnr_nak_state == TID_RNR_NAK_SENT)
			goto done;

		req->state = TID_REQUEST_ACTIVE;
		trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow);
		req->flow_idx = CIRC_NEXT(req->flow_idx, MAX_FLOWS);
		hfi1_add_tid_reap_timer(qp);
		break;

	case TID_REQUEST_RESEND_ACTIVE:
	case TID_REQUEST_RESEND:
		trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow);
		req->flow_idx = CIRC_NEXT(req->flow_idx, MAX_FLOWS);
		if (!CIRC_CNT(req->setup_head, req->flow_idx, MAX_FLOWS))
			req->state = TID_REQUEST_ACTIVE;

		hfi1_mod_tid_reap_timer(qp);
		break;
	}
	flow->flow_state.resp_ib_psn = bth2;
	resp_addr = (void *)flow->tid_entry;
	resp_len = sizeof(*flow->tid_entry) * flow->tidcnt;
	req->cur_seg++;

	memset(&ohdr->u.tid_rdma.w_rsp, 0, sizeof(ohdr->u.tid_rdma.w_rsp));
	epriv->ss.sge.vaddr = resp_addr;
	epriv->ss.sge.sge_length = resp_len;
	epriv->ss.sge.length = epriv->ss.sge.sge_length;
	/*
	 * We can safely zero these out. Since the first SGE covers the
	 * entire packet, nothing else should even look at the MR.
	 */
	epriv->ss.sge.mr = NULL;
	epriv->ss.sge.m = 0;
	epriv->ss.sge.n = 0;

	epriv->ss.sg_list = NULL;
	epriv->ss.total_len = epriv->ss.sge.sge_length;
	epriv->ss.num_sge = 1;

	*ss = &epriv->ss;
	*len = epriv->ss.total_len;

	/* Construct the TID RDMA WRITE RESP packet header */
	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);

	KDETH_RESET(ohdr->u.tid_rdma.w_rsp.kdeth0, KVER, 0x1);
	KDETH_RESET(ohdr->u.tid_rdma.w_rsp.kdeth1, JKEY, remote->jkey);
	ohdr->u.tid_rdma.w_rsp.aeth = rvt_compute_aeth(qp);
	ohdr->u.tid_rdma.w_rsp.tid_flow_psn =
		cpu_to_be32((flow->flow_state.generation <<
			     HFI1_KDETH_BTH_SEQ_SHIFT) |
			    (flow->flow_state.spsn &
			     HFI1_KDETH_BTH_SEQ_MASK));
	ohdr->u.tid_rdma.w_rsp.tid_flow_qp =
		cpu_to_be32(qpriv->tid_rdma.local.qp |
			    ((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) <<
			     TID_RDMA_DESTQP_FLOW_SHIFT) |
			    qpriv->rcd->ctxt);
	ohdr->u.tid_rdma.w_rsp.verbs_qp = cpu_to_be32(qp->remote_qpn);
	*bth1 = remote->qp;
	rcu_read_unlock();
	hdwords = sizeof(ohdr->u.tid_rdma.w_rsp) / sizeof(u32);
	qpriv->pending_tid_w_segs++;
done:
	return hdwords;
}

static void hfi1_add_tid_reap_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = qp->priv;

	lockdep_assert_held(&qp->s_lock);
	if (!(qpriv->s_flags & HFI1_R_TID_RSC_TIMER)) {
		qpriv->s_flags |= HFI1_R_TID_RSC_TIMER;
		qpriv->s_tid_timer.expires = jiffies +
			qpriv->tid_timer_timeout_jiffies;
		add_timer(&qpriv->s_tid_timer);
	}
}

static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = qp->priv;

	lockdep_assert_held(&qp->s_lock);
	qpriv->s_flags |= HFI1_R_TID_RSC_TIMER;
	mod_timer(&qpriv->s_tid_timer, jiffies +
		  qpriv->tid_timer_timeout_jiffies);
}

static int hfi1_stop_tid_reap_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	int rval = 0;

	lockdep_assert_held(&qp->s_lock);
	if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) {
		rval = del_timer(&qpriv->s_tid_timer);
		qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER;
	}
	return rval;
}

void hfi1_del_tid_reap_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *qpriv = qp->priv;

	del_timer_sync(&qpriv->s_tid_timer);
	qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER;
}

static void hfi1_tid_timeout(struct timer_list *t)
{
	struct hfi1_qp_priv *qpriv = from_timer(qpriv, t, s_tid_timer);
	struct rvt_qp *qp = qpriv->owner;
	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
	unsigned long flags;
	u32 i;

	spin_lock_irqsave(&qp->r_lock, flags);
	spin_lock(&qp->s_lock);
	if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) {
		dd_dev_warn(dd_from_ibdev(qp->ibqp.device), "[QP%u] %s %d\n",
			    qp->ibqp.qp_num, __func__, __LINE__);
		trace_hfi1_msg_tid_timeout(/* msg */
			qp, "resource timeout = ",
			(u64)qpriv->tid_timer_timeout_jiffies);
		hfi1_stop_tid_reap_timer(qp);
		/*
		 * Go though the entire ack queue and clear any outstanding
		 * HW flow and RcvArray resources.
		 */
		hfi1_kern_clear_hw_flow(qpriv->rcd, qp);
		for (i = 0; i < rvt_max_atomic(rdi); i++) {
			struct tid_rdma_request *req =
				ack_to_tid_req(&qp->s_ack_queue[i]);

			hfi1_kern_exp_rcv_clear_all(req);
		}
		spin_unlock(&qp->s_lock);
		if (qp->ibqp.event_handler) {
			struct ib_event ev;

			ev.device = qp->ibqp.device;
			ev.element.qp = &qp->ibqp;
			ev.event = IB_EVENT_QP_FATAL;
			qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
		}
		rvt_rc_error(qp, IB_WC_RESP_TIMEOUT_ERR);
		goto unlock_r_lock;
	}
	spin_unlock(&qp->s_lock);
unlock_r_lock:
	spin_unlock_irqrestore(&qp->r_lock, flags);
}

void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet)
{
	/* HANDLER FOR TID RDMA WRITE RESPONSE packet (Requestor side */

	/*
	 * 1. Find matching SWQE
	 * 2. Check that TIDENTRY array has enough space for a complete
	 *    segment. If not, put QP in error state.
	 * 3. Save response data in struct tid_rdma_req and struct tid_rdma_flow
	 * 4. Remove HFI1_S_WAIT_TID_RESP from s_flags.
	 * 5. Set qp->s_state
	 * 6. Kick the send engine (hfi1_schedule_send())
	 */
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct hfi1_ctxtdata *rcd = packet->rcd;
	struct rvt_swqe *wqe;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	enum ib_wc_status status;
	u32 opcode, aeth, psn, flow_psn, i, tidlen = 0, pktlen;
	bool fecn;
	unsigned long flags;

	fecn = process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	aeth = be32_to_cpu(ohdr->u.tid_rdma.w_rsp.aeth);
	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;

	spin_lock_irqsave(&qp->s_lock, flags);

	/* Ignore invalid responses */
	if (cmp_psn(psn, qp->s_next_psn) >= 0)
		goto ack_done;

	/* Ignore duplicate responses. */
	if (unlikely(cmp_psn(psn, qp->s_last_psn) <= 0))
		goto ack_done;

	if (unlikely(qp->s_acked == qp->s_tail))
		goto ack_done;

	/*
	 * If we are waiting for a particular packet sequence number
	 * due to a request being resent, check for it. Otherwise,
	 * ensure that we haven't missed anything.
	 */
	if (qp->r_flags & RVT_R_RDMAR_SEQ) {
		if (cmp_psn(psn, qp->s_last_psn + 1) != 0)
			goto ack_done;
		qp->r_flags &= ~RVT_R_RDMAR_SEQ;
	}

	wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur);
	if (unlikely(wqe->wr.opcode != IB_WR_TID_RDMA_WRITE))
		goto ack_op_err;

	req = wqe_to_tid_req(wqe);
	/*
	 * If we've lost ACKs and our acked_tail pointer is too far
	 * behind, don't overwrite segments. Just drop the packet and
	 * let the reliability protocol take care of it.
	 */
	if (!CIRC_SPACE(req->setup_head, req->acked_tail, MAX_FLOWS))
		goto ack_done;

	/*
	 * The call to do_rc_ack() should be last in the chain of
	 * packet checks because it will end up updating the QP state.
	 * Therefore, anything that would prevent the packet from
	 * being accepted as a successful response should be prior
	 * to it.
	 */
	if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
		goto ack_done;

	trace_hfi1_ack(qp, psn);

	flow = &req->flows[req->setup_head];
	flow->pkt = 0;
	flow->tid_idx = 0;
	flow->tid_offset = 0;
	flow->sent = 0;
	flow->resync_npkts = 0;
	flow->tid_qpn = be32_to_cpu(ohdr->u.tid_rdma.w_rsp.tid_flow_qp);
	flow->idx = (flow->tid_qpn >> TID_RDMA_DESTQP_FLOW_SHIFT) &
		TID_RDMA_DESTQP_FLOW_MASK;
	flow_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.w_rsp.tid_flow_psn));
	flow->flow_state.generation = flow_psn >> HFI1_KDETH_BTH_SEQ_SHIFT;
	flow->flow_state.spsn = flow_psn & HFI1_KDETH_BTH_SEQ_MASK;
	flow->flow_state.resp_ib_psn = psn;
	flow->length = min_t(u32, req->seg_len,
			     (wqe->length - (req->comp_seg * req->seg_len)));

	flow->npkts = rvt_div_round_up_mtu(qp, flow->length);
	flow->flow_state.lpsn = flow->flow_state.spsn +
		flow->npkts - 1;
	/* payload length = packet length - (header length + ICRC length) */
	pktlen = packet->tlen - (packet->hlen + 4);
	if (pktlen > sizeof(flow->tid_entry)) {
		status = IB_WC_LOC_LEN_ERR;
		goto ack_err;
	}
	memcpy(flow->tid_entry, packet->ebuf, pktlen);
	flow->tidcnt = pktlen / sizeof(*flow->tid_entry);
	trace_hfi1_tid_flow_rcv_write_resp(qp, req->setup_head, flow);

	req->comp_seg++;
	trace_hfi1_tid_write_sender_rcv_resp(qp, 0);
	/*
	 * Walk the TID_ENTRY list to make sure we have enough space for a
	 * complete segment.
	 */
	for (i = 0; i < flow->tidcnt; i++) {
		trace_hfi1_tid_entry_rcv_write_resp(/* entry */
			qp, i, flow->tid_entry[i]);
		if (!EXP_TID_GET(flow->tid_entry[i], LEN)) {
			status = IB_WC_LOC_LEN_ERR;
			goto ack_err;
		}
		tidlen += EXP_TID_GET(flow->tid_entry[i], LEN);
	}
	if (tidlen * PAGE_SIZE < flow->length) {
		status = IB_WC_LOC_LEN_ERR;
		goto ack_err;
	}

	trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn,
					  wqe->lpsn, req);
	/*
	 * If this is the first response for this request, set the initial
	 * flow index to the current flow.
	 */
	if (!cmp_psn(psn, wqe->psn)) {
		req->r_last_acked = mask_psn(wqe->psn - 1);
		/* Set acked flow index to head index */
		req->acked_tail = req->setup_head;
	}

	/* advance circular buffer head */
	req->setup_head = CIRC_NEXT(req->setup_head, MAX_FLOWS);
	req->state = TID_REQUEST_ACTIVE;

	/*
	 * If all responses for this TID RDMA WRITE request have been received
	 * advance the pointer to the next one.
	 * Since TID RDMA requests could be mixed in with regular IB requests,
	 * they might not appear sequentially in the queue. Therefore, the
	 * next request needs to be "found".
	 */
	if (qpriv->s_tid_cur != qpriv->s_tid_head &&
	    req->comp_seg == req->total_segs) {
		for (i = qpriv->s_tid_cur + 1; ; i++) {
			if (i == qp->s_size)
				i = 0;
			wqe = rvt_get_swqe_ptr(qp, i);
			if (i == qpriv->s_tid_head)
				break;
			if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
				break;
		}
		qpriv->s_tid_cur = i;
	}
	qp->s_flags &= ~HFI1_S_WAIT_TID_RESP;
	hfi1_schedule_tid_send(qp);
	goto ack_done;

ack_op_err:
	status = IB_WC_LOC_QP_OP_ERR;
ack_err:
	rvt_error_qp(qp, status);
ack_done:
	if (fecn)
		qp->s_flags |= RVT_S_ECN;
	spin_unlock_irqrestore(&qp->s_lock, flags);
}

bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe,
				struct ib_other_headers *ohdr,
				u32 *bth1, u32 *bth2, u32 *len)
{
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_flow *flow = &req->flows[req->clear_tail];
	struct tid_rdma_params *remote;
	struct rvt_qp *qp = req->qp;
	struct hfi1_qp_priv *qpriv = qp->priv;
	u32 tidentry = flow->tid_entry[flow->tid_idx];
	u32 tidlen = EXP_TID_GET(tidentry, LEN) << PAGE_SHIFT;
	struct tid_rdma_write_data *wd = &ohdr->u.tid_rdma.w_data;
	u32 next_offset, om = KDETH_OM_LARGE;
	bool last_pkt;

	if (!tidlen) {
		hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR);
		rvt_error_qp(qp, IB_WC_REM_INV_RD_REQ_ERR);
	}

	*len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset);
	flow->sent += *len;
	next_offset = flow->tid_offset + *len;
	last_pkt = (flow->tid_idx == (flow->tidcnt - 1) &&
		    next_offset >= tidlen) || (flow->sent >= flow->length);
	trace_hfi1_tid_entry_build_write_data(qp, flow->tid_idx, tidentry);
	trace_hfi1_tid_flow_build_write_data(qp, req->clear_tail, flow);

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	KDETH_RESET(wd->kdeth0, KVER, 0x1);
	KDETH_SET(wd->kdeth0, SH, !last_pkt);
	KDETH_SET(wd->kdeth0, INTR, !!(!last_pkt && remote->urg));
	KDETH_SET(wd->kdeth0, TIDCTRL, EXP_TID_GET(tidentry, CTRL));
	KDETH_SET(wd->kdeth0, TID, EXP_TID_GET(tidentry, IDX));
	KDETH_SET(wd->kdeth0, OM, om == KDETH_OM_LARGE);
	KDETH_SET(wd->kdeth0, OFFSET, flow->tid_offset / om);
	KDETH_RESET(wd->kdeth1, JKEY, remote->jkey);
	wd->verbs_qp = cpu_to_be32(qp->remote_qpn);
	rcu_read_unlock();

	*bth1 = flow->tid_qpn;
	*bth2 = mask_psn(((flow->flow_state.spsn + flow->pkt++) &
			 HFI1_KDETH_BTH_SEQ_MASK) |
			 (flow->flow_state.generation <<
			  HFI1_KDETH_BTH_SEQ_SHIFT));
	if (last_pkt) {
		/* PSNs are zero-based, so +1 to count number of packets */
		if (flow->flow_state.lpsn + 1 +
		    rvt_div_round_up_mtu(qp, req->seg_len) >
		    MAX_TID_FLOW_PSN)
			req->state = TID_REQUEST_SYNC;
		*bth2 |= IB_BTH_REQ_ACK;
	}

	if (next_offset >= tidlen) {
		flow->tid_offset = 0;
		flow->tid_idx++;
	} else {
		flow->tid_offset = next_offset;
	}
	return last_pkt;
}

void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)
{
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *priv = qp->priv;
	struct hfi1_ctxtdata *rcd = priv->rcd;
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_ack_entry *e;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
	unsigned long flags;
	u32 psn, next;
	u8 opcode;
	bool fecn;

	fecn = process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;

	/*
	 * All error handling should be done by now. If we are here, the packet
	 * is either good or been accepted by the error handler.
	 */
	spin_lock_irqsave(&qp->s_lock, flags);
	e = &qp->s_ack_queue[priv->r_tid_tail];
	req = ack_to_tid_req(e);
	flow = &req->flows[req->clear_tail];
	if (cmp_psn(psn, full_flow_psn(flow, flow->flow_state.lpsn))) {
		update_r_next_psn_fecn(packet, priv, rcd, flow, fecn);

		if (cmp_psn(psn, flow->flow_state.r_next_psn))
			goto send_nak;

		flow->flow_state.r_next_psn = mask_psn(psn + 1);
		/*
		 * Copy the payload to destination buffer if this packet is
		 * delivered as an eager packet due to RSM rule and FECN.
		 * The RSM rule selects FECN bit in BTH and SH bit in
		 * KDETH header and therefore will not match the last
		 * packet of each segment that has SH bit cleared.
		 */
		if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) {
			struct rvt_sge_state ss;
			u32 len;
			u32 tlen = packet->tlen;
			u16 hdrsize = packet->hlen;
			u8 pad = packet->pad;
			u8 extra_bytes = pad + packet->extra_byte +
				(SIZE_OF_CRC << 2);
			u32 pmtu = qp->pmtu;

			if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
				goto send_nak;
			len = req->comp_seg * req->seg_len;
			len += delta_psn(psn,
				full_flow_psn(flow, flow->flow_state.spsn)) *
				pmtu;
			if (unlikely(req->total_len - len < pmtu))
				goto send_nak;

			/*
			 * The e->rdma_sge field is set when TID RDMA WRITE REQ
			 * is first received and is never modified thereafter.
			 */
			ss.sge = e->rdma_sge;
			ss.sg_list = NULL;
			ss.num_sge = 1;
			ss.total_len = req->total_len;
			rvt_skip_sge(&ss, len, false);
			rvt_copy_sge(qp, &ss, packet->payload, pmtu, false,
				     false);
			/* Raise the sw sequence check flag for next packet */
			priv->r_next_psn_kdeth = mask_psn(psn + 1);
			priv->s_flags |= HFI1_R_TID_SW_PSN;
		}
		goto exit;
	}
	flow->flow_state.r_next_psn = mask_psn(psn + 1);
	hfi1_kern_exp_rcv_clear(req);
	priv->alloc_w_segs--;
	rcd->flows[flow->idx].psn = psn & HFI1_KDETH_BTH_SEQ_MASK;
	req->comp_seg++;
	priv->s_nak_state = 0;

	/*
	 * Release the flow if one of the following conditions has been met:
	 *  - The request has reached a sync point AND all outstanding
	 *    segments have been completed, or
	 *  - The entire request is complete and there are no more requests
	 *    (of any kind) in the queue.
	 */
	trace_hfi1_rsp_rcv_tid_write_data(qp, psn);
	trace_hfi1_tid_req_rcv_write_data(qp, 0, e->opcode, e->psn, e->lpsn,
					  req);
	trace_hfi1_tid_write_rsp_rcv_data(qp);
	if (priv->r_tid_ack == HFI1_QP_WQE_INVALID)
		priv->r_tid_ack = priv->r_tid_tail;

	if (opcode == TID_OP(WRITE_DATA_LAST)) {
		release_rdma_sge_mr(e);
		for (next = priv->r_tid_tail + 1; ; next++) {
			if (next > rvt_size_atomic(&dev->rdi))
				next = 0;
			if (next == priv->r_tid_head)
				break;
			e = &qp->s_ack_queue[next];
			if (e->opcode == TID_OP(WRITE_REQ))
				break;
		}
		priv->r_tid_tail = next;
		if (++qp->s_acked_ack_queue > rvt_size_atomic(&dev->rdi))
			qp->s_acked_ack_queue = 0;
	}

	hfi1_tid_write_alloc_resources(qp, true);

	/*
	 * If we need to generate more responses, schedule the
	 * send engine.
	 */
	if (req->cur_seg < req->total_segs ||
	    qp->s_tail_ack_queue != qp->r_head_ack_queue) {
		qp->s_flags |= RVT_S_RESP_PENDING;
		hfi1_schedule_send(qp);
	}

	priv->pending_tid_w_segs--;
	if (priv->s_flags & HFI1_R_TID_RSC_TIMER) {
		if (priv->pending_tid_w_segs)
			hfi1_mod_tid_reap_timer(req->qp);
		else
			hfi1_stop_tid_reap_timer(req->qp);
	}

done:
	priv->s_flags |= RVT_S_ACK_PENDING;
	hfi1_schedule_tid_send(qp);
exit:
	priv->r_next_psn_kdeth = flow->flow_state.r_next_psn;
	if (fecn)
		qp->s_flags |= RVT_S_ECN;
	spin_unlock_irqrestore(&qp->s_lock, flags);
	return;

send_nak:
	if (!priv->s_nak_state) {
		priv->s_nak_state = IB_NAK_PSN_ERROR;
		priv->s_nak_psn = flow->flow_state.r_next_psn;
		priv->s_flags |= RVT_S_ACK_PENDING;
		if (priv->r_tid_ack == HFI1_QP_WQE_INVALID)
			priv->r_tid_ack = priv->r_tid_tail;
		hfi1_schedule_tid_send(qp);
	}
	goto done;
}

static bool hfi1_tid_rdma_is_resync_psn(u32 psn)
{
	return (bool)((psn & HFI1_KDETH_BTH_SEQ_MASK) ==
		      HFI1_KDETH_BTH_SEQ_MASK);
}

u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e,
				  struct ib_other_headers *ohdr, u16 iflow,
				  u32 *bth1, u32 *bth2)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_flow_state *fs = &qpriv->flow_state;
	struct tid_rdma_request *req = ack_to_tid_req(e);
	struct tid_rdma_flow *flow = &req->flows[iflow];
	struct tid_rdma_params *remote;

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth1, JKEY, remote->jkey);
	ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn);
	*bth1 = remote->qp;
	rcu_read_unlock();

	if (qpriv->resync) {
		*bth2 = mask_psn((fs->generation <<
				  HFI1_KDETH_BTH_SEQ_SHIFT) - 1);
		ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp);
	} else if (qpriv->s_nak_state) {
		*bth2 = mask_psn(qpriv->s_nak_psn);
		ohdr->u.tid_rdma.ack.aeth =
			cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
				    (qpriv->s_nak_state <<
				     IB_AETH_CREDIT_SHIFT));
	} else {
		*bth2 = full_flow_psn(flow, flow->flow_state.lpsn);
		ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp);
	}
	KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth0, KVER, 0x1);
	ohdr->u.tid_rdma.ack.tid_flow_qp =
		cpu_to_be32(qpriv->tid_rdma.local.qp |
			    ((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) <<
			     TID_RDMA_DESTQP_FLOW_SHIFT) |
			    qpriv->rcd->ctxt);

	ohdr->u.tid_rdma.ack.tid_flow_psn = 0;
	ohdr->u.tid_rdma.ack.verbs_psn =
		cpu_to_be32(flow->flow_state.resp_ib_psn);

	if (qpriv->resync) {
		/*
		 * If the PSN before the current expect KDETH PSN is the
		 * RESYNC PSN, then we never received a good TID RDMA WRITE
		 * DATA packet after a previous RESYNC.
		 * In this case, the next expected KDETH PSN stays the same.
		 */
		if (hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1)) {
			ohdr->u.tid_rdma.ack.tid_flow_psn =
				cpu_to_be32(qpriv->r_next_psn_kdeth_save);
		} else {
			/*
			 * Because the KDETH PSNs jump during a RESYNC, it's
			 * not possible to infer (or compute) the previous value
			 * of r_next_psn_kdeth in the case of back-to-back
			 * RESYNC packets. Therefore, we save it.
			 */
			qpriv->r_next_psn_kdeth_save =
				qpriv->r_next_psn_kdeth - 1;
			ohdr->u.tid_rdma.ack.tid_flow_psn =
				cpu_to_be32(qpriv->r_next_psn_kdeth_save);
			qpriv->r_next_psn_kdeth = mask_psn(*bth2 + 1);
		}
		qpriv->resync = false;
	}

	return sizeof(ohdr->u.tid_rdma.ack) / sizeof(u32);
}

void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)
{
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct rvt_swqe *wqe;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	u32 aeth, psn, req_psn, ack_psn, flpsn, resync_psn, ack_kpsn;
	unsigned long flags;
	u16 fidx;

	trace_hfi1_tid_write_sender_rcv_tid_ack(qp, 0);
	process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
	aeth = be32_to_cpu(ohdr->u.tid_rdma.ack.aeth);
	req_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.ack.verbs_psn));
	resync_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.ack.tid_flow_psn));

	spin_lock_irqsave(&qp->s_lock, flags);
	trace_hfi1_rcv_tid_ack(qp, aeth, psn, req_psn, resync_psn);

	/* If we are waiting for an ACK to RESYNC, drop any other packets */
	if ((qp->s_flags & HFI1_S_WAIT_HALT) &&
	    cmp_psn(psn, qpriv->s_resync_psn))
		goto ack_op_err;

	ack_psn = req_psn;
	if (hfi1_tid_rdma_is_resync_psn(psn))
		ack_kpsn = resync_psn;
	else
		ack_kpsn = psn;
	if (aeth >> 29) {
		ack_psn--;
		ack_kpsn--;
	}

	if (unlikely(qp->s_acked == qp->s_tail))
		goto ack_op_err;

	wqe = rvt_get_swqe_ptr(qp, qp->s_acked);

	if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
		goto ack_op_err;

	req = wqe_to_tid_req(wqe);
	trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn,
				       wqe->lpsn, req);
	flow = &req->flows[req->acked_tail];
	trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow);

	/* Drop stale ACK/NAK */
	if (cmp_psn(psn, full_flow_psn(flow, flow->flow_state.spsn)) < 0 ||
	    cmp_psn(req_psn, flow->flow_state.resp_ib_psn) < 0)
		goto ack_op_err;

	while (cmp_psn(ack_kpsn,
		       full_flow_psn(flow, flow->flow_state.lpsn)) >= 0 &&
	       req->ack_seg < req->cur_seg) {
		req->ack_seg++;
		/* advance acked segment pointer */
		req->acked_tail = CIRC_NEXT(req->acked_tail, MAX_FLOWS);
		req->r_last_acked = flow->flow_state.resp_ib_psn;
		trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn,
					       wqe->lpsn, req);
		if (req->ack_seg == req->total_segs) {
			req->state = TID_REQUEST_COMPLETE;
			wqe = do_rc_completion(qp, wqe,
					       to_iport(qp->ibqp.device,
							qp->port_num));
			trace_hfi1_sender_rcv_tid_ack(qp);
			atomic_dec(&qpriv->n_tid_requests);
			if (qp->s_acked == qp->s_tail)
				break;
			if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
				break;
			req = wqe_to_tid_req(wqe);
		}
		flow = &req->flows[req->acked_tail];
		trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow);
	}

	trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn,
				       wqe->lpsn, req);
	switch (aeth >> 29) {
	case 0:         /* ACK */
		if (qpriv->s_flags & RVT_S_WAIT_ACK)
			qpriv->s_flags &= ~RVT_S_WAIT_ACK;
		if (!hfi1_tid_rdma_is_resync_psn(psn)) {
			/* Check if there is any pending TID ACK */
			if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
			    req->ack_seg < req->cur_seg)
				hfi1_mod_tid_retry_timer(qp);
			else
				hfi1_stop_tid_retry_timer(qp);
			hfi1_schedule_send(qp);
		} else {
			u32 spsn, fpsn, last_acked, generation;
			struct tid_rdma_request *rptr;

			/* ACK(RESYNC) */
			hfi1_stop_tid_retry_timer(qp);
			/* Allow new requests (see hfi1_make_tid_rdma_pkt) */
			qp->s_flags &= ~HFI1_S_WAIT_HALT;
			/*
			 * Clear RVT_S_SEND_ONE flag in case that the TID RDMA
			 * ACK is received after the TID retry timer is fired
			 * again. In this case, do not send any more TID
			 * RESYNC request or wait for any more TID ACK packet.
			 */
			qpriv->s_flags &= ~RVT_S_SEND_ONE;
			hfi1_schedule_send(qp);

			if ((qp->s_acked == qpriv->s_tid_tail &&
			     req->ack_seg == req->total_segs) ||
			    qp->s_acked == qp->s_tail) {
				qpriv->s_state = TID_OP(WRITE_DATA_LAST);
				goto done;
			}

			if (req->ack_seg == req->comp_seg) {
				qpriv->s_state = TID_OP(WRITE_DATA);
				goto done;
			}

			/*
			 * The PSN to start with is the next PSN after the
			 * RESYNC PSN.
			 */
			psn = mask_psn(psn + 1);
			generation = psn >> HFI1_KDETH_BTH_SEQ_SHIFT;
			spsn = 0;

			/*
			 * Update to the correct WQE when we get an ACK(RESYNC)
			 * in the middle of a request.
			 */
			if (delta_psn(ack_psn, wqe->lpsn))
				wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
			req = wqe_to_tid_req(wqe);
			flow = &req->flows[req->acked_tail];
			/*
			 * RESYNC re-numbers the PSN ranges of all remaining
			 * segments. Also, PSN's start from 0 in the middle of a
			 * segment and the first segment size is less than the
			 * default number of packets. flow->resync_npkts is used
			 * to track the number of packets from the start of the
			 * real segment to the point of 0 PSN after the RESYNC
			 * in order to later correctly rewind the SGE.
			 */
			fpsn = full_flow_psn(flow, flow->flow_state.spsn);
			req->r_ack_psn = psn;
			flow->resync_npkts +=
				delta_psn(mask_psn(resync_psn + 1), fpsn);
			/*
			 * Renumber all packet sequence number ranges
			 * based on the new generation.
			 */
			last_acked = qp->s_acked;
			rptr = req;
			while (1) {
				/* start from last acked segment */
				for (fidx = rptr->acked_tail;
				     CIRC_CNT(rptr->setup_head, fidx,
					      MAX_FLOWS);
				     fidx = CIRC_NEXT(fidx, MAX_FLOWS)) {
					u32 lpsn;
					u32 gen;

					flow = &rptr->flows[fidx];
					gen = flow->flow_state.generation;
					if (WARN_ON(gen == generation &&
						    flow->flow_state.spsn !=
						     spsn))
						continue;
					lpsn = flow->flow_state.lpsn;
					lpsn = full_flow_psn(flow, lpsn);
					flow->npkts =
						delta_psn(lpsn,
							  mask_psn(resync_psn)
							  );
					flow->flow_state.generation =
						generation;
					flow->flow_state.spsn = spsn;
					flow->flow_state.lpsn =
						flow->flow_state.spsn +
						flow->npkts - 1;
					flow->pkt = 0;
					spsn += flow->npkts;
					resync_psn += flow->npkts;
					trace_hfi1_tid_flow_rcv_tid_ack(qp,
									fidx,
									flow);
				}
				if (++last_acked == qpriv->s_tid_cur + 1)
					break;
				if (last_acked == qp->s_size)
					last_acked = 0;
				wqe = rvt_get_swqe_ptr(qp, last_acked);
				rptr = wqe_to_tid_req(wqe);
			}
			req->cur_seg = req->ack_seg;
			qpriv->s_tid_tail = qp->s_acked;
			qpriv->s_state = TID_OP(WRITE_REQ);
			hfi1_schedule_tid_send(qp);
		}
done:
		qpriv->s_retry = qp->s_retry_cnt;
		break;

	case 3:         /* NAK */
		hfi1_stop_tid_retry_timer(qp);
		switch ((aeth >> IB_AETH_CREDIT_SHIFT) &
			IB_AETH_CREDIT_MASK) {
		case 0: /* PSN sequence error */
			if (!req->flows)
				break;
			flow = &req->flows[req->acked_tail];
			flpsn = full_flow_psn(flow, flow->flow_state.lpsn);
			if (cmp_psn(psn, flpsn) > 0)
				break;
			trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail,
							flow);
			req->r_ack_psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
			req->cur_seg = req->ack_seg;
			qpriv->s_tid_tail = qp->s_acked;
			qpriv->s_state = TID_OP(WRITE_REQ);
			qpriv->s_retry = qp->s_retry_cnt;
			hfi1_schedule_tid_send(qp);
			break;

		default:
			break;
		}
		break;

	default:
		break;
	}

ack_op_err:
	spin_unlock_irqrestore(&qp->s_lock, flags);
}

void hfi1_add_tid_retry_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct ib_qp *ibqp = &qp->ibqp;
	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);

	lockdep_assert_held(&qp->s_lock);
	if (!(priv->s_flags & HFI1_S_TID_RETRY_TIMER)) {
		priv->s_flags |= HFI1_S_TID_RETRY_TIMER;
		priv->s_tid_retry_timer.expires = jiffies +
			priv->tid_retry_timeout_jiffies + rdi->busy_jiffies;
		add_timer(&priv->s_tid_retry_timer);
	}
}

static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct ib_qp *ibqp = &qp->ibqp;
	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);

	lockdep_assert_held(&qp->s_lock);
	priv->s_flags |= HFI1_S_TID_RETRY_TIMER;
	mod_timer(&priv->s_tid_retry_timer, jiffies +
		  priv->tid_retry_timeout_jiffies + rdi->busy_jiffies);
}

static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;
	int rval = 0;

	lockdep_assert_held(&qp->s_lock);
	if (priv->s_flags & HFI1_S_TID_RETRY_TIMER) {
		rval = del_timer(&priv->s_tid_retry_timer);
		priv->s_flags &= ~HFI1_S_TID_RETRY_TIMER;
	}
	return rval;
}

void hfi1_del_tid_retry_timer(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;

	del_timer_sync(&priv->s_tid_retry_timer);
	priv->s_flags &= ~HFI1_S_TID_RETRY_TIMER;
}

static void hfi1_tid_retry_timeout(struct timer_list *t)
{
	struct hfi1_qp_priv *priv = from_timer(priv, t, s_tid_retry_timer);
	struct rvt_qp *qp = priv->owner;
	struct rvt_swqe *wqe;
	unsigned long flags;
	struct tid_rdma_request *req;

	spin_lock_irqsave(&qp->r_lock, flags);
	spin_lock(&qp->s_lock);
	trace_hfi1_tid_write_sender_retry_timeout(qp, 0);
	if (priv->s_flags & HFI1_S_TID_RETRY_TIMER) {
		hfi1_stop_tid_retry_timer(qp);
		if (!priv->s_retry) {
			trace_hfi1_msg_tid_retry_timeout(/* msg */
				qp,
				"Exhausted retries. Tid retry timeout = ",
				(u64)priv->tid_retry_timeout_jiffies);

			wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
			hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
			rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
		} else {
			wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
			req = wqe_to_tid_req(wqe);
			trace_hfi1_tid_req_tid_retry_timeout(/* req */
			   qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req);

			priv->s_flags &= ~RVT_S_WAIT_ACK;
			/* Only send one packet (the RESYNC) */
			priv->s_flags |= RVT_S_SEND_ONE;
			/*
			 * No additional request shall be made by this QP until
			 * the RESYNC has been complete.
			 */
			qp->s_flags |= HFI1_S_WAIT_HALT;
			priv->s_state = TID_OP(RESYNC);
			priv->s_retry--;
			hfi1_schedule_tid_send(qp);
		}
	}
	spin_unlock(&qp->s_lock);
	spin_unlock_irqrestore(&qp->r_lock, flags);
}

u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe,
			       struct ib_other_headers *ohdr, u32 *bth1,
			       u32 *bth2, u16 fidx)
{
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct tid_rdma_params *remote;
	struct tid_rdma_request *req = wqe_to_tid_req(wqe);
	struct tid_rdma_flow *flow = &req->flows[fidx];
	u32 generation;

	rcu_read_lock();
	remote = rcu_dereference(qpriv->tid_rdma.remote);
	KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth1, JKEY, remote->jkey);
	ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn);
	*bth1 = remote->qp;
	rcu_read_unlock();

	generation = kern_flow_generation_next(flow->flow_state.generation);
	*bth2 = mask_psn((generation << HFI1_KDETH_BTH_SEQ_SHIFT) - 1);
	qpriv->s_resync_psn = *bth2;
	*bth2 |= IB_BTH_REQ_ACK;
	KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth0, KVER, 0x1);

	return sizeof(ohdr->u.tid_rdma.resync) / sizeof(u32);
}

void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet)
{
	struct ib_other_headers *ohdr = packet->ohdr;
	struct rvt_qp *qp = packet->qp;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct hfi1_ctxtdata *rcd = qpriv->rcd;
	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
	struct rvt_ack_entry *e;
	struct tid_rdma_request *req;
	struct tid_rdma_flow *flow;
	struct tid_flow_state *fs = &qpriv->flow_state;
	u32 psn, generation, idx, gen_next;
	bool fecn;
	unsigned long flags;

	fecn = process_ecn(qp, packet);
	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));

	generation = mask_psn(psn + 1) >> HFI1_KDETH_BTH_SEQ_SHIFT;
	spin_lock_irqsave(&qp->s_lock, flags);

	gen_next = (fs->generation == KERN_GENERATION_RESERVED) ?
		generation : kern_flow_generation_next(fs->generation);
	/*
	 * RESYNC packet contains the "next" generation and can only be
	 * from the current or previous generations
	 */
	if (generation != mask_generation(gen_next - 1) &&
	    generation != gen_next)
		goto bail;
	/* Already processing a resync */
	if (qpriv->resync)
		goto bail;

	spin_lock(&rcd->exp_lock);
	if (fs->index >= RXE_NUM_TID_FLOWS) {
		/*
		 * If we don't have a flow, save the generation so it can be
		 * applied when a new flow is allocated
		 */
		fs->generation = generation;
	} else {
		/* Reprogram the QP flow with new generation */
		rcd->flows[fs->index].generation = generation;
		fs->generation = kern_setup_hw_flow(rcd, fs->index);
	}
	fs->psn = 0;
	/*
	 * Disable SW PSN checking since a RESYNC is equivalent to a
	 * sync point and the flow has/will be reprogrammed
	 */
	qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
	trace_hfi1_tid_write_rsp_rcv_resync(qp);

	/*
	 * Reset all TID flow information with the new generation.
	 * This is done for all requests and segments after the
	 * last received segment
	 */
	for (idx = qpriv->r_tid_tail; ; idx++) {
		u16 flow_idx;

		if (idx > rvt_size_atomic(&dev->rdi))
			idx = 0;
		e = &qp->s_ack_queue[idx];
		if (e->opcode == TID_OP(WRITE_REQ)) {
			req = ack_to_tid_req(e);
			trace_hfi1_tid_req_rcv_resync(qp, 0, e->opcode, e->psn,
						      e->lpsn, req);

			/* start from last unacked segment */
			for (flow_idx = req->clear_tail;
			     CIRC_CNT(req->setup_head, flow_idx,
				      MAX_FLOWS);
			     flow_idx = CIRC_NEXT(flow_idx, MAX_FLOWS)) {
				u32 lpsn;
				u32 next;

				flow = &req->flows[flow_idx];
				lpsn = full_flow_psn(flow,
						     flow->flow_state.lpsn);
				next = flow->flow_state.r_next_psn;
				flow->npkts = delta_psn(lpsn, next - 1);
				flow->flow_state.generation = fs->generation;
				flow->flow_state.spsn = fs->psn;
				flow->flow_state.lpsn =
					flow->flow_state.spsn + flow->npkts - 1;
				flow->flow_state.r_next_psn =
					full_flow_psn(flow,
						      flow->flow_state.spsn);
				fs->psn += flow->npkts;
				trace_hfi1_tid_flow_rcv_resync(qp, flow_idx,
							       flow);
			}
		}
		if (idx == qp->s_tail_ack_queue)
			break;
	}

	spin_unlock(&rcd->exp_lock);
	qpriv->resync = true;
	/* RESYNC request always gets a TID RDMA ACK. */
	qpriv->s_nak_state = 0;
	qpriv->s_flags |= RVT_S_ACK_PENDING;
	hfi1_schedule_tid_send(qp);
bail:
	if (fecn)
		qp->s_flags |= RVT_S_ECN;
	spin_unlock_irqrestore(&qp->s_lock, flags);
}

/*
 * Call this function when the last TID RDMA WRITE DATA packet for a request
 * is built.
 */
static void update_tid_tail(struct rvt_qp *qp)
	__must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv = qp->priv;
	u32 i;
	struct rvt_swqe *wqe;

	lockdep_assert_held(&qp->s_lock);
	/* Can't move beyond s_tid_cur */
	if (priv->s_tid_tail == priv->s_tid_cur)
		return;
	for (i = priv->s_tid_tail + 1; ; i++) {
		if (i == qp->s_size)
			i = 0;

		if (i == priv->s_tid_cur)
			break;
		wqe = rvt_get_swqe_ptr(qp, i);
		if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
			break;
	}
	priv->s_tid_tail = i;
	priv->s_state = TID_OP(WRITE_RESP);
}

int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
	__must_hold(&qp->s_lock)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct rvt_swqe *wqe;
	u32 bth1 = 0, bth2 = 0, hwords = 5, len, middle = 0;
	struct ib_other_headers *ohdr;
	struct rvt_sge_state *ss = &qp->s_sge;
	struct rvt_ack_entry *e = &qp->s_ack_queue[qp->s_tail_ack_queue];
	struct tid_rdma_request *req = ack_to_tid_req(e);
	bool last = false;
	u8 opcode = TID_OP(WRITE_DATA);

	lockdep_assert_held(&qp->s_lock);
	trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0);
	/*
	 * Prioritize the sending of the requests and responses over the
	 * sending of the TID RDMA data packets.
	 */
	if (((atomic_read(&priv->n_tid_requests) < HFI1_TID_RDMA_WRITE_CNT) &&
	     atomic_read(&priv->n_requests) &&
	     !(qp->s_flags & (RVT_S_BUSY | RVT_S_WAIT_ACK |
			     HFI1_S_ANY_WAIT_IO))) ||
	    (e->opcode == TID_OP(WRITE_REQ) && req->cur_seg < req->alloc_seg &&
	     !(qp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT_IO)))) {
		struct iowait_work *iowork;

		iowork = iowait_get_ib_work(&priv->s_iowait);
		ps->s_txreq = get_waiting_verbs_txreq(iowork);
		if (ps->s_txreq || hfi1_make_rc_req(qp, ps)) {
			priv->s_flags |= HFI1_S_TID_BUSY_SET;
			return 1;
		}
	}

	ps->s_txreq = get_txreq(ps->dev, qp);
	if (!ps->s_txreq)
		goto bail_no_tx;

	ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;

	if ((priv->s_flags & RVT_S_ACK_PENDING) &&
	    make_tid_rdma_ack(qp, ohdr, ps))
		return 1;

	/*
	 * Bail out if we can't send data.
	 * Be reminded that this check must been done after the call to
	 * make_tid_rdma_ack() because the responding QP could be in
	 * RTR state where it can send TID RDMA ACK, not TID RDMA WRITE DATA.
	 */
	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK))
		goto bail;

	if (priv->s_flags & RVT_S_WAIT_ACK)
		goto bail;

	/* Check whether there is anything to do. */
	if (priv->s_tid_tail == HFI1_QP_WQE_INVALID)
		goto bail;
	wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail);
	req = wqe_to_tid_req(wqe);
	trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn,
					wqe->lpsn, req);
	switch (priv->s_state) {
	case TID_OP(WRITE_REQ):
	case TID_OP(WRITE_RESP):
		priv->tid_ss.sge = wqe->sg_list[0];
		priv->tid_ss.sg_list = wqe->sg_list + 1;
		priv->tid_ss.num_sge = wqe->wr.num_sge;
		priv->tid_ss.total_len = wqe->length;

		if (priv->s_state == TID_OP(WRITE_REQ))
			hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
		priv->s_state = TID_OP(WRITE_DATA);
		/* fall through */

	case TID_OP(WRITE_DATA):
		/*
		 * 1. Check whether TID RDMA WRITE RESP available.
		 * 2. If no:
		 *    2.1 If have more segments and no TID RDMA WRITE RESP,
		 *        set HFI1_S_WAIT_TID_RESP
		 *    2.2 Return indicating no progress made.
		 * 3. If yes:
		 *    3.1 Build TID RDMA WRITE DATA packet.
		 *    3.2 If last packet in segment:
		 *        3.2.1 Change KDETH header bits
		 *        3.2.2 Advance RESP pointers.
		 *    3.3 Return indicating progress made.
		 */
		trace_hfi1_sender_make_tid_pkt(qp);
		trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0);
		wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail);
		req = wqe_to_tid_req(wqe);
		len = wqe->length;

		if (!req->comp_seg || req->cur_seg == req->comp_seg)
			goto bail;

		trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode,
						wqe->psn, wqe->lpsn, req);
		last = hfi1_build_tid_rdma_packet(wqe, ohdr, &bth1, &bth2,
						  &len);

		if (last) {
			/* move pointer to next flow */
			req->clear_tail = CIRC_NEXT(req->clear_tail,
						    MAX_FLOWS);
			if (++req->cur_seg < req->total_segs) {
				if (!CIRC_CNT(req->setup_head, req->clear_tail,
					      MAX_FLOWS))
					qp->s_flags |= HFI1_S_WAIT_TID_RESP;
			} else {
				priv->s_state = TID_OP(WRITE_DATA_LAST);
				opcode = TID_OP(WRITE_DATA_LAST);

				/* Advance the s_tid_tail now */
				update_tid_tail(qp);
			}
		}
		hwords += sizeof(ohdr->u.tid_rdma.w_data) / sizeof(u32);
		ss = &priv->tid_ss;
		break;

	case TID_OP(RESYNC):
		trace_hfi1_sender_make_tid_pkt(qp);
		/* Use generation from the most recently received response */
		wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur);
		req = wqe_to_tid_req(wqe);
		/* If no responses for this WQE look at the previous one */
		if (!req->comp_seg) {
			wqe = rvt_get_swqe_ptr(qp,
					       (!priv->s_tid_cur ? qp->s_size :
						priv->s_tid_cur) - 1);
			req = wqe_to_tid_req(wqe);
		}
		hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1,
						     &bth2,
						     CIRC_PREV(req->setup_head,
							       MAX_FLOWS));
		ss = NULL;
		len = 0;
		opcode = TID_OP(RESYNC);
		break;

	default:
		goto bail;
	}
	if (priv->s_flags & RVT_S_SEND_ONE) {
		priv->s_flags &= ~RVT_S_SEND_ONE;
		priv->s_flags |= RVT_S_WAIT_ACK;
		bth2 |= IB_BTH_REQ_ACK;
	}
	qp->s_len -= len;
	ps->s_txreq->hdr_dwords = hwords;
	ps->s_txreq->sde = priv->s_sde;
	ps->s_txreq->ss = ss;
	ps->s_txreq->s_cur_size = len;
	hfi1_make_ruc_header(qp, ohdr, (opcode << 24), bth1, bth2,
			     middle, ps);
	return 1;
bail:
	hfi1_put_txreq(ps->s_txreq);
bail_no_tx:
	ps->s_txreq = NULL;
	priv->s_flags &= ~RVT_S_BUSY;
	/*
	 * If we didn't get a txreq, the QP will be woken up later to try
	 * again, set the flags to the the wake up which work item to wake
	 * up.
	 * (A better algorithm should be found to do this and generalize the
	 * sleep/wakeup flags.)
	 */
	iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_TID);
	return 0;
}

static int make_tid_rdma_ack(struct rvt_qp *qp,
			     struct ib_other_headers *ohdr,
			     struct hfi1_pkt_state *ps)
{
	struct rvt_ack_entry *e;
	struct hfi1_qp_priv *qpriv = qp->priv;
	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
	u32 hwords, next;
	u32 len = 0;
	u32 bth1 = 0, bth2 = 0;
	int middle = 0;
	u16 flow;
	struct tid_rdma_request *req, *nreq;

	trace_hfi1_tid_write_rsp_make_tid_ack(qp);
	/* Don't send an ACK if we aren't supposed to. */
	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
		goto bail;

	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
	hwords = 5;

	e = &qp->s_ack_queue[qpriv->r_tid_ack];
	req = ack_to_tid_req(e);
	/*
	 * In the RESYNC case, we are exactly one segment past the
	 * previously sent ack or at the previously sent NAK. So to send
	 * the resync ack, we go back one segment (which might be part of
	 * the previous request) and let the do-while loop execute again.
	 * The advantage of executing the do-while loop is that any data
	 * received after the previous ack is automatically acked in the
	 * RESYNC ack. It turns out that for the do-while loop we only need
	 * to pull back qpriv->r_tid_ack, not the segment
	 * indices/counters. The scheme works even if the previous request
	 * was not a TID WRITE request.
	 */
	if (qpriv->resync) {
		if (!req->ack_seg || req->ack_seg == req->total_segs)
			qpriv->r_tid_ack = !qpriv->r_tid_ack ?
				rvt_size_atomic(&dev->rdi) :
				qpriv->r_tid_ack - 1;
		e = &qp->s_ack_queue[qpriv->r_tid_ack];
		req = ack_to_tid_req(e);
	}

	trace_hfi1_rsp_make_tid_ack(qp, e->psn);
	trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn,
					req);
	/*
	 * If we've sent all the ACKs that we can, we are done
	 * until we get more segments...
	 */
	if (!qpriv->s_nak_state && !qpriv->resync &&
	    req->ack_seg == req->comp_seg)
		goto bail;

	do {
		/*
		 * To deal with coalesced ACKs, the acked_tail pointer
		 * into the flow array is used. The distance between it
		 * and the clear_tail is the number of flows that are
		 * being ACK'ed.
		 */
		req->ack_seg +=
			/* Get up-to-date value */
			CIRC_CNT(req->clear_tail, req->acked_tail,
				 MAX_FLOWS);
		/* Advance acked index */
		req->acked_tail = req->clear_tail;

		/*
		 * req->clear_tail points to the segment currently being
		 * received. So, when sending an ACK, the previous
		 * segment is being ACK'ed.
		 */
		flow = CIRC_PREV(req->acked_tail, MAX_FLOWS);
		if (req->ack_seg != req->total_segs)
			break;
		req->state = TID_REQUEST_COMPLETE;

		next = qpriv->r_tid_ack + 1;
		if (next > rvt_size_atomic(&dev->rdi))
			next = 0;
		qpriv->r_tid_ack = next;
		if (qp->s_ack_queue[next].opcode != TID_OP(WRITE_REQ))
			break;
		nreq = ack_to_tid_req(&qp->s_ack_queue[next]);
		if (!nreq->comp_seg || nreq->ack_seg == nreq->comp_seg)
			break;

		/* Move to the next ack entry now */
		e = &qp->s_ack_queue[qpriv->r_tid_ack];
		req = ack_to_tid_req(e);
	} while (1);

	/*
	 * At this point qpriv->r_tid_ack == qpriv->r_tid_tail but e and
	 * req could be pointing at the previous ack queue entry
	 */
	if (qpriv->s_nak_state ||
	    (qpriv->resync &&
	     !hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1) &&
	     (cmp_psn(qpriv->r_next_psn_kdeth - 1,
		      full_flow_psn(&req->flows[flow],
				    req->flows[flow].flow_state.lpsn)) > 0))) {
		/*
		 * A NAK will implicitly acknowledge all previous TID RDMA
		 * requests. Therefore, we NAK with the req->acked_tail
		 * segment for the request at qpriv->r_tid_ack (same at
		 * this point as the req->clear_tail segment for the
		 * qpriv->r_tid_tail request)
		 */
		e = &qp->s_ack_queue[qpriv->r_tid_ack];
		req = ack_to_tid_req(e);
		flow = req->acked_tail;
	} else if (req->ack_seg == req->total_segs &&
		   qpriv->s_flags & HFI1_R_TID_WAIT_INTERLCK)
		qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK;

	trace_hfi1_tid_write_rsp_make_tid_ack(qp);
	trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn,
					req);
	hwords += hfi1_build_tid_rdma_write_ack(qp, e, ohdr, flow, &bth1,
						&bth2);
	len = 0;
	qpriv->s_flags &= ~RVT_S_ACK_PENDING;
	ps->s_txreq->hdr_dwords = hwords;
	ps->s_txreq->sde = qpriv->s_sde;
	ps->s_txreq->s_cur_size = len;
	ps->s_txreq->ss = NULL;
	hfi1_make_ruc_header(qp, ohdr, (TID_OP(ACK) << 24), bth1, bth2, middle,
			     ps);
	ps->s_txreq->txreq.flags |= SDMA_TXREQ_F_VIP;
	return 1;
bail:
	/*
	 * Ensure s_rdma_ack_cnt changes are committed prior to resetting
	 * RVT_S_RESP_PENDING
	 */
	smp_wmb();
	qpriv->s_flags &= ~RVT_S_ACK_PENDING;
	return 0;
}

static int hfi1_send_tid_ok(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;

	return !(priv->s_flags & RVT_S_BUSY ||
		 qp->s_flags & HFI1_S_ANY_WAIT_IO) &&
		(verbs_txreq_queued(iowait_get_tid_work(&priv->s_iowait)) ||
		 (priv->s_flags & RVT_S_RESP_PENDING) ||
		 !(qp->s_flags & HFI1_S_ANY_TID_WAIT_SEND));
}

void _hfi1_do_tid_send(struct work_struct *work)
{
	struct iowait_work *w = container_of(work, struct iowait_work, iowork);
	struct rvt_qp *qp = iowait_to_qp(w->iow);

	hfi1_do_tid_send(qp);
}

static void hfi1_do_tid_send(struct rvt_qp *qp)
{
	struct hfi1_pkt_state ps;
	struct hfi1_qp_priv *priv = qp->priv;

	ps.dev = to_idev(qp->ibqp.device);
	ps.ibp = to_iport(qp->ibqp.device, qp->port_num);
	ps.ppd = ppd_from_ibp(ps.ibp);
	ps.wait = iowait_get_tid_work(&priv->s_iowait);
	ps.in_thread = false;
	ps.timeout_int = qp->timeout_jiffies / 8;

	trace_hfi1_rc_do_tid_send(qp, false);
	spin_lock_irqsave(&qp->s_lock, ps.flags);

	/* Return if we are already busy processing a work request. */
	if (!hfi1_send_tid_ok(qp)) {
		if (qp->s_flags & HFI1_S_ANY_WAIT_IO)
			iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_TID);
		spin_unlock_irqrestore(&qp->s_lock, ps.flags);
		return;
	}

	priv->s_flags |= RVT_S_BUSY;

	ps.timeout = jiffies + ps.timeout_int;
	ps.cpu = priv->s_sde ? priv->s_sde->cpu :
		cpumask_first(cpumask_of_node(ps.ppd->dd->node));
	ps.pkts_sent = false;

	/* insure a pre-built packet is handled  */
	ps.s_txreq = get_waiting_verbs_txreq(ps.wait);
	do {
		/* Check for a constructed packet to be sent. */
		if (ps.s_txreq) {
			if (priv->s_flags & HFI1_S_TID_BUSY_SET) {
				qp->s_flags |= RVT_S_BUSY;
				ps.wait = iowait_get_ib_work(&priv->s_iowait);
			}
			spin_unlock_irqrestore(&qp->s_lock, ps.flags);

			/*
			 * If the packet cannot be sent now, return and
			 * the send tasklet will be woken up later.
			 */
			if (hfi1_verbs_send(qp, &ps))
				return;

			/* allow other tasks to run */
			if (hfi1_schedule_send_yield(qp, &ps, true))
				return;

			spin_lock_irqsave(&qp->s_lock, ps.flags);
			if (priv->s_flags & HFI1_S_TID_BUSY_SET) {
				qp->s_flags &= ~RVT_S_BUSY;
				priv->s_flags &= ~HFI1_S_TID_BUSY_SET;
				ps.wait = iowait_get_tid_work(&priv->s_iowait);
				if (iowait_flag_set(&priv->s_iowait,
						    IOWAIT_PENDING_IB))
					hfi1_schedule_send(qp);
			}
		}
	} while (hfi1_make_tid_rdma_pkt(qp, &ps));
	iowait_starve_clear(ps.pkts_sent, &priv->s_iowait);
	spin_unlock_irqrestore(&qp->s_lock, ps.flags);
}

static bool _hfi1_schedule_tid_send(struct rvt_qp *qp)
{
	struct hfi1_qp_priv *priv = qp->priv;
	struct hfi1_ibport *ibp =
		to_iport(qp->ibqp.device, qp->port_num);
	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);

	return iowait_tid_schedule(&priv->s_iowait, ppd->hfi1_wq,
				   priv->s_sde ?
				   priv->s_sde->cpu :
				   cpumask_first(cpumask_of_node(dd->node)));
}

/**
 * hfi1_schedule_tid_send - schedule progress on TID RDMA state machine
 * @qp: the QP
 *
 * This schedules qp progress on the TID RDMA state machine. Caller
 * should hold the s_lock.
 * Unlike hfi1_schedule_send(), this cannot use hfi1_send_ok() because
 * the two state machines can step on each other with respect to the
 * RVT_S_BUSY flag.
 * Therefore, a modified test is used.
 * @return true if the second leg is scheduled;
 *  false if the second leg is not scheduled.
 */
bool hfi1_schedule_tid_send(struct rvt_qp *qp)
{
	lockdep_assert_held(&qp->s_lock);
	if (hfi1_send_tid_ok(qp)) {
		/*
		 * The following call returns true if the qp is not on the
		 * queue and false if the qp is already on the queue before
		 * this call. Either way, the qp will be on the queue when the
		 * call returns.
		 */
		_hfi1_schedule_tid_send(qp);
		return true;
	}
	if (qp->s_flags & HFI1_S_ANY_WAIT_IO)
		iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait,
				IOWAIT_PENDING_TID);
	return false;
}

bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e)
{
	struct rvt_ack_entry *prev;
	struct tid_rdma_request *req;
	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
	struct hfi1_qp_priv *priv = qp->priv;
	u32 s_prev;

	s_prev = qp->s_tail_ack_queue == 0 ? rvt_size_atomic(&dev->rdi) :
		(qp->s_tail_ack_queue - 1);
	prev = &qp->s_ack_queue[s_prev];

	if ((e->opcode == TID_OP(READ_REQ) ||
	     e->opcode == OP(RDMA_READ_REQUEST)) &&
	    prev->opcode == TID_OP(WRITE_REQ)) {
		req = ack_to_tid_req(prev);
		if (req->ack_seg != req->total_segs) {
			priv->s_flags |= HFI1_R_TID_WAIT_INTERLCK;
			return true;
		}
	}
	return false;
}

static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx)
{
	u64 reg;

	/*
	 * The only sane way to get the amount of
	 * progress is to read the HW flow state.
	 */
	reg = read_uctxt_csr(dd, ctxt, RCV_TID_FLOW_TABLE + (8 * fidx));
	return mask_psn(reg);
}

static void tid_rdma_rcv_err(struct hfi1_packet *packet,
			     struct ib_other_headers *ohdr,
			     struct rvt_qp *qp, u32 psn, int diff, bool fecn)
{
	unsigned long flags;

	tid_rdma_rcv_error(packet, ohdr, qp, psn, diff);
	if (fecn) {
		spin_lock_irqsave(&qp->s_lock, flags);
		qp->s_flags |= RVT_S_ECN;
		spin_unlock_irqrestore(&qp->s_lock, flags);
	}
}

static void update_r_next_psn_fecn(struct hfi1_packet *packet,
				   struct hfi1_qp_priv *priv,
				   struct hfi1_ctxtdata *rcd,
				   struct tid_rdma_flow *flow,
				   bool fecn)
{
	/*
	 * If a start/middle packet is delivered here due to
	 * RSM rule and FECN, we need to update the r_next_psn.
	 */
	if (fecn && packet->etype == RHF_RCV_TYPE_EAGER &&
	    !(priv->s_flags & HFI1_R_TID_SW_PSN)) {
		struct hfi1_devdata *dd = rcd->dd;

		flow->flow_state.r_next_psn =
			read_r_next_psn(dd, rcd->ctxt, flow->idx);
	}
}