1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
4  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
5  */
6 
7 #include "efct_driver.h"
8 #include "efct_hw.h"
9 #include "efct_unsol.h"
10 
11 int
12 efct_hw_init_queues(struct efct_hw *hw)
13 {
14 	struct hw_eq *eq = NULL;
15 	struct hw_cq *cq = NULL;
16 	struct hw_wq *wq = NULL;
17 	struct hw_mq *mq = NULL;
18 
19 	struct hw_eq *eqs[EFCT_HW_MAX_NUM_EQ];
20 	struct hw_cq *cqs[EFCT_HW_MAX_NUM_EQ];
21 	struct hw_rq *rqs[EFCT_HW_MAX_NUM_EQ];
22 	u32 i = 0, j;
23 
24 	hw->eq_count = 0;
25 	hw->cq_count = 0;
26 	hw->mq_count = 0;
27 	hw->wq_count = 0;
28 	hw->rq_count = 0;
29 	hw->hw_rq_count = 0;
30 	INIT_LIST_HEAD(&hw->eq_list);
31 
32 	for (i = 0; i < hw->config.n_eq; i++) {
33 		/* Create EQ */
34 		eq = efct_hw_new_eq(hw, EFCT_HW_EQ_DEPTH);
35 		if (!eq) {
36 			efct_hw_queue_teardown(hw);
37 			return -ENOMEM;
38 		}
39 
40 		eqs[i] = eq;
41 
42 		/* Create one MQ */
43 		if (!i) {
44 			cq = efct_hw_new_cq(eq,
45 					    hw->num_qentries[SLI4_QTYPE_CQ]);
46 			if (!cq) {
47 				efct_hw_queue_teardown(hw);
48 				return -ENOMEM;
49 			}
50 
51 			mq = efct_hw_new_mq(cq, EFCT_HW_MQ_DEPTH);
52 			if (!mq) {
53 				efct_hw_queue_teardown(hw);
54 				return -ENOMEM;
55 			}
56 		}
57 
58 		/* Create WQ */
59 		cq = efct_hw_new_cq(eq, hw->num_qentries[SLI4_QTYPE_CQ]);
60 		if (!cq) {
61 			efct_hw_queue_teardown(hw);
62 			return -ENOMEM;
63 		}
64 
65 		wq = efct_hw_new_wq(cq, hw->num_qentries[SLI4_QTYPE_WQ]);
66 		if (!wq) {
67 			efct_hw_queue_teardown(hw);
68 			return -ENOMEM;
69 		}
70 	}
71 
72 	/* Create CQ set */
73 	if (efct_hw_new_cq_set(eqs, cqs, i, hw->num_qentries[SLI4_QTYPE_CQ])) {
74 		efct_hw_queue_teardown(hw);
75 		return -EIO;
76 	}
77 
78 	/* Create RQ set */
79 	if (efct_hw_new_rq_set(cqs, rqs, i, EFCT_HW_RQ_ENTRIES_DEF)) {
80 		efct_hw_queue_teardown(hw);
81 		return -EIO;
82 	}
83 
84 	for (j = 0; j < i ; j++) {
85 		rqs[j]->filter_mask = 0;
86 		rqs[j]->is_mrq = true;
87 		rqs[j]->base_mrq_id = rqs[0]->hdr->id;
88 	}
89 
90 	hw->hw_mrq_count = i;
91 
92 	return 0;
93 }
94 
95 int
96 efct_hw_map_wq_cpu(struct efct_hw *hw)
97 {
98 	struct efct *efct = hw->os;
99 	u32 cpu = 0, i;
100 
101 	/* Init cpu_map array */
102 	hw->wq_cpu_array = kcalloc(num_possible_cpus(), sizeof(void *),
103 				   GFP_KERNEL);
104 	if (!hw->wq_cpu_array)
105 		return -ENOMEM;
106 
107 	for (i = 0; i < hw->config.n_eq; i++) {
108 		const struct cpumask *maskp;
109 
110 		/* Get a CPU mask for all CPUs affinitized to this vector */
111 		maskp = pci_irq_get_affinity(efct->pci, i);
112 		if (!maskp) {
113 			efc_log_debug(efct, "maskp null for vector:%d\n", i);
114 			continue;
115 		}
116 
117 		/* Loop through all CPUs associated with vector idx */
118 		for_each_cpu_and(cpu, maskp, cpu_present_mask) {
119 			efc_log_debug(efct, "CPU:%d irq vector:%d\n", cpu, i);
120 			hw->wq_cpu_array[cpu] = hw->hw_wq[i];
121 		}
122 	}
123 
124 	return 0;
125 }
126 
127 struct hw_eq *
128 efct_hw_new_eq(struct efct_hw *hw, u32 entry_count)
129 {
130 	struct hw_eq *eq = kzalloc(sizeof(*eq), GFP_KERNEL);
131 
132 	if (!eq)
133 		return NULL;
134 
135 	eq->type = SLI4_QTYPE_EQ;
136 	eq->hw = hw;
137 	eq->entry_count = entry_count;
138 	eq->instance = hw->eq_count++;
139 	eq->queue = &hw->eq[eq->instance];
140 	INIT_LIST_HEAD(&eq->cq_list);
141 
142 	if (sli_queue_alloc(&hw->sli, SLI4_QTYPE_EQ, eq->queue,	entry_count,
143 			    NULL)) {
144 		efc_log_err(hw->os, "EQ[%d] alloc failure\n", eq->instance);
145 		kfree(eq);
146 		return NULL;
147 	}
148 
149 	sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8);
150 	hw->hw_eq[eq->instance] = eq;
151 	INIT_LIST_HEAD(&eq->list_entry);
152 	list_add_tail(&eq->list_entry, &hw->eq_list);
153 	efc_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance,
154 		      eq->queue->id, eq->entry_count);
155 	return eq;
156 }
157 
158 struct hw_cq *
159 efct_hw_new_cq(struct hw_eq *eq, u32 entry_count)
160 {
161 	struct efct_hw *hw = eq->hw;
162 	struct hw_cq *cq = kzalloc(sizeof(*cq), GFP_KERNEL);
163 
164 	if (!cq)
165 		return NULL;
166 
167 	cq->eq = eq;
168 	cq->type = SLI4_QTYPE_CQ;
169 	cq->instance = eq->hw->cq_count++;
170 	cq->entry_count = entry_count;
171 	cq->queue = &hw->cq[cq->instance];
172 
173 	INIT_LIST_HEAD(&cq->q_list);
174 
175 	if (sli_queue_alloc(&hw->sli, SLI4_QTYPE_CQ, cq->queue,
176 			    cq->entry_count, eq->queue)) {
177 		efc_log_err(hw->os, "CQ[%d] allocation failure len=%d\n",
178 			    eq->instance, eq->entry_count);
179 		kfree(cq);
180 		return NULL;
181 	}
182 
183 	hw->hw_cq[cq->instance] = cq;
184 	INIT_LIST_HEAD(&cq->list_entry);
185 	list_add_tail(&cq->list_entry, &eq->cq_list);
186 	efc_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance,
187 		      cq->queue->id, cq->entry_count);
188 	return cq;
189 }
190 
191 u32
192 efct_hw_new_cq_set(struct hw_eq *eqs[], struct hw_cq *cqs[],
193 		   u32 num_cqs, u32 entry_count)
194 {
195 	u32 i;
196 	struct efct_hw *hw = eqs[0]->hw;
197 	struct sli4 *sli4 = &hw->sli;
198 	struct hw_cq *cq = NULL;
199 	struct sli4_queue *qs[SLI4_MAX_CQ_SET_COUNT];
200 	struct sli4_queue *assefct[SLI4_MAX_CQ_SET_COUNT];
201 
202 	/* Initialise CQS pointers to NULL */
203 	for (i = 0; i < num_cqs; i++)
204 		cqs[i] = NULL;
205 
206 	for (i = 0; i < num_cqs; i++) {
207 		cq = kzalloc(sizeof(*cq), GFP_KERNEL);
208 		if (!cq)
209 			goto error;
210 
211 		cqs[i]          = cq;
212 		cq->eq          = eqs[i];
213 		cq->type        = SLI4_QTYPE_CQ;
214 		cq->instance    = hw->cq_count++;
215 		cq->entry_count = entry_count;
216 		cq->queue       = &hw->cq[cq->instance];
217 		qs[i]           = cq->queue;
218 		assefct[i]       = eqs[i]->queue;
219 		INIT_LIST_HEAD(&cq->q_list);
220 	}
221 
222 	if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assefct)) {
223 		efc_log_err(hw->os, "Failed to create CQ Set.\n");
224 		goto error;
225 	}
226 
227 	for (i = 0; i < num_cqs; i++) {
228 		hw->hw_cq[cqs[i]->instance] = cqs[i];
229 		INIT_LIST_HEAD(&cqs[i]->list_entry);
230 		list_add_tail(&cqs[i]->list_entry, &cqs[i]->eq->cq_list);
231 	}
232 
233 	return 0;
234 
235 error:
236 	for (i = 0; i < num_cqs; i++) {
237 		kfree(cqs[i]);
238 		cqs[i] = NULL;
239 	}
240 	return -EIO;
241 }
242 
243 struct hw_mq *
244 efct_hw_new_mq(struct hw_cq *cq, u32 entry_count)
245 {
246 	struct efct_hw *hw = cq->eq->hw;
247 	struct hw_mq *mq = kzalloc(sizeof(*mq), GFP_KERNEL);
248 
249 	if (!mq)
250 		return NULL;
251 
252 	mq->cq = cq;
253 	mq->type = SLI4_QTYPE_MQ;
254 	mq->instance = cq->eq->hw->mq_count++;
255 	mq->entry_count = entry_count;
256 	mq->entry_size = EFCT_HW_MQ_DEPTH;
257 	mq->queue = &hw->mq[mq->instance];
258 
259 	if (sli_queue_alloc(&hw->sli, SLI4_QTYPE_MQ, mq->queue, mq->entry_size,
260 			    cq->queue)) {
261 		efc_log_err(hw->os, "MQ allocation failure\n");
262 		kfree(mq);
263 		return NULL;
264 	}
265 
266 	hw->hw_mq[mq->instance] = mq;
267 	INIT_LIST_HEAD(&mq->list_entry);
268 	list_add_tail(&mq->list_entry, &cq->q_list);
269 	efc_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance,
270 		      mq->queue->id, mq->entry_count);
271 	return mq;
272 }
273 
274 struct hw_wq *
275 efct_hw_new_wq(struct hw_cq *cq, u32 entry_count)
276 {
277 	struct efct_hw *hw = cq->eq->hw;
278 	struct hw_wq *wq = kzalloc(sizeof(*wq), GFP_KERNEL);
279 
280 	if (!wq)
281 		return NULL;
282 
283 	wq->hw = cq->eq->hw;
284 	wq->cq = cq;
285 	wq->type = SLI4_QTYPE_WQ;
286 	wq->instance = cq->eq->hw->wq_count++;
287 	wq->entry_count = entry_count;
288 	wq->queue = &hw->wq[wq->instance];
289 	wq->wqec_set_count = EFCT_HW_WQEC_SET_COUNT;
290 	wq->wqec_count = wq->wqec_set_count;
291 	wq->free_count = wq->entry_count - 1;
292 	INIT_LIST_HEAD(&wq->pending_list);
293 
294 	if (sli_queue_alloc(&hw->sli, SLI4_QTYPE_WQ, wq->queue,
295 			    wq->entry_count, cq->queue)) {
296 		efc_log_err(hw->os, "WQ allocation failure\n");
297 		kfree(wq);
298 		return NULL;
299 	}
300 
301 	hw->hw_wq[wq->instance] = wq;
302 	INIT_LIST_HEAD(&wq->list_entry);
303 	list_add_tail(&wq->list_entry, &cq->q_list);
304 	efc_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d\n",
305 		      wq->instance, wq->queue->id, wq->entry_count, wq->class);
306 	return wq;
307 }
308 
309 u32
310 efct_hw_new_rq_set(struct hw_cq *cqs[], struct hw_rq *rqs[],
311 		   u32 num_rq_pairs, u32 entry_count)
312 {
313 	struct efct_hw *hw = cqs[0]->eq->hw;
314 	struct hw_rq *rq = NULL;
315 	struct sli4_queue *qs[SLI4_MAX_RQ_SET_COUNT * 2] = { NULL };
316 	u32 i, q_count, size;
317 
318 	/* Initialise RQS pointers */
319 	for (i = 0; i < num_rq_pairs; i++)
320 		rqs[i] = NULL;
321 
322 	/*
323 	 * Allocate an RQ object SET, where each element in set
324 	 * encapsulates 2 SLI queues (for rq pair)
325 	 */
326 	for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) {
327 		rq = kzalloc(sizeof(*rq), GFP_KERNEL);
328 		if (!rq)
329 			goto error;
330 
331 		rqs[i] = rq;
332 		rq->instance = hw->hw_rq_count++;
333 		rq->cq = cqs[i];
334 		rq->type = SLI4_QTYPE_RQ;
335 		rq->entry_count = entry_count;
336 
337 		/* Header RQ */
338 		rq->hdr = &hw->rq[hw->rq_count];
339 		rq->hdr_entry_size = EFCT_HW_RQ_HEADER_SIZE;
340 		hw->hw_rq_lookup[hw->rq_count] = rq->instance;
341 		hw->rq_count++;
342 		qs[q_count] = rq->hdr;
343 
344 		/* Data RQ */
345 		rq->data = &hw->rq[hw->rq_count];
346 		rq->data_entry_size = hw->config.rq_default_buffer_size;
347 		hw->hw_rq_lookup[hw->rq_count] = rq->instance;
348 		hw->rq_count++;
349 		qs[q_count + 1] = rq->data;
350 
351 		rq->rq_tracker = NULL;
352 	}
353 
354 	if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs,
355 				cqs[0]->queue->id,
356 			    rqs[0]->entry_count,
357 			    rqs[0]->hdr_entry_size,
358 			    rqs[0]->data_entry_size)) {
359 		efc_log_err(hw->os, "RQ Set alloc failure for base CQ=%d\n",
360 			    cqs[0]->queue->id);
361 		goto error;
362 	}
363 
364 	for (i = 0; i < num_rq_pairs; i++) {
365 		hw->hw_rq[rqs[i]->instance] = rqs[i];
366 		INIT_LIST_HEAD(&rqs[i]->list_entry);
367 		list_add_tail(&rqs[i]->list_entry, &cqs[i]->q_list);
368 		size = sizeof(struct efc_hw_sequence *) * rqs[i]->entry_count;
369 		rqs[i]->rq_tracker = kzalloc(size, GFP_KERNEL);
370 		if (!rqs[i]->rq_tracker)
371 			goto error;
372 	}
373 
374 	return 0;
375 
376 error:
377 	for (i = 0; i < num_rq_pairs; i++) {
378 		if (rqs[i]) {
379 			kfree(rqs[i]->rq_tracker);
380 			kfree(rqs[i]);
381 		}
382 	}
383 
384 	return -EIO;
385 }
386 
387 void
388 efct_hw_del_eq(struct hw_eq *eq)
389 {
390 	struct hw_cq *cq;
391 	struct hw_cq *cq_next;
392 
393 	if (!eq)
394 		return;
395 
396 	list_for_each_entry_safe(cq, cq_next, &eq->cq_list, list_entry)
397 		efct_hw_del_cq(cq);
398 	list_del(&eq->list_entry);
399 	eq->hw->hw_eq[eq->instance] = NULL;
400 	kfree(eq);
401 }
402 
403 void
404 efct_hw_del_cq(struct hw_cq *cq)
405 {
406 	struct hw_q *q;
407 	struct hw_q *q_next;
408 
409 	if (!cq)
410 		return;
411 
412 	list_for_each_entry_safe(q, q_next, &cq->q_list, list_entry) {
413 		switch (q->type) {
414 		case SLI4_QTYPE_MQ:
415 			efct_hw_del_mq((struct hw_mq *)q);
416 			break;
417 		case SLI4_QTYPE_WQ:
418 			efct_hw_del_wq((struct hw_wq *)q);
419 			break;
420 		case SLI4_QTYPE_RQ:
421 			efct_hw_del_rq((struct hw_rq *)q);
422 			break;
423 		default:
424 			break;
425 		}
426 	}
427 	list_del(&cq->list_entry);
428 	cq->eq->hw->hw_cq[cq->instance] = NULL;
429 	kfree(cq);
430 }
431 
432 void
433 efct_hw_del_mq(struct hw_mq *mq)
434 {
435 	if (!mq)
436 		return;
437 
438 	list_del(&mq->list_entry);
439 	mq->cq->eq->hw->hw_mq[mq->instance] = NULL;
440 	kfree(mq);
441 }
442 
443 void
444 efct_hw_del_wq(struct hw_wq *wq)
445 {
446 	if (!wq)
447 		return;
448 
449 	list_del(&wq->list_entry);
450 	wq->cq->eq->hw->hw_wq[wq->instance] = NULL;
451 	kfree(wq);
452 }
453 
454 void
455 efct_hw_del_rq(struct hw_rq *rq)
456 {
457 	struct efct_hw *hw = NULL;
458 
459 	if (!rq)
460 		return;
461 	/* Free RQ tracker */
462 	kfree(rq->rq_tracker);
463 	rq->rq_tracker = NULL;
464 	list_del(&rq->list_entry);
465 	hw = rq->cq->eq->hw;
466 	hw->hw_rq[rq->instance] = NULL;
467 	kfree(rq);
468 }
469 
470 void
471 efct_hw_queue_teardown(struct efct_hw *hw)
472 {
473 	struct hw_eq *eq;
474 	struct hw_eq *eq_next;
475 
476 	if (!hw->eq_list.next)
477 		return;
478 
479 	list_for_each_entry_safe(eq, eq_next, &hw->eq_list, list_entry)
480 		efct_hw_del_eq(eq);
481 }
482 
483 static inline int
484 efct_hw_rqpair_find(struct efct_hw *hw, u16 rq_id)
485 {
486 	return efct_hw_queue_hash_find(hw->rq_hash, rq_id);
487 }
488 
489 static struct efc_hw_sequence *
490 efct_hw_rqpair_get(struct efct_hw *hw, u16 rqindex, u16 bufindex)
491 {
492 	struct sli4_queue *rq_hdr = &hw->rq[rqindex];
493 	struct efc_hw_sequence *seq = NULL;
494 	struct hw_rq *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]];
495 	unsigned long flags = 0;
496 
497 	if (bufindex >= rq_hdr->length) {
498 		efc_log_err(hw->os,
499 			    "RQidx %d bufidx %d exceed ring len %d for id %d\n",
500 			    rqindex, bufindex, rq_hdr->length, rq_hdr->id);
501 		return NULL;
502 	}
503 
504 	/* rq_hdr lock also covers rqindex+1 queue */
505 	spin_lock_irqsave(&rq_hdr->lock, flags);
506 
507 	seq = rq->rq_tracker[bufindex];
508 	rq->rq_tracker[bufindex] = NULL;
509 
510 	if (!seq) {
511 		efc_log_err(hw->os,
512 			    "RQbuf NULL, rqidx %d, bufidx %d, cur q idx = %d\n",
513 			    rqindex, bufindex, rq_hdr->index);
514 	}
515 
516 	spin_unlock_irqrestore(&rq_hdr->lock, flags);
517 	return seq;
518 }
519 
520 int
521 efct_hw_rqpair_process_rq(struct efct_hw *hw, struct hw_cq *cq,
522 			  u8 *cqe)
523 {
524 	u16 rq_id;
525 	u32 index;
526 	int rqindex;
527 	int rq_status;
528 	u32 h_len;
529 	u32 p_len;
530 	struct efc_hw_sequence *seq;
531 	struct hw_rq *rq;
532 
533 	rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe,
534 					      &rq_id, &index);
535 	if (rq_status != 0) {
536 		switch (rq_status) {
537 		case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED:
538 		case SLI4_FC_ASYNC_RQ_DMA_FAILURE:
539 			/* just get RQ buffer then return to chip */
540 			rqindex = efct_hw_rqpair_find(hw, rq_id);
541 			if (rqindex < 0) {
542 				efc_log_debug(hw->os,
543 					      "status=%#x: lookup fail id=%#x\n",
544 					     rq_status, rq_id);
545 				break;
546 			}
547 
548 			/* get RQ buffer */
549 			seq = efct_hw_rqpair_get(hw, rqindex, index);
550 
551 			/* return to chip */
552 			if (efct_hw_rqpair_sequence_free(hw, seq)) {
553 				efc_log_debug(hw->os,
554 					      "status=%#x,fail rtrn buf to RQ\n",
555 					     rq_status);
556 				break;
557 			}
558 			break;
559 		case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED:
560 		case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC:
561 			/*
562 			 * since RQ buffers were not consumed, cannot return
563 			 * them to chip
564 			 */
565 			efc_log_debug(hw->os, "Warning: RCQE status=%#x,\n",
566 				      rq_status);
567 			fallthrough;
568 		default:
569 			break;
570 		}
571 		return -EIO;
572 	}
573 
574 	rqindex = efct_hw_rqpair_find(hw, rq_id);
575 	if (rqindex < 0) {
576 		efc_log_debug(hw->os, "Error: rq_id lookup failed for id=%#x\n",
577 			      rq_id);
578 		return -EIO;
579 	}
580 
581 	rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]];
582 	rq->use_count++;
583 
584 	seq = efct_hw_rqpair_get(hw, rqindex, index);
585 	if (WARN_ON(!seq))
586 		return -EIO;
587 
588 	seq->hw = hw;
589 
590 	sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len);
591 	seq->header->dma.len = h_len;
592 	seq->payload->dma.len = p_len;
593 	seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe);
594 	seq->hw_priv = cq->eq;
595 
596 	efct_unsolicited_cb(hw->os, seq);
597 
598 	return 0;
599 }
600 
601 static int
602 efct_hw_rqpair_put(struct efct_hw *hw, struct efc_hw_sequence *seq)
603 {
604 	struct sli4_queue *rq_hdr = &hw->rq[seq->header->rqindex];
605 	struct sli4_queue *rq_payload = &hw->rq[seq->payload->rqindex];
606 	u32 hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex];
607 	struct hw_rq *rq = hw->hw_rq[hw_rq_index];
608 	u32 phys_hdr[2];
609 	u32 phys_payload[2];
610 	int qindex_hdr;
611 	int qindex_payload;
612 	unsigned long flags = 0;
613 
614 	/* Update the RQ verification lookup tables */
615 	phys_hdr[0] = upper_32_bits(seq->header->dma.phys);
616 	phys_hdr[1] = lower_32_bits(seq->header->dma.phys);
617 	phys_payload[0] = upper_32_bits(seq->payload->dma.phys);
618 	phys_payload[1] = lower_32_bits(seq->payload->dma.phys);
619 
620 	/* rq_hdr lock also covers payload / header->rqindex+1 queue */
621 	spin_lock_irqsave(&rq_hdr->lock, flags);
622 
623 	/*
624 	 * Note: The header must be posted last for buffer pair mode because
625 	 *       posting on the header queue posts the payload queue as well.
626 	 *       We do not ring the payload queue independently in RQ pair mode.
627 	 */
628 	qindex_payload = sli_rq_write(&hw->sli, rq_payload,
629 				      (void *)phys_payload);
630 	qindex_hdr = sli_rq_write(&hw->sli, rq_hdr, (void *)phys_hdr);
631 	if (qindex_hdr < 0 ||
632 	    qindex_payload < 0) {
633 		efc_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id);
634 		spin_unlock_irqrestore(&rq_hdr->lock, flags);
635 		return -EIO;
636 	}
637 
638 	/* ensure the indexes are the same */
639 	WARN_ON(qindex_hdr != qindex_payload);
640 
641 	/* Update the lookup table */
642 	if (!rq->rq_tracker[qindex_hdr]) {
643 		rq->rq_tracker[qindex_hdr] = seq;
644 	} else {
645 		efc_log_debug(hw->os,
646 			      "expected rq_tracker[%d][%d] buffer to be NULL\n",
647 			      hw_rq_index, qindex_hdr);
648 	}
649 
650 	spin_unlock_irqrestore(&rq_hdr->lock, flags);
651 	return 0;
652 }
653 
654 int
655 efct_hw_rqpair_sequence_free(struct efct_hw *hw, struct efc_hw_sequence *seq)
656 {
657 	int rc = 0;
658 
659 	/*
660 	 * Post the data buffer first. Because in RQ pair mode, ringing the
661 	 * doorbell of the header ring will post the data buffer as well.
662 	 */
663 	if (efct_hw_rqpair_put(hw, seq)) {
664 		efc_log_err(hw->os, "error writing buffers\n");
665 		return -EIO;
666 	}
667 
668 	return rc;
669 }
670 
671 int
672 efct_efc_hw_sequence_free(struct efc *efc, struct efc_hw_sequence *seq)
673 {
674 	struct efct *efct = efc->base;
675 
676 	return efct_hw_rqpair_sequence_free(&efct->hw, seq);
677 }
678