1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /*          Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6 
7 #include <linux/errno.h>
8 #include <linux/pci.h>
9 #include <linux/types.h>
10 
11 #include "erdma.h"
12 #include "erdma_hw.h"
13 #include "erdma_verbs.h"
14 
15 #define MAX_POLL_CHUNK_SIZE 16
16 
17 void notify_eq(struct erdma_eq *eq)
18 {
19 	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |
20 		      FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
21 
22 	*eq->db_record = db_data;
23 	writeq(db_data, eq->db_addr);
24 
25 	atomic64_inc(&eq->notify_num);
26 }
27 
28 void *get_next_valid_eqe(struct erdma_eq *eq)
29 {
30 	u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);
31 	u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe));
32 
33 	return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;
34 }
35 
36 void erdma_aeq_event_handler(struct erdma_dev *dev)
37 {
38 	struct erdma_aeqe *aeqe;
39 	u32 cqn, qpn;
40 	struct erdma_qp *qp;
41 	struct erdma_cq *cq;
42 	struct ib_event event;
43 	u32 poll_cnt = 0;
44 
45 	memset(&event, 0, sizeof(event));
46 
47 	while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
48 		aeqe = get_next_valid_eqe(&dev->aeq);
49 		if (!aeqe)
50 			break;
51 
52 		dma_rmb();
53 
54 		dev->aeq.ci++;
55 		atomic64_inc(&dev->aeq.event_num);
56 		poll_cnt++;
57 
58 		if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK,
59 			      le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) {
60 			cqn = le32_to_cpu(aeqe->event_data0);
61 			cq = find_cq_by_cqn(dev, cqn);
62 			if (!cq)
63 				continue;
64 
65 			event.device = cq->ibcq.device;
66 			event.element.cq = &cq->ibcq;
67 			event.event = IB_EVENT_CQ_ERR;
68 			if (cq->ibcq.event_handler)
69 				cq->ibcq.event_handler(&event,
70 						       cq->ibcq.cq_context);
71 		} else {
72 			qpn = le32_to_cpu(aeqe->event_data0);
73 			qp = find_qp_by_qpn(dev, qpn);
74 			if (!qp)
75 				continue;
76 
77 			event.device = qp->ibqp.device;
78 			event.element.qp = &qp->ibqp;
79 			event.event = IB_EVENT_QP_FATAL;
80 			if (qp->ibqp.event_handler)
81 				qp->ibqp.event_handler(&event,
82 						       qp->ibqp.qp_context);
83 		}
84 	}
85 
86 	notify_eq(&dev->aeq);
87 }
88 
89 int erdma_aeq_init(struct erdma_dev *dev)
90 {
91 	struct erdma_eq *eq = &dev->aeq;
92 	u32 buf_size;
93 
94 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
95 	buf_size = eq->depth << EQE_SHIFT;
96 
97 	eq->qbuf =
98 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
99 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
100 	if (!eq->qbuf)
101 		return -ENOMEM;
102 
103 	spin_lock_init(&eq->lock);
104 	atomic64_set(&eq->event_num, 0);
105 	atomic64_set(&eq->notify_num, 0);
106 
107 	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
108 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
109 
110 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
111 			  upper_32_bits(eq->qbuf_dma_addr));
112 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG,
113 			  lower_32_bits(eq->qbuf_dma_addr));
114 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);
115 	erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG,
116 			  eq->qbuf_dma_addr + buf_size);
117 
118 	return 0;
119 }
120 
121 void erdma_aeq_destroy(struct erdma_dev *dev)
122 {
123 	struct erdma_eq *eq = &dev->aeq;
124 
125 	dma_free_coherent(&dev->pdev->dev,
126 			  WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf,
127 			  eq->qbuf_dma_addr);
128 }
129 
130 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)
131 {
132 	struct erdma_dev *dev = ceq_cb->dev;
133 	struct erdma_cq *cq;
134 	u32 poll_cnt = 0;
135 	u64 *ceqe;
136 	int cqn;
137 
138 	if (!ceq_cb->ready)
139 		return;
140 
141 	while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
142 		ceqe = get_next_valid_eqe(&ceq_cb->eq);
143 		if (!ceqe)
144 			break;
145 
146 		dma_rmb();
147 		ceq_cb->eq.ci++;
148 		poll_cnt++;
149 		cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe));
150 
151 		cq = find_cq_by_cqn(dev, cqn);
152 		if (!cq)
153 			continue;
154 
155 		if (rdma_is_kernel_res(&cq->ibcq.res))
156 			cq->kern_cq.cmdsn++;
157 
158 		if (cq->ibcq.comp_handler)
159 			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
160 	}
161 
162 	notify_eq(&ceq_cb->eq);
163 }
164 
165 static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)
166 {
167 	struct erdma_eq_cb *ceq_cb = data;
168 
169 	tasklet_schedule(&ceq_cb->tasklet);
170 
171 	return IRQ_HANDLED;
172 }
173 
174 static void erdma_intr_ceq_task(unsigned long data)
175 {
176 	erdma_ceq_completion_handler((struct erdma_eq_cb *)data);
177 }
178 
179 static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)
180 {
181 	struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
182 	int err;
183 
184 	snprintf(eqc->irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn,
185 		 pci_name(dev->pdev));
186 	eqc->irq.msix_vector = pci_irq_vector(dev->pdev, ceqn + 1);
187 
188 	tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task,
189 		     (unsigned long)&dev->ceqs[ceqn]);
190 
191 	cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node),
192 			&eqc->irq.affinity_hint_mask);
193 
194 	err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0,
195 			  eqc->irq.name, eqc);
196 	if (err) {
197 		dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err);
198 		return err;
199 	}
200 
201 	irq_set_affinity_hint(eqc->irq.msix_vector,
202 			      &eqc->irq.affinity_hint_mask);
203 
204 	return 0;
205 }
206 
207 static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn)
208 {
209 	struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
210 
211 	irq_set_affinity_hint(eqc->irq.msix_vector, NULL);
212 	free_irq(eqc->irq.msix_vector, eqc);
213 }
214 
215 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)
216 {
217 	struct erdma_cmdq_create_eq_req req;
218 	dma_addr_t db_info_dma_addr;
219 
220 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
221 				CMDQ_OPCODE_CREATE_EQ);
222 	req.eqn = eqn;
223 	req.depth = ilog2(eq->depth);
224 	req.qbuf_addr = eq->qbuf_dma_addr;
225 	req.qtype = ERDMA_EQ_TYPE_CEQ;
226 	/* Vector index is the same as EQN. */
227 	req.vector_idx = eqn;
228 	db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT);
229 	req.db_dma_addr_l = lower_32_bits(db_info_dma_addr);
230 	req.db_dma_addr_h = upper_32_bits(db_info_dma_addr);
231 
232 	return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req,
233 				   sizeof(struct erdma_cmdq_create_eq_req),
234 				   NULL, NULL);
235 }
236 
237 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
238 {
239 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
240 	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
241 	int ret;
242 
243 	eq->qbuf =
244 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
245 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
246 	if (!eq->qbuf)
247 		return -ENOMEM;
248 
249 	spin_lock_init(&eq->lock);
250 	atomic64_set(&eq->event_num, 0);
251 	atomic64_set(&eq->notify_num, 0);
252 
253 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
254 	eq->db_addr =
255 		(u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
256 				(ceqn + 1) * ERDMA_DB_SIZE);
257 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
258 	eq->ci = 0;
259 	dev->ceqs[ceqn].dev = dev;
260 
261 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
262 	ret = create_eq_cmd(dev, ceqn + 1, eq);
263 	dev->ceqs[ceqn].ready = ret ? false : true;
264 
265 	return ret;
266 }
267 
268 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn)
269 {
270 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
271 	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
272 	struct erdma_cmdq_destroy_eq_req req;
273 	int err;
274 
275 	dev->ceqs[ceqn].ready = 0;
276 
277 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
278 				CMDQ_OPCODE_DESTROY_EQ);
279 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
280 	req.eqn = ceqn + 1;
281 	req.qtype = ERDMA_EQ_TYPE_CEQ;
282 	req.vector_idx = ceqn + 1;
283 
284 	err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL,
285 				  NULL);
286 	if (err)
287 		return;
288 
289 	dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf,
290 			  eq->qbuf_dma_addr);
291 }
292 
293 int erdma_ceqs_init(struct erdma_dev *dev)
294 {
295 	u32 i, j;
296 	int err;
297 
298 	for (i = 0; i < dev->attrs.irq_num - 1; i++) {
299 		err = erdma_ceq_init_one(dev, i);
300 		if (err)
301 			goto out_err;
302 
303 		err = erdma_set_ceq_irq(dev, i);
304 		if (err) {
305 			erdma_ceq_uninit_one(dev, i);
306 			goto out_err;
307 		}
308 	}
309 
310 	return 0;
311 
312 out_err:
313 	for (j = 0; j < i; j++) {
314 		erdma_free_ceq_irq(dev, j);
315 		erdma_ceq_uninit_one(dev, j);
316 	}
317 
318 	return err;
319 }
320 
321 void erdma_ceqs_uninit(struct erdma_dev *dev)
322 {
323 	u32 i;
324 
325 	for (i = 0; i < dev->attrs.irq_num - 1; i++) {
326 		erdma_free_ceq_irq(dev, i);
327 		erdma_ceq_uninit_one(dev, i);
328 	}
329 }
330