1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /*          Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6 
7 #include "erdma_verbs.h"
8 
9 #define MAX_POLL_CHUNK_SIZE 16
10 
11 void notify_eq(struct erdma_eq *eq)
12 {
13 	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |
14 		      FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
15 
16 	*eq->db_record = db_data;
17 	writeq(db_data, eq->db_addr);
18 
19 	atomic64_inc(&eq->notify_num);
20 }
21 
22 void *get_next_valid_eqe(struct erdma_eq *eq)
23 {
24 	u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);
25 	u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe));
26 
27 	return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;
28 }
29 
30 void erdma_aeq_event_handler(struct erdma_dev *dev)
31 {
32 	struct erdma_aeqe *aeqe;
33 	u32 cqn, qpn;
34 	struct erdma_qp *qp;
35 	struct erdma_cq *cq;
36 	struct ib_event event;
37 	u32 poll_cnt = 0;
38 
39 	memset(&event, 0, sizeof(event));
40 
41 	while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
42 		aeqe = get_next_valid_eqe(&dev->aeq);
43 		if (!aeqe)
44 			break;
45 
46 		dma_rmb();
47 
48 		dev->aeq.ci++;
49 		atomic64_inc(&dev->aeq.event_num);
50 		poll_cnt++;
51 
52 		if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK,
53 			      le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) {
54 			cqn = le32_to_cpu(aeqe->event_data0);
55 			cq = find_cq_by_cqn(dev, cqn);
56 			if (!cq)
57 				continue;
58 
59 			event.device = cq->ibcq.device;
60 			event.element.cq = &cq->ibcq;
61 			event.event = IB_EVENT_CQ_ERR;
62 			if (cq->ibcq.event_handler)
63 				cq->ibcq.event_handler(&event,
64 						       cq->ibcq.cq_context);
65 		} else {
66 			qpn = le32_to_cpu(aeqe->event_data0);
67 			qp = find_qp_by_qpn(dev, qpn);
68 			if (!qp)
69 				continue;
70 
71 			event.device = qp->ibqp.device;
72 			event.element.qp = &qp->ibqp;
73 			event.event = IB_EVENT_QP_FATAL;
74 			if (qp->ibqp.event_handler)
75 				qp->ibqp.event_handler(&event,
76 						       qp->ibqp.qp_context);
77 		}
78 	}
79 
80 	notify_eq(&dev->aeq);
81 }
82 
83 int erdma_aeq_init(struct erdma_dev *dev)
84 {
85 	struct erdma_eq *eq = &dev->aeq;
86 	u32 buf_size;
87 
88 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
89 	buf_size = eq->depth << EQE_SHIFT;
90 
91 	eq->qbuf =
92 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
93 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
94 	if (!eq->qbuf)
95 		return -ENOMEM;
96 
97 	spin_lock_init(&eq->lock);
98 	atomic64_set(&eq->event_num, 0);
99 	atomic64_set(&eq->notify_num, 0);
100 
101 	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
102 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
103 
104 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
105 			  upper_32_bits(eq->qbuf_dma_addr));
106 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG,
107 			  lower_32_bits(eq->qbuf_dma_addr));
108 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);
109 	erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG,
110 			  eq->qbuf_dma_addr + buf_size);
111 
112 	return 0;
113 }
114 
115 void erdma_aeq_destroy(struct erdma_dev *dev)
116 {
117 	struct erdma_eq *eq = &dev->aeq;
118 
119 	dma_free_coherent(&dev->pdev->dev,
120 			  WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf,
121 			  eq->qbuf_dma_addr);
122 }
123 
124 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb)
125 {
126 	struct erdma_dev *dev = ceq_cb->dev;
127 	struct erdma_cq *cq;
128 	u32 poll_cnt = 0;
129 	u64 *ceqe;
130 	int cqn;
131 
132 	if (!ceq_cb->ready)
133 		return;
134 
135 	while (poll_cnt < MAX_POLL_CHUNK_SIZE) {
136 		ceqe = get_next_valid_eqe(&ceq_cb->eq);
137 		if (!ceqe)
138 			break;
139 
140 		dma_rmb();
141 		ceq_cb->eq.ci++;
142 		poll_cnt++;
143 		cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe));
144 
145 		cq = find_cq_by_cqn(dev, cqn);
146 		if (!cq)
147 			continue;
148 
149 		if (rdma_is_kernel_res(&cq->ibcq.res))
150 			cq->kern_cq.cmdsn++;
151 
152 		if (cq->ibcq.comp_handler)
153 			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
154 	}
155 
156 	notify_eq(&ceq_cb->eq);
157 }
158 
159 static irqreturn_t erdma_intr_ceq_handler(int irq, void *data)
160 {
161 	struct erdma_eq_cb *ceq_cb = data;
162 
163 	tasklet_schedule(&ceq_cb->tasklet);
164 
165 	return IRQ_HANDLED;
166 }
167 
168 static void erdma_intr_ceq_task(unsigned long data)
169 {
170 	erdma_ceq_completion_handler((struct erdma_eq_cb *)data);
171 }
172 
173 static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn)
174 {
175 	struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
176 	int err;
177 
178 	snprintf(eqc->irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn,
179 		 pci_name(dev->pdev));
180 	eqc->irq.msix_vector = pci_irq_vector(dev->pdev, ceqn + 1);
181 
182 	tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task,
183 		     (unsigned long)&dev->ceqs[ceqn]);
184 
185 	cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node),
186 			&eqc->irq.affinity_hint_mask);
187 
188 	err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0,
189 			  eqc->irq.name, eqc);
190 	if (err) {
191 		dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err);
192 		return err;
193 	}
194 
195 	irq_set_affinity_hint(eqc->irq.msix_vector,
196 			      &eqc->irq.affinity_hint_mask);
197 
198 	return 0;
199 }
200 
201 static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn)
202 {
203 	struct erdma_eq_cb *eqc = &dev->ceqs[ceqn];
204 
205 	irq_set_affinity_hint(eqc->irq.msix_vector, NULL);
206 	free_irq(eqc->irq.msix_vector, eqc);
207 }
208 
209 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)
210 {
211 	struct erdma_cmdq_create_eq_req req;
212 	dma_addr_t db_info_dma_addr;
213 
214 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
215 				CMDQ_OPCODE_CREATE_EQ);
216 	req.eqn = eqn;
217 	req.depth = ilog2(eq->depth);
218 	req.qbuf_addr = eq->qbuf_dma_addr;
219 	req.qtype = ERDMA_EQ_TYPE_CEQ;
220 	/* Vector index is the same as EQN. */
221 	req.vector_idx = eqn;
222 	db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT);
223 	req.db_dma_addr_l = lower_32_bits(db_info_dma_addr);
224 	req.db_dma_addr_h = upper_32_bits(db_info_dma_addr);
225 
226 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
227 }
228 
229 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
230 {
231 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
232 	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
233 	int ret;
234 
235 	eq->qbuf =
236 		dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size),
237 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);
238 	if (!eq->qbuf)
239 		return -ENOMEM;
240 
241 	spin_lock_init(&eq->lock);
242 	atomic64_set(&eq->event_num, 0);
243 	atomic64_set(&eq->notify_num, 0);
244 
245 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
246 	eq->db_addr =
247 		(u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
248 				(ceqn + 1) * ERDMA_DB_SIZE);
249 	eq->db_record = (u64 *)(eq->qbuf + buf_size);
250 	eq->ci = 0;
251 	dev->ceqs[ceqn].dev = dev;
252 
253 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
254 	ret = create_eq_cmd(dev, ceqn + 1, eq);
255 	dev->ceqs[ceqn].ready = ret ? false : true;
256 
257 	return ret;
258 }
259 
260 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn)
261 {
262 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;
263 	u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT;
264 	struct erdma_cmdq_destroy_eq_req req;
265 	int err;
266 
267 	dev->ceqs[ceqn].ready = 0;
268 
269 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
270 				CMDQ_OPCODE_DESTROY_EQ);
271 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */
272 	req.eqn = ceqn + 1;
273 	req.qtype = ERDMA_EQ_TYPE_CEQ;
274 	req.vector_idx = ceqn + 1;
275 
276 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
277 	if (err)
278 		return;
279 
280 	dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf,
281 			  eq->qbuf_dma_addr);
282 }
283 
284 int erdma_ceqs_init(struct erdma_dev *dev)
285 {
286 	u32 i, j;
287 	int err;
288 
289 	for (i = 0; i < dev->attrs.irq_num - 1; i++) {
290 		err = erdma_ceq_init_one(dev, i);
291 		if (err)
292 			goto out_err;
293 
294 		err = erdma_set_ceq_irq(dev, i);
295 		if (err) {
296 			erdma_ceq_uninit_one(dev, i);
297 			goto out_err;
298 		}
299 	}
300 
301 	return 0;
302 
303 out_err:
304 	for (j = 0; j < i; j++) {
305 		erdma_free_ceq_irq(dev, j);
306 		erdma_ceq_uninit_one(dev, j);
307 	}
308 
309 	return err;
310 }
311 
312 void erdma_ceqs_uninit(struct erdma_dev *dev)
313 {
314 	u32 i;
315 
316 	for (i = 0; i < dev->attrs.irq_num - 1; i++) {
317 		erdma_free_ceq_irq(dev, i);
318 		erdma_ceq_uninit_one(dev, i);
319 	}
320 }
321