1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * SolidRun DPU driver for control plane
4  *
5  * Copyright (C) 2022-2023 SolidRun
6  *
7  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
8  *
9  */
10 
11 #include <linux/iopoll.h>
12 
13 #include "snet_vdpa.h"
14 
15 enum snet_ctrl_opcodes {
16 	SNET_CTRL_OP_DESTROY = 1,
17 	SNET_CTRL_OP_READ_VQ_STATE,
18 	SNET_CTRL_OP_SUSPEND,
19 	SNET_CTRL_OP_RESUME,
20 };
21 
22 #define SNET_CTRL_TIMEOUT	        2000000
23 
24 #define SNET_CTRL_DATA_SIZE_MASK	0x0000FFFF
25 #define SNET_CTRL_IN_PROCESS_MASK	0x00010000
26 #define SNET_CTRL_CHUNK_RDY_MASK	0x00020000
27 #define SNET_CTRL_ERROR_MASK		0x0FFC0000
28 
29 #define SNET_VAL_TO_ERR(val)		(-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
30 #define SNET_EMPTY_CTRL(val)		(((val) & SNET_CTRL_ERROR_MASK) || \
31 						!((val) & SNET_CTRL_IN_PROCESS_MASK))
32 #define SNET_DATA_READY(val)		((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
33 
34 /* Control register used to read data from the DPU */
35 struct snet_ctrl_reg_ctrl {
36 	/* Chunk size in 4B words */
37 	u16 data_size;
38 	/* We are in the middle of a command */
39 	u16 in_process:1;
40 	/* A data chunk is ready and can be consumed */
41 	u16 chunk_ready:1;
42 	/* Error code */
43 	u16 error:10;
44 	/* Saved for future usage */
45 	u16 rsvd:4;
46 };
47 
48 /* Opcode register */
49 struct snet_ctrl_reg_op {
50 	u16 opcode;
51 	/* Only if VQ index is relevant for the command */
52 	u16 vq_idx;
53 };
54 
55 struct snet_ctrl_regs {
56 	struct snet_ctrl_reg_op op;
57 	struct snet_ctrl_reg_ctrl ctrl;
58 	u32 rsvd;
59 	u32 data[];
60 };
61 
62 static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
63 {
64 	return snet->bar + snet->psnet->cfg.ctrl_off;
65 }
66 
67 static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
68 {
69 	u32 val;
70 
71 	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
72 				  SNET_CTRL_TIMEOUT);
73 }
74 
75 static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
76 {
77 	u32 val;
78 
79 	return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
80 }
81 
82 static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
83 {
84 	u32 val;
85 
86 	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
87 				  SNET_CTRL_TIMEOUT);
88 }
89 
90 static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
91 {
92 	return ioread32(&ctrl_regs->data[word_idx]);
93 }
94 
95 static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
96 {
97 	return ioread32(&ctrl_regs->ctrl);
98 }
99 
100 static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
101 {
102 	iowrite32(val, &ctrl_regs->ctrl);
103 }
104 
105 static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
106 {
107 	iowrite32(val, &ctrl_regs->op);
108 }
109 
110 static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
111 {
112 	/* Wait until the DPU finishes completely.
113 	 * It will clear the opcode register.
114 	 */
115 	return snet_wait_for_empty_op(ctrl_regs);
116 }
117 
118 /* Reading ctrl from the DPU:
119  * buf_size must be 4B aligned
120  *
121  * Steps:
122  *
123  * (1) Verify that the DPU is not in the middle of another operation by
124  *     reading the in_process and error bits in the control register.
125  * (2) Write the request opcode and the VQ idx in the opcode register
126  *     and write the buffer size in the control register.
127  * (3) Start readind chunks of data, chunk_ready bit indicates that a
128  *     data chunk is available, we signal that we read the data by clearing the bit.
129  * (4) Detect that the transfer is completed when the in_process bit
130  *     in the control register is cleared or when the an error appears.
131  */
132 static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
133 				   u32 buf_size)
134 {
135 	struct pci_dev *pdev = snet->pdev;
136 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
137 	u32 *bfr_ptr = (u32 *)buffer;
138 	u32 val;
139 	u16 buf_words;
140 	int ret;
141 	u16 words, i, tot_words = 0;
142 
143 	/* Supported for config 2+ */
144 	if (!SNET_CFG_VER(snet, 2))
145 		return -EOPNOTSUPP;
146 
147 	if (!IS_ALIGNED(buf_size, 4))
148 		return -EINVAL;
149 
150 	mutex_lock(&snet->ctrl_lock);
151 
152 	buf_words = buf_size / 4;
153 
154 	/* Make sure control register is empty */
155 	ret = snet_wait_for_empty_ctrl(regs);
156 	if (ret) {
157 		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
158 		goto exit;
159 	}
160 
161 	/* We need to write the buffer size in the control register, and the opcode + vq index in
162 	 * the opcode register.
163 	 * We use a spinlock to serialize the writes.
164 	 */
165 	spin_lock(&snet->ctrl_spinlock);
166 
167 	snet_write_ctrl(regs, buf_words);
168 	snet_write_op(regs, opcode | (vq_idx << 16));
169 
170 	spin_unlock(&snet->ctrl_spinlock);
171 
172 	while (buf_words != tot_words) {
173 		ret = snet_wait_for_data(regs);
174 		if (ret) {
175 			SNET_WARN(pdev, "Timeout waiting for control data\n");
176 			goto exit;
177 		}
178 
179 		val = snet_read_ctrl(regs);
180 
181 		/* Error? */
182 		if (val & SNET_CTRL_ERROR_MASK) {
183 			ret = SNET_VAL_TO_ERR(val);
184 			SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
185 			goto exit;
186 		}
187 
188 		words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
189 
190 		for (i = 0; i < words; i++) {
191 			*bfr_ptr = snet_read32_word(regs, i);
192 			bfr_ptr++;
193 		}
194 
195 		tot_words += words;
196 
197 		/* Is the job completed? */
198 		if (!(val & SNET_CTRL_IN_PROCESS_MASK))
199 			break;
200 
201 		/* Clear the chunk ready bit and continue */
202 		val &= ~SNET_CTRL_CHUNK_RDY_MASK;
203 		snet_write_ctrl(regs, val);
204 	}
205 
206 	ret = snet_wait_for_dpu_completion(regs);
207 	if (ret)
208 		SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
209 
210 exit:
211 	mutex_unlock(&snet->ctrl_lock);
212 	return ret;
213 }
214 
215 /* Send a control message to the DPU using the old mechanism
216  * used with config version 1.
217  */
218 static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
219 {
220 	struct pci_dev *pdev = snet->pdev;
221 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
222 	int ret;
223 
224 	mutex_lock(&snet->ctrl_lock);
225 
226 	/* Old mechanism uses just 1 register, the opcode register.
227 	 * Make sure that the opcode register is empty, and that the DPU isn't
228 	 * processing an old message.
229 	 */
230 	ret = snet_wait_for_empty_op(regs);
231 	if (ret) {
232 		SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
233 		goto exit;
234 	}
235 
236 	/* Write the message */
237 	snet_write_op(regs, opcode);
238 
239 	/* DPU ACKs the message by clearing the opcode register */
240 	ret = snet_wait_for_empty_op(regs);
241 	if (ret)
242 		SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
243 
244 exit:
245 	mutex_unlock(&snet->ctrl_lock);
246 	return ret;
247 }
248 
249 /* Send a control message to the DPU.
250  * A control message is a message without payload.
251  */
252 static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
253 {
254 	struct pci_dev *pdev = snet->pdev;
255 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
256 	u32 val;
257 	int ret;
258 
259 	/* If config version is not 2+, use the old mechanism */
260 	if (!SNET_CFG_VER(snet, 2))
261 		return snet_send_ctrl_msg_old(snet, opcode);
262 
263 	mutex_lock(&snet->ctrl_lock);
264 
265 	/* Make sure control register is empty */
266 	ret = snet_wait_for_empty_ctrl(regs);
267 	if (ret) {
268 		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
269 		goto exit;
270 	}
271 
272 	/* We need to clear the control register and write the opcode + vq index in the opcode
273 	 * register.
274 	 * We use a spinlock to serialize the writes.
275 	 */
276 	spin_lock(&snet->ctrl_spinlock);
277 
278 	snet_write_ctrl(regs, 0);
279 	snet_write_op(regs, opcode | (vq_idx << 16));
280 
281 	spin_unlock(&snet->ctrl_spinlock);
282 
283 	/* The DPU ACKs control messages by setting the chunk ready bit
284 	 * without data.
285 	 */
286 	ret = snet_wait_for_data(regs);
287 	if (ret) {
288 		SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
289 		goto exit;
290 	}
291 
292 	/* Check for errors */
293 	val = snet_read_ctrl(regs);
294 	ret = SNET_VAL_TO_ERR(val);
295 
296 	/* Clear the chunk ready bit */
297 	val &= ~SNET_CTRL_CHUNK_RDY_MASK;
298 	snet_write_ctrl(regs, val);
299 
300 	ret = snet_wait_for_dpu_completion(regs);
301 	if (ret)
302 		SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
303 			  ret);
304 
305 exit:
306 	mutex_unlock(&snet->ctrl_lock);
307 	return ret;
308 }
309 
310 void snet_ctrl_clear(struct snet *snet)
311 {
312 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
313 
314 	snet_write_op(regs, 0);
315 }
316 
317 int snet_destroy_dev(struct snet *snet)
318 {
319 	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
320 }
321 
322 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
323 {
324 	return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
325 				       sizeof(*state));
326 }
327 
328 int snet_suspend_dev(struct snet *snet)
329 {
330 	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
331 }
332 
333 int snet_resume_dev(struct snet *snet)
334 {
335 	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);
336 }
337