1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * SolidRun DPU driver for control plane
4  *
5  * Copyright (C) 2022-2023 SolidRun
6  *
7  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
8  *
9  */
10 
11 #include <linux/iopoll.h>
12 
13 #include "snet_vdpa.h"
14 
15 enum snet_ctrl_opcodes {
16 	SNET_CTRL_OP_DESTROY = 1,
17 	SNET_CTRL_OP_READ_VQ_STATE,
18 	SNET_CTRL_OP_SUSPEND,
19 };
20 
21 #define SNET_CTRL_TIMEOUT	        2000000
22 
23 #define SNET_CTRL_DATA_SIZE_MASK	0x0000FFFF
24 #define SNET_CTRL_IN_PROCESS_MASK	0x00010000
25 #define SNET_CTRL_CHUNK_RDY_MASK	0x00020000
26 #define SNET_CTRL_ERROR_MASK		0x0FFC0000
27 
28 #define SNET_VAL_TO_ERR(val)		(-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
29 #define SNET_EMPTY_CTRL(val)		(((val) & SNET_CTRL_ERROR_MASK) || \
30 						!((val) & SNET_CTRL_IN_PROCESS_MASK))
31 #define SNET_DATA_READY(val)		((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
32 
33 /* Control register used to read data from the DPU */
34 struct snet_ctrl_reg_ctrl {
35 	/* Chunk size in 4B words */
36 	u16 data_size;
37 	/* We are in the middle of a command */
38 	u16 in_process:1;
39 	/* A data chunk is ready and can be consumed */
40 	u16 chunk_ready:1;
41 	/* Error code */
42 	u16 error:10;
43 	/* Saved for future usage */
44 	u16 rsvd:4;
45 };
46 
47 /* Opcode register */
48 struct snet_ctrl_reg_op {
49 	u16 opcode;
50 	/* Only if VQ index is relevant for the command */
51 	u16 vq_idx;
52 };
53 
54 struct snet_ctrl_regs {
55 	struct snet_ctrl_reg_op op;
56 	struct snet_ctrl_reg_ctrl ctrl;
57 	u32 rsvd;
58 	u32 data[];
59 };
60 
61 static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
62 {
63 	return snet->bar + snet->psnet->cfg.ctrl_off;
64 }
65 
66 static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
67 {
68 	u32 val;
69 
70 	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
71 				  SNET_CTRL_TIMEOUT);
72 }
73 
74 static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
75 {
76 	u32 val;
77 
78 	return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
79 }
80 
81 static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
82 {
83 	u32 val;
84 
85 	return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
86 				  SNET_CTRL_TIMEOUT);
87 }
88 
89 static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
90 {
91 	return ioread32(&ctrl_regs->data[word_idx]);
92 }
93 
94 static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
95 {
96 	return ioread32(&ctrl_regs->ctrl);
97 }
98 
99 static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
100 {
101 	iowrite32(val, &ctrl_regs->ctrl);
102 }
103 
104 static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
105 {
106 	iowrite32(val, &ctrl_regs->op);
107 }
108 
109 static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
110 {
111 	/* Wait until the DPU finishes completely.
112 	 * It will clear the opcode register.
113 	 */
114 	return snet_wait_for_empty_op(ctrl_regs);
115 }
116 
117 /* Reading ctrl from the DPU:
118  * buf_size must be 4B aligned
119  *
120  * Steps:
121  *
122  * (1) Verify that the DPU is not in the middle of another operation by
123  *     reading the in_process and error bits in the control register.
124  * (2) Write the request opcode and the VQ idx in the opcode register
125  *     and write the buffer size in the control register.
126  * (3) Start readind chunks of data, chunk_ready bit indicates that a
127  *     data chunk is available, we signal that we read the data by clearing the bit.
128  * (4) Detect that the transfer is completed when the in_process bit
129  *     in the control register is cleared or when the an error appears.
130  */
131 static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
132 				   u32 buf_size)
133 {
134 	struct pci_dev *pdev = snet->pdev;
135 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
136 	u32 *bfr_ptr = (u32 *)buffer;
137 	u32 val;
138 	u16 buf_words;
139 	int ret;
140 	u16 words, i, tot_words = 0;
141 
142 	/* Supported for config 2+ */
143 	if (!SNET_CFG_VER(snet, 2))
144 		return -EOPNOTSUPP;
145 
146 	if (!IS_ALIGNED(buf_size, 4))
147 		return -EINVAL;
148 
149 	mutex_lock(&snet->ctrl_lock);
150 
151 	buf_words = buf_size / 4;
152 
153 	/* Make sure control register is empty */
154 	ret = snet_wait_for_empty_ctrl(regs);
155 	if (ret) {
156 		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
157 		goto exit;
158 	}
159 
160 	/* We need to write the buffer size in the control register, and the opcode + vq index in
161 	 * the opcode register.
162 	 * We use a spinlock to serialize the writes.
163 	 */
164 	spin_lock(&snet->ctrl_spinlock);
165 
166 	snet_write_ctrl(regs, buf_words);
167 	snet_write_op(regs, opcode | (vq_idx << 16));
168 
169 	spin_unlock(&snet->ctrl_spinlock);
170 
171 	while (buf_words != tot_words) {
172 		ret = snet_wait_for_data(regs);
173 		if (ret) {
174 			SNET_WARN(pdev, "Timeout waiting for control data\n");
175 			goto exit;
176 		}
177 
178 		val = snet_read_ctrl(regs);
179 
180 		/* Error? */
181 		if (val & SNET_CTRL_ERROR_MASK) {
182 			ret = SNET_VAL_TO_ERR(val);
183 			SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
184 			goto exit;
185 		}
186 
187 		words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
188 
189 		for (i = 0; i < words; i++) {
190 			*bfr_ptr = snet_read32_word(regs, i);
191 			bfr_ptr++;
192 		}
193 
194 		tot_words += words;
195 
196 		/* Is the job completed? */
197 		if (!(val & SNET_CTRL_IN_PROCESS_MASK))
198 			break;
199 
200 		/* Clear the chunk ready bit and continue */
201 		val &= ~SNET_CTRL_CHUNK_RDY_MASK;
202 		snet_write_ctrl(regs, val);
203 	}
204 
205 	ret = snet_wait_for_dpu_completion(regs);
206 	if (ret)
207 		SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
208 
209 exit:
210 	mutex_unlock(&snet->ctrl_lock);
211 	return ret;
212 }
213 
214 /* Send a control message to the DPU using the old mechanism
215  * used with config version 1.
216  */
217 static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
218 {
219 	struct pci_dev *pdev = snet->pdev;
220 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
221 	int ret;
222 
223 	mutex_lock(&snet->ctrl_lock);
224 
225 	/* Old mechanism uses just 1 register, the opcode register.
226 	 * Make sure that the opcode register is empty, and that the DPU isn't
227 	 * processing an old message.
228 	 */
229 	ret = snet_wait_for_empty_op(regs);
230 	if (ret) {
231 		SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
232 		goto exit;
233 	}
234 
235 	/* Write the message */
236 	snet_write_op(regs, opcode);
237 
238 	/* DPU ACKs the message by clearing the opcode register */
239 	ret = snet_wait_for_empty_op(regs);
240 	if (ret)
241 		SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
242 
243 exit:
244 	mutex_unlock(&snet->ctrl_lock);
245 	return ret;
246 }
247 
248 /* Send a control message to the DPU.
249  * A control message is a message without payload.
250  */
251 static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
252 {
253 	struct pci_dev *pdev = snet->pdev;
254 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
255 	u32 val;
256 	int ret;
257 
258 	/* If config version is not 2+, use the old mechanism */
259 	if (!SNET_CFG_VER(snet, 2))
260 		return snet_send_ctrl_msg_old(snet, opcode);
261 
262 	mutex_lock(&snet->ctrl_lock);
263 
264 	/* Make sure control register is empty */
265 	ret = snet_wait_for_empty_ctrl(regs);
266 	if (ret) {
267 		SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
268 		goto exit;
269 	}
270 
271 	/* We need to clear the control register and write the opcode + vq index in the opcode
272 	 * register.
273 	 * We use a spinlock to serialize the writes.
274 	 */
275 	spin_lock(&snet->ctrl_spinlock);
276 
277 	snet_write_ctrl(regs, 0);
278 	snet_write_op(regs, opcode | (vq_idx << 16));
279 
280 	spin_unlock(&snet->ctrl_spinlock);
281 
282 	/* The DPU ACKs control messages by setting the chunk ready bit
283 	 * without data.
284 	 */
285 	ret = snet_wait_for_data(regs);
286 	if (ret) {
287 		SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
288 		goto exit;
289 	}
290 
291 	/* Check for errors */
292 	val = snet_read_ctrl(regs);
293 	ret = SNET_VAL_TO_ERR(val);
294 
295 	/* Clear the chunk ready bit */
296 	val &= ~SNET_CTRL_CHUNK_RDY_MASK;
297 	snet_write_ctrl(regs, val);
298 
299 	ret = snet_wait_for_dpu_completion(regs);
300 	if (ret)
301 		SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
302 			  ret);
303 
304 exit:
305 	mutex_unlock(&snet->ctrl_lock);
306 	return ret;
307 }
308 
309 void snet_ctrl_clear(struct snet *snet)
310 {
311 	struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
312 
313 	snet_write_op(regs, 0);
314 }
315 
316 int snet_destroy_dev(struct snet *snet)
317 {
318 	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
319 }
320 
321 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
322 {
323 	return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
324 				       sizeof(*state));
325 }
326 
327 int snet_suspend_dev(struct snet *snet)
328 {
329 	return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
330 }
331