1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
3
4 #include <linux/io.h>
5 #include <linux/types.h>
6 #include <linux/delay.h>
7
8 #include <linux/pds/pds_common.h>
9 #include <linux/pds/pds_core_if.h>
10 #include <linux/pds/pds_adminq.h>
11
12 #include "vfio_dev.h"
13 #include "cmds.h"
14
15 #define SUSPEND_TIMEOUT_S 5
16 #define SUSPEND_CHECK_INTERVAL_MS 1
17
pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device * pds_vfio,union pds_core_adminq_cmd * req,union pds_core_adminq_comp * resp,bool fast_poll)18 static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio,
19 union pds_core_adminq_cmd *req,
20 union pds_core_adminq_comp *resp,
21 bool fast_poll)
22 {
23 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
24 union pds_core_adminq_cmd cmd = {};
25 struct pdsc *pdsc;
26 int err;
27
28 /* Wrap the client request */
29 cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD;
30 cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id);
31 memcpy(cmd.client_request.client_cmd, req,
32 sizeof(cmd.client_request.client_cmd));
33
34 pdsc = pdsc_get_pf_struct(pdev);
35 if (IS_ERR(pdsc))
36 return PTR_ERR(pdsc);
37
38 err = pdsc_adminq_post(pdsc, &cmd, resp, fast_poll);
39 if (err && err != -EAGAIN)
40 dev_err(pds_vfio_to_dev(pds_vfio),
41 "client admin cmd failed: %pe\n", ERR_PTR(err));
42
43 return err;
44 }
45
pds_vfio_register_client_cmd(struct pds_vfio_pci_device * pds_vfio)46 int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio)
47 {
48 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
49 char devname[PDS_DEVNAME_LEN];
50 struct pdsc *pdsc;
51 int ci;
52
53 snprintf(devname, sizeof(devname), "%s.%d-%u", PDS_VFIO_LM_DEV_NAME,
54 pci_domain_nr(pdev->bus),
55 PCI_DEVID(pdev->bus->number, pdev->devfn));
56
57 pdsc = pdsc_get_pf_struct(pdev);
58 if (IS_ERR(pdsc))
59 return PTR_ERR(pdsc);
60
61 ci = pds_client_register(pdsc, devname);
62 if (ci < 0)
63 return ci;
64
65 pds_vfio->client_id = ci;
66
67 return 0;
68 }
69
pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device * pds_vfio)70 void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio)
71 {
72 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
73 struct pdsc *pdsc;
74 int err;
75
76 pdsc = pdsc_get_pf_struct(pdev);
77 if (IS_ERR(pdsc))
78 return;
79
80 err = pds_client_unregister(pdsc, pds_vfio->client_id);
81 if (err)
82 dev_err(&pdev->dev, "unregister from DSC failed: %pe\n",
83 ERR_PTR(err));
84
85 pds_vfio->client_id = 0;
86 }
87
88 static int
pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)89 pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
90 {
91 union pds_core_adminq_cmd cmd = {
92 .lm_suspend_status = {
93 .opcode = PDS_LM_CMD_SUSPEND_STATUS,
94 .vf_id = cpu_to_le16(pds_vfio->vf_id),
95 .type = type,
96 },
97 };
98 struct device *dev = pds_vfio_to_dev(pds_vfio);
99 union pds_core_adminq_comp comp = {};
100 unsigned long time_limit;
101 unsigned long time_start;
102 unsigned long time_done;
103 int err;
104
105 time_start = jiffies;
106 time_limit = time_start + HZ * SUSPEND_TIMEOUT_S;
107 do {
108 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
109 if (err != -EAGAIN)
110 break;
111
112 msleep(SUSPEND_CHECK_INTERVAL_MS);
113 } while (time_before(jiffies, time_limit));
114
115 time_done = jiffies;
116 dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n", __func__,
117 pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start));
118
119 /* Check the results */
120 if (time_after_eq(time_done, time_limit)) {
121 dev_err(dev, "%s: vf%u: Suspend comp timeout\n", __func__,
122 pds_vfio->vf_id);
123 err = -ETIMEDOUT;
124 }
125
126 return err;
127 }
128
pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)129 int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
130 {
131 union pds_core_adminq_cmd cmd = {
132 .lm_suspend = {
133 .opcode = PDS_LM_CMD_SUSPEND,
134 .vf_id = cpu_to_le16(pds_vfio->vf_id),
135 .type = type,
136 },
137 };
138 struct device *dev = pds_vfio_to_dev(pds_vfio);
139 union pds_core_adminq_comp comp = {};
140 int err;
141
142 dev_dbg(dev, "vf%u: Suspend device\n", pds_vfio->vf_id);
143
144 /*
145 * The initial suspend request to the firmware starts the device suspend
146 * operation and the firmware returns success if it's started
147 * successfully.
148 */
149 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
150 if (err) {
151 dev_err(dev, "vf%u: Suspend failed: %pe\n", pds_vfio->vf_id,
152 ERR_PTR(err));
153 return err;
154 }
155
156 /*
157 * The subsequent suspend status request(s) check if the firmware has
158 * completed the device suspend process.
159 */
160 return pds_vfio_suspend_wait_device_cmd(pds_vfio, type);
161 }
162
pds_vfio_resume_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)163 int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
164 {
165 union pds_core_adminq_cmd cmd = {
166 .lm_resume = {
167 .opcode = PDS_LM_CMD_RESUME,
168 .vf_id = cpu_to_le16(pds_vfio->vf_id),
169 .type = type,
170 },
171 };
172 struct device *dev = pds_vfio_to_dev(pds_vfio);
173 union pds_core_adminq_comp comp = {};
174
175 dev_dbg(dev, "vf%u: Resume device\n", pds_vfio->vf_id);
176
177 return pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
178 }
179
pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device * pds_vfio,u64 * size)180 int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size)
181 {
182 union pds_core_adminq_cmd cmd = {
183 .lm_state_size = {
184 .opcode = PDS_LM_CMD_STATE_SIZE,
185 .vf_id = cpu_to_le16(pds_vfio->vf_id),
186 },
187 };
188 struct device *dev = pds_vfio_to_dev(pds_vfio);
189 union pds_core_adminq_comp comp = {};
190 int err;
191
192 dev_dbg(dev, "vf%u: Get migration status\n", pds_vfio->vf_id);
193
194 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
195 if (err)
196 return err;
197
198 *size = le64_to_cpu(comp.lm_state_size.size);
199 return 0;
200 }
201
pds_vfio_dma_map_lm_file(struct device * dev,enum dma_data_direction dir,struct pds_vfio_lm_file * lm_file)202 static int pds_vfio_dma_map_lm_file(struct device *dev,
203 enum dma_data_direction dir,
204 struct pds_vfio_lm_file *lm_file)
205 {
206 struct pds_lm_sg_elem *sgl, *sge;
207 struct scatterlist *sg;
208 dma_addr_t sgl_addr;
209 size_t sgl_size;
210 int err;
211 int i;
212
213 if (!lm_file)
214 return -EINVAL;
215
216 /* dma map file pages */
217 err = dma_map_sgtable(dev, &lm_file->sg_table, dir, 0);
218 if (err)
219 return err;
220
221 lm_file->num_sge = lm_file->sg_table.nents;
222
223 /* alloc sgl */
224 sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem);
225 sgl = kzalloc(sgl_size, GFP_KERNEL);
226 if (!sgl) {
227 err = -ENOMEM;
228 goto out_unmap_sgtable;
229 }
230
231 /* fill sgl */
232 sge = sgl;
233 for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) {
234 sge->addr = cpu_to_le64(sg_dma_address(sg));
235 sge->len = cpu_to_le32(sg_dma_len(sg));
236 dev_dbg(dev, "addr = %llx, len = %u\n", sge->addr, sge->len);
237 sge++;
238 }
239
240 sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE);
241 if (dma_mapping_error(dev, sgl_addr)) {
242 err = -EIO;
243 goto out_free_sgl;
244 }
245
246 lm_file->sgl = sgl;
247 lm_file->sgl_addr = sgl_addr;
248
249 return 0;
250
251 out_free_sgl:
252 kfree(sgl);
253 out_unmap_sgtable:
254 lm_file->num_sge = 0;
255 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
256 return err;
257 }
258
pds_vfio_dma_unmap_lm_file(struct device * dev,enum dma_data_direction dir,struct pds_vfio_lm_file * lm_file)259 static void pds_vfio_dma_unmap_lm_file(struct device *dev,
260 enum dma_data_direction dir,
261 struct pds_vfio_lm_file *lm_file)
262 {
263 if (!lm_file)
264 return;
265
266 /* free sgl */
267 if (lm_file->sgl) {
268 dma_unmap_single(dev, lm_file->sgl_addr,
269 lm_file->num_sge * sizeof(*lm_file->sgl),
270 DMA_TO_DEVICE);
271 kfree(lm_file->sgl);
272 lm_file->sgl = NULL;
273 lm_file->sgl_addr = DMA_MAPPING_ERROR;
274 lm_file->num_sge = 0;
275 }
276
277 /* dma unmap file pages */
278 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
279 }
280
pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device * pds_vfio)281 int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
282 {
283 union pds_core_adminq_cmd cmd = {
284 .lm_save = {
285 .opcode = PDS_LM_CMD_SAVE,
286 .vf_id = cpu_to_le16(pds_vfio->vf_id),
287 },
288 };
289 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
290 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
291 union pds_core_adminq_comp comp = {};
292 struct pds_vfio_lm_file *lm_file;
293 int err;
294
295 dev_dbg(&pdev->dev, "vf%u: Get migration state\n", pds_vfio->vf_id);
296
297 lm_file = pds_vfio->save_file;
298
299 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
300 if (err) {
301 dev_err(&pdev->dev, "failed to map save migration file: %pe\n",
302 ERR_PTR(err));
303 return err;
304 }
305
306 cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
307 cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge);
308
309 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
310 if (err)
311 dev_err(&pdev->dev, "failed to get migration state: %pe\n",
312 ERR_PTR(err));
313
314 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
315
316 return err;
317 }
318
pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device * pds_vfio)319 int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
320 {
321 union pds_core_adminq_cmd cmd = {
322 .lm_restore = {
323 .opcode = PDS_LM_CMD_RESTORE,
324 .vf_id = cpu_to_le16(pds_vfio->vf_id),
325 },
326 };
327 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
328 struct device *pdsc_dev = &pci_physfn(pdev)->dev;
329 union pds_core_adminq_comp comp = {};
330 struct pds_vfio_lm_file *lm_file;
331 int err;
332
333 dev_dbg(&pdev->dev, "vf%u: Set migration state\n", pds_vfio->vf_id);
334
335 lm_file = pds_vfio->restore_file;
336
337 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
338 if (err) {
339 dev_err(&pdev->dev,
340 "failed to map restore migration file: %pe\n",
341 ERR_PTR(err));
342 return err;
343 }
344
345 cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
346 cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge);
347
348 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
349 if (err)
350 dev_err(&pdev->dev, "failed to set migration state: %pe\n",
351 ERR_PTR(err));
352
353 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
354
355 return err;
356 }
357
pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device * pds_vfio,enum pds_lm_host_vf_status vf_status)358 void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio,
359 enum pds_lm_host_vf_status vf_status)
360 {
361 union pds_core_adminq_cmd cmd = {
362 .lm_host_vf_status = {
363 .opcode = PDS_LM_CMD_HOST_VF_STATUS,
364 .vf_id = cpu_to_le16(pds_vfio->vf_id),
365 .status = vf_status,
366 },
367 };
368 struct device *dev = pds_vfio_to_dev(pds_vfio);
369 union pds_core_adminq_comp comp = {};
370 int err;
371
372 dev_dbg(dev, "vf%u: Set host VF LM status: %u", pds_vfio->vf_id,
373 vf_status);
374 if (vf_status != PDS_LM_STA_IN_PROGRESS &&
375 vf_status != PDS_LM_STA_NONE) {
376 dev_warn(dev, "Invalid host VF migration status, %d\n",
377 vf_status);
378 return;
379 }
380
381 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
382 if (err)
383 dev_warn(dev, "failed to send host VF migration status: %pe\n",
384 ERR_PTR(err));
385 }
386
pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device * pds_vfio,u64 regions_dma,u8 * max_regions,u8 * num_regions)387 int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio,
388 u64 regions_dma, u8 *max_regions, u8 *num_regions)
389 {
390 union pds_core_adminq_cmd cmd = {
391 .lm_dirty_status = {
392 .opcode = PDS_LM_CMD_DIRTY_STATUS,
393 .vf_id = cpu_to_le16(pds_vfio->vf_id),
394 },
395 };
396 struct device *dev = pds_vfio_to_dev(pds_vfio);
397 union pds_core_adminq_comp comp = {};
398 int err;
399
400 dev_dbg(dev, "vf%u: Dirty status\n", pds_vfio->vf_id);
401
402 cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma);
403 cmd.lm_dirty_status.max_regions = *max_regions;
404
405 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
406 if (err) {
407 dev_err(dev, "failed to get dirty status: %pe\n", ERR_PTR(err));
408 return err;
409 }
410
411 /* only support seq_ack approach for now */
412 if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) &
413 BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) {
414 dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n");
415 return -EOPNOTSUPP;
416 }
417
418 *num_regions = comp.lm_dirty_status.num_regions;
419 *max_regions = comp.lm_dirty_status.max_regions;
420
421 dev_dbg(dev,
422 "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n",
423 *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK");
424
425 return 0;
426 }
427
pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device * pds_vfio,u64 regions_dma,u8 num_regions)428 int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio,
429 u64 regions_dma, u8 num_regions)
430 {
431 union pds_core_adminq_cmd cmd = {
432 .lm_dirty_enable = {
433 .opcode = PDS_LM_CMD_DIRTY_ENABLE,
434 .vf_id = cpu_to_le16(pds_vfio->vf_id),
435 .regions_dma = cpu_to_le64(regions_dma),
436 .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK,
437 .num_regions = num_regions,
438 },
439 };
440 struct device *dev = pds_vfio_to_dev(pds_vfio);
441 union pds_core_adminq_comp comp = {};
442 int err;
443
444 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
445 if (err) {
446 dev_err(dev, "failed dirty tracking enable: %pe\n",
447 ERR_PTR(err));
448 return err;
449 }
450
451 return 0;
452 }
453
pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device * pds_vfio)454 int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio)
455 {
456 union pds_core_adminq_cmd cmd = {
457 .lm_dirty_disable = {
458 .opcode = PDS_LM_CMD_DIRTY_DISABLE,
459 .vf_id = cpu_to_le16(pds_vfio->vf_id),
460 },
461 };
462 struct device *dev = pds_vfio_to_dev(pds_vfio);
463 union pds_core_adminq_comp comp = {};
464 int err;
465
466 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
467 if (err || comp.lm_dirty_status.num_regions != 0) {
468 /* in case num_regions is still non-zero after disable */
469 err = err ? err : -EIO;
470 dev_err(dev,
471 "failed dirty tracking disable: %pe, num_regions %d\n",
472 ERR_PTR(err), comp.lm_dirty_status.num_regions);
473 return err;
474 }
475
476 return 0;
477 }
478
pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device * pds_vfio,u64 sgl_dma,u16 num_sge,u32 offset,u32 total_len,bool read_seq)479 int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio,
480 u64 sgl_dma, u16 num_sge, u32 offset,
481 u32 total_len, bool read_seq)
482 {
483 const char *cmd_type_str = read_seq ? "read_seq" : "write_ack";
484 union pds_core_adminq_cmd cmd = {
485 .lm_dirty_seq_ack = {
486 .vf_id = cpu_to_le16(pds_vfio->vf_id),
487 .len_bytes = cpu_to_le32(total_len),
488 .off_bytes = cpu_to_le32(offset),
489 .sgl_addr = cpu_to_le64(sgl_dma),
490 .num_sge = cpu_to_le16(num_sge),
491 },
492 };
493 struct device *dev = pds_vfio_to_dev(pds_vfio);
494 union pds_core_adminq_comp comp = {};
495 int err;
496
497 if (read_seq)
498 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ;
499 else
500 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK;
501
502 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
503 if (err) {
504 dev_err(dev, "failed cmd Page Tracking %s: %pe\n", cmd_type_str,
505 ERR_PTR(err));
506 return err;
507 }
508
509 return 0;
510 }
511