163f77a71SBrett Creeley // SPDX-License-Identifier: GPL-2.0
263f77a71SBrett Creeley /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
363f77a71SBrett Creeley
463f77a71SBrett Creeley #include <linux/io.h>
563f77a71SBrett Creeley #include <linux/types.h>
6bb500dbeSBrett Creeley #include <linux/delay.h>
763f77a71SBrett Creeley
863f77a71SBrett Creeley #include <linux/pds/pds_common.h>
963f77a71SBrett Creeley #include <linux/pds/pds_core_if.h>
1063f77a71SBrett Creeley #include <linux/pds/pds_adminq.h>
1163f77a71SBrett Creeley
1263f77a71SBrett Creeley #include "vfio_dev.h"
1363f77a71SBrett Creeley #include "cmds.h"
1463f77a71SBrett Creeley
15bb500dbeSBrett Creeley #define SUSPEND_TIMEOUT_S 5
16bb500dbeSBrett Creeley #define SUSPEND_CHECK_INTERVAL_MS 1
17bb500dbeSBrett Creeley
pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device * pds_vfio,union pds_core_adminq_cmd * req,union pds_core_adminq_comp * resp,bool fast_poll)18bb500dbeSBrett Creeley static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio,
19bb500dbeSBrett Creeley union pds_core_adminq_cmd *req,
20bb500dbeSBrett Creeley union pds_core_adminq_comp *resp,
21bb500dbeSBrett Creeley bool fast_poll)
22bb500dbeSBrett Creeley {
23bb500dbeSBrett Creeley struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
24bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {};
25bb500dbeSBrett Creeley struct pdsc *pdsc;
26bb500dbeSBrett Creeley int err;
27bb500dbeSBrett Creeley
28bb500dbeSBrett Creeley /* Wrap the client request */
29bb500dbeSBrett Creeley cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD;
30bb500dbeSBrett Creeley cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id);
31bb500dbeSBrett Creeley memcpy(cmd.client_request.client_cmd, req,
32bb500dbeSBrett Creeley sizeof(cmd.client_request.client_cmd));
33bb500dbeSBrett Creeley
34bb500dbeSBrett Creeley pdsc = pdsc_get_pf_struct(pdev);
35bb500dbeSBrett Creeley if (IS_ERR(pdsc))
36bb500dbeSBrett Creeley return PTR_ERR(pdsc);
37bb500dbeSBrett Creeley
38bb500dbeSBrett Creeley err = pdsc_adminq_post(pdsc, &cmd, resp, fast_poll);
39bb500dbeSBrett Creeley if (err && err != -EAGAIN)
40bb500dbeSBrett Creeley dev_err(pds_vfio_to_dev(pds_vfio),
41bb500dbeSBrett Creeley "client admin cmd failed: %pe\n", ERR_PTR(err));
42bb500dbeSBrett Creeley
43bb500dbeSBrett Creeley return err;
44bb500dbeSBrett Creeley }
45bb500dbeSBrett Creeley
pds_vfio_register_client_cmd(struct pds_vfio_pci_device * pds_vfio)4663f77a71SBrett Creeley int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio)
4763f77a71SBrett Creeley {
4863f77a71SBrett Creeley struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
4963f77a71SBrett Creeley char devname[PDS_DEVNAME_LEN];
5063f77a71SBrett Creeley struct pdsc *pdsc;
5163f77a71SBrett Creeley int ci;
5263f77a71SBrett Creeley
5363f77a71SBrett Creeley snprintf(devname, sizeof(devname), "%s.%d-%u", PDS_VFIO_LM_DEV_NAME,
5463f77a71SBrett Creeley pci_domain_nr(pdev->bus),
5563f77a71SBrett Creeley PCI_DEVID(pdev->bus->number, pdev->devfn));
5663f77a71SBrett Creeley
5763f77a71SBrett Creeley pdsc = pdsc_get_pf_struct(pdev);
5863f77a71SBrett Creeley if (IS_ERR(pdsc))
5963f77a71SBrett Creeley return PTR_ERR(pdsc);
6063f77a71SBrett Creeley
6163f77a71SBrett Creeley ci = pds_client_register(pdsc, devname);
6263f77a71SBrett Creeley if (ci < 0)
6363f77a71SBrett Creeley return ci;
6463f77a71SBrett Creeley
6563f77a71SBrett Creeley pds_vfio->client_id = ci;
6663f77a71SBrett Creeley
6763f77a71SBrett Creeley return 0;
6863f77a71SBrett Creeley }
6963f77a71SBrett Creeley
pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device * pds_vfio)7063f77a71SBrett Creeley void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio)
7163f77a71SBrett Creeley {
7263f77a71SBrett Creeley struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
7363f77a71SBrett Creeley struct pdsc *pdsc;
7463f77a71SBrett Creeley int err;
7563f77a71SBrett Creeley
7663f77a71SBrett Creeley pdsc = pdsc_get_pf_struct(pdev);
7763f77a71SBrett Creeley if (IS_ERR(pdsc))
7863f77a71SBrett Creeley return;
7963f77a71SBrett Creeley
8063f77a71SBrett Creeley err = pds_client_unregister(pdsc, pds_vfio->client_id);
8163f77a71SBrett Creeley if (err)
8263f77a71SBrett Creeley dev_err(&pdev->dev, "unregister from DSC failed: %pe\n",
8363f77a71SBrett Creeley ERR_PTR(err));
8463f77a71SBrett Creeley
8563f77a71SBrett Creeley pds_vfio->client_id = 0;
8663f77a71SBrett Creeley }
87bb500dbeSBrett Creeley
88bb500dbeSBrett Creeley static int
pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)89*642265e2SBrett Creeley pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
90bb500dbeSBrett Creeley {
91bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
92bb500dbeSBrett Creeley .lm_suspend_status = {
93bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_SUSPEND_STATUS,
94bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
95*642265e2SBrett Creeley .type = type,
96bb500dbeSBrett Creeley },
97bb500dbeSBrett Creeley };
98bb500dbeSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
99bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
100bb500dbeSBrett Creeley unsigned long time_limit;
101bb500dbeSBrett Creeley unsigned long time_start;
102bb500dbeSBrett Creeley unsigned long time_done;
103bb500dbeSBrett Creeley int err;
104bb500dbeSBrett Creeley
105bb500dbeSBrett Creeley time_start = jiffies;
106bb500dbeSBrett Creeley time_limit = time_start + HZ * SUSPEND_TIMEOUT_S;
107bb500dbeSBrett Creeley do {
108bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
109bb500dbeSBrett Creeley if (err != -EAGAIN)
110bb500dbeSBrett Creeley break;
111bb500dbeSBrett Creeley
112bb500dbeSBrett Creeley msleep(SUSPEND_CHECK_INTERVAL_MS);
113bb500dbeSBrett Creeley } while (time_before(jiffies, time_limit));
114bb500dbeSBrett Creeley
115bb500dbeSBrett Creeley time_done = jiffies;
116bb500dbeSBrett Creeley dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n", __func__,
117bb500dbeSBrett Creeley pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start));
118bb500dbeSBrett Creeley
119bb500dbeSBrett Creeley /* Check the results */
120bb500dbeSBrett Creeley if (time_after_eq(time_done, time_limit)) {
121bb500dbeSBrett Creeley dev_err(dev, "%s: vf%u: Suspend comp timeout\n", __func__,
122bb500dbeSBrett Creeley pds_vfio->vf_id);
123bb500dbeSBrett Creeley err = -ETIMEDOUT;
124bb500dbeSBrett Creeley }
125bb500dbeSBrett Creeley
126bb500dbeSBrett Creeley return err;
127bb500dbeSBrett Creeley }
128bb500dbeSBrett Creeley
pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)129bb500dbeSBrett Creeley int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
130bb500dbeSBrett Creeley {
131bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
132bb500dbeSBrett Creeley .lm_suspend = {
133bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_SUSPEND,
134bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
135bb500dbeSBrett Creeley .type = type,
136bb500dbeSBrett Creeley },
137bb500dbeSBrett Creeley };
138bb500dbeSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
139bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
140bb500dbeSBrett Creeley int err;
141bb500dbeSBrett Creeley
142bb500dbeSBrett Creeley dev_dbg(dev, "vf%u: Suspend device\n", pds_vfio->vf_id);
143bb500dbeSBrett Creeley
144bb500dbeSBrett Creeley /*
145bb500dbeSBrett Creeley * The initial suspend request to the firmware starts the device suspend
146bb500dbeSBrett Creeley * operation and the firmware returns success if it's started
147bb500dbeSBrett Creeley * successfully.
148bb500dbeSBrett Creeley */
149bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
150bb500dbeSBrett Creeley if (err) {
151bb500dbeSBrett Creeley dev_err(dev, "vf%u: Suspend failed: %pe\n", pds_vfio->vf_id,
152bb500dbeSBrett Creeley ERR_PTR(err));
153bb500dbeSBrett Creeley return err;
154bb500dbeSBrett Creeley }
155bb500dbeSBrett Creeley
156bb500dbeSBrett Creeley /*
157bb500dbeSBrett Creeley * The subsequent suspend status request(s) check if the firmware has
158bb500dbeSBrett Creeley * completed the device suspend process.
159bb500dbeSBrett Creeley */
160*642265e2SBrett Creeley return pds_vfio_suspend_wait_device_cmd(pds_vfio, type);
161bb500dbeSBrett Creeley }
162bb500dbeSBrett Creeley
pds_vfio_resume_device_cmd(struct pds_vfio_pci_device * pds_vfio,u8 type)163bb500dbeSBrett Creeley int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
164bb500dbeSBrett Creeley {
165bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
166bb500dbeSBrett Creeley .lm_resume = {
167bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_RESUME,
168bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
169bb500dbeSBrett Creeley .type = type,
170bb500dbeSBrett Creeley },
171bb500dbeSBrett Creeley };
172bb500dbeSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
173bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
174bb500dbeSBrett Creeley
175bb500dbeSBrett Creeley dev_dbg(dev, "vf%u: Resume device\n", pds_vfio->vf_id);
176bb500dbeSBrett Creeley
177bb500dbeSBrett Creeley return pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
178bb500dbeSBrett Creeley }
179bb500dbeSBrett Creeley
pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device * pds_vfio,u64 * size)180bb500dbeSBrett Creeley int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size)
181bb500dbeSBrett Creeley {
182bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
183bb500dbeSBrett Creeley .lm_state_size = {
184bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_STATE_SIZE,
185bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
186bb500dbeSBrett Creeley },
187bb500dbeSBrett Creeley };
188bb500dbeSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
189bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
190bb500dbeSBrett Creeley int err;
191bb500dbeSBrett Creeley
192bb500dbeSBrett Creeley dev_dbg(dev, "vf%u: Get migration status\n", pds_vfio->vf_id);
193bb500dbeSBrett Creeley
194bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
195bb500dbeSBrett Creeley if (err)
196bb500dbeSBrett Creeley return err;
197bb500dbeSBrett Creeley
198bb500dbeSBrett Creeley *size = le64_to_cpu(comp.lm_state_size.size);
199bb500dbeSBrett Creeley return 0;
200bb500dbeSBrett Creeley }
201bb500dbeSBrett Creeley
pds_vfio_dma_map_lm_file(struct device * dev,enum dma_data_direction dir,struct pds_vfio_lm_file * lm_file)202bb500dbeSBrett Creeley static int pds_vfio_dma_map_lm_file(struct device *dev,
203bb500dbeSBrett Creeley enum dma_data_direction dir,
204bb500dbeSBrett Creeley struct pds_vfio_lm_file *lm_file)
205bb500dbeSBrett Creeley {
206bb500dbeSBrett Creeley struct pds_lm_sg_elem *sgl, *sge;
207bb500dbeSBrett Creeley struct scatterlist *sg;
208bb500dbeSBrett Creeley dma_addr_t sgl_addr;
209bb500dbeSBrett Creeley size_t sgl_size;
210bb500dbeSBrett Creeley int err;
211bb500dbeSBrett Creeley int i;
212bb500dbeSBrett Creeley
213bb500dbeSBrett Creeley if (!lm_file)
214bb500dbeSBrett Creeley return -EINVAL;
215bb500dbeSBrett Creeley
216bb500dbeSBrett Creeley /* dma map file pages */
217bb500dbeSBrett Creeley err = dma_map_sgtable(dev, &lm_file->sg_table, dir, 0);
218bb500dbeSBrett Creeley if (err)
219bb500dbeSBrett Creeley return err;
220bb500dbeSBrett Creeley
221bb500dbeSBrett Creeley lm_file->num_sge = lm_file->sg_table.nents;
222bb500dbeSBrett Creeley
223bb500dbeSBrett Creeley /* alloc sgl */
224bb500dbeSBrett Creeley sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem);
225bb500dbeSBrett Creeley sgl = kzalloc(sgl_size, GFP_KERNEL);
226bb500dbeSBrett Creeley if (!sgl) {
227bb500dbeSBrett Creeley err = -ENOMEM;
228bb500dbeSBrett Creeley goto out_unmap_sgtable;
229bb500dbeSBrett Creeley }
230bb500dbeSBrett Creeley
231bb500dbeSBrett Creeley /* fill sgl */
232bb500dbeSBrett Creeley sge = sgl;
233bb500dbeSBrett Creeley for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) {
234bb500dbeSBrett Creeley sge->addr = cpu_to_le64(sg_dma_address(sg));
235bb500dbeSBrett Creeley sge->len = cpu_to_le32(sg_dma_len(sg));
236bb500dbeSBrett Creeley dev_dbg(dev, "addr = %llx, len = %u\n", sge->addr, sge->len);
237bb500dbeSBrett Creeley sge++;
238bb500dbeSBrett Creeley }
239bb500dbeSBrett Creeley
240bb500dbeSBrett Creeley sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE);
241bb500dbeSBrett Creeley if (dma_mapping_error(dev, sgl_addr)) {
242bb500dbeSBrett Creeley err = -EIO;
243bb500dbeSBrett Creeley goto out_free_sgl;
244bb500dbeSBrett Creeley }
245bb500dbeSBrett Creeley
246bb500dbeSBrett Creeley lm_file->sgl = sgl;
247bb500dbeSBrett Creeley lm_file->sgl_addr = sgl_addr;
248bb500dbeSBrett Creeley
249bb500dbeSBrett Creeley return 0;
250bb500dbeSBrett Creeley
251bb500dbeSBrett Creeley out_free_sgl:
252bb500dbeSBrett Creeley kfree(sgl);
253bb500dbeSBrett Creeley out_unmap_sgtable:
254bb500dbeSBrett Creeley lm_file->num_sge = 0;
255bb500dbeSBrett Creeley dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
256bb500dbeSBrett Creeley return err;
257bb500dbeSBrett Creeley }
258bb500dbeSBrett Creeley
pds_vfio_dma_unmap_lm_file(struct device * dev,enum dma_data_direction dir,struct pds_vfio_lm_file * lm_file)259bb500dbeSBrett Creeley static void pds_vfio_dma_unmap_lm_file(struct device *dev,
260bb500dbeSBrett Creeley enum dma_data_direction dir,
261bb500dbeSBrett Creeley struct pds_vfio_lm_file *lm_file)
262bb500dbeSBrett Creeley {
263bb500dbeSBrett Creeley if (!lm_file)
264bb500dbeSBrett Creeley return;
265bb500dbeSBrett Creeley
266bb500dbeSBrett Creeley /* free sgl */
267bb500dbeSBrett Creeley if (lm_file->sgl) {
268bb500dbeSBrett Creeley dma_unmap_single(dev, lm_file->sgl_addr,
269bb500dbeSBrett Creeley lm_file->num_sge * sizeof(*lm_file->sgl),
270bb500dbeSBrett Creeley DMA_TO_DEVICE);
271bb500dbeSBrett Creeley kfree(lm_file->sgl);
272bb500dbeSBrett Creeley lm_file->sgl = NULL;
273bb500dbeSBrett Creeley lm_file->sgl_addr = DMA_MAPPING_ERROR;
274bb500dbeSBrett Creeley lm_file->num_sge = 0;
275bb500dbeSBrett Creeley }
276bb500dbeSBrett Creeley
277bb500dbeSBrett Creeley /* dma unmap file pages */
278bb500dbeSBrett Creeley dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
279bb500dbeSBrett Creeley }
280bb500dbeSBrett Creeley
pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device * pds_vfio)281bb500dbeSBrett Creeley int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
282bb500dbeSBrett Creeley {
283bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
284bb500dbeSBrett Creeley .lm_save = {
285bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_SAVE,
286bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
287bb500dbeSBrett Creeley },
288bb500dbeSBrett Creeley };
289bb500dbeSBrett Creeley struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
290bb500dbeSBrett Creeley struct device *pdsc_dev = &pci_physfn(pdev)->dev;
291bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
292bb500dbeSBrett Creeley struct pds_vfio_lm_file *lm_file;
293bb500dbeSBrett Creeley int err;
294bb500dbeSBrett Creeley
295bb500dbeSBrett Creeley dev_dbg(&pdev->dev, "vf%u: Get migration state\n", pds_vfio->vf_id);
296bb500dbeSBrett Creeley
297bb500dbeSBrett Creeley lm_file = pds_vfio->save_file;
298bb500dbeSBrett Creeley
299bb500dbeSBrett Creeley err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
300bb500dbeSBrett Creeley if (err) {
301bb500dbeSBrett Creeley dev_err(&pdev->dev, "failed to map save migration file: %pe\n",
302bb500dbeSBrett Creeley ERR_PTR(err));
303bb500dbeSBrett Creeley return err;
304bb500dbeSBrett Creeley }
305bb500dbeSBrett Creeley
306bb500dbeSBrett Creeley cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
307bb500dbeSBrett Creeley cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge);
308bb500dbeSBrett Creeley
309bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
310bb500dbeSBrett Creeley if (err)
311bb500dbeSBrett Creeley dev_err(&pdev->dev, "failed to get migration state: %pe\n",
312bb500dbeSBrett Creeley ERR_PTR(err));
313bb500dbeSBrett Creeley
314bb500dbeSBrett Creeley pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
315bb500dbeSBrett Creeley
316bb500dbeSBrett Creeley return err;
317bb500dbeSBrett Creeley }
318bb500dbeSBrett Creeley
pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device * pds_vfio)319bb500dbeSBrett Creeley int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
320bb500dbeSBrett Creeley {
321bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
322bb500dbeSBrett Creeley .lm_restore = {
323bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_RESTORE,
324bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
325bb500dbeSBrett Creeley },
326bb500dbeSBrett Creeley };
327bb500dbeSBrett Creeley struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
328bb500dbeSBrett Creeley struct device *pdsc_dev = &pci_physfn(pdev)->dev;
329bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
330bb500dbeSBrett Creeley struct pds_vfio_lm_file *lm_file;
331bb500dbeSBrett Creeley int err;
332bb500dbeSBrett Creeley
333bb500dbeSBrett Creeley dev_dbg(&pdev->dev, "vf%u: Set migration state\n", pds_vfio->vf_id);
334bb500dbeSBrett Creeley
335bb500dbeSBrett Creeley lm_file = pds_vfio->restore_file;
336bb500dbeSBrett Creeley
337bb500dbeSBrett Creeley err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
338bb500dbeSBrett Creeley if (err) {
339bb500dbeSBrett Creeley dev_err(&pdev->dev,
340bb500dbeSBrett Creeley "failed to map restore migration file: %pe\n",
341bb500dbeSBrett Creeley ERR_PTR(err));
342bb500dbeSBrett Creeley return err;
343bb500dbeSBrett Creeley }
344bb500dbeSBrett Creeley
345bb500dbeSBrett Creeley cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
346bb500dbeSBrett Creeley cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge);
347bb500dbeSBrett Creeley
348bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
349bb500dbeSBrett Creeley if (err)
350bb500dbeSBrett Creeley dev_err(&pdev->dev, "failed to set migration state: %pe\n",
351bb500dbeSBrett Creeley ERR_PTR(err));
352bb500dbeSBrett Creeley
353bb500dbeSBrett Creeley pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
354bb500dbeSBrett Creeley
355bb500dbeSBrett Creeley return err;
356bb500dbeSBrett Creeley }
357bb500dbeSBrett Creeley
pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device * pds_vfio,enum pds_lm_host_vf_status vf_status)358bb500dbeSBrett Creeley void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio,
359bb500dbeSBrett Creeley enum pds_lm_host_vf_status vf_status)
360bb500dbeSBrett Creeley {
361bb500dbeSBrett Creeley union pds_core_adminq_cmd cmd = {
362bb500dbeSBrett Creeley .lm_host_vf_status = {
363bb500dbeSBrett Creeley .opcode = PDS_LM_CMD_HOST_VF_STATUS,
364bb500dbeSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
365bb500dbeSBrett Creeley .status = vf_status,
366bb500dbeSBrett Creeley },
367bb500dbeSBrett Creeley };
368bb500dbeSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
369bb500dbeSBrett Creeley union pds_core_adminq_comp comp = {};
370bb500dbeSBrett Creeley int err;
371bb500dbeSBrett Creeley
372bb500dbeSBrett Creeley dev_dbg(dev, "vf%u: Set host VF LM status: %u", pds_vfio->vf_id,
373bb500dbeSBrett Creeley vf_status);
374bb500dbeSBrett Creeley if (vf_status != PDS_LM_STA_IN_PROGRESS &&
375bb500dbeSBrett Creeley vf_status != PDS_LM_STA_NONE) {
376bb500dbeSBrett Creeley dev_warn(dev, "Invalid host VF migration status, %d\n",
377bb500dbeSBrett Creeley vf_status);
378bb500dbeSBrett Creeley return;
379bb500dbeSBrett Creeley }
380bb500dbeSBrett Creeley
381bb500dbeSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
382bb500dbeSBrett Creeley if (err)
383bb500dbeSBrett Creeley dev_warn(dev, "failed to send host VF migration status: %pe\n",
384bb500dbeSBrett Creeley ERR_PTR(err));
385bb500dbeSBrett Creeley }
386f232836aSBrett Creeley
pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device * pds_vfio,u64 regions_dma,u8 * max_regions,u8 * num_regions)387f232836aSBrett Creeley int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio,
388f232836aSBrett Creeley u64 regions_dma, u8 *max_regions, u8 *num_regions)
389f232836aSBrett Creeley {
390f232836aSBrett Creeley union pds_core_adminq_cmd cmd = {
391f232836aSBrett Creeley .lm_dirty_status = {
392f232836aSBrett Creeley .opcode = PDS_LM_CMD_DIRTY_STATUS,
393f232836aSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
394f232836aSBrett Creeley },
395f232836aSBrett Creeley };
396f232836aSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
397f232836aSBrett Creeley union pds_core_adminq_comp comp = {};
398f232836aSBrett Creeley int err;
399f232836aSBrett Creeley
400f232836aSBrett Creeley dev_dbg(dev, "vf%u: Dirty status\n", pds_vfio->vf_id);
401f232836aSBrett Creeley
402f232836aSBrett Creeley cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma);
403f232836aSBrett Creeley cmd.lm_dirty_status.max_regions = *max_regions;
404f232836aSBrett Creeley
405f232836aSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
406f232836aSBrett Creeley if (err) {
407f232836aSBrett Creeley dev_err(dev, "failed to get dirty status: %pe\n", ERR_PTR(err));
408f232836aSBrett Creeley return err;
409f232836aSBrett Creeley }
410f232836aSBrett Creeley
411f232836aSBrett Creeley /* only support seq_ack approach for now */
412f232836aSBrett Creeley if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) &
413f232836aSBrett Creeley BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) {
414f232836aSBrett Creeley dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n");
415f232836aSBrett Creeley return -EOPNOTSUPP;
416f232836aSBrett Creeley }
417f232836aSBrett Creeley
418f232836aSBrett Creeley *num_regions = comp.lm_dirty_status.num_regions;
419f232836aSBrett Creeley *max_regions = comp.lm_dirty_status.max_regions;
420f232836aSBrett Creeley
421f232836aSBrett Creeley dev_dbg(dev,
422f232836aSBrett Creeley "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n",
423f232836aSBrett Creeley *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK");
424f232836aSBrett Creeley
425f232836aSBrett Creeley return 0;
426f232836aSBrett Creeley }
427f232836aSBrett Creeley
pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device * pds_vfio,u64 regions_dma,u8 num_regions)428f232836aSBrett Creeley int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio,
429f232836aSBrett Creeley u64 regions_dma, u8 num_regions)
430f232836aSBrett Creeley {
431f232836aSBrett Creeley union pds_core_adminq_cmd cmd = {
432f232836aSBrett Creeley .lm_dirty_enable = {
433f232836aSBrett Creeley .opcode = PDS_LM_CMD_DIRTY_ENABLE,
434f232836aSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
435f232836aSBrett Creeley .regions_dma = cpu_to_le64(regions_dma),
436f232836aSBrett Creeley .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK,
437f232836aSBrett Creeley .num_regions = num_regions,
438f232836aSBrett Creeley },
439f232836aSBrett Creeley };
440f232836aSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
441f232836aSBrett Creeley union pds_core_adminq_comp comp = {};
442f232836aSBrett Creeley int err;
443f232836aSBrett Creeley
444f232836aSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
445f232836aSBrett Creeley if (err) {
446f232836aSBrett Creeley dev_err(dev, "failed dirty tracking enable: %pe\n",
447f232836aSBrett Creeley ERR_PTR(err));
448f232836aSBrett Creeley return err;
449f232836aSBrett Creeley }
450f232836aSBrett Creeley
451f232836aSBrett Creeley return 0;
452f232836aSBrett Creeley }
453f232836aSBrett Creeley
pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device * pds_vfio)454f232836aSBrett Creeley int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio)
455f232836aSBrett Creeley {
456f232836aSBrett Creeley union pds_core_adminq_cmd cmd = {
457f232836aSBrett Creeley .lm_dirty_disable = {
458f232836aSBrett Creeley .opcode = PDS_LM_CMD_DIRTY_DISABLE,
459f232836aSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
460f232836aSBrett Creeley },
461f232836aSBrett Creeley };
462f232836aSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
463f232836aSBrett Creeley union pds_core_adminq_comp comp = {};
464f232836aSBrett Creeley int err;
465f232836aSBrett Creeley
466f232836aSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
467f232836aSBrett Creeley if (err || comp.lm_dirty_status.num_regions != 0) {
468f232836aSBrett Creeley /* in case num_regions is still non-zero after disable */
469f232836aSBrett Creeley err = err ? err : -EIO;
470f232836aSBrett Creeley dev_err(dev,
471f232836aSBrett Creeley "failed dirty tracking disable: %pe, num_regions %d\n",
472f232836aSBrett Creeley ERR_PTR(err), comp.lm_dirty_status.num_regions);
473f232836aSBrett Creeley return err;
474f232836aSBrett Creeley }
475f232836aSBrett Creeley
476f232836aSBrett Creeley return 0;
477f232836aSBrett Creeley }
478f232836aSBrett Creeley
pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device * pds_vfio,u64 sgl_dma,u16 num_sge,u32 offset,u32 total_len,bool read_seq)479f232836aSBrett Creeley int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio,
480f232836aSBrett Creeley u64 sgl_dma, u16 num_sge, u32 offset,
481f232836aSBrett Creeley u32 total_len, bool read_seq)
482f232836aSBrett Creeley {
483f232836aSBrett Creeley const char *cmd_type_str = read_seq ? "read_seq" : "write_ack";
484f232836aSBrett Creeley union pds_core_adminq_cmd cmd = {
485f232836aSBrett Creeley .lm_dirty_seq_ack = {
486f232836aSBrett Creeley .vf_id = cpu_to_le16(pds_vfio->vf_id),
487f232836aSBrett Creeley .len_bytes = cpu_to_le32(total_len),
488f232836aSBrett Creeley .off_bytes = cpu_to_le32(offset),
489f232836aSBrett Creeley .sgl_addr = cpu_to_le64(sgl_dma),
490f232836aSBrett Creeley .num_sge = cpu_to_le16(num_sge),
491f232836aSBrett Creeley },
492f232836aSBrett Creeley };
493f232836aSBrett Creeley struct device *dev = pds_vfio_to_dev(pds_vfio);
494f232836aSBrett Creeley union pds_core_adminq_comp comp = {};
495f232836aSBrett Creeley int err;
496f232836aSBrett Creeley
497f232836aSBrett Creeley if (read_seq)
498f232836aSBrett Creeley cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ;
499f232836aSBrett Creeley else
500f232836aSBrett Creeley cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK;
501f232836aSBrett Creeley
502f232836aSBrett Creeley err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
503f232836aSBrett Creeley if (err) {
504f232836aSBrett Creeley dev_err(dev, "failed cmd Page Tracking %s: %pe\n", cmd_type_str,
505f232836aSBrett Creeley ERR_PTR(err));
506f232836aSBrett Creeley return err;
507f232836aSBrett Creeley }
508f232836aSBrett Creeley
509f232836aSBrett Creeley return 0;
510f232836aSBrett Creeley }
511